diff options
Diffstat (limited to 'pkgtools')
70 files changed, 7517 insertions, 4798 deletions
diff --git a/pkgtools/pkglint/Makefile b/pkgtools/pkglint/Makefile index 48832998fec..9f86db77413 100644 --- a/pkgtools/pkglint/Makefile +++ b/pkgtools/pkglint/Makefile @@ -1,10 +1,10 @@ -# $NetBSD: Makefile,v 1.474 2015/12/30 04:16:56 dholland Exp $ +# $NetBSD: Makefile,v 1.475 2016/01/12 01:02:48 rillig Exp $ -PKGNAME= pkglint-5.2.2.2 +PKGNAME= pkglint-5.3 DISTFILES= # none CATEGORIES= pkgtools -OWNER= wiz@NetBSD.org +OWNER= rillig@NetBSD.org HOMEPAGE= http://www.NetBSD.org/docs/pkgsrc/ COMMENT= Verifier for NetBSD packages LICENSE= 2-clause-bsd diff --git a/pkgtools/pkglint/files/buildlink3.go b/pkgtools/pkglint/files/buildlink3.go index b68dd697c93..7efef6a43a6 100644 --- a/pkgtools/pkglint/files/buildlink3.go +++ b/pkgtools/pkglint/files/buildlink3.go @@ -1,161 +1,202 @@ package main import ( - "regexp" "strings" ) -func checklinesBuildlink3Mk(lines []*Line) { - defer tracecall("checklinesBuildlink3Mk", lines[0].fname)() +func ChecklinesBuildlink3Mk(mklines *MkLines) { + if G.opts.DebugTrace { + defer tracecall1(mklines.lines[0].Fname)() + } - ParselinesMk(lines) - ChecklinesMk(lines) + mklines.Check() - exp := NewExpecter(lines) + exp := NewExpecter(mklines.lines) - for exp.advanceIfMatches(`^#`) != nil { - if hasPrefix(exp.previousLine().text, "# XXX") { - exp.previousLine().notef("Please read this comment and remove it if appropriate.") + for exp.AdvanceIfPrefix("#") { + line := exp.PreviousLine() + // See pkgtools/createbuildlink/files/createbuildlink + if hasPrefix(line.Text, "# XXX This file was created automatically") { + line.Error0("This comment indicates unfinished work (url2pkg).") } } - exp.expectEmptyLine() + exp.ExpectEmptyLine() - if exp.advanceIfMatches(`^BUILDLINK_DEPMETHOD\.(\S+)\?=.*$`) != nil { - exp.previousLine().warnf("This line belongs inside the .ifdef block.") - for exp.advanceIfMatches(`^$`) != nil { + if exp.AdvanceIfMatches(`^BUILDLINK_DEPMETHOD\.(\S+)\?=.*$`) { + exp.PreviousLine().Warn0("This line belongs inside the .ifdef block.") + for exp.AdvanceIfEquals("") { } } - pkgbaseLine, pkgbase := (*Line)(nil), "" - pkgidLine, pkgid := exp.currentLine(), "" - abiLine, abiPkg, abiVersion := (*Line)(nil), "", "" - apiLine, apiPkg, apiVersion := (*Line)(nil), "", "" + pkgbaseLine, pkgbase := exp.CurrentLine(), "" + var abiLine, apiLine *Line + var abi, api *DependencyPattern // First paragraph: Introduction of the package identifier - if m := exp.advanceIfMatches(`^BUILDLINK_TREE\+=\s*(\S+)$`); m != nil { - pkgid = m[1] - } else { - exp.currentLine().warnf("Expected a BUILDLINK_TREE line.") + if !exp.AdvanceIfMatches(`^BUILDLINK_TREE\+=\s*(\S+)$`) { + exp.CurrentLine().Warn0("Expected a BUILDLINK_TREE line.") return } - exp.expectEmptyLine() + pkgbase = exp.m[1] + if containsVarRef(pkgbase) { + warned := false + for _, pair := range []struct{ varuse, simple string }{ + {"${PYPKGPREFIX}", "py"}, + {"${RUBY_BASE}", "ruby"}, + {"${RUBY_PKGPREFIX}", "ruby"}, + {"${PHP_PKG_PREFIX}", "php"}, + } { + if contains(pkgbase, pair.varuse) && !pkgbaseLine.AutofixReplace(pair.varuse, pair.simple) { + pkgbaseLine.Warn2("Please use %q instead of %q.", pair.simple, pair.varuse) + warned = true + } + } + if !warned { + if m, varuse := match1(pkgbase, `(\$\{\w+\})`); m { + pkgbaseLine.Warn1("Please replace %q with a simple string.", varuse) + warned = true + } + } + if warned { + Explain( + "Having variable package names in the BUILDLINK_TREE is not", + "necessary, since other packages depend on this package only for", + "a specific version of Python, Ruby or PHP. Since these", + "package identifiers are only used at build time, they should", + "not include the specific version of the language interpreter.") + } + } + + exp.ExpectEmptyLine() // Second paragraph: multiple inclusion protection and introduction // of the uppercase package identifier. - if m := exp.advanceIfMatches(`^\.if !defined\((\S+)_BUILDLINK3_MK\)$`); m != nil { - pkgbaseLine = exp.previousLine() - pkgbase = m[1] - } else { + if !exp.AdvanceIfMatches(`^\.if !defined\((\S+)_BUILDLINK3_MK\)$`) { return } - if !exp.expectText(pkgbase + "_BUILDLINK3_MK:=") { - exp.currentLine().errorf("Expected the multiple-inclusion guard.") + pkgupperLine, pkgupper := exp.PreviousLine(), exp.m[1] + + if !exp.ExpectText(pkgupper + "_BUILDLINK3_MK:=") { return } - exp.expectEmptyLine() + exp.ExpectEmptyLine() - ucPkgid := strings.ToUpper(strings.Replace(pkgid, "-", "_", -1)) - if ucPkgid != pkgbase { - pkgbaseLine.errorf("Package name mismatch between %q ...", pkgbase) - pkgidLine.errorf("... and %q.", pkgid) + // See pkgtools/createbuildlink/files/createbuildlink, keyword PKGUPPER + ucPkgbase := strings.ToUpper(strings.Replace(pkgbase, "-", "_", -1)) + if ucPkgbase != pkgupper && !containsVarRef(pkgbase) { + pkgupperLine.Error2("Package name mismatch between multiple-inclusion guard %q (expected %q) ...", pkgupper, ucPkgbase) + pkgbaseLine.Error1("... and package name %q.", pkgbase) } - if G.pkgContext != nil { - if mkbase := G.pkgContext.effectivePkgbase; mkbase != "" && mkbase != pkgid { - pkgidLine.errorf("Package name mismatch between %q ...", pkgid) - G.pkgContext.effectivePkgnameLine.errorf("... and %q.", mkbase) + if G.Pkg != nil { + if mkbase := G.Pkg.EffectivePkgbase; mkbase != "" && mkbase != pkgbase { + pkgbaseLine.Error1("Package name mismatch between %q in this file ...", pkgbase) + G.Pkg.EffectivePkgnameLine.Line.Error1("... and %q from the package Makefile.", mkbase) } } // Third paragraph: Package information. indentLevel := 1 // The first .if is from the second paragraph. for { - if exp.eof() { - exp.currentLine().warnf("Expected .endif") + if exp.EOF() { + exp.CurrentLine().Warn0("Expected .endif") return } - line := exp.currentLine() + line := exp.CurrentLine() + mkline := mklines.mklines[exp.index] - if m := exp.advanceIfMatches(reVarassign); m != nil { - varname, value := m[1], m[3] + if mkline.IsVarassign() { + exp.Advance() + varname, value := mkline.Varname(), mkline.Value() doCheck := false - if varname == "BUILDLINK_ABI_DEPENDS."+pkgid { + const ( + reDependencyCmp = `^((?:\$\{[\w_]+\}|[\w_\.+]|-[^\d])+)[<>]=?(\d[^-*?\[\]]*)$` + reDependencyWildcard = `^(-(?:\[0-9\]\*|\d[^-]*)$` + ) + + if varname == "BUILDLINK_ABI_DEPENDS."+pkgbase { abiLine = line - if m, p, v := match2(value, reDependencyCmp); m { - abiPkg, abiVersion = p, v - } else if m, p := match1(value, reDependencyWildcard); m { - abiPkg, abiVersion = p, "" - } else { - _ = G.opts.DebugUnchecked && line.debugf("Unchecked dependency pattern %q.", value) + parser := NewParser(value) + if dp := parser.Dependency(); dp != nil && parser.EOF() { + abi = dp } doCheck = true } - if varname == "BUILDLINK_API_DEPENDS."+pkgid { + if varname == "BUILDLINK_API_DEPENDS."+pkgbase { apiLine = line - if m, p, v := match2(value, reDependencyCmp); m { - apiPkg, apiVersion = p, v - } else if m, p := match1(value, reDependencyWildcard); m { - apiPkg, apiVersion = p, "" - } else { - _ = G.opts.DebugUnchecked && line.debugf("Unchecked dependency pattern %q.", value) + parser := NewParser(value) + if dp := parser.Dependency(); dp != nil && parser.EOF() { + api = dp } doCheck = true } - if doCheck && abiPkg != "" && apiPkg != "" && abiPkg != apiPkg { - abiLine.warnf("Package name mismatch between %q ...", abiPkg) - apiLine.warnf("... and %q.", apiPkg) + if doCheck && abi != nil && api != nil && abi.pkgbase != api.pkgbase && !hasPrefix(api.pkgbase, "{") { + abiLine.Warn1("Package name mismatch between ABI %q ...", abi.pkgbase) + apiLine.Warn1("... and API %q.", api.pkgbase) } - if doCheck && abiVersion != "" && apiVersion != "" && pkgverCmp(abiVersion, apiVersion) < 0 { - abiLine.warnf("ABI version (%s) should be at least ...", abiVersion) - apiLine.warnf("... API version (%s).", apiVersion) + if doCheck { + if abi != nil && abi.lower != "" && !containsVarRef(abi.lower) { + if api != nil && api.lower != "" && !containsVarRef(api.lower) { + if pkgverCmp(abi.lower, api.lower) < 0 { + abiLine.Warn1("ABI version %q should be at least ...", abi.lower) + apiLine.Warn1("... API version %q.", api.lower) + } + } + } } - if m, varparam := match1(varname, `^BUILDLINK_[\w_]+\.(.*)$`); m { - if varparam != pkgid { - line.warnf("Only buildlink variables for %q, not %q may be set in this file.", pkgid, varparam) + if varparam := mkline.Varparam(); varparam != "" && varparam != pkgbase { + if hasPrefix(varname, "BUILDLINK_") && mkline.Varcanon() != "BUILDLINK_API_DEPENDS.*" { + line.Warn2("Only buildlink variables for %q, not %q may be set in this file.", pkgbase, varparam) } } if varname == "pkgbase" { - exp.advanceIfMatches(`^\.\s*include "../../mk/pkg-build-options\.mk"$`) + exp.AdvanceIfMatches(`^\.\s*include "../../mk/pkg-build-options\.mk"$`) } - } else if exp.advanceIfMatches(`^(?:#.*)?$`) != nil { + } else if exp.AdvanceIfEquals("") || exp.AdvanceIfPrefix("#") { // Comments and empty lines are fine here. - } else if exp.advanceIfMatches(`^\.\s*include "\.\./\.\./([^/]+/[^/]+)/buildlink3\.mk"$`) != nil || - exp.advanceIfMatches(`^\.\s*include "\.\./\.\./mk/(\S+)\.buildlink3\.mk"$`) != nil { + } else if exp.AdvanceIfMatches(`^\.\s*include "\.\./\.\./([^/]+/[^/]+)/buildlink3\.mk"$`) || + exp.AdvanceIfMatches(`^\.\s*include "\.\./\.\./mk/(\S+)\.buildlink3\.mk"$`) { // TODO: Maybe check dependency lines. - } else if exp.advanceIfMatches(`^\.if\s`) != nil { + } else if exp.AdvanceIfMatches(`^\.if\s`) { indentLevel++ - } else if exp.advanceIfMatches(`^\.endif.*$`) != nil { + } else if exp.AdvanceIfMatches(`^\.endif.*$`) { indentLevel-- if indentLevel == 0 { break } } else { - _ = G.opts.DebugUnchecked && exp.currentLine().warnf("Unchecked line in third paragraph.") - exp.advance() + if G.opts.DebugUnchecked { + exp.CurrentLine().Debugf("Unchecked line in third paragraph.") + } + exp.Advance() } } if apiLine == nil { - exp.currentLine().warnf("Definition of BUILDLINK_API_DEPENDS is missing.") + exp.CurrentLine().Warn0("Definition of BUILDLINK_API_DEPENDS is missing.") } - exp.expectEmptyLine() + exp.ExpectEmptyLine() // Fourth paragraph: Cleanup, corresponding to the first paragraph. - if exp.advanceIfMatches(`^BUILDLINK_TREE\+=\s*-`+regexp.QuoteMeta(pkgid)+`$`) == nil { - exp.currentLine().warnf("Expected BUILDLINK_TREE line.") + if !exp.ExpectText("BUILDLINK_TREE+=\t-" + pkgbase) { + return + } + + if !exp.EOF() { + exp.CurrentLine().Warn0("The file should end here.") } - if !exp.eof() { - exp.currentLine().warnf("The file should end here.") + if G.Pkg != nil { + G.Pkg.checklinesBuildlink3Inclusion(mklines) } - checklinesBuildlink3Inclusion(lines) + SaveAutofixChanges(mklines.lines) } diff --git a/pkgtools/pkglint/files/buildlink3_test.go b/pkgtools/pkglint/files/buildlink3_test.go index a643f078752..9bceff8b036 100644 --- a/pkgtools/pkglint/files/buildlink3_test.go +++ b/pkgtools/pkglint/files/buildlink3_test.go @@ -6,9 +6,9 @@ import ( func (s *Suite) TestChecklinesBuildlink3(c *check.C) { G.globalData.InitVartypes() - lines := s.NewLines("buildlink3.mk", + mklines := s.NewMkLines("buildlink3.mk", "# $"+"NetBSD$", - "# XXX automatically generated", + "# XXX This file was created automatically using createbuildlink-@PKGVERSION@", "", "BUILDLINK_TREE+= Xbae", "", @@ -25,23 +25,24 @@ func (s *Suite) TestChecklinesBuildlink3(c *check.C) { "", "BUILDLINK_TREE+= -Xbae") - checklinesBuildlink3Mk(lines) + ChecklinesBuildlink3Mk(mklines) c.Check(s.Output(), equals, ""+ + "ERROR: buildlink3.mk:12: \"/x11/Xbae\" does not exist.\n"+ + "ERROR: buildlink3.mk:12: There is no package in \"x11/Xbae\".\n"+ "ERROR: buildlink3.mk:14: \"/mk/motif.buildlink3.mk\" does not exist.\n"+ - "NOTE: buildlink3.mk:2: Please read this comment and remove it if appropriate.\n") + "ERROR: buildlink3.mk:2: This comment indicates unfinished work (url2pkg).\n") } -// The mismatch reported here is a false positive. The mk/haskell.mk file -// takes care of constructing the correct PKGNAME, but pkglint doesn’t -// look at that file. +// Before version 5.3, pkglint wrongly warned here. +// The mk/haskell.mk file takes care of constructing the correct PKGNAME, +// but pkglint had not looked at that file. func (s *Suite) TestChecklinesBuildlink3_NameMismatch(c *check.C) { - s.UseCommandLine(c, "-Wall", "-Call") G.globalData.InitVartypes() - G.pkgContext = newPkgContext("x11/hs-X11") - G.pkgContext.effectivePkgbase = "X11" - G.pkgContext.effectivePkgnameLine = NewLine("Makefile", "3", "DISTNAME=\tX11-1.0", nil) - lines := s.NewLines("buildlink3.mk", + G.Pkg = NewPackage("x11/hs-X11") + G.Pkg.EffectivePkgbase = "X11" + G.Pkg.EffectivePkgnameLine = NewMkLine(NewLine("Makefile", 3, "DISTNAME=\tX11-1.0", nil)) + mklines := s.NewMkLines("buildlink3.mk", "# $"+"NetBSD$", "", "BUILDLINK_TREE+=\ths-X11", @@ -56,20 +57,111 @@ func (s *Suite) TestChecklinesBuildlink3_NameMismatch(c *check.C) { "", "BUILDLINK_TREE+=\t-hs-X11") - checklinesBuildlink3Mk(lines) + ChecklinesBuildlink3Mk(mklines) c.Check(s.Output(), equals, ""+ - "ERROR: buildlink3.mk:3: Package name mismatch between \"hs-X11\" ...\n"+ - "ERROR: Makefile:3: ... and \"X11\".\n") + "ERROR: buildlink3.mk:3: Package name mismatch between \"hs-X11\" in this file ...\n"+ + "ERROR: Makefile:3: ... and \"X11\" from the package Makefile.\n") } -func (s *Suite) TestChecklinesBuildlink3_NoBuildlinkTree(c *check.C) { - s.UseCommandLine(c, "-Wall", "-Call") +func (s *Suite) TestChecklinesBuildlink3_NameMismatchMultipleInclusion(c *check.C) { G.globalData.InitVartypes() - lines := s.NewLines("buildlink3.mk", + mklines := s.NewMkLines("buildlink3.mk", + "# $"+"NetBSD$", + "", + "BUILDLINK_TREE+=\tpkgbase1", + "", + ".if !defined(PKGBASE2_BUILDLINK3_MK)", + "PKGBASE2_BUILDLINK3_MK:=", + "", + ".endif", + "", + "BUILDLINK_TREE+=\t-pkgbase1") + + ChecklinesBuildlink3Mk(mklines) + + c.Check(s.Output(), equals, ""+ + "ERROR: buildlink3.mk:5: Package name mismatch between multiple-inclusion guard \"PKGBASE2\" (expected \"PKGBASE1\") ...\n"+ + "ERROR: buildlink3.mk:3: ... and package name \"pkgbase1\".\n"+ + "WARN: buildlink3.mk:9: Definition of BUILDLINK_API_DEPENDS is missing.\n") +} + +func (s *Suite) TestChecklinesBuildlink3_NameMismatchAbiApi(c *check.C) { + G.globalData.InitVartypes() + mklines := s.NewMkLines("buildlink3.mk", + "# $"+"NetBSD$", + "", + "BUILDLINK_TREE+=\ths-X11", + "", + ".if !defined(HS_X11_BUILDLINK3_MK)", + "HS_X11_BUILDLINK3_MK:=", + "", + "BUILDLINK_API_DEPENDS.hs-X11+=\ths-X11>=1.6.1", + "BUILDLINK_ABI_DEPENDS.hs-X11+=\ths-X12>=1.6.1.2nb2", + "", + ".endif\t# HS_X11_BUILDLINK3_MK", + "", + "BUILDLINK_TREE+=\t-hs-X11") + + ChecklinesBuildlink3Mk(mklines) + + c.Check(s.Output(), equals, ""+ + "WARN: buildlink3.mk:9: Package name mismatch between ABI \"hs-X12\" ...\n"+ + "WARN: buildlink3.mk:8: ... and API \"hs-X11\".\n") +} + +func (s *Suite) TestChecklinesBuildlink3_AbiApiVersions(c *check.C) { + G.globalData.InitVartypes() + mklines := s.NewMkLines("buildlink3.mk", + "# $"+"NetBSD$", + "", + "BUILDLINK_TREE+=\ths-X11", + "", + ".if !defined(HS_X11_BUILDLINK3_MK)", + "HS_X11_BUILDLINK3_MK:=", + "", + "BUILDLINK_API_DEPENDS.hs-X11+=\ths-X11>=1.6.1", + "BUILDLINK_ABI_DEPENDS.hs-X11+=\ths-X11>=1.6.0", + "", + ".endif\t# HS_X11_BUILDLINK3_MK", + "", + "BUILDLINK_TREE+=\t-hs-X11") + + ChecklinesBuildlink3Mk(mklines) + + c.Check(s.Output(), equals, ""+ + "WARN: buildlink3.mk:9: ABI version \"1.6.0\" should be at least ...\n"+ + "WARN: buildlink3.mk:8: ... API version \"1.6.1\".\n") +} + +func (s *Suite) TestChecklinesBuildlink3_NoBuildlinkTreeAtBeginning(c *check.C) { + G.globalData.InitVartypes() + mklines := s.NewMkLines("buildlink3.mk", + "# $"+"NetBSD$", + "", + ".if !defined(HS_X11_BUILDLINK3_MK)", + "HS_X11_BUILDLINK3_MK:=", + "", + "BUILDLINK_DEPMETHOD.hs-X11?=\tfull", + "BUILDLINK_API_DEPENDS.hs-X11+=\ths-X11>=1.6.1", + "BUILDLINK_ABI_DEPENDS.hs-X11+=\ths-X11>=1.6.1.2nb2", + "", + ".endif\t# HS_X11_BUILDLINK3_MK", + "", + "BUILDLINK_TREE+=\t-hs-X11") + + ChecklinesBuildlink3Mk(mklines) + + c.Check(s.Output(), equals, "WARN: buildlink3.mk:3: Expected a BUILDLINK_TREE line.\n") +} + +func (s *Suite) TestChecklinesBuildlink3_NoBuildlinkTreeAtEnd(c *check.C) { + G.globalData.InitVartypes() + mklines := s.NewMkLines("buildlink3.mk", "# $"+"NetBSD$", "", "BUILDLINK_DEPMETHOD.hs-X11?=\tfull", + "", "BUILDLINK_TREE+=\ths-X11", "", ".if !defined(HS_X11_BUILDLINK3_MK)", @@ -83,10 +175,112 @@ func (s *Suite) TestChecklinesBuildlink3_NoBuildlinkTree(c *check.C) { "# needless comment", "BUILDLINK_TREE+=\t-hs-X11") - checklinesBuildlink3Mk(lines) + ChecklinesBuildlink3Mk(mklines) c.Check(s.Output(), equals, ""+ "WARN: buildlink3.mk:3: This line belongs inside the .ifdef block.\n"+ - "WARN: buildlink3.mk:14: Expected BUILDLINK_TREE line.\n"+ - "WARN: buildlink3.mk:14: The file should end here.\n") + "WARN: buildlink3.mk:15: This line should contain the following text: BUILDLINK_TREE+=\t-hs-X11\n") +} + +func (s *Suite) TestChecklinesBuildlink3_MultipleInclusionWrong(c *check.C) { + G.globalData.InitVartypes() + mklines := s.NewMkLines("buildlink3.mk", + "# $"+"NetBSD$", + "", + "BUILDLINK_TREE+=\ths-X11", + "", + ".if !defined(HS_X11_BUILDLINK3_MK)", + "UNRELATED_BUILDLINK3_MK:=") + + ChecklinesBuildlink3Mk(mklines) + + c.Check(s.Output(), equals, ""+ + "WARN: buildlink3.mk:6: UNRELATED_BUILDLINK3_MK is defined but not used. Spelling mistake?\n"+ + "WARN: buildlink3.mk:6: This line should contain the following text: HS_X11_BUILDLINK3_MK:=\n") +} + +func (s *Suite) TestChecklinesBuildlink3_EndIfMissing(c *check.C) { + G.globalData.InitVartypes() + mklines := s.NewMkLines("buildlink3.mk", + "# $"+"NetBSD$", + "", + "BUILDLINK_TREE+=\tpkgbase1", + "", + ".if !defined(PKGBASE1_BUILDLINK3_MK)", + "PKGBASE1_BUILDLINK3_MK:=") + + ChecklinesBuildlink3Mk(mklines) + + c.Check(s.Output(), equals, "WARN: buildlink3.mk:EOF: Expected .endif\n") +} + +func (s *Suite) TestChecklinesBuildlink3_UnknownDependencyPatterns(c *check.C) { + G.globalData.InitVartypes() + mklines := s.NewMkLines("buildlink3.mk", + "# $"+"NetBSD$", + "", + "BUILDLINK_TREE+= hs-X11", + "", + ".if !defined(HS_X11_BUILDLINK3_MK)", + "HS_X11_BUILDLINK3_MK:=", + "", + "BUILDLINK_DEPMETHOD.hs-X11?=\tfull", + "BUILDLINK_API_DEPENDS.hs-X11+=\ths-X11!=1.6.1", + "BUILDLINK_ABI_DEPENDS.hs-X11+=\ths-X11!=1.6.1.2nb2", + "", + ".endif\t# HS_X11_BUILDLINK3_MK", + "", + "BUILDLINK_TREE+=\t-hs-X11") + + ChecklinesBuildlink3Mk(mklines) + + c.Check(s.Output(), equals, ""+ + "WARN: buildlink3.mk:9: Unknown dependency pattern \"hs-X11!=1.6.1\".\n"+ + "WARN: buildlink3.mk:10: Unknown dependency pattern \"hs-X11!=1.6.1.2nb2\".\n") +} + +func (s *Suite) TestChecklinesBuildlink3_PkgbaseWithVariable(c *check.C) { + G.globalData.InitVartypes() + mklines := s.NewMkLines("buildlink3.mk", + "# $"+"NetBSD$", + "", + "BUILDLINK_TREE+=\t${PYPKGPREFIX}-wxWidgets", + "", + ".if !defined(PY_WXWIDGETS_BUILDLINK3_MK)", + "PY_WXWIDGETS_BUILDLINK3_MK:=", + "", + "BUILDLINK_API_DEPENDS.${PYPKGPREFIX}-wxWidgets+=\t${PYPKGPREFIX}-wxWidgets>=2.6.1.0", + "BUILDLINK_ABI_DEPENDS.${PYPKGPREFIX}-wxWidgets+=\t${PYPKGPREFIX}-wxWidgets>=2.8.10.1nb26", + "", + ".endif", + "", + "BUILDLINK_TREE+=\t-${PYPKGPREFIX}-wxWidgets") + + ChecklinesBuildlink3Mk(mklines) + + c.Check(s.Output(), equals, "WARN: buildlink3.mk:3: Please use \"py\" instead of \"${PYPKGPREFIX}\".\n") +} + +func (s *Suite) TestChecklinesBuildlink3_PkgbaseWithUnknownVariable(c *check.C) { + G.globalData.InitVartypes() + mklines := s.NewMkLines("buildlink3.mk", + "# $"+"NetBSD$", + "", + "BUILDLINK_TREE+=\t${LICENSE}-wxWidgets", + "", + ".if !defined(LICENSE_BUILDLINK3_MK)", + "LICENSE_BUILDLINK3_MK:=", + "", + "BUILDLINK_API_DEPENDS.${LICENSE}-wxWidgets+=\t${PYPKGPREFIX}-wxWidgets>=2.6.1.0", + "BUILDLINK_ABI_DEPENDS.${LICENSE}-wxWidgets+=\t${PYPKGPREFIX}-wxWidgets>=2.8.10.1nb26", + "", + ".endif", + "", + "BUILDLINK_TREE+=\t-${PYPKGPREFIX}-wxWidgets") + + ChecklinesBuildlink3Mk(mklines) + + c.Check(s.Output(), equals, ""+ + "WARN: buildlink3.mk:3: Please replace \"${LICENSE}\" with a simple string.\n"+ + "WARN: buildlink3.mk:13: This line should contain the following text: BUILDLINK_TREE+=\t-${LICENSE}-wxWidgets\n") } diff --git a/pkgtools/pkglint/files/category.go b/pkgtools/pkglint/files/category.go index 4c2e5e8665b..cbdfe60a2fd 100644 --- a/pkgtools/pkglint/files/category.go +++ b/pkgtools/pkglint/files/category.go @@ -4,76 +4,75 @@ import ( "sort" ) -type subdir struct { - name string - line *Line - active bool -} - -func checkdirCategory() { - defer tracecall("checkdirCategory", G.currentDir)() +func CheckdirCategory() { + if G.opts.DebugTrace { + defer tracecall1(G.CurrentDir)() + } - fname := G.currentDir + "/Makefile" - lines := LoadNonemptyLines(fname, true) + lines := LoadNonemptyLines(G.CurrentDir+"/Makefile", true) if lines == nil { return } - ParselinesMk(lines) + + mklines := NewMkLines(lines) + mklines.Check() exp := NewExpecter(lines) - if checklineRcsid(exp.currentLine(), `#\s+`, "# ") { - exp.advance() + for exp.AdvanceIfPrefix("#") { } + exp.ExpectEmptyLine() - for !exp.eof() && exp.advanceIfMatches(`^#`) != nil { + if exp.AdvanceIfMatches(`^COMMENT=\t*(.*)`) { + mklines.mklines[exp.index-1].CheckValidCharactersInValue(`[- '(),/0-9A-Za-z]`) + } else { + exp.CurrentLine().Error0("COMMENT= line expected.") } - exp.expectEmptyLine() + exp.ExpectEmptyLine() - if exp.advanceIfMatches(`^COMMENT=\t*(.*)`) != nil { - checklineValidCharactersInValue(exp.previousLine(), `[- '(),/0-9A-Za-z]`) - } else { - exp.currentLine().errorf("COMMENT= line expected.") + type subdir struct { + name string + line *Line + active bool } - exp.expectEmptyLine() // And now to the most complicated part of the category Makefiles, // the (hopefully) sorted list of SUBDIRs. The first step is to // collect the SUBDIRs in the Makefile and in the file system. - fSubdirs := getSubdirs(G.currentDir) + fSubdirs := getSubdirs(G.CurrentDir) sort.Sort(sort.StringSlice(fSubdirs)) var mSubdirs []subdir prevSubdir := "" - for !exp.eof() { - line := exp.currentLine() - text := line.text + for !exp.EOF() { + line := exp.CurrentLine() + text := line.Text if m, commentFlag, indentation, name, comment := match4(text, `^(#?)SUBDIR\+=(\s*)(\S+)\s*(?:#\s*(.*?)\s*|)$`); m { commentedOut := commentFlag == "#" if commentedOut && comment == "" { - line.warnf("%q commented out without giving a reason.", name) + line.Warn1("%q commented out without giving a reason.", name) } if indentation != "\t" { - line.warnf("Indentation should be a single tab character.") + line.Warn0("Indentation should be a single tab character.") } if name == prevSubdir { - line.errorf("%q must only appear once.", name) + line.Error1("%q must only appear once.", name) } else if name < prevSubdir { - line.warnf("%q should come before %q.", name, prevSubdir) + line.Warn2("%q should come before %q.", name, prevSubdir) } else { // correctly ordered } mSubdirs = append(mSubdirs, subdir{name, line, !commentedOut}) prevSubdir = name - exp.advance() + exp.Advance() } else { - if line.text != "" { - line.errorf("SUBDIR+= line or empty line expected.") + if line.Text != "" { + line.Error0("SUBDIR+= line or empty line expected.") } break } @@ -104,7 +103,7 @@ func checkdirCategory() { mNeednext = false if mIndex >= len(mSubdirs) { mAtend = true - line = exp.currentLine() + line = exp.CurrentLine() continue } else { mCurrent = mSubdirs[mIndex].name @@ -127,15 +126,17 @@ func checkdirCategory() { if !fAtend && (mAtend || fCurrent < mCurrent) { if !mCheck[fCurrent] { - line.errorf("%q exists in the file system, but not in the Makefile.", fCurrent) - line.insertBefore("SUBDIR+=\t" + fCurrent) + if !line.AutofixInsertBefore("SUBDIR+=\t" + fCurrent) { + line.Error1("%q exists in the file system, but not in the Makefile.", fCurrent) + } } fNeednext = true } else if !mAtend && (fAtend || mCurrent < fCurrent) { if !fCheck[mCurrent] { - line.errorf("%q exists in the Makefile, but not in the file system.", mCurrent) - line.delete() + if !line.AutofixDelete() { + line.Error1("%q exists in the Makefile, but not in the file system.", mCurrent) + } } mNeednext = true @@ -143,34 +144,26 @@ func checkdirCategory() { fNeednext = true mNeednext = true if mActive { - subdirs = append(subdirs, G.currentDir+"/"+mCurrent) + subdirs = append(subdirs, G.CurrentDir+"/"+mCurrent) } } } // the pkgsrc-wip category Makefile defines its own targets for // generating indexes and READMEs. Just skip them. - if G.isWip { + if G.Wip { exp.index = len(exp.lines) - 2 } - exp.expectEmptyLine() - - if exp.currentLine().text == ".include \"../mk/bsd.pkg.subdir.mk\"" { - exp.advance() - } else { - exp.expectText(".include \"../mk/misc/category.mk\"") + exp.ExpectEmptyLine() + exp.ExpectText(".include \"../mk/misc/category.mk\"") + if !exp.EOF() { + exp.CurrentLine().Error0("The file should end here.") } - if !exp.eof() { - exp.currentLine().errorf("The file should end here.") - } - - ChecklinesMk(lines) - - saveAutofixChanges(lines) + SaveAutofixChanges(lines) if G.opts.Recursive { - G.todo = append(append([]string(nil), subdirs...), G.todo...) + G.Todo = append(append([]string(nil), subdirs...), G.Todo...) } } diff --git a/pkgtools/pkglint/files/category_test.go b/pkgtools/pkglint/files/category_test.go new file mode 100644 index 00000000000..01dc8f63897 --- /dev/null +++ b/pkgtools/pkglint/files/category_test.go @@ -0,0 +1,53 @@ +package main + +import ( + check "gopkg.in/check.v1" +) + +func (s *Suite) TestCheckdirCategory_TotallyBroken(c *check.C) { + G.globalData.InitVartypes() + s.CreateTmpFile(c, "archivers/Makefile", ""+ + "# $\n"+ + "SUBDIR+=pkg1\n"+ + "SUBDIR+=\u0020aaaaa\n"+ + "SUBDIR-=unknown #doesn’t work\n"+ + "\n"+ + ".include \"../mk/category.mk\"\n") + + G.CurrentDir = s.tmpdir + "/archivers" + CheckdirCategory() + + c.Check(s.OutputCleanTmpdir(), equals, ""+ + "ERROR: ~/archivers/Makefile:1: Expected \"# $"+"NetBSD$\".\n"+ + "WARN: ~/archivers/Makefile:4: Line contains invalid characters (U+2019).\n"+ + "WARN: ~/archivers/Makefile:4: SUBDIR- is defined but not used. Spelling mistake?\n"+ + "ERROR: ~/archivers/Makefile:6: \"../mk/category.mk\" does not exist.\n"+ + "ERROR: ~/archivers/Makefile:2: COMMENT= line expected.\n"+ + "WARN: ~/archivers/Makefile:2: Indentation should be a single tab character.\n"+ + "WARN: ~/archivers/Makefile:3: Indentation should be a single tab character.\n"+ + "WARN: ~/archivers/Makefile:3: \"aaaaa\" should come before \"pkg1\".\n"+ + "ERROR: ~/archivers/Makefile:4: SUBDIR+= line or empty line expected.\n"+ + "ERROR: ~/archivers/Makefile:2: \"pkg1\" exists in the Makefile, but not in the file system.\n"+ + "ERROR: ~/archivers/Makefile:3: \"aaaaa\" exists in the Makefile, but not in the file system.\n"+ + "WARN: ~/archivers/Makefile:4: This line should contain the following text: .include \"../mk/misc/category.mk\"\n"+ + "ERROR: ~/archivers/Makefile:4: The file should end here.\n") +} + +func (s *Suite) TestCheckdirCategory_InvalidComment(c *check.C) { + G.globalData.InitVartypes() + s.CreateTmpFile(c, "archivers/Makefile", ""+ + "# $"+"NetBSD$\n"+ + "COMMENT=\t\\Make $$$$ fast\"\n"+ + "\n"+ + "SUBDIR+=\tpackage\n"+ + "\n"+ + ".include \"../mk/misc/category.mk\"\n") + s.CreateTmpFile(c, "archivers/package/Makefile", "# dummy\n") + s.CreateTmpFile(c, "mk/misc/category.mk", "# dummy\n") + G.CurrentDir = s.tmpdir + "/archivers" + G.CurPkgsrcdir = ".." + + CheckdirCategory() + + c.Check(s.OutputCleanTmpdir(), equals, "WARN: ~/archivers/Makefile:2: COMMENT contains invalid characters (U+005C U+0024 U+0024 U+0024 U+0024 U+0022).\n") +} diff --git a/pkgtools/pkglint/files/check_test.go b/pkgtools/pkglint/files/check_test.go index 74b8e51a44e..8e63d260c2b 100644 --- a/pkgtools/pkglint/files/check_test.go +++ b/pkgtools/pkglint/files/check_test.go @@ -2,10 +2,12 @@ package main import ( "bytes" + "fmt" "io/ioutil" "os" "path" "path/filepath" + "strings" "testing" check "gopkg.in/check.v1" @@ -35,14 +37,51 @@ func (s *Suite) Output() string { return s.Stdout() + s.Stderr() } -func (s *Suite) NewLines(fname string, lines ...string) []*Line { - result := make([]*Line, len(lines)) - for i, line := range lines { - result[i] = NewLine(fname, sprintf("%d", i+1), line, []*RawLine{{i + 1, line + "\n"}}) +func (s *Suite) OutputCleanTmpdir() string { + if s.tmpdir == "" { + return "error: OutputCleanTmpdir must only be called when s.tmpdir is actually set." + } + return strings.Replace(s.Output(), s.tmpdir, "~", -1) +} + +// Arguments are either (lineno, orignl) or (lineno, orignl, textnl). +func (s *Suite) NewRawLines(args ...interface{}) []*RawLine { + rawlines := make([]*RawLine, len(args)/2) + j := 0 + for i := 0; i < len(args); i += 2 { + lineno := args[i].(int) + orignl := args[i+1].(string) + textnl := orignl + if i+2 < len(args) { + if s, ok := args[i+2].(string); ok { + textnl = s + i++ + } + } + rawlines[j] = &RawLine{lineno, orignl, textnl} + j++ + } + return rawlines[:j] +} + +func (s *Suite) NewLines(fname string, texts ...string) []*Line { + result := make([]*Line, len(texts)) + for i, text := range texts { + textnl := text + "\n" + result[i] = NewLine(fname, i+1, text, s.NewRawLines(i+1, textnl)) } return result } +func (s *Suite) NewMkLines(fname string, lines ...string) *MkLines { + return NewMkLines(s.NewLines(fname, lines...)) +} + +func (s *Suite) DebugToStdout() { + G.debugOut = os.Stdout + G.opts.DebugTrace = true +} + func (s *Suite) UseCommandLine(c *check.C, args ...string) { exitcode := new(Pkglint).ParseCommandLine(append([]string{"pkglint"}, args...)) if exitcode != nil && *exitcode != 0 { @@ -51,27 +90,37 @@ func (s *Suite) UseCommandLine(c *check.C, args ...string) { } func (s *Suite) RegisterTool(toolname, varname string, varRequired bool) { - if G.globalData.tools == nil { - G.globalData.tools = make(map[string]bool) - G.globalData.vartools = make(map[string]string) + if G.globalData.Tools == nil { + G.globalData.Tools = make(map[string]bool) + G.globalData.Vartools = make(map[string]string) G.globalData.toolsVarRequired = make(map[string]bool) + G.globalData.PredefinedTools = make(map[string]bool) } - G.globalData.tools[toolname] = true - G.globalData.vartools[toolname] = varname + G.globalData.Tools[toolname] = true + G.globalData.Vartools[toolname] = varname if varRequired { G.globalData.toolsVarRequired[toolname] = true } + G.globalData.PredefinedTools[toolname] = true } -func (s *Suite) CreateTmpFile(c *check.C, fname, content string) { +func (s *Suite) CreateTmpFile(c *check.C, relFname, content string) (absFname string) { if s.tmpdir == "" { s.tmpdir = filepath.ToSlash(c.MkDir()) } - err := os.MkdirAll(s.tmpdir+"/"+path.Dir(fname), 0777) - c.Check(err, check.IsNil) + absFname = s.tmpdir + "/" + relFname + err := os.MkdirAll(path.Dir(absFname), 0777) + c.Assert(err, check.IsNil) - err = ioutil.WriteFile(s.tmpdir+"/"+fname, []byte(content), 0666) + err = ioutil.WriteFile(absFname, []byte(content), 0666) c.Check(err, check.IsNil) + return +} + +func (s *Suite) LoadTmpFile(c *check.C, relFname string) string { + bytes, err := ioutil.ReadFile(s.tmpdir + "/" + relFname) + c.Assert(err, check.IsNil) + return string(bytes) } func (s *Suite) ExpectFatalError(action func()) { @@ -85,14 +134,15 @@ func (s *Suite) ExpectFatalError(action func()) { } func (s *Suite) SetUpTest(c *check.C) { - G = new(GlobalVars) - G.logOut, G.logErr, G.traceOut = &s.stdout, &s.stderr, &s.stdout + G = GlobalVars{TestingData: &TestingData{VerifiedBits: make(map[string]bool)}} + G.logOut, G.logErr, G.debugOut = &s.stdout, &s.stderr, &s.stdout + s.UseCommandLine(c /* no arguments */) } func (s *Suite) TearDownTest(c *check.C) { - G = nil + G = GlobalVars{} if out := s.Output(); out != "" { - c.Logf("Unchecked output; check with: c.Check(s.Output(), equals, %q)", out) + fmt.Fprintf(os.Stderr, "Unchecked output in %q; check with: c.Check(s.Output(), equals, %q)", c.TestName(), out) } s.tmpdir = "" } diff --git a/pkgtools/pkglint/files/deprecated.go b/pkgtools/pkglint/files/deprecated.go deleted file mode 100644 index e286b9a1475..00000000000 --- a/pkgtools/pkglint/files/deprecated.go +++ /dev/null @@ -1,163 +0,0 @@ -package main - -// This file contains names of Makefile variables and a short explanation -// what to do to make the warning disappear. Entries should only be removed -// if the explanation changes, in which case the new explanation should -// be added to the current date. - -func getDeprecatedVars() map[string]string { - return map[string]string{ - - // December 2003 - "FIX_RPATH": "It has been removed from pkgsrc in 2003.", - - // February 2005 - "LIB_DEPENDS": "Use DEPENDS instead.", - "ONLY_FOR_ARCHS": "Use ONLY_FOR_PLATFORM instead.", - "NOT_FOR_ARCHS": "Use NOT_FOR_PLATFORM instead.", - "ONLY_FOR_OPSYS": "Use ONLY_FOR_PLATFORM instead.", - "NOT_FOR_OPSYS": "Use NOT_FOR_PLATFORM instead.", - - // May 2005 - "ALL_TARGET": "Use BUILD_TARGET instead.", - "DIGEST_FILE": "Use DISTINFO_FILE instead.", - "IGNORE": "Use PKG_FAIL_REASON or PKG_SKIP_REASON instead.", - "IS_INTERACTIVE": "Use INTERACTIVE_STAGE instead.", - "KERBEROS": "Use the PKG_OPTIONS framework instead.", - "MASTER_SITE_SUBDIR": "Use some form of MASTER_SITES instead.", - "MD5_FILE": "Use DISTINFO_FILE instead.", - "MIRROR_DISTFILE": "Use NO_BIN_ON_FTP and/or NO_SRC_ON_FTP instead.", - "NO_CDROM": "Use NO_BIN_ON_CDROM and/or NO_SRC_ON_CDROM instead.", - "NO_PATCH": "You can just remove it.", - "NO_WRKSUBDIR": "Use WRKSRC=${WRKDIR} instead.", - "PATCH_SITE_SUBDIR": "Use some form of PATCHES_SITES instead.", - "PATCH_SUM_FILE": "Use DISTINFO_FILE instead.", - "PKG_JVM": "Use PKG_DEFAULT_JVM instead.", - "USE_BUILDLINK2": "You can just remove it.", - "USE_BUILDLINK3": "You can just remove it.", - "USE_CANNA": "Use the PKG_OPTIONS framework instead.", - "USE_DB4": "Use the PKG_OPTIONS framework instead.", - "USE_DIRS": "You can just remove it.", - "USE_ESOUND": "Use the PKG_OPTIONS framework instead.", - "USE_GIF": "Use the PKG_OPTIONS framework instead.", - "USE_GMAKE": "Use USE_TOOLS+=gmake instead.", - "USE_GNU_TOOLS": "Use USE_TOOLS instead.", - "USE_IDEA": "Use the PKG_OPTIONS framework instead.", - "USE_LIBCRACK": "Use the PKG_OPTIONS framework instead.", - "USE_MMX": "Use the PKG_OPTIONS framework instead.", - "USE_PKGLIBTOOL": "Use USE_LIBTOOL instead.", - "USE_SSL": "Include \"../../security/openssl/buildlink3.mk\" instead.", - - // July 2005 - "USE_PERL5": "Use USE_TOOLS+=perl or USE_TOOLS+=perl:run instead.", - - // October 2005 - "NO_TOOLS": "You can just remove it.", - "NO_WRAPPER": "You can just remove it.", - - // November 2005 - "ALLFILES": "Use CKSUMFILES instead.", - "DEPENDS_TARGET": "Use DEPENDS instead.", - "FETCH_DEPENDS": "Use DEPENDS instead.", - "RUN_DEPENDS": "Use DEPENDS instead.", - - // December 2005 - "USE_CUPS": "Use the PKG_OPTIONS framework (option cups) instead.", - "USE_I586": "Use the PKG_OPTIONS framework (option i586) instead.", - "USE_INN": "Use the PKG_OPTIONS framework instead.", - "USE_OPENLDAP": "Use the PKG_OPTIONS framework (option openldap) instead.", - "USE_OSS": "Use the PKG_OPTIONS framework (option oss) instead.", - "USE_RSAREF2": "Use the PKG_OPTIONS framework (option rsaref) instead.", - "USE_SASL": "Use the PKG_OPTIONS framework (option sasl) instead.", - "USE_SASL2": "Use the PKG_OPTIONS framework (option sasl) instead.", - "USE_SJ3": "Use the PKG_OPTIONS framework (option sj3) instead.", - "USE_SOCKS": "Use the PKG_OPTIONS framework (socks4 and socks5 options) instead.", - "USE_WNN4": "Use the PKG_OPTIONS framework (option wnn4) instead.", - "USE_XFACE": "Use the PKG_OPTIONS framework instead.", - - // February 2006 - "TOOLS_DEPMETHOD": "Use the :build or :run modifiers in USE_TOOLS instead.", - "MANDIR": "Please use ${PREFIX}/${PKGMANDIR} instead.", - "DOWNLOADED_DISTFILE": "Use the shell variable $$extract_file instead.", - "DECOMPRESS_CMD": "Use EXTRACT_CMD instead.", - - // March 2006 - "INSTALL_EXTRA_TMPL": "Use INSTALL_TEMPLATE instead.", - "DEINSTALL_EXTRA_TMPL": "Use DEINSTALL_TEMPLATE instead.", - - // April 2006 - "RECOMMENDED": "Use ABI_DEPENDS instead.", - "BUILD_USES_MSGFMT": "Use USE_TOOLS+=msgfmt instead.", - "USE_MSGFMT_PLURALS": "Use USE_TOOLS+=msgfmt instead.", - - // May 2006 - "EXTRACT_USING_PAX": "Use \"EXTRACT_OPTS=-t pax\" instead.", - "NO_EXTRACT": "It doesn't exist anymore.", - "_FETCH_MESSAGE": "Use FETCH_MESSAGE (different format) instead.", - "BUILDLINK_DEPENDS.*": "Use BUILDLINK_API_DEPENDS.* instead.", - "BUILDLINK_RECOMMENDED.*": "Use BUILDLINK_ABI_DEPENDS.* instead.", - "SHLIB_HANDLING": "Use CHECK_SHLIBS_SUPPORTED instead.", - "USE_RMAN": "It has been removed.", - - // June 2006 - "DEINSTALL_SRC": "Use the pkginstall framework instead.", - "INSTALL_SRC": "Use the pkginstall framework instead.", - "DEINSTALL_TEMPLATE": "Use DEINSTALL_TEMPLATES instead.", - "INSTALL_TEMPLATE": "Use INSTALL_TEMPLATES instead.", - "HEADER_TEMPLATE": "Use HEADER_TEMPLATES instead.", - "_REPLACE.*": "Use REPLACE.* instead.", - "_REPLACE_FILES.*": "Use REPLACE_FILES.* instead.", - "MESSAGE": "Use MESSAGE_SRC instead.", - "INSTALL_FILE": "It may only be used internally by pkgsrc.", - "DEINSTALL_FILE": "It may only be used internally by pkgsrc.", - - // July 2006 - "USE_DIGEST": "You can just remove it.", - "LTCONFIG_OVERRIDE": "You can just remove it.", - "USE_GNU_GETTEXT": "You can just remove it.", - "BUILD_ENV": "Use PKGSRC_MAKE_ENV instead.", - "DYNAMIC_MASTER_SITES": "You can just remove it.", - - // September 2006 - "MAKEFILE": "Use MAKE_FILE instead.", - - // November 2006 - "SKIP_PORTABILITY_CHECK": "Use CHECK_PORTABILITY_SKIP (a list of patterns) instead.", - "PKG_SKIP_REASON": "Use PKG_FAIL_REASON instead.", - - // January 2007 - "BUILDLINK_TRANSFORM.*": "Use BUILDLINK_FNAME_TRANSFORM.* instead.", - - // March 2007 - "SCRIPTDIR": "You can just remove it.", - "NO_PKG_REGISTER": "You can just remove it.", - "NO_DEPENDS": "You can just remove it.", - - // October 2007 - "_PKG_SILENT": "Use RUN (with more error checking) instead.", - "_PKG_DEBUG": "Use RUN (with more error checking) instead.", - "LICENCE": "Use LICENSE instead.", - - // November 2007 - //USE_NCURSES Include "../../devel/ncurses/buildlink3.mk" instead. - - // December 2007 - "INSTALLATION_DIRS_FROM_PLIST": "Use AUTO_MKDIRS instead.", - - // April 2009 - "NO_PACKAGE": "It doesn't exist anymore.", - "NO_MTREE": "You can just remove it.", - - // July 2012 - "SETGIDGAME": "Use USE_GAMESGROUP instead.", - "GAMEGRP": "Use GAMES_GROUP instead.", - "GAMEOWN": "Use GAMES_USER instead.", - - // July 2013 - "USE_GNU_READLINE": "Include \"../../devel/readline/buildlink3.mk\" instead.", - - // October 2014 - "SVR4_PKGNAME": "Just remove it.", - "PKG_INSTALLATION_TYPES": "Just remove it.", - } -} diff --git a/pkgtools/pkglint/files/deprecated_test.go b/pkgtools/pkglint/files/deprecated_test.go deleted file mode 100644 index a3415a79307..00000000000 --- a/pkgtools/pkglint/files/deprecated_test.go +++ /dev/null @@ -1,14 +0,0 @@ -package main - -import ( - check "gopkg.in/check.v1" -) - -func (s *Suite) TestDeprecated(c *check.C) { - G.globalData.deprecated = getDeprecatedVars() - - line := NewLine("Makefile", "5", "USE_PERL5=\tyes", nil) - NewMkLine(line).checkVarassign() - - c.Check(s.Output(), equals, "WARN: Makefile:5: Definition of USE_PERL5 is deprecated. Use USE_TOOLS+=perl or USE_TOOLS+=perl:run instead.\n") -} diff --git a/pkgtools/pkglint/files/descr.go b/pkgtools/pkglint/files/descr.go deleted file mode 100644 index ca51a75e34a..00000000000 --- a/pkgtools/pkglint/files/descr.go +++ /dev/null @@ -1,27 +0,0 @@ -package main - -func checklinesDescr(lines []*Line) { - defer tracecall("checklinesDescr", lines[0].fname)() - - for _, line := range lines { - checklineLength(line, 80) - checklineTrailingWhitespace(line) - checklineValidCharacters(line, `[\t -~]`) - if contains(line.text, "${") { - line.notef("Variables are not expanded in the DESCR file.") - } - } - checklinesTrailingEmptyLines(lines) - - if maxlines := 24; len(lines) > maxlines { - line := lines[maxlines] - - line.warnf("File too long (should be no more than %d lines).", maxlines) - line.explain( - "A common terminal size is 80x25 characters. The DESCR file should", - "fit on one screen. It is also intended to give a _brief_ summary", - "about the package's contents.") - } - - saveAutofixChanges(lines) -} diff --git a/pkgtools/pkglint/files/descr_test.go b/pkgtools/pkglint/files/descr_test.go deleted file mode 100644 index 16c5e6b01f5..00000000000 --- a/pkgtools/pkglint/files/descr_test.go +++ /dev/null @@ -1,22 +0,0 @@ -package main - -import ( - check "gopkg.in/check.v1" - "strings" -) - -func (s *Suite) TestChecklinesDescr(c *check.C) { - lines := s.NewLines("DESCR", - strings.Repeat("X", 90), - "", "", "", "", "", "", "", "", "10", - "Try ${PREFIX}", - "", "", "", "", "", "", "", "", "20", - "", "", "", "", "", "", "", "", "", "30") - - checklinesDescr(lines) - - c.Check(s.Output(), equals, ""+ - "WARN: DESCR:1: Line too long (should be no more than 80 characters).\n"+ - "NOTE: DESCR:11: Variables are not expanded in the DESCR file.\n"+ - "WARN: DESCR:25: File too long (should be no more than 24 lines).\n") -} diff --git a/pkgtools/pkglint/files/dir.go b/pkgtools/pkglint/files/dir.go index 4c7832c67bb..d62385020f1 100644 --- a/pkgtools/pkglint/files/dir.go +++ b/pkgtools/pkglint/files/dir.go @@ -6,23 +6,25 @@ import ( ) func CheckDirent(fname string) { - defer tracecall("CheckDirent", fname)() + if G.opts.DebugTrace { + defer tracecall1(fname)() + } st, err := os.Lstat(fname) if err != nil || !st.Mode().IsDir() && !st.Mode().IsRegular() { - errorf(fname, noLines, "No such file or directory.") + Errorf(fname, noLines, "No such file or directory.") return } isDir := st.Mode().IsDir() isReg := st.Mode().IsRegular() - G.currentDir = ifelseStr(isReg, path.Dir(fname), fname) - absCurrentDir := abspath(G.currentDir) - G.isWip = !G.opts.Import && matches(absCurrentDir, `/wip/|/wip$`) - G.isInfrastructure = matches(absCurrentDir, `/mk/|/mk$`) - G.curPkgsrcdir = findPkgsrcTopdir(G.currentDir) - if G.curPkgsrcdir == "" { - errorf(fname, noLines, "Cannot determine the pkgsrc root directory for %q.", G.currentDir) + G.CurrentDir = ifelseStr(isReg, path.Dir(fname), fname) + absCurrentDir := abspath(G.CurrentDir) + G.Wip = !G.opts.Import && matches(absCurrentDir, `/wip/|/wip$`) + G.Infrastructure = matches(absCurrentDir, `/mk/|/mk$`) + G.CurPkgsrcdir = findPkgsrcTopdir(G.CurrentDir) + if G.CurPkgsrcdir == "" { + Errorf(fname, noLines, "Cannot determine the pkgsrc root directory for %q.", G.CurrentDir) return } @@ -30,18 +32,18 @@ func CheckDirent(fname string) { case isDir && isEmptyDir(fname): return case isReg: - checkfile(fname) + Checkfile(fname) return } - switch G.curPkgsrcdir { + switch G.CurPkgsrcdir { case "../..": - checkdirPackage(relpath(G.globalData.pkgsrcdir, G.currentDir)) + checkdirPackage(relpath(G.globalData.Pkgsrcdir, G.CurrentDir)) case "..": - checkdirCategory() + CheckdirCategory() case ".": - checkdirToplevel() + CheckdirToplevel() default: - errorf(fname, noLines, "Cannot check directories outside a pkgsrc tree.") + Errorf(fname, noLines, "Cannot check directories outside a pkgsrc tree.") } } diff --git a/pkgtools/pkglint/files/dir_test.go b/pkgtools/pkglint/files/dir_test.go new file mode 100644 index 00000000000..39bf5ba59d0 --- /dev/null +++ b/pkgtools/pkglint/files/dir_test.go @@ -0,0 +1,37 @@ +package main + +import ( + check "gopkg.in/check.v1" +) + +func (s *Suite) TestCheckDirent_outside(c *check.C) { + s.CreateTmpFile(c, "empty", "") + + CheckDirent(s.tmpdir) + + c.Check(s.OutputCleanTmpdir(), equals, "ERROR: ~: Cannot determine the pkgsrc root directory for \"~\".\n") +} + +func (s *Suite) TestCheckDirent(c *check.C) { + s.CreateTmpFile(c, "mk/bsd.pkg.mk", "") + s.CreateTmpFile(c, "category/package/Makefile", "") + s.CreateTmpFile(c, "category/Makefile", "") + s.CreateTmpFile(c, "Makefile", "") + G.globalData.Pkgsrcdir = s.tmpdir + + CheckDirent(s.tmpdir) + + c.Check(s.OutputCleanTmpdir(), equals, "ERROR: ~/Makefile: Must not be empty.\n") + + CheckDirent(s.tmpdir + "/category") + + c.Check(s.OutputCleanTmpdir(), equals, "ERROR: ~/category/Makefile: Must not be empty.\n") + + CheckDirent(s.tmpdir + "/category/package") + + c.Check(s.OutputCleanTmpdir(), equals, "ERROR: ~/category/package/Makefile: Must not be empty.\n") + + CheckDirent(s.tmpdir + "/category/package/nonexistent") + + c.Check(s.OutputCleanTmpdir(), equals, "ERROR: ~/category/package/nonexistent: No such file or directory.\n") +} diff --git a/pkgtools/pkglint/files/distinfo.go b/pkgtools/pkglint/files/distinfo.go index 7f280e674c6..8607fc30e11 100644 --- a/pkgtools/pkglint/files/distinfo.go +++ b/pkgtools/pkglint/files/distinfo.go @@ -3,29 +3,34 @@ package main import ( "bytes" "crypto/sha1" + "fmt" "io/ioutil" "strings" ) -func checklinesDistinfo(lines []*Line) { - defer tracecall("checklinesDistinfo", lines[0].fname)() +func ChecklinesDistinfo(lines []*Line) { + if G.opts.DebugTrace { + defer tracecall1(lines[0].Fname)() + } - fname := lines[0].fname + fname := lines[0].Fname var patchesDir = "patches" - if G.pkgContext != nil && dirExists(G.currentDir+"/"+G.pkgContext.patchdir) { - patchesDir = G.pkgContext.patchdir + if G.Pkg != nil && hasSuffix(fname, "/lang/php55/distinfo") { + patchesDir = G.CurPkgsrcdir + "/lang/php55/patches" + } else if G.Pkg != nil && dirExists(G.CurrentDir+"/"+G.Pkg.Patchdir) { + patchesDir = G.Pkg.Patchdir } - if G.pkgContext != nil && hasSuffix(fname, "/lang/php54/distinfo") { - patchesDir = G.curPkgsrcdir + "/lang/php54/patches" + if G.opts.DebugMisc { + Debugf(fname, noLines, "patchesDir=%q", patchesDir) } - _ = G.opts.DebugMisc && debugf(fname, noLines, "patchesDir=%q", patchesDir) ck := &distinfoLinesChecker{ fname, patchesDir, isCommitted(fname), - make(map[string]bool), "", false, nil} + make(map[string]bool), nil, "", false, nil} ck.checkLines(lines) - checklinesTrailingEmptyLines(lines) + ChecklinesTrailingEmptyLines(lines) ck.checkUnrecordedPatches() + SaveAutofixChanges(lines) } type distinfoLinesChecker struct { @@ -34,28 +39,29 @@ type distinfoLinesChecker struct { distinfoIsCommitted bool patches map[string]bool // "patch-aa" => true - previousFilename string + currentFirstLine *Line + currentFilename string isPatch bool algorithms []string } func (ck *distinfoLinesChecker) checkLines(lines []*Line) { - checklineRcsid(lines[0], ``, "") - if 1 < len(lines) && lines[1].text != "" { - lines[1].notef("Empty line expected.") + lines[0].CheckRcsid(``, "") + if 1 < len(lines) && lines[1].Text != "" { + lines[1].Note0("Empty line expected.") } for i, line := range lines { if i < 2 { continue } - m, alg, filename, hash := match3(line.text, `^(\w+) \((\w[^)]*)\) = (.*)(?: bytes)?$`) + m, alg, filename, hash := match3(line.Text, `^(\w+) \((\w[^)]*)\) = (.*)(?: bytes)?$`) if !m { - line.errorf("Invalid line.") + line.Error0("Invalid line.") continue } - if filename != ck.previousFilename { + if filename != ck.currentFilename { ck.onFilenameChange(line, filename) } ck.algorithms = append(ck.algorithms, alg) @@ -63,33 +69,40 @@ func (ck *distinfoLinesChecker) checkLines(lines []*Line) { ck.checkGlobalMismatch(line, filename, alg, hash) ck.checkUncommittedPatch(line, filename, hash) } - ck.onFilenameChange(NewLine(ck.distinfoFilename, "EOF", "", nil), "") + ck.onFilenameChange(NewLineEOF(ck.distinfoFilename), "") } func (ck *distinfoLinesChecker) onFilenameChange(line *Line, nextFname string) { - prevFname := ck.previousFilename - if prevFname != "" { + currentFname := ck.currentFilename + if currentFname != "" { algorithms := strings.Join(ck.algorithms, ", ") if ck.isPatch { if algorithms != "SHA1" { - line.errorf("Expected SHA1 hash for %s, got %s.", prevFname, algorithms) - } - } else { - if algorithms != "SHA1, RMD160, Size" && algorithms != "SHA1, RMD160, SHA512, Size" { - line.errorf("Expected SHA1, RMD160, SHA512, Size checksums for %q, got %s.", prevFname, algorithms) + line.Error2("Expected SHA1 hash for %s, got %s.", currentFname, algorithms) } + } else if hasPrefix(currentFname, "patch-") && algorithms == "SHA1" { + ck.currentFirstLine.Warn2("Patch file %q does not exist in directory %q.", currentFname, cleanpath(ck.patchdir)) + Explain( + "If the patches directory looks correct, the patch may have been", + "removed without updating the distinfo file. In such a case please", + "update the distinfo file.", + "", + "If the patches directory looks wrong, pkglint needs to be improved.") + } else if algorithms != "SHA1, RMD160, Size" && algorithms != "SHA1, RMD160, SHA512, Size" { + line.Error2("Expected SHA1, RMD160, SHA512, Size checksums for %q, got %s.", currentFname, algorithms) } } - ck.isPatch = matches(nextFname, `^patch-.+$`) && fileExists(G.currentDir+"/"+ck.patchdir+"/"+nextFname) - ck.previousFilename = nextFname + ck.isPatch = hasPrefix(nextFname, "patch-") && fileExists(G.CurrentDir+"/"+ck.patchdir+"/"+nextFname) + ck.currentFilename = nextFname + ck.currentFirstLine = line ck.algorithms = nil } func (ck *distinfoLinesChecker) checkPatchSha1(line *Line, patchFname, distinfoSha1Hex string) { - patchBytes, err := ioutil.ReadFile(G.currentDir + "/" + patchFname) + patchBytes, err := ioutil.ReadFile(G.CurrentDir + "/" + patchFname) if err != nil { - line.errorf("%s does not exist.", patchFname) + line.Error1("%s does not exist.", patchFname) return } @@ -100,39 +113,43 @@ func (ck *distinfoLinesChecker) checkPatchSha1(line *Line, patchFname, distinfoS h.Write(patchLine) } } - fileSha1Hex := sprintf("%x", h.Sum(nil)) + fileSha1Hex := fmt.Sprintf("%x", h.Sum(nil)) if distinfoSha1Hex != fileSha1Hex { - line.errorf("%s hash of %s differs (distinfo has %s, patch file has %s). Run \"%s makepatchsum\".", "SHA1", patchFname, distinfoSha1Hex, fileSha1Hex, confMake) + if !line.AutofixReplace(distinfoSha1Hex, fileSha1Hex) { + line.Errorf("%s hash of %s differs (distinfo has %s, patch file has %s). Run \"%s makepatchsum\".", "SHA1", patchFname, distinfoSha1Hex, fileSha1Hex, confMake) + } } } func (ck *distinfoLinesChecker) checkUnrecordedPatches() { - files, err := ioutil.ReadDir(G.currentDir + "/" + ck.patchdir) + files, err := ioutil.ReadDir(G.CurrentDir + "/" + ck.patchdir) if err != nil { - _ = G.opts.DebugUnchecked && debugf(ck.distinfoFilename, noLines, "Cannot read patchesDir %q: %s", ck.patchdir, err) + if G.opts.DebugUnchecked { + Debugf(ck.distinfoFilename, noLines, "Cannot read patchesDir %q: %s", ck.patchdir, err) + } return } for _, file := range files { patch := file.Name() if file.Mode().IsRegular() && !ck.patches[patch] { - errorf(ck.distinfoFilename, noLines, "patch %q is not recorded. Run \"%s makepatchsum\".", ck.patchdir+"/"+patch, confMake) + Errorf(ck.distinfoFilename, noLines, "patch %q is not recorded. Run \"%s makepatchsum\".", ck.patchdir+"/"+patch, confMake) } } } // Inter-package check for differing distfile checksums. func (ck *distinfoLinesChecker) checkGlobalMismatch(line *Line, fname, alg, hash string) { - if G.ipcDistinfo != nil && !ck.isPatch { + if G.Hash != nil && !hasPrefix(fname, "patch-") { // Intentionally checking the filename instead of ck.isPatch key := alg + ":" + fname - otherHash := G.ipcDistinfo[key] + otherHash := G.Hash[key] if otherHash != nil { if otherHash.hash != hash { - line.errorf("The hash %s for %s is %s, ...", alg, fname, hash) - otherHash.line.errorf("... which differs from %s.", otherHash.hash) + line.Errorf("The hash %s for %s is %s, ...", alg, fname, hash) + otherHash.line.Error1("... which differs from %s.", otherHash.hash) } } else { - G.ipcDistinfo[key] = &Hash{hash, line} + G.Hash[key] = &Hash{hash, line} } } } @@ -140,8 +157,8 @@ func (ck *distinfoLinesChecker) checkGlobalMismatch(line *Line, fname, alg, hash func (ck *distinfoLinesChecker) checkUncommittedPatch(line *Line, patchName, sha1Hash string) { if ck.isPatch { patchFname := ck.patchdir + "/" + patchName - if ck.distinfoIsCommitted && !isCommitted(G.currentDir+"/"+patchFname) { - line.warnf("%s is registered in distinfo but not added to CVS.", patchFname) + if ck.distinfoIsCommitted && !isCommitted(G.CurrentDir+"/"+patchFname) { + line.Warn1("%s is registered in distinfo but not added to CVS.", patchFname) } ck.checkPatchSha1(line, patchFname, sha1Hash) ck.patches[patchName] = true diff --git a/pkgtools/pkglint/files/distinfo_test.go b/pkgtools/pkglint/files/distinfo_test.go index c01fb45ba27..0f59e9b8c3b 100644 --- a/pkgtools/pkglint/files/distinfo_test.go +++ b/pkgtools/pkglint/files/distinfo_test.go @@ -8,27 +8,32 @@ func (s *Suite) TestChecklinesDistinfo(c *check.C) { s.CreateTmpFile(c, "patches/patch-aa", ""+ "$"+"NetBSD$ line is ignored\n"+ "patch contents\n") - G.currentDir = s.tmpdir + s.CreateTmpFile(c, "patches/patch-ab", ""+ + "patch contents\n") + G.CurrentDir = s.tmpdir - checklinesDistinfo(s.NewLines("distinfo", + ChecklinesDistinfo(s.NewLines("distinfo", "should be the RCS ID", "should be empty", "MD5 (distfile.tar.gz) = 12345678901234567890123456789012", "SHA1 (distfile.tar.gz) = 1234567890123456789012345678901234567890", - "SHA1 (patch-aa) = 6b98dd609f85a9eb9c4c1e4e7055a6aaa62b7cc7")) + "SHA1 (patch-aa) = 6b98dd609f85a9eb9c4c1e4e7055a6aaa62b7cc7", + "SHA1 (patch-ab) = 6b98dd609f85a9eb9c4c1e4e7055a6aaa62b7cc7", + "SHA1 (patch-nonexistent) = 1234")) c.Check(s.Output(), equals, ""+ "ERROR: distinfo:1: Expected \"$"+"NetBSD$\".\n"+ "NOTE: distinfo:2: Empty line expected.\n"+ - "ERROR: distinfo:5: Expected SHA1, RMD160, SHA512, Size checksums for \"distfile.tar.gz\", got MD5, SHA1.\n") + "ERROR: distinfo:5: Expected SHA1, RMD160, SHA512, Size checksums for \"distfile.tar.gz\", got MD5, SHA1.\n"+ + "WARN: distinfo:7: Patch file \"patch-nonexistent\" does not exist in directory \"patches\".\n") } func (s *Suite) TestChecklinesDistinfo_GlobalHashMismatch(c *check.C) { - otherLine := NewLine("other/distinfo", "7", "dummy", nil) - G.ipcDistinfo = make(map[string]*Hash) - G.ipcDistinfo["SHA512:pkgname-1.0.tar.gz"] = &Hash{"asdfasdf", otherLine} + otherLine := NewLine("other/distinfo", 7, "dummy", nil) + G.Hash = make(map[string]*Hash) + G.Hash["SHA512:pkgname-1.0.tar.gz"] = &Hash{"asdfasdf", otherLine} - checklinesDistinfo(s.NewLines("distinfo", + ChecklinesDistinfo(s.NewLines("distinfo", "$"+"NetBSD$", "", "SHA512 (pkgname-1.0.tar.gz) = 12341234")) @@ -50,23 +55,24 @@ func (s *Suite) TestChecklinesDistinfo_UncommittedPatch(c *check.C) { "+new\n") s.CreateTmpFile(c, "CVS/Entries", "/distinfo/...\n") - G.currentDir = s.tmpdir + G.CurrentDir = s.tmpdir - checklinesDistinfo(s.NewLines(s.tmpdir+"/distinfo", + ChecklinesDistinfo(s.NewLines(s.tmpdir+"/distinfo", "$"+"NetBSD$", "", "SHA1 (patch-aa) = 5ad1fb9b3c328fff5caa1a23e8f330e707dd50c0")) - c.Check(s.Output(), equals, "WARN: "+s.tmpdir+"/distinfo:3: patches/patch-aa is registered in distinfo but not added to CVS.\n") + c.Check(s.OutputCleanTmpdir(), equals, ""+ + "WARN: ~/distinfo:3: patches/patch-aa is registered in distinfo but not added to CVS.\n") } func (s *Suite) TestChecklinesDistinfo_UnrecordedPatches(c *check.C) { s.CreateTmpFile(c, "patches/CVS/Entries", "") s.CreateTmpFile(c, "patches/patch-aa", "") s.CreateTmpFile(c, "patches/patch-src-Makefile", "") - G.currentDir = s.tmpdir + G.CurrentDir = s.tmpdir - checklinesDistinfo(s.NewLines(s.tmpdir+"/distinfo", + ChecklinesDistinfo(s.NewLines(s.tmpdir+"/distinfo", "$"+"NetBSD$", "", "SHA1 (distfile.tar.gz) = ...", @@ -74,8 +80,7 @@ func (s *Suite) TestChecklinesDistinfo_UnrecordedPatches(c *check.C) { "SHA512 (distfile.tar.gz) = ...", "Size (distfile.tar.gz) = 1024 bytes")) - c.Check(s.Output(), equals, sprintf(""+ - "ERROR: %[1]s: patch \"patches/patch-aa\" is not recorded. Run \"%s makepatchsum\".\n"+ - "ERROR: %[1]s: patch \"patches/patch-src-Makefile\" is not recorded. Run \"%s makepatchsum\".\n", - s.tmpdir+"/distinfo", confMake)) + c.Check(s.OutputCleanTmpdir(), equals, ""+ + "ERROR: ~/distinfo: patch \"patches/patch-aa\" is not recorded. Run \""+confMake+" makepatchsum\".\n"+ + "ERROR: ~/distinfo: patch \"patches/patch-src-Makefile\" is not recorded. Run \""+confMake+" makepatchsum\".\n") } diff --git a/pkgtools/pkglint/files/expecter.go b/pkgtools/pkglint/files/expecter.go index d84273d333b..f7ceb9470d8 100644 --- a/pkgtools/pkglint/files/expecter.go +++ b/pkgtools/pkglint/files/expecter.go @@ -4,58 +4,90 @@ package main type Expecter struct { lines []*Line index int + m []string } func NewExpecter(lines []*Line) *Expecter { - return &Expecter{lines, 0} + return &Expecter{lines, 0, nil} } -func (ctx *Expecter) currentLine() *Line { - if ctx.index < len(ctx.lines) { - return ctx.lines[ctx.index] +func (exp *Expecter) CurrentLine() *Line { + if exp.index < len(exp.lines) { + return exp.lines[exp.index] } - return NewLine(ctx.lines[0].fname, "EOF", "", nil) // dummy + return NewLineEOF(exp.lines[0].Fname) } -func (ctx *Expecter) previousLine() *Line { - return ctx.lines[ctx.index-1] +func (exp *Expecter) PreviousLine() *Line { + return exp.lines[exp.index-1] } -func (ctx *Expecter) eof() bool { - return !(ctx.index < len(ctx.lines)) +func (exp *Expecter) EOF() bool { + return !(exp.index < len(exp.lines)) } -func (ctx *Expecter) advance() { - ctx.index++ + +func (exp *Expecter) Advance() bool { + exp.index++ + exp.m = nil + return true +} + +func (exp *Expecter) StepBack() { + exp.index-- } -func (ctx *Expecter) advanceIfMatches(re string) []string { - defer tracecall("Expecter.advanceIfMatches", ctx.currentLine().text, re)() +func (exp *Expecter) AdvanceIfMatches(re string) bool { + if G.opts.DebugTrace { + defer tracecall2(exp.CurrentLine().Text, re)() + } - if ctx.index < len(ctx.lines) { - if m := match(ctx.lines[ctx.index].text, re); m != nil { - ctx.index++ - return m + if !exp.EOF() { + if m := match(exp.lines[exp.index].Text, re); m != nil { + exp.index++ + exp.m = m + return true } } - return nil + return false } -func (ctx *Expecter) expectEmptyLine() bool { - if ctx.advanceIfMatches(`^$`) != nil { +func (exp *Expecter) AdvanceIfPrefix(prefix string) bool { + if G.opts.DebugTrace { + defer tracecall2(exp.CurrentLine().Text, prefix)() + } + + return !exp.EOF() && hasPrefix(exp.lines[exp.index].Text, prefix) && exp.Advance() +} + +func (exp *Expecter) AdvanceIfEquals(text string) bool { + if G.opts.DebugTrace { + defer tracecall2(exp.CurrentLine().Text, text)() + } + + return !exp.EOF() && exp.lines[exp.index].Text == text && exp.Advance() +} + +func (exp *Expecter) ExpectEmptyLine() bool { + if exp.AdvanceIfEquals("") { return true } - _ = G.opts.WarnSpace && ctx.currentLine().notef("Empty line expected.") + if G.opts.WarnSpace { + if !exp.CurrentLine().AutofixInsertBefore("") { + exp.CurrentLine().Note0("Empty line expected.") + } + } return false } -func (ctx *Expecter) expectText(text string) bool { - if ctx.index < len(ctx.lines) && ctx.lines[ctx.index].text == text { - ctx.index++ +func (exp *Expecter) ExpectText(text string) bool { + if !exp.EOF() && exp.lines[exp.index].Text == text { + exp.index++ + exp.m = nil return true } - ctx.currentLine().warnf("This line should contain the following text: %s", text) + exp.CurrentLine().Warn1("This line should contain the following text: %s", text) return false } diff --git a/pkgtools/pkglint/files/files.go b/pkgtools/pkglint/files/files.go index ef1cda33461..7006aee75bc 100644 --- a/pkgtools/pkglint/files/files.go +++ b/pkgtools/pkglint/files/files.go @@ -3,17 +3,18 @@ package main import ( "io/ioutil" "os" + "strconv" "strings" ) func LoadNonemptyLines(fname string, joinContinuationLines bool) []*Line { lines, err := readLines(fname, joinContinuationLines) if err != nil { - errorf(fname, noLines, "Cannot be read.") + Errorf(fname, noLines, "Cannot be read.") return nil } if len(lines) == 0 { - errorf(fname, noLines, "Must not be empty.") + Errorf(fname, noLines, "Must not be empty.") return nil } return lines @@ -22,18 +23,28 @@ func LoadNonemptyLines(fname string, joinContinuationLines bool) []*Line { func LoadExistingLines(fname string, foldBackslashLines bool) []*Line { lines, err := readLines(fname, foldBackslashLines) if err != nil { - fatalf(fname, noLines, "Cannot be read.") + Fatalf(fname, noLines, "Cannot be read.") } if lines == nil { - fatalf(fname, noLines, "Must not be empty.") + Fatalf(fname, noLines, "Must not be empty.") } return lines } func getLogicalLine(fname string, rawLines []*RawLine, pindex *int) *Line { + { // Handle the common case efficiently + index := *pindex + rawLine := rawLines[*pindex] + textnl := rawLine.textnl + if hasSuffix(textnl, "\n") && !hasSuffix(textnl, "\\\n") { + *pindex = index + 1 + return NewLine(fname, rawLine.Lineno, textnl[:len(textnl)-1], rawLines[index:index+1]) + } + } + text := "" index := *pindex - firstlineno := rawLines[index].lineno + firstlineno := rawLines[index].Lineno var lineRawLines []*RawLine interestingRawLines := rawLines[index:] @@ -46,7 +57,7 @@ func getLogicalLine(fname string, rawLines []*RawLine, pindex *int) *Line { text += rawText lineRawLines = append(lineRawLines, rawLine) - if cont == "\\" && i != len(interestingRawLines)-1 { + if cont != "" && i != len(interestingRawLines)-1 { text += " " index++ } else { @@ -55,22 +66,40 @@ func getLogicalLine(fname string, rawLines []*RawLine, pindex *int) *Line { } } - lastlineno := rawLines[index].lineno + lastlineno := rawLines[index].Lineno *pindex = index + 1 - if firstlineno == lastlineno { - return NewLine(fname, sprintf("%d", firstlineno), text, lineRawLines) - } else { - return NewLine(fname, sprintf("%d--%d", firstlineno, lastlineno), text, lineRawLines) - } + return NewLineMulti(fname, firstlineno, lastlineno, text, lineRawLines) } func splitRawLine(textnl string) (leadingWhitespace, text, trailingWhitespace, cont string) { - m1234 := strings.TrimSuffix(textnl, "\n") - m234 := strings.TrimLeft(m1234, " \t") - m23 := strings.TrimSuffix(m234, "\\") - m2 := strings.TrimRight(m23, " \t") - return m1234[:len(m1234)-len(m234)], m2, m23[len(m2):], m234[len(m23):] + i, m := 0, len(textnl) + + if m > i && textnl[m-1] == '\n' { + m-- + } + + if m > i && textnl[m-1] == '\\' { + m-- + cont = textnl[m : m+1] + } + + trailingEnd := m + for m > i && (textnl[m-1] == ' ' || textnl[m-1] == '\t') { + m-- + } + trailingStart := m + trailingWhitespace = textnl[trailingStart:trailingEnd] + + leadingStart := i + for i < m && (textnl[i] == ' ' || textnl[i] == '\t') { + i++ + } + leadingEnd := i + leadingWhitespace = textnl[leadingStart:leadingEnd] + + text = textnl[leadingEnd:trailingStart] + return } func readLines(fname string, joinContinuationLines bool) ([]*Line, error) { @@ -86,7 +115,7 @@ func convertToLogicalLines(fname string, rawText string, joinContinuationLines b var rawLines []*RawLine for lineno, rawLine := range strings.SplitAfter(rawText, "\n") { if rawLine != "" { - rawLines = append(rawLines, &RawLine{1 + lineno, rawLine}) + rawLines = append(rawLines, &RawLine{1 + lineno, rawLine, rawLine}) } } @@ -97,19 +126,26 @@ func convertToLogicalLines(fname string, rawText string, joinContinuationLines b } } else { for _, rawLine := range rawLines { - loglines = append(loglines, NewLine(fname, sprintf("%d", rawLine.lineno), strings.TrimSuffix(rawLine.textnl, "\n"), []*RawLine{rawLine})) + text := strings.TrimSuffix(rawLine.textnl, "\n") + logline := NewLine(fname, rawLine.Lineno, text, []*RawLine{rawLine}) + loglines = append(loglines, logline) } } if 0 < len(rawLines) && !hasSuffix(rawLines[len(rawLines)-1].textnl, "\n") { - errorf(fname, sprintf("%d", rawLines[len(rawLines)-1].lineno), "File must end with a newline.") + Errorf(fname, strconv.Itoa(rawLines[len(rawLines)-1].Lineno), "File must end with a newline.") } return loglines } -func saveAutofixChanges(lines []*Line) { +func SaveAutofixChanges(lines []*Line) (autofixed bool) { if !G.opts.Autofix { + for _, line := range lines { + if line.changed { + G.autofixAvailable = true + } + } return } @@ -117,9 +153,9 @@ func saveAutofixChanges(lines []*Line) { changed := make(map[string]bool) for _, line := range lines { if line.changed { - changed[line.fname] = true + changed[line.Fname] = true } - changes[line.fname] = append(changes[line.fname], line.rawLines()...) + changes[line.Fname] = append(changes[line.Fname], line.rawLines()...) } for fname := range changed { @@ -129,16 +165,18 @@ func saveAutofixChanges(lines []*Line) { for _, rawLine := range rawLines { text += rawLine.textnl } - err := ioutil.WriteFile(tmpname, []byte(text), 0777) + err := ioutil.WriteFile(tmpname, []byte(text), 0666) if err != nil { - errorf(tmpname, noLines, "Cannot write.") + Errorf(tmpname, noLines, "Cannot write.") continue } err = os.Rename(tmpname, fname) if err != nil { - errorf(fname, noLines, "Cannot overwrite with auto-fixed content.") + Errorf(fname, noLines, "Cannot overwrite with auto-fixed content.") continue } - notef(fname, noLines, "Has been auto-fixed. Please re-run pkglint.") + autofixf(fname, noLines, "Has been auto-fixed. Please re-run pkglint.") + autofixed = true } + return } diff --git a/pkgtools/pkglint/files/files_test.go b/pkgtools/pkglint/files/files_test.go index 0c3a2c4881c..57bfb5bf801 100644 --- a/pkgtools/pkglint/files/files_test.go +++ b/pkgtools/pkglint/files/files_test.go @@ -2,8 +2,6 @@ package main import ( check "gopkg.in/check.v1" - "io/ioutil" - "path/filepath" ) func (s *Suite) TestConvertToLogicalLines_nocont(c *check.C) { @@ -58,27 +56,41 @@ func (s *Suite) TestSplitRawLine(c *check.C) { c.Check(continuation, equals, "\\") } -func (s *Suite) TestAutofix(c *check.C) { +func (s *Suite) TestAutofix_show(c *check.C) { s.UseCommandLine(c, "--show-autofix") - tmpdir := c.MkDir() - fname := filepath.ToSlash(tmpdir + "/Makefile") - lines := s.NewLines(fname, - "line1", - "line2", - "line3") - lines[1].replaceRegex(`.`, "X") - - saveAutofixChanges(lines) - - c.Assert(fileExists(fname), equals, false) - c.Check(s.Output(), equals, "NOTE: "+fname+":2: Autofix: replacing regular expression \".\" with \"X\".\n") + fname := s.CreateTmpFile(c, "Makefile", ""+ + "line1\n"+ + "line2\n"+ + "line3\n") + lines := LoadExistingLines(fname, true) + + if !lines[1].AutofixReplaceRegexp(`.`, "X") { + lines[1].Warn0("Something's wrong here.") // Prints the autofix NOTE afterwards + } + SaveAutofixChanges(lines) + + c.Check(lines[1].raw[0].textnl, equals, "XXXXX\n") + c.Check(s.LoadTmpFile(c, "Makefile"), equals, "line1\nline2\nline3\n") + c.Check(s.OutputCleanTmpdir(), equals, ""+ + "WARN: ~/Makefile:2: Something's wrong here.\n"+ + "AUTOFIX: ~/Makefile:2: Replacing regular expression \".\" with \"X\".\n") +} +func (s *Suite) TestAutofix_fix(c *check.C) { s.UseCommandLine(c, "--autofix") - - saveAutofixChanges(lines) - - content, err := ioutil.ReadFile(fname) - c.Assert(err, check.IsNil) - c.Check(string(content), equals, "line1\nXXXXX\nline3\n") - c.Check(s.Output(), equals, "NOTE: "+fname+": Has been auto-fixed. Please re-run pkglint.\n") + fname := s.CreateTmpFile(c, "Makefile", ""+ + "line1\n"+ + "line2\n"+ + "line3\n") + lines := LoadExistingLines(fname, true) + + if !lines[1].AutofixReplaceRegexp(`.`, "X") { + lines[1].Warn0("Something's wrong here.") // Prints the autofix NOTE afterwards + } + SaveAutofixChanges(lines) + + c.Check(s.LoadTmpFile(c, "Makefile"), equals, "line1\nXXXXX\nline3\n") + c.Check(s.OutputCleanTmpdir(), equals, ""+ + "AUTOFIX: ~/Makefile:2: Replacing regular expression \".\" with \"X\".\n"+ + "AUTOFIX: ~/Makefile: Has been auto-fixed. Please re-run pkglint.\n") } diff --git a/pkgtools/pkglint/files/getopt.go b/pkgtools/pkglint/files/getopt.go index 884cee58eff..7f3eae73366 100644 --- a/pkgtools/pkglint/files/getopt.go +++ b/pkgtools/pkglint/files/getopt.go @@ -19,6 +19,8 @@ func NewOptions() *Options { } func (o *Options) AddFlagGroup(shortName rune, longName, argDescription, description string) *FlagGroup { + switch { // prevent inlining + } grp := new(FlagGroup) opt := &option{shortName, longName, argDescription, description, grp} o.options = append(o.options, opt) @@ -26,6 +28,8 @@ func (o *Options) AddFlagGroup(shortName rune, longName, argDescription, descrip } func (o *Options) AddFlagVar(shortName rune, longName string, pflag *bool, defval bool, description string) { + switch { // prevent inlining + } *pflag = defval opt := &option{shortName, longName, "", description, pflag} o.options = append(o.options, opt) @@ -123,14 +127,14 @@ optchar: case *FlagGroup: argarg := optchars[ai+utf8.RuneLen(optchar):] if argarg != "" { - return 0, data.parse(sprintf("-%c", optchar), argarg) + return 0, data.parse(string([]rune{'-', optchar}), argarg) } else { - return 1, data.parse(sprintf("-%c", optchar), args[i+1]) + return 1, data.parse(string([]rune{'-', optchar}), args[i+1]) } } } } - return 0, optErr(sprintf("unknown option: -%c", optchar)) + return 0, optErr("unknown option: -" + string([]rune{optchar})) } return 0, nil } @@ -138,8 +142,8 @@ optchar: func (o *Options) Help(out io.Writer, generalUsage string) { wr := tabwriter.NewWriter(out, 1, 0, 2, ' ', tabwriter.TabIndent) - fmt.Fprintf(wr, "usage: %s\n", generalUsage) - fmt.Fprintln(wr) + io.WriteString(wr, "usage: "+generalUsage+"\n") + io.WriteString(wr, "\n") wr.Flush() for _, opt := range o.options { @@ -158,10 +162,10 @@ func (o *Options) Help(out io.Writer, generalUsage string) { switch flagGroup := opt.data.(type) { case *FlagGroup: hasFlagGroups = true - fmt.Fprintln(wr) + io.WriteString(wr, "\n") fmt.Fprintf(wr, " Flags for -%c, --%s:\n", opt.shortName, opt.longName) - fmt.Fprintf(wr, " all\t all of the following\n") - fmt.Fprintf(wr, " none\t none of the following\n") + io.WriteString(wr, " all\t all of the following\n") + io.WriteString(wr, " none\t none of the following\n") for _, flag := range flagGroup.flags { fmt.Fprintf(wr, " %s\t %s (%v)\n", flag.name, flag.help, ifelseStr(*flag.value, "enabled", "disabled")) } @@ -169,8 +173,8 @@ func (o *Options) Help(out io.Writer, generalUsage string) { } } if hasFlagGroups { - fmt.Fprintln(wr) - fmt.Fprint(wr, " (Prefix a flag with \"no-\" to disable it.)\n") + io.WriteString(wr, "\n") + io.WriteString(wr, " (Prefix a flag with \"no-\" to disable it.)\n") wr.Flush() } } @@ -188,6 +192,8 @@ type FlagGroup struct { } func (fg *FlagGroup) AddFlagVar(name string, flag *bool, defval bool, help string) { + switch { // prevent inlining + } opt := &groupFlag{name, flag, help} fg.flags = append(fg.flags, opt) *flag = defval diff --git a/pkgtools/pkglint/files/globaldata.go b/pkgtools/pkglint/files/globaldata.go index 5d6ae10f4a3..adb55f0c447 100644 --- a/pkgtools/pkglint/files/globaldata.go +++ b/pkgtools/pkglint/files/globaldata.go @@ -4,56 +4,57 @@ import ( "io/ioutil" "path" "sort" + "strings" ) // GlobalData contains data describing pkgsrc as a whole. type GlobalData struct { - pkgsrcdir string // Relative to the current working directory. - masterSiteUrls map[string]string // "https://github.com/" => "MASTER_SITE_GITHUB" - masterSiteVars map[string]bool // "MASTER_SITE_GITHUB" => true - pkgOptions map[string]string // "x11" => "Provides X11 support" - tools map[string]bool // Known tool names, e.g. "sed" and "gm4". - vartools map[string]string // Maps tool names to their respective variable, e.g. "sed" => "SED", "gzip" => "GZIP_CMD". - predefinedTools map[string]bool // Tools that a package does not need to add to USE_TOOLS explicitly because they are used by the pkgsrc infrastructure, too. - varnameToToolname map[string]string // Maps the tool variable names to the tool name they use, e.g. "GZIP_CMD" => "gzip" and "SED" => "sed". - systemBuildDefs map[string]bool // The set of user-defined variables that are added to BUILD_DEFS within the bsd.pkg.mk file. + Pkgsrcdir string // Relative to the current working directory. + MasterSiteUrls map[string]string // "https://github.com/" => "MASTER_SITE_GITHUB" + MasterSiteVars map[string]bool // "MASTER_SITE_GITHUB" => true + PkgOptions map[string]string // "x11" => "Provides X11 support" + Tools map[string]bool // Known tool names, e.g. "sed" and "gm4". + Vartools map[string]string // Maps tool names to their respective variable, e.g. "sed" => "SED", "gzip" => "GZIP_CMD". + PredefinedTools map[string]bool // Tools that a package does not need to add to USE_TOOLS explicitly because they are used by the pkgsrc infrastructure, too. + VarnameToToolname map[string]string // Maps the tool variable names to the tool name they use, e.g. "GZIP_CMD" => "gzip" and "SED" => "sed". + SystemBuildDefs map[string]bool // The set of user-defined variables that are added to BUILD_DEFS within the bsd.pkg.mk file. toolvarsVarRequired map[string]bool // Tool variable names that may not be converted to their "direct" form, that is: ${CP} may not be written as cp. - toolsVarRequired map[string]bool // Tools that need to be written in variable form, e.g. echo => ${ECHO}. + toolsVarRequired map[string]bool // Tools that need to be written in variable form, e.g. "echo"; see Vartools. suggestedUpdates []SuggestedUpdate // suggestedWipUpdates []SuggestedUpdate // - lastChange map[string]*Change // - userDefinedVars map[string]*Line // varname => line (after calling parselineMk on it) - deprecated map[string]string // + LastChange map[string]*Change // + UserDefinedVars map[string]*MkLine // varname => line + Deprecated map[string]string // vartypes map[string]*Vartype // varcanon => type } // Change is a change entry from the `doc/CHANGES-*` files. type Change struct { - line *Line - action string - pkgpath string - version string - author string - date string + Line *Line + Action string + Pkgpath string + Version string + Author string + Date string } // SuggestedUpdate is from the `doc/TODO` file. type SuggestedUpdate struct { - line *Line - pkgname string - version string - comment string + Line *Line + Pkgname string + Version string + Comment string } func (gd *GlobalData) Initialize() { - firstArg := G.todo[0] + firstArg := G.Todo[0] if fileExists(firstArg) { firstArg = path.Dir(firstArg) } if relTopdir := findPkgsrcTopdir(firstArg); relTopdir != "" { - gd.pkgsrcdir = firstArg + "/" + relTopdir + gd.Pkgsrcdir = firstArg + "/" + relTopdir } else { - dummyLine.fatalf("%q is not inside a pkgsrc tree.", firstArg) + dummyLine.Fatalf("%q is not inside a pkgsrc tree.", firstArg) } gd.vartypes = make(map[string]*Vartype) @@ -64,17 +65,17 @@ func (gd *GlobalData) Initialize() { gd.loadSuggestedUpdates() gd.loadUserDefinedVars() gd.loadTools() - gd.deprecated = getDeprecatedVars() + gd.loadDeprecatedVars() } func (gd *GlobalData) loadDistSites() { - fname := gd.pkgsrcdir + "/mk/fetch/sites.mk" + fname := gd.Pkgsrcdir + "/mk/fetch/sites.mk" lines := LoadExistingLines(fname, true) names := make(map[string]bool) url2name := make(map[string]string) for _, line := range lines { - if m, varname, _, urls, _ := matchVarassign(line.text); m { + if m, varname, _, urls, _ := MatchVarassign(line.Text); m { if hasPrefix(varname, "MASTER_SITE_") && varname != "MASTER_SITE_BACKUP" { names[varname] = true for _, url := range splitOnSpace(urls) { @@ -90,21 +91,23 @@ func (gd *GlobalData) loadDistSites() { names["MASTER_SITE_SUSE_UPD"] = true names["MASTER_SITE_LOCAL"] = true - _ = G.opts.DebugMisc && debugf(fname, noLines, "Loaded %d MASTER_SITE_* URLs.", len(url2name)) - gd.masterSiteUrls = url2name - gd.masterSiteVars = names + if G.opts.DebugMisc { + Debugf(fname, noLines, "Loaded %d MASTER_SITE_* URLs.", len(url2name)) + } + gd.MasterSiteUrls = url2name + gd.MasterSiteVars = names } func (gd *GlobalData) loadPkgOptions() { - fname := gd.pkgsrcdir + "/mk/defaults/options.description" + fname := gd.Pkgsrcdir + "/mk/defaults/options.description" lines := LoadExistingLines(fname, false) - gd.pkgOptions = make(map[string]string) + gd.PkgOptions = make(map[string]string) for _, line := range lines { - if m, optname, optdescr := match2(line.text, `^([-0-9a-z_+]+)(?:\s+(.*))?$`); m { - gd.pkgOptions[optname] = optdescr + if m, optname, optdescr := match2(line.Text, `^([-0-9a-z_+]+)(?:\s+(.*))?$`); m { + gd.PkgOptions[optname] = optdescr } else { - line.fatalf("Unknown line format.") + line.Fatalf("Unknown line format.") } } } @@ -112,10 +115,10 @@ func (gd *GlobalData) loadPkgOptions() { func (gd *GlobalData) loadTools() { toolFiles := []string{"defaults.mk"} { - fname := G.globalData.pkgsrcdir + "/mk/tools/bsd.tools.mk" + fname := G.globalData.Pkgsrcdir + "/mk/tools/bsd.tools.mk" lines := LoadExistingLines(fname, true) for _, line := range lines { - if m, _, includefile := match2(line.text, reMkInclude); m { + if m, _, includefile := match2(line.Text, reMkInclude); m { if !contains(includefile, "/") { toolFiles = append(toolFiles, includefile) } @@ -123,7 +126,7 @@ func (gd *GlobalData) loadTools() { } } if len(toolFiles) <= 1 { - fatalf(toolFiles[0], noLines, "Too few tool files.") + Fatalf(toolFiles[0], noLines, "Too few tool files.") } tools := make(map[string]bool) @@ -133,10 +136,10 @@ func (gd *GlobalData) loadTools() { systemBuildDefs := make(map[string]bool) for _, basename := range toolFiles { - fname := G.globalData.pkgsrcdir + "/mk/tools/" + basename + fname := G.globalData.Pkgsrcdir + "/mk/tools/" + basename lines := LoadExistingLines(fname, true) for _, line := range lines { - if m, varname, _, value, _ := matchVarassign(line.text); m { + if m, varname, _, value, _ := MatchVarassign(line.Text); m { if varname == "TOOLS_CREATE" && (value == "[" || matches(value, `^?[-\w.]+$`)) { tools[value] = true } else if m, toolname := match1(varname, `^(?:_TOOLS_VARNAME)\.([-\w.]+|\[)$`); m { @@ -159,16 +162,18 @@ func (gd *GlobalData) loadTools() { { basename := "bsd.pkg.mk" - fname := G.globalData.pkgsrcdir + "/mk/" + basename + fname := G.globalData.Pkgsrcdir + "/mk/" + basename condDepth := 0 lines := LoadExistingLines(fname, true) for _, line := range lines { - text := line.text + text := line.Text - if m, varname, _, value, _ := matchVarassign(text); m { + if m, varname, _, value, _ := MatchVarassign(text); m { if varname == "USE_TOOLS" { - _ = G.opts.DebugTools && line.debugf("[condDepth=%d] %s", condDepth, value) + if G.opts.DebugTools { + line.Debugf("[condDepth=%d] %s", condDepth, value) + } if condDepth == 0 { for _, tool := range splitOnSpace(value) { if !containsVarRef(tool) && tools[tool] { @@ -184,15 +189,11 @@ func (gd *GlobalData) loadTools() { } } - } else if m, _, cond, _ := match3(text, reMkCond); m { + } else if m, _, cond, _ := matchMkCond(text); m { switch cond { - case "if": - case "ifdef": - case "ifndef": - case "for": + case "if", "ifdef", "ifndef", "for": condDepth++ - case "endif": - case "endfor": + case "endif", "endfor": condDepth-- } } @@ -200,12 +201,14 @@ func (gd *GlobalData) loadTools() { } if G.opts.DebugTools { - dummyLine.debugf("tools: %v", stringBoolMapKeys(tools)) - dummyLine.debugf("vartools: %v", stringStringMapKeys(vartools)) - dummyLine.debugf("predefinedTools: %v", stringBoolMapKeys(predefinedTools)) - dummyLine.debugf("varnameToToolname: %v", stringStringMapKeys(varnameToToolname)) + dummyLine.Debugf("tools: %v", stringBoolMapKeys(tools)) + dummyLine.Debugf("vartools: %v", stringStringMapKeys(vartools)) + dummyLine.Debugf("predefinedTools: %v", stringBoolMapKeys(predefinedTools)) + dummyLine.Debugf("varnameToToolname: %v", stringStringMapKeys(varnameToToolname)) + } + if G.opts.DebugMisc { + dummyLine.Debugf("systemBuildDefs: %v", systemBuildDefs) } - _ = G.opts.DebugMisc && dummyLine.debugf("systemBuildDefs: %v", systemBuildDefs) // Some user-defined variables do not influence the binary // package at all and therefore do not have to be added to @@ -221,11 +224,11 @@ func (gd *GlobalData) loadTools() { systemBuildDefs["GAMEOWN"] = true systemBuildDefs["GAMEGRP"] = true - gd.tools = tools - gd.vartools = vartools - gd.predefinedTools = predefinedTools - gd.varnameToToolname = varnameToToolname - gd.systemBuildDefs = systemBuildDefs + gd.Tools = tools + gd.Vartools = vartools + gd.PredefinedTools = predefinedTools + gd.VarnameToToolname = varnameToToolname + gd.SystemBuildDefs = systemBuildDefs gd.toolvarsVarRequired = map[string]bool{ "ECHO": true, "ECHO_N": true, @@ -250,7 +253,7 @@ func parselinesSuggestedUpdates(lines []*Line) []SuggestedUpdate { var updates []SuggestedUpdate state := 0 for _, line := range lines { - text := line.text + text := line.Text if state == 0 && text == "Suggested package updates" { state = 1 @@ -267,10 +270,10 @@ func parselinesSuggestedUpdates(lines []*Line) []SuggestedUpdate { if m, pkgbase, pkgversion := match2(pkgname, rePkgname); m { updates = append(updates, SuggestedUpdate{line, pkgbase, pkgversion, comment}) } else { - line.warnf("Invalid package name %q", pkgname) + line.Warn1("Invalid package name %q", pkgname) } } else { - line.warnf("Invalid line format %q", text) + line.Warn1("Invalid line format %q", text) } } } @@ -278,47 +281,60 @@ func parselinesSuggestedUpdates(lines []*Line) []SuggestedUpdate { } func (gd *GlobalData) loadSuggestedUpdates() { - gd.suggestedUpdates = loadSuggestedUpdates(G.globalData.pkgsrcdir + "/doc/TODO") - if wipFilename := G.globalData.pkgsrcdir + "/wip/TODO"; fileExists(wipFilename) { + gd.suggestedUpdates = loadSuggestedUpdates(G.globalData.Pkgsrcdir + "/doc/TODO") + if wipFilename := G.globalData.Pkgsrcdir + "/wip/TODO"; fileExists(wipFilename) { gd.suggestedWipUpdates = loadSuggestedUpdates(wipFilename) } } -func (gd *GlobalData) loadDocChangesFromFile(fname string) []Change { +func (gd *GlobalData) loadDocChangesFromFile(fname string) []*Change { lines := LoadExistingLines(fname, false) - var changes []Change - for _, line := range lines { - text := line.text - if !(hasPrefix(text, "\t") && matches(text, `^\t[A-Z]`)) { - continue + parseChange := func(line *Line) *Change { + text := line.Text + if !hasPrefix(line.Text, "\t") { + return nil } - if m, action, pkgpath, version, author, date := match5(text, `^\t(Updated) (\S+) to (\S+) \[(\S+) (\d\d\d\d-\d\d-\d\d)\]$`); m { - changes = append(changes, Change{line, action, pkgpath, version, author, date}) - - } else if m, action, pkgpath, version, author, date := match5(text, `^\t(Added) (\S+) version (\S+) \[(\S+) (\d\d\d\d-\d\d-\d\d)\]$`); m { - changes = append(changes, Change{line, action, pkgpath, version, author, date}) - - } else if m, action, pkgpath, author, date := match4(text, `^\t(Removed) (\S+) (?:successor \S+ )?\[(\S+) (\d\d\d\d-\d\d-\d\d)\]$`); m { - changes = append(changes, Change{line, action, pkgpath, "", author, date}) - - } else if m, action, pkgpath, version, author, date := match5(text, `^\t(Downgraded) (\S+) to (\S+) \[(\S+) (\d\d\d\d-\d\d-\d\d)\]$`); m { - changes = append(changes, Change{line, action, pkgpath, version, author, date}) + f := strings.Fields(text) + n := len(f) + if n != 4 && n != 6 { + return nil + } - } else if m, action, pkgpath, version, author, date := match5(text, `^\t(Renamed|Moved) (\S+) to (\S+) \[(\S+) (\d\d\d\d-\d\d-\d\d)\]$`); m { - changes = append(changes, Change{line, action, pkgpath, version, author, date}) + action, pkgpath, author, date := f[0], f[1], f[len(f)-2], f[len(f)-1] + if !hasPrefix(author, "[") || !hasSuffix(date, "]") { + return nil + } + author, date = author[1:], date[:len(date)-1] + + switch { + case action == "Added" && f[2] == "version" && n == 6: + return &Change{line, action, pkgpath, f[3], author, date} + case (action == "Updated" || action == "Downgraded") && f[2] == "to" && n == 6: + return &Change{line, action, pkgpath, f[3], author, date} + case action == "Removed" && (n == 6 && f[2] == "successor" || n == 4): + return &Change{line, action, pkgpath, "", author, date} + case (action == "Renamed" || action == "Moved") && f[2] == "to" && n == 6: + return &Change{line, action, pkgpath, "", author, date} + } + return nil + } - } else { - line.warnf("Unknown doc/CHANGES line: %q", text) - line.explain("See mk/misc/developer.mk for the rules.") + var changes []*Change + for _, line := range lines { + if change := parseChange(line); change != nil { + changes = append(changes, change) + } else if len(line.Text) >= 2 && line.Text[0] == '\t' && 'A' <= line.Text[1] && line.Text[1] <= 'Z' { + line.Warn1("Unknown doc/CHANGES line: %q", line.Text) + Explain1("See mk/misc/developer.mk for the rules.") } } return changes } -func (gd *GlobalData) getSuggestedPackageUpdates() []SuggestedUpdate { - if G.isWip { +func (gd *GlobalData) GetSuggestedPackageUpdates() []SuggestedUpdate { + if G.Wip { return gd.suggestedWipUpdates } else { return gd.suggestedUpdates @@ -326,10 +342,10 @@ func (gd *GlobalData) getSuggestedPackageUpdates() []SuggestedUpdate { } func (gd *GlobalData) loadDocChanges() { - docdir := G.globalData.pkgsrcdir + "/doc" + docdir := G.globalData.Pkgsrcdir + "/doc" files, err := ioutil.ReadDir(docdir) if err != nil { - fatalf(docdir, noLines, "Cannot be read.") + Fatalf(docdir, noLines, "Cannot be read.") } var fnames []string @@ -341,24 +357,180 @@ func (gd *GlobalData) loadDocChanges() { } sort.Strings(fnames) - gd.lastChange = make(map[string]*Change) + gd.LastChange = make(map[string]*Change) for _, fname := range fnames { changes := gd.loadDocChangesFromFile(docdir + "/" + fname) for _, change := range changes { - c := change - gd.lastChange[change.pkgpath] = &c + gd.LastChange[change.Pkgpath] = change } } } func (gd *GlobalData) loadUserDefinedVars() { - lines := LoadExistingLines(G.globalData.pkgsrcdir+"/mk/defaults/mk.conf", true) + lines := LoadExistingLines(G.globalData.Pkgsrcdir+"/mk/defaults/mk.conf", true) + mklines := NewMkLines(lines) - gd.userDefinedVars = make(map[string]*Line) - for _, line := range lines { - parselineMk(line) - if varname, ok := line.extra["varname"].(string); ok { - gd.userDefinedVars[varname] = line + gd.UserDefinedVars = make(map[string]*MkLine) + for _, mkline := range mklines.mklines { + if mkline.IsVarassign() { + gd.UserDefinedVars[mkline.Varname()] = mkline } } } + +func (gd *GlobalData) loadDeprecatedVars() { + gd.Deprecated = map[string]string{ + + // December 2003 + "FIX_RPATH": "It has been removed from pkgsrc in 2003.", + + // February 2005 + "LIB_DEPENDS": "Use DEPENDS instead.", + "ONLY_FOR_ARCHS": "Use ONLY_FOR_PLATFORM instead.", + "NOT_FOR_ARCHS": "Use NOT_FOR_PLATFORM instead.", + "ONLY_FOR_OPSYS": "Use ONLY_FOR_PLATFORM instead.", + "NOT_FOR_OPSYS": "Use NOT_FOR_PLATFORM instead.", + + // May 2005 + "ALL_TARGET": "Use BUILD_TARGET instead.", + "DIGEST_FILE": "Use DISTINFO_FILE instead.", + "IGNORE": "Use PKG_FAIL_REASON or PKG_SKIP_REASON instead.", + "IS_INTERACTIVE": "Use INTERACTIVE_STAGE instead.", + "KERBEROS": "Use the PKG_OPTIONS framework instead.", + "MASTER_SITE_SUBDIR": "Use some form of MASTER_SITES instead.", + "MD5_FILE": "Use DISTINFO_FILE instead.", + "MIRROR_DISTFILE": "Use NO_BIN_ON_FTP and/or NO_SRC_ON_FTP instead.", + "NO_CDROM": "Use NO_BIN_ON_CDROM and/or NO_SRC_ON_CDROM instead.", + "NO_PATCH": "You can just remove it.", + "NO_WRKSUBDIR": "Use WRKSRC=${WRKDIR} instead.", + "PATCH_SITE_SUBDIR": "Use some form of PATCHES_SITES instead.", + "PATCH_SUM_FILE": "Use DISTINFO_FILE instead.", + "PKG_JVM": "Use PKG_DEFAULT_JVM instead.", + "USE_BUILDLINK2": "You can just remove it.", + "USE_BUILDLINK3": "You can just remove it.", + "USE_CANNA": "Use the PKG_OPTIONS framework instead.", + "USE_DB4": "Use the PKG_OPTIONS framework instead.", + "USE_DIRS": "You can just remove it.", + "USE_ESOUND": "Use the PKG_OPTIONS framework instead.", + "USE_GIF": "Use the PKG_OPTIONS framework instead.", + "USE_GMAKE": "Use USE_TOOLS+=gmake instead.", + "USE_GNU_TOOLS": "Use USE_TOOLS instead.", + "USE_IDEA": "Use the PKG_OPTIONS framework instead.", + "USE_LIBCRACK": "Use the PKG_OPTIONS framework instead.", + "USE_MMX": "Use the PKG_OPTIONS framework instead.", + "USE_PKGLIBTOOL": "Use USE_LIBTOOL instead.", + "USE_SSL": "Include \"../../security/openssl/buildlink3.mk\" instead.", + + // July 2005 + "USE_PERL5": "Use USE_TOOLS+=perl or USE_TOOLS+=perl:run instead.", + + // October 2005 + "NO_TOOLS": "You can just remove it.", + "NO_WRAPPER": "You can just remove it.", + + // November 2005 + "ALLFILES": "Use CKSUMFILES instead.", + "DEPENDS_TARGET": "Use DEPENDS instead.", + "FETCH_DEPENDS": "Use DEPENDS instead.", + "RUN_DEPENDS": "Use DEPENDS instead.", + + // December 2005 + "USE_CUPS": "Use the PKG_OPTIONS framework (option cups) instead.", + "USE_I586": "Use the PKG_OPTIONS framework (option i586) instead.", + "USE_INN": "Use the PKG_OPTIONS framework instead.", + "USE_OPENLDAP": "Use the PKG_OPTIONS framework (option openldap) instead.", + "USE_OSS": "Use the PKG_OPTIONS framework (option oss) instead.", + "USE_RSAREF2": "Use the PKG_OPTIONS framework (option rsaref) instead.", + "USE_SASL": "Use the PKG_OPTIONS framework (option sasl) instead.", + "USE_SASL2": "Use the PKG_OPTIONS framework (option sasl) instead.", + "USE_SJ3": "Use the PKG_OPTIONS framework (option sj3) instead.", + "USE_SOCKS": "Use the PKG_OPTIONS framework (socks4 and socks5 options) instead.", + "USE_WNN4": "Use the PKG_OPTIONS framework (option wnn4) instead.", + "USE_XFACE": "Use the PKG_OPTIONS framework instead.", + + // February 2006 + "TOOLS_DEPMETHOD": "Use the :build or :run modifiers in USE_TOOLS instead.", + "MANDIR": "Please use ${PREFIX}/${PKGMANDIR} instead.", + "DOWNLOADED_DISTFILE": "Use the shell variable $$extract_file instead.", + "DECOMPRESS_CMD": "Use EXTRACT_CMD instead.", + + // March 2006 + "INSTALL_EXTRA_TMPL": "Use INSTALL_TEMPLATE instead.", + "DEINSTALL_EXTRA_TMPL": "Use DEINSTALL_TEMPLATE instead.", + + // April 2006 + "RECOMMENDED": "Use ABI_DEPENDS instead.", + "BUILD_USES_MSGFMT": "Use USE_TOOLS+=msgfmt instead.", + "USE_MSGFMT_PLURALS": "Use USE_TOOLS+=msgfmt instead.", + + // May 2006 + "EXTRACT_USING_PAX": "Use \"EXTRACT_OPTS=-t pax\" instead.", + "NO_EXTRACT": "It doesn't exist anymore.", + "_FETCH_MESSAGE": "Use FETCH_MESSAGE (different format) instead.", + "BUILDLINK_DEPENDS.*": "Use BUILDLINK_API_DEPENDS.* instead.", + "BUILDLINK_RECOMMENDED.*": "Use BUILDLINK_ABI_DEPENDS.* instead.", + "SHLIB_HANDLING": "Use CHECK_SHLIBS_SUPPORTED instead.", + "USE_RMAN": "It has been removed.", + + // June 2006 + "DEINSTALL_SRC": "Use the pkginstall framework instead.", + "INSTALL_SRC": "Use the pkginstall framework instead.", + "DEINSTALL_TEMPLATE": "Use DEINSTALL_TEMPLATES instead.", + "INSTALL_TEMPLATE": "Use INSTALL_TEMPLATES instead.", + "HEADER_TEMPLATE": "Use HEADER_TEMPLATES instead.", + "_REPLACE.*": "Use REPLACE.* instead.", + "_REPLACE_FILES.*": "Use REPLACE_FILES.* instead.", + "MESSAGE": "Use MESSAGE_SRC instead.", + "INSTALL_FILE": "It may only be used internally by pkgsrc.", + "DEINSTALL_FILE": "It may only be used internally by pkgsrc.", + + // July 2006 + "USE_DIGEST": "You can just remove it.", + "LTCONFIG_OVERRIDE": "You can just remove it.", + "USE_GNU_GETTEXT": "You can just remove it.", + "BUILD_ENV": "Use PKGSRC_MAKE_ENV instead.", + "DYNAMIC_MASTER_SITES": "You can just remove it.", + + // September 2006 + "MAKEFILE": "Use MAKE_FILE instead.", + + // November 2006 + "SKIP_PORTABILITY_CHECK": "Use CHECK_PORTABILITY_SKIP (a list of patterns) instead.", + "PKG_SKIP_REASON": "Use PKG_FAIL_REASON instead.", + + // January 2007 + "BUILDLINK_TRANSFORM.*": "Use BUILDLINK_FNAME_TRANSFORM.* instead.", + + // March 2007 + "SCRIPTDIR": "You can just remove it.", + "NO_PKG_REGISTER": "You can just remove it.", + "NO_DEPENDS": "You can just remove it.", + + // October 2007 + "_PKG_SILENT": "Use RUN (with more error checking) instead.", + "_PKG_DEBUG": "Use RUN (with more error checking) instead.", + "LICENCE": "Use LICENSE instead.", + + // November 2007 + //USE_NCURSES Include "../../devel/ncurses/buildlink3.mk" instead. + + // December 2007 + "INSTALLATION_DIRS_FROM_PLIST": "Use AUTO_MKDIRS instead.", + + // April 2009 + "NO_PACKAGE": "It doesn't exist anymore.", + "NO_MTREE": "You can just remove it.", + + // July 2012 + "SETGIDGAME": "Use USE_GAMESGROUP instead.", + "GAMEGRP": "Use GAMES_GROUP instead.", + "GAMEOWN": "Use GAMES_USER instead.", + + // July 2013 + "USE_GNU_READLINE": "Include \"../../devel/readline/buildlink3.mk\" instead.", + + // October 2014 + "SVR4_PKGNAME": "Just remove it.", + "PKG_INSTALLATION_TYPES": "Just remove it.", + } +} diff --git a/pkgtools/pkglint/files/globaldata_test.go b/pkgtools/pkglint/files/globaldata_test.go index cc80a557cb4..c5b6483be3e 100644 --- a/pkgtools/pkglint/files/globaldata_test.go +++ b/pkgtools/pkglint/files/globaldata_test.go @@ -45,9 +45,9 @@ func (s *Suite) TestGlobalData_LoadTools(c *check.C) { "USE_TOOLS+=msgfmt\n"+ "TOOLS_CREATE+=msgfmt\n") s.CreateTmpFile(c, "mk/bsd.pkg.mk", "# empty\n") - G.globalData.pkgsrcdir = s.tmpdir - G.currentDir = s.tmpdir - G.curPkgsrcdir = "." + G.globalData.Pkgsrcdir = s.tmpdir + G.CurrentDir = s.tmpdir + G.CurPkgsrcdir = "." G.globalData.loadTools() @@ -57,3 +57,34 @@ func (s *Suite) TestGlobalData_LoadTools(c *check.C) { "DEBUG: predefinedTools: []\n"+ "DEBUG: varnameToToolname: [AWK CHOWN MV]\n") } + +func (s *Suite) TestGlobalData_loadDocChanges(c *check.C) { + s.CreateTmpFile(c, "doc/CHANGES-2015", ""+ + "\tAdded category/package version 1.0 [author1 2015-01-01]\n"+ + "\tUpdated category/package to 1.5 [author2 2015-01-02]\n"+ + "\tRenamed category/package to category/pkg [author3 2015-01-03]\n"+ + "\tMoved category/package to other/package [author4 2015-01-04]\n"+ + "\tRemoved category/package [author5 2015-01-05]\n"+ + "\tRemoved category/package successor category/package2 [author6 2015-01-06]\n"+ + "\tDowngraded category/package to 1.2 [author7 2015-01-07]\n") + + changes := G.globalData.loadDocChangesFromFile(s.tmpdir + "/doc/CHANGES-2015") + + c.Assert(len(changes), equals, 7) + c.Check(*changes[0], equals, Change{changes[0].Line, "Added", "category/package", "1.0", "author1", "2015-01-01"}) + c.Check(*changes[1], equals, Change{changes[1].Line, "Updated", "category/package", "1.5", "author2", "2015-01-02"}) + c.Check(*changes[2], equals, Change{changes[2].Line, "Renamed", "category/package", "", "author3", "2015-01-03"}) + c.Check(*changes[3], equals, Change{changes[3].Line, "Moved", "category/package", "", "author4", "2015-01-04"}) + c.Check(*changes[4], equals, Change{changes[4].Line, "Removed", "category/package", "", "author5", "2015-01-05"}) + c.Check(*changes[5], equals, Change{changes[5].Line, "Removed", "category/package", "", "author6", "2015-01-06"}) + c.Check(*changes[6], equals, Change{changes[6].Line, "Downgraded", "category/package", "1.2", "author7", "2015-01-07"}) +} + +func (s *Suite) TestGlobalData_deprecated(c *check.C) { + G.globalData.loadDeprecatedVars() + + line := NewLine("Makefile", 5, "USE_PERL5=\tyes", nil) + NewMkLine(line).CheckVarassign() + + c.Check(s.Output(), equals, "WARN: Makefile:5: Definition of USE_PERL5 is deprecated. Use USE_TOOLS+=perl or USE_TOOLS+=perl:run instead.\n") +} diff --git a/pkgtools/pkglint/files/globalvars.go b/pkgtools/pkglint/files/globalvars.go index 2bab3cdf38d..51dc69905e5 100644 --- a/pkgtools/pkglint/files/globalvars.go +++ b/pkgtools/pkglint/files/globalvars.go @@ -6,19 +6,20 @@ import ( ) type GlobalVars struct { - opts CmdOpts - globalData GlobalData - pkgContext *PkgContext - mkContext *MkContext + opts CmdOpts // + globalData GlobalData // + Pkg *Package // The package that is currently checked. + Mk *MkLines // The Makefile (or fragment) that is currently checked. - todo []string // The items that still need to be checked. - currentDir string // The currently checked directory, relative to the cwd - curPkgsrcdir string // The pkgsrc directory, relative to currentDir - isWip bool // Is the currently checked directory from pkgsrc-wip? - isInfrastructure bool // Is the currently checked item from the pkgsrc infrastructure? + Todo []string // The files or directories that still need to be checked. + CurrentDir string // The currently checked directory, relative to the cwd + CurPkgsrcdir string // The pkgsrc directory, relative to currentDir + Wip bool // Is the currently checked directory from pkgsrc-wip? + Infrastructure bool // Is the currently checked item from the pkgsrc infrastructure? + TestingData *TestingData // Is pkglint in self-testing mode (only during development)? - ipcDistinfo map[string]*Hash // Maps "alg:fname" => "checksum". - ipcUsedLicenses map[string]bool // Maps "license name" => true + Hash map[string]*Hash // Maps "alg:fname" => hash (inter-package check). + UsedLicenses map[string]bool // Maps "license name" => true (inter-package check). errors int warnings int @@ -28,11 +29,13 @@ type GlobalVars struct { traceDepth int logOut io.Writer logErr io.Writer - traceOut io.Writer + debugOut io.Writer res map[string]*regexp.Regexp rematch *Histogram renomatch *Histogram + retime *Histogram + loghisto *Histogram } type CmdOpts struct { @@ -94,4 +97,8 @@ type Hash struct { line *Line } -var G *GlobalVars +type TestingData struct { + VerifiedBits map[string]bool +} + +var G GlobalVars diff --git a/pkgtools/pkglint/files/licenses.go b/pkgtools/pkglint/files/licenses.go index d04ca062263..b4459d414ae 100644 --- a/pkgtools/pkglint/files/licenses.go +++ b/pkgtools/pkglint/files/licenses.go @@ -12,41 +12,41 @@ func parseLicenses(licenses string) []string { } func checktoplevelUnusedLicenses() { - if G.ipcUsedLicenses == nil { + if G.UsedLicenses == nil { return } - licensedir := G.globalData.pkgsrcdir + "/licenses" + licensedir := G.globalData.Pkgsrcdir + "/licenses" files, _ := ioutil.ReadDir(licensedir) for _, licensefile := range files { licensename := licensefile.Name() licensepath := licensedir + "/" + licensename if fileExists(licensepath) { - if !G.ipcUsedLicenses[licensename] { - warnf(licensepath, noLines, "This license seems to be unused.") + if !G.UsedLicenses[licensename] { + Warnf(licensepath, noLines, "This license seems to be unused.") } } } } -func checklineLicense(line *Line, value string) { +func checklineLicense(line *MkLine, value string) { licenses := parseLicenses(value) for _, license := range licenses { var licenseFile string - if pkg := G.pkgContext; pkg != nil { - if licenseFileValue, ok := pkg.varValue("LICENSE_FILE"); ok { - licenseFile = G.currentDir + "/" + resolveVarsInRelativePath(licenseFileValue, false) + if G.Pkg != nil { + if licenseFileValue, ok := G.Pkg.varValue("LICENSE_FILE"); ok { + licenseFile = G.CurrentDir + "/" + resolveVarsInRelativePath(licenseFileValue, false) } } if licenseFile == "" { - licenseFile = G.globalData.pkgsrcdir + "/licenses/" + license - if G.ipcUsedLicenses != nil { - G.ipcUsedLicenses[license] = true + licenseFile = G.globalData.Pkgsrcdir + "/licenses/" + license + if G.UsedLicenses != nil { + G.UsedLicenses[license] = true } } if !fileExists(licenseFile) { - line.warnf("License file %s does not exist.", cleanpath(licenseFile)) + line.Warn1("License file %s does not exist.", cleanpath(licenseFile)) } switch license { @@ -55,7 +55,7 @@ func checklineLicense(line *Line, value string) { "no-profit", "no-redistribution", "shareware": - line.warnf("License %q is deprecated.", license) + line.Warn1("License %q is deprecated.", license) } } } diff --git a/pkgtools/pkglint/files/licenses_test.go b/pkgtools/pkglint/files/licenses_test.go index 912b8469b05..1452876529b 100644 --- a/pkgtools/pkglint/files/licenses_test.go +++ b/pkgtools/pkglint/files/licenses_test.go @@ -8,3 +8,26 @@ func (s *Suite) TestParseLicenses(c *check.C) { c.Check(parseLicenses("gnu-gpl-v2"), check.DeepEquals, []string{"gnu-gpl-v2"}) c.Check(parseLicenses("AND artistic"), check.DeepEquals, []string{"artistic"}) } + +func (s *Suite) TestChecklineLicense(c *check.C) { + s.CreateTmpFile(c, "licenses/gnu-gpl-v2", "Most software \u2026") + mkline := NewMkLine(NewLine("Makefile", 7, "LICENSE=dummy", nil)) + G.globalData.Pkgsrcdir = s.tmpdir + G.CurrentDir = s.tmpdir + + checklineLicense(mkline, "gpl-v2") + + c.Check(s.OutputCleanTmpdir(), equals, "WARN: Makefile:7: License file ~/licenses/gpl-v2 does not exist.\n") + + checklineLicense(mkline, "no-profit shareware") + + c.Check(s.OutputCleanTmpdir(), equals, ""+ + "WARN: Makefile:7: License file ~/licenses/no-profit does not exist.\n"+ + "WARN: Makefile:7: License \"no-profit\" is deprecated.\n"+ + "WARN: Makefile:7: License file ~/licenses/shareware does not exist.\n"+ + "WARN: Makefile:7: License \"shareware\" is deprecated.\n") + + checklineLicense(mkline, "gnu-gpl-v2") + + c.Check(s.Output(), equals, "") +} diff --git a/pkgtools/pkglint/files/line.go b/pkgtools/pkglint/files/line.go index 153277db4a9..4cefd8024b7 100644 --- a/pkgtools/pkglint/files/line.go +++ b/pkgtools/pkglint/files/line.go @@ -16,137 +16,216 @@ package main import ( "fmt" "io" + "strconv" "strings" ) type RawLine struct { - lineno int + Lineno int + orignl string textnl string } func (rline *RawLine) String() string { - return sprintf("%d:%s", rline.lineno, rline.textnl) + return strconv.Itoa(rline.Lineno) + ":" + rline.textnl } type Line struct { - fname string - lines string - text string - raw []*RawLine - changed bool - before []*RawLine - after []*RawLine - extra map[string]interface{} + Fname string + firstLine int32 // Zero means not applicable, -1 means EOF + lastLine int32 // Usually the same as firstLine, may differ in Makefiles + Text string + raw []*RawLine + changed bool + before []*RawLine + after []*RawLine + autofixMessage *string } -func NewLine(fname, linenos, text string, rawLines []*RawLine) *Line { - return &Line{fname, linenos, text, rawLines, false, nil, nil, make(map[string]interface{})} +func NewLine(fname string, lineno int, text string, rawLines []*RawLine) *Line { + return NewLineMulti(fname, lineno, lineno, text, rawLines) } -func (ln *Line) rawLines() []*RawLine { - return append(append(append([]*RawLine(nil), ln.before...), ln.raw...), ln.after...) +// NewLineMulti is for logical Makefile lines that end with backslash. +func NewLineMulti(fname string, firstLine, lastLine int, text string, rawLines []*RawLine) *Line { + return &Line{fname, int32(firstLine), int32(lastLine), text, rawLines, false, nil, nil, nil} } -func (ln *Line) printSource(out io.Writer) { +// NewLineEOF creates a dummy line for logging, with the “line number” EOF. +func NewLineEOF(fname string) *Line { + return NewLineMulti(fname, -1, 0, "", nil) +} + +func (line *Line) rawLines() []*RawLine { + switch { // prevent inlining + } + return append(append(append([]*RawLine(nil), line.before...), line.raw...), line.after...) +} + +func (line *Line) linenos() string { + switch { + case line.firstLine == -1: + return "EOF" + case line.firstLine == 0: + return "" + case line.firstLine == line.lastLine: + return strconv.Itoa(int(line.firstLine)) + default: + return strconv.Itoa(int(line.firstLine)) + "--" + strconv.Itoa(int(line.lastLine)) + } +} + +func (line *Line) ReferenceFrom(other *Line) string { + if line.Fname != other.Fname { + return line.Fname + ":" + line.linenos() + } + return "line " + line.linenos() +} + +func (line *Line) IsMultiline() bool { + return line.firstLine > 0 && line.firstLine != line.lastLine +} + +func (line *Line) printSource(out io.Writer) { if G.opts.PrintSource { io.WriteString(out, "\n") - for _, rawLine := range ln.rawLines() { - fmt.Fprintf(out, "> %s", rawLine.textnl) + for _, rawLine := range line.rawLines() { + if rawLine.textnl != rawLine.orignl { + if rawLine.orignl != "" { + io.WriteString(out, "- "+rawLine.orignl) + } + if rawLine.textnl != "" { + io.WriteString(out, "+ "+rawLine.textnl) + } + } else { + io.WriteString(out, "> "+rawLine.orignl) + } } } } -func (ln *Line) fatalf(format string, args ...interface{}) { - ln.printSource(G.logErr) - fatalf(ln.fname, ln.lines, format, args...) -} -func (ln *Line) errorf(format string, args ...interface{}) bool { - ln.printSource(G.logOut) - return errorf(ln.fname, ln.lines, format, args...) +func (line *Line) Fatalf(format string, args ...interface{}) { + line.printSource(G.logErr) + Fatalf(line.Fname, line.linenos(), format, args...) } -func (ln *Line) warnf(format string, args ...interface{}) bool { - ln.printSource(G.logOut) - return warnf(ln.fname, ln.lines, format, args...) + +func (line *Line) Errorf(format string, args ...interface{}) { + line.printSource(G.logOut) + Errorf(line.Fname, line.linenos(), format, args...) + line.logAutofix() } -func (ln *Line) notef(format string, args ...interface{}) bool { - ln.printSource(G.logOut) - return notef(ln.fname, ln.lines, format, args...) +func (line *Line) Error0(format string) { line.Errorf(format) } +func (line *Line) Error1(format, arg1 string) { line.Errorf(format, arg1) } +func (line *Line) Error2(format, arg1, arg2 string) { line.Errorf(format, arg1, arg2) } + +func (line *Line) Warnf(format string, args ...interface{}) { + line.printSource(G.logOut) + Warnf(line.Fname, line.linenos(), format, args...) + line.logAutofix() } -func (ln *Line) debugf(format string, args ...interface{}) bool { - ln.printSource(G.logOut) - return debugf(ln.fname, ln.lines, format, args...) +func (line *Line) Warn0(format string) { line.Warnf(format) } +func (line *Line) Warn1(format, arg1 string) { line.Warnf(format, arg1) } +func (line *Line) Warn2(format, arg1, arg2 string) { line.Warnf(format, arg1, arg2) } + +func (line *Line) Notef(format string, args ...interface{}) { + line.printSource(G.logOut) + Notef(line.Fname, line.linenos(), format, args...) + line.logAutofix() } +func (line *Line) Note0(format string) { line.Notef(format) } +func (line *Line) Note1(format, arg1 string) { line.Notef(format, arg1) } +func (line *Line) Note2(format, arg1, arg2 string) { line.Notef(format, arg1, arg2) } -func (ln *Line) explain(explanation ...string) { - if G.opts.Explain { - complete := strings.Join(explanation, "\n") - if G.explanationsGiven[complete] { - return - } - if G.explanationsGiven == nil { - G.explanationsGiven = make(map[string]bool) - G.explanationsGiven[complete] = true - } +func (line *Line) Debugf(format string, args ...interface{}) { + line.printSource(G.logOut) + Debugf(line.Fname, line.linenos(), format, args...) + line.logAutofix() +} +func (line *Line) Debug1(format, arg1 string) { line.Debugf(format, arg1) } +func (line *Line) Debug2(format, arg1, arg2 string) { line.Debugf(format, arg1, arg2) } - io.WriteString(G.logOut, "\n") - for _, explanationLine := range explanation { - io.WriteString(G.logOut, "\t"+explanationLine+"\n") - } - io.WriteString(G.logOut, "\n") - } - G.explanationsAvailable = true +func (line *Line) String() string { + return line.Fname + ":" + line.linenos() + ": " + line.Text } -func (ln *Line) String() string { - return ln.fname + ":" + ln.lines + ": " + ln.text +func (line *Line) logAutofix() { + if line.autofixMessage != nil { + autofixf(line.Fname, line.linenos(), "%s", *line.autofixMessage) + line.autofixMessage = nil + } } -func (ln *Line) insertBefore(line string) { - ln.before = append(ln.before, &RawLine{0, line + "\n"}) - ln.noteAutofix("Autofix: inserting a line %q before this line.", line) +func (line *Line) AutofixInsertBefore(text string) bool { + if G.opts.PrintAutofix || G.opts.Autofix { + line.before = append(line.before, &RawLine{0, "", text + "\n"}) + } + return line.RememberAutofix("Inserting a line %q before this line.", text) } -func (ln *Line) insertAfter(line string) { - ln.after = append(ln.after, &RawLine{0, line + "\n"}) - ln.noteAutofix("Autofix: inserting a line %q after this line.", line) +func (line *Line) AutofixInsertAfter(text string) bool { + if G.opts.PrintAutofix || G.opts.Autofix { + line.after = append(line.after, &RawLine{0, "", text + "\n"}) + } + return line.RememberAutofix("Inserting a line %q after this line.", text) } -func (ln *Line) delete() { - ln.raw = nil - ln.changed = true +func (line *Line) AutofixDelete() bool { + if G.opts.PrintAutofix || G.opts.Autofix { + for _, rawLine := range line.raw { + rawLine.textnl = "" + } + } + return line.RememberAutofix("Deleting this line.") } -func (ln *Line) replace(from, to string) { - for _, rawLine := range ln.raw { - if rawLine.lineno != 0 { +func (line *Line) AutofixReplace(from, to string) bool { + for _, rawLine := range line.raw { + if rawLine.Lineno != 0 { if replaced := strings.Replace(rawLine.textnl, from, to, 1); replaced != rawLine.textnl { - rawLine.textnl = replaced - ln.noteAutofix("Autofix: replacing %q with %q.", from, to) + if G.opts.PrintAutofix || G.opts.Autofix { + rawLine.textnl = replaced + } + return line.RememberAutofix("Replacing %q with %q.", from, to) } } } + return false } -func (ln *Line) replaceRegex(from, to string) { - for _, rawLine := range ln.raw { - if rawLine.lineno != 0 { + +func (line *Line) AutofixReplaceRegexp(from, to string) bool { + for _, rawLine := range line.raw { + if rawLine.Lineno != 0 { if replaced := regcomp(from).ReplaceAllString(rawLine.textnl, to); replaced != rawLine.textnl { - rawLine.textnl = replaced - ln.noteAutofix("Autofix: replacing regular expression %q with %q.", from, to) + if G.opts.PrintAutofix || G.opts.Autofix { + rawLine.textnl = replaced + } + return line.RememberAutofix("Replacing regular expression %q with %q.", from, to) } } } + return false } -func (ln *Line) noteAutofix(format string, args ...interface{}) { - ln.changed = true - if G.opts.Autofix || G.opts.PrintAutofix { - ln.notef(format, args...) +func (line *Line) RememberAutofix(format string, args ...interface{}) (hasBeenFixed bool) { + if line.firstLine < 1 { + return false } - G.autofixAvailable = true + line.changed = true + if G.opts.Autofix { + autofixf(line.Fname, line.linenos(), format, args...) + return true + } + if G.opts.PrintAutofix { + msg := fmt.Sprintf(format, args...) + line.autofixMessage = &msg + } + return false } -func (ln *Line) checkAbsolutePathname(text string) { - defer tracecall("Line.checkAbsolutePathname", text)() +func (line *Line) CheckAbsolutePathname(text string) { + if G.opts.DebugTrace { + defer tracecall1(text)() + } // In the GNU coding standards, DESTDIR is defined as a (usually // empty) prefix that can be used to install files to a different @@ -157,7 +236,58 @@ func (ln *Line) checkAbsolutePathname(text string) { // assignments like "bindir=/bin". if m, path := match1(text, `(?:^|\$[{(]DESTDIR[)}]|[\w_]+\s*=\s*)(/(?:[^"'\s]|"[^"*]"|'[^']*')*)`); m { if matches(path, `^/\w`) { - checkwordAbsolutePathname(ln, path) + checkwordAbsolutePathname(line, path) } } } + +func (line *Line) CheckLength(maxlength int) { + if len(line.Text) > maxlength { + line.Warnf("Line too long (should be no more than %d characters).", maxlength) + Explain3( + "Back in the old time, terminals with 80x25 characters were common.", + "And this is still the default size of many terminal emulators.", + "Moderately short lines also make reading easier.") + } +} + +func (line *Line) CheckValidCharacters(reChar string) { + rest := regcomp(reChar).ReplaceAllString(line.Text, "") + if rest != "" { + uni := "" + for _, c := range rest { + uni += fmt.Sprintf(" %U", c) + } + line.Warn1("Line contains invalid characters (%s).", uni[1:]) + } +} + +func (line *Line) CheckTrailingWhitespace() { + if hasSuffix(line.Text, " ") || hasSuffix(line.Text, "\t") { + if !line.AutofixReplaceRegexp(`\s+\n$`, "\n") { + line.Note0("Trailing white-space.") + Explain2( + "When a line ends with some white-space, that space is in most cases", + "irrelevant and can be removed.") + } + } +} + +func (line *Line) CheckRcsid(prefixRe, suggestedPrefix string) bool { + if G.opts.DebugTrace { + defer tracecall2(prefixRe, suggestedPrefix)() + } + + if matches(line.Text, `^`+prefixRe+`\$`+`NetBSD(?::[^\$]+)?\$$`) { + return true + } + + if !line.AutofixInsertBefore(suggestedPrefix + "$" + "NetBSD$") { + line.Error1("Expected %q.", suggestedPrefix+"$"+"NetBSD$") + Explain3( + "Several files in pkgsrc must contain the CVS Id, so that their", + "current version can be traced back later from a binary package.", + "This is to ensure reproducible builds, for example for finding bugs.") + } + return false +} diff --git a/pkgtools/pkglint/files/line_test.go b/pkgtools/pkglint/files/line_test.go index afb5ddd69e2..864269fcfbf 100644 --- a/pkgtools/pkglint/files/line_test.go +++ b/pkgtools/pkglint/files/line_test.go @@ -5,56 +5,119 @@ import ( ) func (s *Suite) TestLineModify(c *check.C) { - line := NewLine("fname", "1", "dummy", []*RawLine{{1, "original\n"}}) + s.UseCommandLine(c, "--show-autofix") + + line := NewLine("fname", 1, "dummy", s.NewRawLines(1, "original\n")) c.Check(line.changed, equals, false) - c.Check(line.rawLines(), check.DeepEquals, []*RawLine{{1, "original\n"}}) + c.Check(line.rawLines(), check.DeepEquals, s.NewRawLines(1, "original\n")) - line.replaceRegex(`(.)(.*)(.)`, "$3$2$1") + line.AutofixReplaceRegexp(`(.)(.*)(.)`, "$3$2$1") c.Check(line.changed, equals, true) - c.Check(line.rawLines(), check.DeepEquals, []*RawLine{{1, "lriginao\n"}}) + c.Check(line.rawLines(), check.DeepEquals, s.NewRawLines(1, "original\n", "lriginao\n")) line.changed = false - line.replace("i", "u") + line.AutofixReplace("i", "u") c.Check(line.changed, equals, true) - c.Check(line.rawLines(), check.DeepEquals, []*RawLine{{1, "lruginao\n"}}) + c.Check(line.rawLines(), check.DeepEquals, s.NewRawLines(1, "original\n", "lruginao\n")) c.Check(line.raw[0].textnl, equals, "lruginao\n") line.changed = false - line.replace("lruginao", "middle") + line.AutofixReplace("lruginao", "middle") c.Check(line.changed, equals, true) - c.Check(line.rawLines(), check.DeepEquals, []*RawLine{{1, "middle\n"}}) + c.Check(line.rawLines(), check.DeepEquals, s.NewRawLines(1, "original\n", "middle\n")) c.Check(line.raw[0].textnl, equals, "middle\n") - line.insertBefore("before") - line.insertBefore("between before and middle") - line.insertAfter("between middle and after") - line.insertAfter("after") - - c.Check(line.rawLines(), check.DeepEquals, []*RawLine{ - {0, "before\n"}, - {0, "between before and middle\n"}, - {1, "middle\n"}, - {0, "between middle and after\n"}, - {0, "after\n"}}) - - line.delete() - - c.Check(line.rawLines(), check.DeepEquals, []*RawLine{ - {0, "before\n"}, - {0, "between before and middle\n"}, - {0, "between middle and after\n"}, - {0, "after\n"}}) + line.AutofixInsertBefore("before") + line.AutofixInsertBefore("between before and middle") + line.AutofixInsertAfter("between middle and after") + line.AutofixInsertAfter("after") + + c.Check(line.rawLines(), check.DeepEquals, s.NewRawLines( + 0, "", "before\n", + 0, "", "between before and middle\n", + 1, "original\n", "middle\n", + 0, "", "between middle and after\n", + 0, "", "after\n")) + + line.AutofixDelete() + + c.Check(line.rawLines(), check.DeepEquals, s.NewRawLines( + 0, "", "before\n", + 0, "", "between before and middle\n", + 1, "original\n", "", + 0, "", "between middle and after\n", + 0, "", "after\n")) } func (s *Suite) TestLine_CheckAbsolutePathname(c *check.C) { - line := NewLine("Makefile", "1", "# dummy", nil) + line := NewLine("Makefile", 1, "# dummy", nil) - line.checkAbsolutePathname("bindir=/bin") - line.checkAbsolutePathname("bindir=/../lib") + line.CheckAbsolutePathname("bindir=/bin") + line.CheckAbsolutePathname("bindir=/../lib") c.Check(s.Output(), equals, "WARN: Makefile:1: Found absolute pathname: /bin\n") } + +func (s *Suite) TestShowAutofix_replace(c *check.C) { + s.UseCommandLine(c, "--show-autofix", "--source") + line := NewLineMulti("Makefile", 27, 29, "# old", s.NewRawLines( + 27, "before\n", + 28, "The old song\n", + 29, "after\n")) + + if !line.AutofixReplace("old", "new") { + line.Warn0("Using \"old\" is deprecated.") + } + + c.Check(s.Output(), equals, ""+ + "\n"+ + "> before\n"+ + "- The old song\n"+ + "+ The new song\n"+ + "> after\n"+ + "WARN: Makefile:27--29: Using \"old\" is deprecated.\n"+ + "AUTOFIX: Makefile:27--29: Replacing \"old\" with \"new\".\n") +} + +func (s *Suite) TestShowAutofix_insert(c *check.C) { + s.UseCommandLine(c, "--show-autofix", "--source") + line := NewLine("Makefile", 30, "original", s.NewRawLines(30, "original\n")) + + if !line.AutofixInsertBefore("inserted") { + line.Warn0("Dummy") + } + + c.Check(s.Output(), equals, ""+ + "\n"+ + "+ inserted\n"+ + "> original\n"+ + "WARN: Makefile:30: Dummy\n"+ + "AUTOFIX: Makefile:30: Inserting a line \"inserted\" before this line.\n") +} + +func (s *Suite) TestShowAutofix_delete(c *check.C) { + s.UseCommandLine(c, "--show-autofix", "--source") + line := NewLine("Makefile", 30, "to be deleted", s.NewRawLines(30, "to be deleted\n")) + + if !line.AutofixDelete() { + line.Warn0("Dummy") + } + + c.Check(s.Output(), equals, ""+ + "\n"+ + "- to be deleted\n"+ + "WARN: Makefile:30: Dummy\n"+ + "AUTOFIX: Makefile:30: Deleting this line.\n") +} + +func (s *Suite) TestLine_CheckTrailingWhitespace(c *check.C) { + line := NewLine("Makefile", 32, "The line must go on ", nil) + + line.CheckTrailingWhitespace() + + c.Check(s.Output(), equals, "NOTE: Makefile:32: Trailing white-space.\n") +} diff --git a/pkgtools/pkglint/files/logging.go b/pkgtools/pkglint/files/logging.go index 48db8890f61..071470cb313 100644 --- a/pkgtools/pkglint/files/logging.go +++ b/pkgtools/pkglint/files/logging.go @@ -1,26 +1,29 @@ package main import ( + "fmt" "io" + "strings" ) const noFile = "" const noLines = "" type LogLevel struct { - traditionalName string - gccName string + TraditionalName string + GccName string } var ( - llFatal = &LogLevel{"FATAL", "fatal"} - llError = &LogLevel{"ERROR", "error"} - llWarn = &LogLevel{"WARN", "warning"} - llNote = &LogLevel{"NOTE", "note"} - llDebug = &LogLevel{"DEBUG", "debug"} + llFatal = &LogLevel{"FATAL", "fatal"} + llError = &LogLevel{"ERROR", "error"} + llWarn = &LogLevel{"WARN", "warning"} + llNote = &LogLevel{"NOTE", "note"} + llDebug = &LogLevel{"DEBUG", "debug"} + llAutofix = &LogLevel{"AUTOFIX", "autofix"} ) -var dummyLine = NewLine(noFile, noLines, "", nil) +var dummyLine = NewLine(noFile, 0, "", nil) func logf(out io.Writer, level *LogLevel, fname, lineno, format string, args ...interface{}) bool { if fname != noFile { @@ -29,7 +32,7 @@ func logf(out io.Writer, level *LogLevel, fname, lineno, format string, args ... var text, sep string if !G.opts.GccOutput { - text += sep + level.traditionalName + ":" + text += sep + level.TraditionalName + ":" sep = " " } if fname != noFile { @@ -40,31 +43,72 @@ func logf(out io.Writer, level *LogLevel, fname, lineno, format string, args ... } } if G.opts.GccOutput { - text += sep + level.gccName + ":" + text += sep + level.GccName + ":" sep = " " } - text += sep + sprintf(format, args...) + "\n" + if G.opts.Profiling { + G.loghisto.Add(format, 1) + } + text += sep + fmt.Sprintf(format, args...) + "\n" io.WriteString(out, text) return true } -func fatalf(fname, lineno, format string, args ...interface{}) { +func Fatalf(fname, lineno, format string, args ...interface{}) { logf(G.logErr, llFatal, fname, lineno, format, args...) panic(pkglintFatal{}) } -func errorf(fname, lineno, format string, args ...interface{}) bool { +func Errorf(fname, lineno, format string, args ...interface{}) bool { G.errors++ return logf(G.logOut, llError, fname, lineno, format, args...) } -func warnf(fname, lineno, format string, args ...interface{}) bool { +func Warnf(fname, lineno, format string, args ...interface{}) bool { G.warnings++ return logf(G.logOut, llWarn, fname, lineno, format, args...) } -func notef(fname, lineno, format string, args ...interface{}) bool { +func Notef(fname, lineno, format string, args ...interface{}) bool { return logf(G.logOut, llNote, fname, lineno, format, args...) } -func debugf(fname, lineno, format string, args ...interface{}) bool { - return logf(G.logOut, llDebug, fname, lineno, format, args...) +func autofixf(fname, lineno, format string, args ...interface{}) bool { + return logf(G.logOut, llAutofix, fname, lineno, format, args...) +} +func Debugf(fname, lineno, format string, args ...interface{}) bool { + return logf(G.debugOut, llDebug, fname, lineno, format, args...) +} + +func Explain(explanation ...string) { + if G.opts.Explain { + complete := strings.Join(explanation, "\n") + if G.explanationsGiven[complete] { + return + } + if G.explanationsGiven == nil { + G.explanationsGiven = make(map[string]bool) + } + G.explanationsGiven[complete] = true + + io.WriteString(G.logOut, "\n") + for _, explanationLine := range explanation { + io.WriteString(G.logOut, "\t"+explanationLine+"\n") + } + io.WriteString(G.logOut, "\n") + } else if G.TestingData != nil { + for _, s := range explanation { + if l := tabLength(s); l > 68 && contains(s, " ") { + print(fmt.Sprintf("Long explanation line (%d): %s\n", l, s)) + } + if m, before := match1(s, `(.+)\. [^ ]`); m { + if !matches(before, `\d$`) { + print(fmt.Sprintf("Short space after period: %s\n", s)) + } + } + } + } + G.explanationsAvailable = true } +func Explain1(e1 string) { Explain(e1) } +func Explain2(e1, e2 string) { Explain(e1, e2) } +func Explain3(e1, e2, e3 string) { Explain(e1, e2, e3) } +func Explain4(e1, e2, e3, e4 string) { Explain(e1, e2, e3, e4) } type pkglintFatal struct{} diff --git a/pkgtools/pkglint/files/main.go b/pkgtools/pkglint/files/main.go index 28005ae658c..4386416ba5f 100644 --- a/pkgtools/pkglint/files/main.go +++ b/pkgtools/pkglint/files/main.go @@ -12,14 +12,13 @@ const confMake = "@BMAKE@" const confVersion = "@VERSION@" func main() { - G = new(GlobalVars) - G.logOut, G.logErr, G.traceOut = os.Stdout, os.Stderr, os.Stdout + G.logOut, G.logErr, G.debugOut = os.Stdout, os.Stderr, os.Stdout os.Exit(new(Pkglint).Main(os.Args...)) } type Pkglint struct{} -func (p *Pkglint) Main(args ...string) (exitcode int) { +func (pkglint *Pkglint) Main(args ...string) (exitcode int) { defer func() { if r := recover(); r != nil { if _, ok := r.(pkglintFatal); ok { @@ -30,7 +29,7 @@ func (p *Pkglint) Main(args ...string) (exitcode int) { } }() - if exitcode := p.ParseCommandLine(args); exitcode != nil { + if exitcode := pkglint.ParseCommandLine(args); exitcode != nil { return *exitcode } @@ -42,34 +41,38 @@ func (p *Pkglint) Main(args ...string) (exitcode int) { if G.opts.Profiling { f, err := os.Create("pkglint.pprof") if err != nil { - dummyLine.fatalf("Cannot create profiling file: %s", err) + dummyLine.Fatalf("Cannot create profiling file: %s", err) } pprof.StartCPUProfile(f) defer pprof.StopCPUProfile() G.rematch = NewHistogram() G.renomatch = NewHistogram() + G.retime = NewHistogram() + G.loghisto = NewHistogram() } for _, arg := range G.opts.args { - G.todo = append(G.todo, filepath.ToSlash(arg)) + G.Todo = append(G.Todo, filepath.ToSlash(arg)) } - if len(G.todo) == 0 { - G.todo = []string{"."} + if len(G.Todo) == 0 { + G.Todo = []string{"."} } G.globalData.Initialize() - for len(G.todo) != 0 { - item := G.todo[0] - G.todo = G.todo[1:] + for len(G.Todo) != 0 { + item := G.Todo[0] + G.Todo = G.Todo[1:] CheckDirent(item) } checktoplevelUnusedLicenses() - printSummary() + pkglint.PrintSummary() if G.opts.Profiling { - G.rematch.printStats("rematch", G.logOut) - G.renomatch.printStats("renomatch", G.logOut) + G.loghisto.PrintStats("loghisto", G.logOut, 0) + G.rematch.PrintStats("rematch", G.logOut, 10) + G.renomatch.PrintStats("renomatch", G.logOut, 10) + G.retime.PrintStats("retime", G.logOut, 10) } if G.errors != 0 { return 1 @@ -77,7 +80,7 @@ func (p *Pkglint) Main(args ...string) (exitcode int) { return 0 } -func (p *Pkglint) ParseCommandLine(args []string) *int { +func (pkglint *Pkglint) ParseCommandLine(args []string) *int { gopts := &G.opts opts := NewOptions() @@ -152,20 +155,20 @@ func (p *Pkglint) ParseCommandLine(args []string) *int { return nil } -func printSummary() { +func (pkglint *Pkglint) PrintSummary() { if !G.opts.Quiet { if G.errors != 0 || G.warnings != 0 { fmt.Fprintf(G.logOut, "%d %s and %d %s found.\n", G.errors, ifelseStr(G.errors == 1, "error", "errors"), G.warnings, ifelseStr(G.warnings == 1, "warning", "warnings")) if G.explanationsAvailable && !G.opts.Explain { - fmt.Fprint(G.logOut, "(Run pkglint with the -e option to show explanations.)\n") + fmt.Fprint(G.logOut, "(Run \"pkglint -e\" to show explanations.)\n") } if G.autofixAvailable && !G.opts.PrintAutofix && !G.opts.Autofix { - fmt.Fprint(G.logOut, "(Run pkglint with the -f option to show what can be fixed automatically.)\n") + fmt.Fprint(G.logOut, "(Run \"pkglint -fs\" to show what can be fixed automatically.)\n") } if G.autofixAvailable && !G.opts.Autofix { - fmt.Fprint(G.logOut, "(Run pkglint with the -F option to automatically fix some issues.)\n") + fmt.Fprint(G.logOut, "(Run \"pkglint -F\" to automatically fix some issues.)\n") } } else { io.WriteString(G.logOut, "looks fine.\n") diff --git a/pkgtools/pkglint/files/main_test.go b/pkgtools/pkglint/files/main_test.go index 1881ebfbf7b..b33d0c9df4e 100644 --- a/pkgtools/pkglint/files/main_test.go +++ b/pkgtools/pkglint/files/main_test.go @@ -1,6 +1,8 @@ package main import ( + "os" + check "gopkg.in/check.v1" ) @@ -15,13 +17,24 @@ func (s *Suite) TestMainVersion(c *check.C) { exitcode := new(Pkglint).Main("pkglint", "--version") c.Check(exitcode, equals, 0) - c.Check(s.Output(), check.Matches, `(?:@VERSION@|\d+\.\d+)\n`) + c.Check(s.Output(), equals, confVersion+"\n") } func (s *Suite) TestMainNoArgs(c *check.C) { - defer s.ExpectFatalError(func() { - c.Check(s.Stderr(), equals, "FATAL: \".\" is not inside a pkgsrc tree.\n") - }) + exitcode := new(Pkglint).Main("pkglint") + + c.Check(exitcode, equals, 1) + c.Check(s.Stderr(), equals, "FATAL: \".\" is not inside a pkgsrc tree.\n") +} - new(Pkglint).Main("pkglint") +// go test -c -covermode count +// pkgsrcdir=... +// env PKGLINT_TESTCMDLINE="$pkgsrcdir -r" ./pkglint.test -test.coverprofile pkglint.cov -check.f TestRunPkglint +// go tool cover -html=pkglint.cov -o coverage.html +func (s *Suite) TestRunPkglint(c *check.C) { + cmdline := os.Getenv("PKGLINT_TESTCMDLINE") + if cmdline != "" { + G.logOut, G.logErr, G.debugOut = os.Stdout, os.Stderr, os.Stdout + new(Pkglint).Main(append([]string{"pkglint"}, splitOnSpace(cmdline)...)...) + } } diff --git a/pkgtools/pkglint/files/makefiles.go b/pkgtools/pkglint/files/makefiles.go deleted file mode 100644 index 318cb21d0b1..00000000000 --- a/pkgtools/pkglint/files/makefiles.go +++ /dev/null @@ -1,528 +0,0 @@ -package main - -import ( - "path" - "strings" -) - -const ( - reMkDependency = `^([^\s:]+(?:\s*[^\s:]+)*)(\s*):\s*([^#]*?)(?:\s*#.*)?$` - reMkSysinclude = `^\.\s*s?include\s+<([^>]+)>\s*(?:#.*)?$` -) - -func readMakefile(fname string, mainLines *[]*Line, allLines *[]*Line) bool { - defer tracecall("readMakefile", fname)() - - fileLines := LoadNonemptyLines(fname, true) - if fileLines == nil { - return false - } - - ParselinesMk(fileLines) - isMainMakefile := len(*mainLines) == 0 - - for _, line := range fileLines { - text := line.text - - if isMainMakefile { - *mainLines = append(*mainLines, line) - } - *allLines = append(*allLines, line) - - var includeFile, incDir, incBase string - if hasPrefix(text, ".") && hasSuffix(text, "\"") { - if m, inc := match1(text, `^\.\s*include\s+\"(.*)\"$`); m { - includeFile = resolveVariableRefs(resolveVarsInRelativePath(inc, true)) - if containsVarRef(includeFile) { - if !contains(fname, "/mk/") { - line.notef("Skipping include file %q. This may result in false warnings.", includeFile) - } - includeFile = "" - } - incDir, incBase = path.Split(includeFile) - } - } - - if includeFile != "" { - if path.Base(fname) != "buildlink3.mk" { - if m, bl3File := match1(includeFile, `^\.\./\.\./(.*)/buildlink3\.mk$`); m { - G.pkgContext.bl3[bl3File] = line - _ = G.opts.DebugMisc && line.debugf("Buildlink3 file in package: %q", bl3File) - } - } - } - - if includeFile != "" && G.pkgContext.included[includeFile] == nil { - G.pkgContext.included[includeFile] = line - - if matches(includeFile, `^\.\./[^./][^/]*/[^/]+`) { - line.warnf("References to other packages should look like \"../../category/package\", not \"../package\".") - explainRelativeDirs(line) - } - - if !hasPrefix(incDir, "../../mk/") && incBase != "buildlink3.mk" && incBase != "builtin.mk" && incBase != "options.mk" { - _ = G.opts.DebugInclude && line.debugf("Including %q sets seenMakefileCommon.", includeFile) - G.pkgContext.seenMakefileCommon = true - } - - if !contains(incDir, "/mk/") { - dirname, _ := path.Split(fname) - dirname = cleanpath(dirname) - - // Only look in the directory relative to the - // current file and in the current working directory. - // Pkglint doesn’t have an include dir list, like make(1) does. - if !fileExists(dirname + "/" + includeFile) { - dirname = G.currentDir - } - if !fileExists(dirname + "/" + includeFile) { - line.errorf("Cannot read %q.", dirname+"/"+includeFile) - return false - } - - _ = G.opts.DebugInclude && line.debugf("Including %q.", dirname+"/"+includeFile) - lengthBeforeInclude := len(*allLines) - if !readMakefile(dirname+"/"+includeFile, mainLines, allLines) { - return false - } - - if incBase == "Makefile.common" && incDir != "" { - makefileCommonLines := (*allLines)[lengthBeforeInclude:] - checkForUsedComment(makefileCommonLines, relpath(G.globalData.pkgsrcdir, fname)) - } - } - } - - if line.extra["is_varassign"] != nil { - varname, op, value := line.extra["varname"].(string), line.extra["op"].(string), line.extra["value"].(string) - - if op != "?=" || G.pkgContext.vardef[varname] == nil { - _ = G.opts.DebugMisc && line.debugf("varassign(%q, %q, %q)", varname, op, value) - G.pkgContext.vardef[varname] = line - } - } - } - - return true -} - -func checkForUsedComment(lines []*Line, relativeName string) { - if len(lines) < 3 { - return - } - - expected := "# used by " + relativeName - for _, line := range lines { - if line.text == expected { - return - } - } - - i := 0 - for i < 2 && matches(lines[i].text, `^\s*#(.*)$`) { - i++ - } - - insertLine := lines[i] - insertLine.warnf("Please add a line %q here.", expected) - insertLine.explain( - "Since Makefile.common files usually don't have any comments and", - "therefore not a clearly defined interface, they should at least contain", - "references to all files that include them, so that it is easier to see", - "what effects future changes may have.", - "", - "If there are more than five packages that use a Makefile.common,", - "you should think about giving it a proper name (maybe plugin.mk) and", - "documenting its interface.") - insertLine.insertBefore(expected) - saveAutofixChanges(lines) -} - -func resolveVarsInRelativePath(relpath string, adjustDepth bool) string { - - tmp := relpath - tmp = strings.Replace(tmp, "${PKGSRCDIR}", G.curPkgsrcdir, -1) - tmp = strings.Replace(tmp, "${.CURDIR}", ".", -1) - tmp = strings.Replace(tmp, "${.PARSEDIR}", ".", -1) - tmp = strings.Replace(tmp, "${LUA_PKGSRCDIR}", "../../lang/lua52", -1) - tmp = strings.Replace(tmp, "${PHPPKGSRCDIR}", "../../lang/php55", -1) - tmp = strings.Replace(tmp, "${SUSE_DIR_PREFIX}", "suse100", -1) - tmp = strings.Replace(tmp, "${PYPKGSRCDIR}", "../../lang/python27", -1) - if G.pkgContext != nil { - tmp = strings.Replace(tmp, "${FILESDIR}", G.pkgContext.filesdir, -1) - } - if G.pkgContext != nil { - tmp = strings.Replace(tmp, "${PKGDIR}", G.pkgContext.pkgdir, -1) - } - - if adjustDepth { - if m, pkgpath := match1(tmp, `^\.\./\.\./([^.].*)$`); m { - tmp = G.curPkgsrcdir + "/" + pkgpath - } - } - - _ = G.opts.DebugMisc && dummyLine.debugf("resolveVarsInRelativePath: %q => %q", relpath, tmp) - return tmp -} - -func parselineMk(line *Line) { - defer tracecall("parselineMk", line.text)() - - if len(line.extra) != 0 { - return - } - - text := line.text - - if m, varname, op, value, comment := matchVarassign(text); m { - value = strings.Replace(value, "\\#", "#", -1) - varparam := varnameParam(varname) - - line.extra["is_varassign"] = true - line.extra["varname"] = varname - line.extra["varcanon"] = varnameCanon(varname) - line.extra["varparam"] = varparam - line.extra["op"] = op - line.extra["value"] = value - line.extra["comment"] = comment - return - } - - if hasPrefix(text, "\t") { - line.extra["is_shellcmd"] = true - line.extra["shellcmd"] = text[1:] - return - } - - if index := strings.IndexByte(text, '#'); index != -1 && strings.TrimSpace(text[:index]) == "" { - line.extra["is_comment"] = true - line.extra["comment"] = text[index+1:] - return - } - - if strings.TrimSpace(text) == "" { - line.extra["is_empty"] = true - return - } - - if m, indent, directive, args := match3(text, reMkCond); m { - line.extra["is_cond"] = true - line.extra["indent"] = indent - line.extra["directive"] = directive - line.extra["args"] = args - return - } - - if m, _, includefile := match2(text, reMkInclude); m { - line.extra["is_include"] = true - line.extra["includefile"] = includefile - return - } - - if m, includefile, comment := match2(text, reMkSysinclude); m { - line.extra["is_sysinclude"] = true - line.extra["includefile"] = includefile - line.extra["comment"] = comment - return - } - - if m, targets, whitespace, sources := match3(text, reMkDependency); m { - line.extra["is_dependency"] = true - line.extra["targets"] = targets - line.extra["sources"] = sources - if whitespace != "" { - line.warnf("Space before colon in dependency line.") - } - return - } - - if matches(text, `^(<<<<<<<|=======|>>>>>>>)`) { - return - } - - line.errorf("Unknown Makefile line format.") -} - -func ParselinesMk(lines []*Line) { - for _, line := range lines { - parselineMk(line) - } -} - -func ChecklinesMk(lines []*Line) { - defer tracecall("ChecklinesMk", lines[0].fname)() - - allowedTargets := make(map[string]bool) - substcontext := new(SubstContext) - - ctx := newMkContext() - G.mkContext = ctx - defer func() { G.mkContext = nil }() - - determineUsedVariables(lines) - - prefixes := splitOnSpace("pre do post") - actions := splitOnSpace("fetch extract patch tools wrapper configure build test install package clean") - for _, prefix := range prefixes { - for _, action := range actions { - allowedTargets[prefix+"-"+action] = true - } - } - - // In the first pass, all additions to BUILD_DEFS and USE_TOOLS - // are collected to make the order of the definitions irrelevant. - - for _, line := range lines { - if line.extra["is_varassign"] == nil { - continue - } - - varcanon := line.extra["varcanon"].(string) - switch varcanon { - case "BUILD_DEFS", "PKG_GROUPS_VARS", "PKG_USERS_VARS": - for _, varname := range splitOnSpace(line.extra["value"].(string)) { - G.mkContext.buildDefs[varname] = true - _ = G.opts.DebugMisc && line.debugf("%q is added to BUILD_DEFS.", varname) - } - - case "PLIST_VARS": - for _, id := range splitOnSpace(line.extra["value"].(string)) { - G.mkContext.plistVars["PLIST."+id] = true - _ = G.opts.DebugMisc && line.debugf("PLIST.%s is added to PLIST_VARS.", id) - useVar(line, "PLIST."+id) - } - - case "USE_TOOLS": - for _, tool := range splitOnSpace(line.extra["value"].(string)) { - tool = strings.Split(tool, ":")[0] - G.mkContext.tools[tool] = true - _ = G.opts.DebugMisc && line.debugf("%s is added to USE_TOOLS.", tool) - } - - case "SUBST_VARS.*": - for _, svar := range splitOnSpace(line.extra["value"].(string)) { - useVar(line, varnameCanon(svar)) - _ = G.opts.DebugMisc && line.debugf("varuse %s", svar) - } - - case "OPSYSVARS": - for _, osvar := range splitOnSpace(line.extra["value"].(string)) { - useVar(line, osvar+".*") - defineVar(line, osvar) - } - } - } - - // In the second pass, the actual checks are done. - - checklineRcsid(lines[0], `#\s+`, "# ") - - for _, line := range lines { - text := line.text - - checklineTrailingWhitespace(line) - - if line.extra["is_empty"] != nil { - substcontext.Finish(NewMkLine(line)) - - } else if line.extra["is_comment"] != nil { - // No further checks. - - } else if line.extra["is_varassign"] != nil { - ml := NewMkLine(line) - ml.checkVaralign() - ml.checkVarassign() - substcontext.Varassign(NewMkLine(line)) - - } else if hasPrefix(text, "\t") { - shellcmd := text[1:] - NewMkLine(line).checkText(shellcmd) - NewMkShellLine(line).checkShelltext(shellcmd) - - } else if m, include, includefile := match2(text, reMkInclude); m { - _ = G.opts.DebugInclude && line.debugf("includefile=%s", includefile) - checklineRelativePath(line, includefile, include == "include") - - if hasSuffix(includefile, "../Makefile") { - line.errorf("Other Makefiles must not be included directly.") - line.explain( - "If you want to include portions of another Makefile, extract", - "the common parts and put them into a Makefile.common. After", - "that, both this one and the other package should include the", - "Makefile.common.") - } - - if includefile == "../../mk/bsd.prefs.mk" { - if path.Base(line.fname) == "buildlink3.mk" { - line.notef("For efficiency reasons, please include bsd.fast.prefs.mk instead of bsd.prefs.mk.") - } - if G.pkgContext != nil { - G.pkgContext.seenBsdPrefsMk = true - } - } else if includefile == "../../mk/bsd.fast.prefs.mk" { - if G.pkgContext != nil { - G.pkgContext.seenBsdPrefsMk = true - } - } - - if matches(includefile, `/x11-links/buildlink3\.mk$`) { - line.errorf("%s must not be included directly. Include \"../../mk/x11.buildlink3.mk\" instead.", includefile) - } - if matches(includefile, `/jpeg/buildlink3\.mk$`) { - line.errorf("%s must not be included directly. Include \"../../mk/jpeg.buildlink3.mk\" instead.", includefile) - } - if matches(includefile, `/intltool/buildlink3\.mk$`) { - line.warnf("Please write \"USE_TOOLS+= intltool\" instead of this line.") - } - if m, dir := match1(includefile, `(.*)/builtin\.mk$`); m { - line.errorf("%s must not be included directly. Include \"%s/buildlink3.mk\" instead.", includefile, dir) - } - - } else if matches(text, reMkSysinclude) { - - } else if m, indent, directive, args := match3(text, reMkCond); m { - if matches(directive, `^(?:endif|endfor|elif|else)$`) { - if len(ctx.indentation) > 1 { - ctx.popIndent() - } else { - line.errorf("Unmatched .%s.", directive) - } - } - - // Check the indentation - if indent != strings.Repeat(" ", ctx.indentDepth()) { - _ = G.opts.WarnSpace && line.notef("This directive should be indented by %d spaces.", ctx.indentDepth()) - } - - if directive == "if" && matches(args, `^!defined\([\w]+_MK\)$`) { - ctx.pushIndent(ctx.indentDepth()) - - } else if matches(directive, `^(?:if|ifdef|ifndef|for|elif|else)$`) { - ctx.pushIndent(ctx.indentDepth() + 2) - } - - reDirectivesWithArgs := `^(?:if|ifdef|ifndef|elif|for|undef)$` - if matches(directive, reDirectivesWithArgs) && args == "" { - line.errorf("\".%s\" requires arguments.", directive) - - } else if !matches(directive, reDirectivesWithArgs) && args != "" { - line.errorf("\".%s\" does not take arguments.", directive) - - if directive == "else" { - line.notef("If you meant \"else if\", use \".elif\".") - } - - } else if directive == "if" || directive == "elif" { - NewMkLine(line).checkIf() - - } else if directive == "ifdef" || directive == "ifndef" { - if matches(args, `\s`) { - line.errorf("The \".%s\" directive can only handle _one_ argument.", directive) - } else { - line.warnf("The \".%s\" directive is deprecated. Please use \".if %sdefined(%s)\" instead.", - directive, ifelseStr(directive == "ifdef", "", "!"), args) - } - - } else if directive == "for" { - if m, vars, values := match2(args, `^(\S+(?:\s*\S+)*?)\s+in\s+(.*)$`); m { - for _, forvar := range splitOnSpace(vars) { - if !G.isInfrastructure && hasPrefix(forvar, "_") { - line.warnf("Variable names starting with an underscore (%s) are reserved for internal pkgsrc use.", forvar) - } - - if matches(forvar, `^[_a-z][_a-z0-9]*$`) { - // Fine. - } else if matches(forvar, `[A-Z]`) { - line.warnf(".for variable names should not contain uppercase letters.") - } else { - line.errorf("Invalid variable name %q.", forvar) - } - - ctx.forVars[forvar] = true - } - - // Check if any of the value's types is not guessed. - guessed := guGuessed - for _, value := range splitOnSpace(values) { - if m, vname := match1(value, `^\$\{(.*)\}`); m { - vartype := getVariableType(line, vname) - if vartype != nil && !vartype.guessed { - guessed = guNotGuessed - } - } - } - - forLoopType := &Vartype{lkSpace, CheckvarUnchecked, []AclEntry{{"*", "pu"}}, guessed} - forLoopContext := &VarUseContext{ - vucTimeParse, - forLoopType, - vucQuotFor, - vucExtentWord, - } - for _, fvar := range extractUsedVariables(line, values) { - NewMkLine(line).checkVaruse(fvar, "", forLoopContext) - } - } - - } else if directive == "undef" && args != "" { - for _, uvar := range splitOnSpace(args) { - if ctx.forVars[uvar] { - line.notef("Using \".undef\" after a \".for\" loop is unnecessary.") - } - } - } - - } else if m, targets, _, dependencies := match3(text, reMkDependency); m { - _ = G.opts.DebugMisc && line.debugf("targets=%q, dependencies=%q", targets, dependencies) - ctx.target = targets - - for _, source := range splitOnSpace(dependencies) { - if source == ".PHONY" { - for _, target := range splitOnSpace(targets) { - allowedTargets[target] = true - } - } - } - - for _, target := range splitOnSpace(targets) { - if target == ".PHONY" { - for _, dep := range splitOnSpace(dependencies) { - allowedTargets[dep] = true - } - - } else if target == ".ORDER" { - // TODO: Check for spelling mistakes. - - } else if !allowedTargets[target] { - line.warnf("Unusual target %q.", target) - line.explain( - "If you want to define your own targets, you can \"declare\"", - "them by inserting a \".PHONY: my-target\" line before this line. This", - "will tell make(1) to not interpret this target's name as a filename.") - } - } - - } else if m, directive := match1(text, `^\.\s*(\S*)`); m { - line.errorf("Unknown directive \".%s\".", directive) - - } else if hasPrefix(text, " ") { - line.warnf("Makefile lines should not start with space characters.") - line.explain( - "If you want this line to contain a shell program, use a tab", - "character for indentation. Otherwise please remove the leading", - "white-space.") - - } else { - _ = G.opts.DebugMisc && line.debugf("Unknown line format") - } - } - substcontext.Finish(NewMkLine(lines[len(lines)-1])) - - checklinesTrailingEmptyLines(lines) - - if len(ctx.indentation) != 1 { - lines[len(lines)-1].errorf("Directive indentation is not 0, but %d.", ctx.indentDepth()) - } - - G.mkContext = nil -} diff --git a/pkgtools/pkglint/files/makefiles_test.go b/pkgtools/pkglint/files/makefiles_test.go deleted file mode 100644 index bd241b0b689..00000000000 --- a/pkgtools/pkglint/files/makefiles_test.go +++ /dev/null @@ -1,69 +0,0 @@ -package main - -import ( - check "gopkg.in/check.v1" -) - -// In variable assignments, a plain '#' introduces a line comment, unless -// it is escaped by a backslash. In shell commands, on the other hand, it -// is interpreted literally. -func (s *Suite) TestParselineMk_VarAssign(c *check.C) { - line := NewLine("fname", "1", "SED_CMD=\t's,\\#,hash,g'", nil) - - parselineMk(line) - - c.Check(line.extra["varname"], equals, "SED_CMD") - c.Check(line.extra["value"], equals, "'s,#,hash,g'") -} - -func (s *Suite) TestCheckForUsedComment_OK(c *check.C) { - lines := s.NewLines("Makefile.common", - "# $"+"NetBSD$", - "", - "# used by sysutils/mc") - - checkForUsedComment(lines, "sysutils/mc") -} - -func (s *Suite) TestCheckForUsedComment_ShortFile0(c *check.C) { - lines := s.NewLines("Makefile.common") - - checkForUsedComment(lines, "category/package") -} - -func (s *Suite) TestCheckForUsedComment_ShortFile1(c *check.C) { - lines := s.NewLines("Makefile.common", - "# $"+"NetBSD$") - - checkForUsedComment(lines, "category/package") -} - -func (s *Suite) TestCheckForUsedComment_ShortFile2(c *check.C) { - lines := s.NewLines("Makefile.common", - "# $"+"NetBSD$", - "") - - checkForUsedComment(lines, "category/package") -} - -func (s *Suite) TestCheckForUsedComment_NotMentioned(c *check.C) { - lines := s.NewLines("Makefile.common", - "# $"+"NetBSD$", - "", - "VARNAME=\tvalue") - - checkForUsedComment(lines, "category/package") - - c.Check(s.Output(), equals, "WARN: Makefile.common:2: Please add a line \"# used by category/package\" here.\n") -} - -func (s *Suite) TestCheckForUsedComment_OnlyComments(c *check.C) { - lines := s.NewLines("Makefile.common", - "# $"+"NetBSD$", - "#", - "#") - - checkForUsedComment(lines, "category/package") - - c.Check(s.Output(), equals, "WARN: Makefile.common:3: Please add a line \"# used by category/package\" here.\n") -} diff --git a/pkgtools/pkglint/files/mkcond.go b/pkgtools/pkglint/files/mkcond.go deleted file mode 100644 index 0907105f686..00000000000 --- a/pkgtools/pkglint/files/mkcond.go +++ /dev/null @@ -1,30 +0,0 @@ -package main - -func parseMkCond(line *Line, cond string) *Tree { - defer tracecall("parseMkCond", cond)() - - const ( - repartVarname = `[A-Z_][A-Z0-9_]*(?:\.[\w_+\-]+)?` - reDefined = `^defined\((` + repartVarname + `)\)` - reEmpty = `^empty\((` + repartVarname + `)\)` - reEmptyMatch = `^empty\((` + repartVarname + `):M([^\$:{})]+)\)` - reCompare = `^\$\{(` + repartVarname + `)\}\s+(==|!=)\s+"([^"\$\\]*)"` - ) - - if m, rest := replaceFirst(cond, `^!`, ""); m != nil { - return NewTree("not", parseMkCond(line, rest)) - } - if m, rest := replaceFirst(cond, reDefined, ""); m != nil { - return NewTree("defined", parseMkCond(line, rest)) - } - if m, _ := replaceFirst(cond, reEmpty, ""); m != nil { - return NewTree("empty", m[1]) - } - if m, _ := replaceFirst(cond, reEmptyMatch, ""); m != nil { - return NewTree("empty", NewTree("match", m[1], m[2])) - } - if m, _ := replaceFirst(cond, reCompare, ""); m != nil { - return NewTree("compareVarStr", m[1], m[2], m[3]) - } - return NewTree("unknown", cond) -} diff --git a/pkgtools/pkglint/files/mkcond_test.go b/pkgtools/pkglint/files/mkcond_test.go deleted file mode 100644 index 2272e939c43..00000000000 --- a/pkgtools/pkglint/files/mkcond_test.go +++ /dev/null @@ -1,40 +0,0 @@ -package main - -import ( - check "gopkg.in/check.v1" -) - -func (s *Suite) TestParseMkCond_NotEmptyMatch(c *check.C) { - line := NewLine("fname", "1", "dummy", nil) - - cond := parseMkCond(line, "!empty(USE_LIBTOOL:M[Yy][Ee][Ss])") - - c.Check(cond, check.DeepEquals, NewTree("not", NewTree("empty", NewTree("match", "USE_LIBTOOL", "[Yy][Ee][Ss]")))) -} - -func (s *Suite) TestParseMkCond_Compare(c *check.C) { - line := NewLine("fname", "1", "dummy", nil) - - cond := parseMkCond(line, "${VARNAME} != \"Value\"") - - c.Check(cond, check.DeepEquals, NewTree("compareVarStr", "VARNAME", "!=", "Value")) -} - -func (s *Suite) TestChecklineMkCondition(c *check.C) { - s.UseCommandLine(c, "-Wtypes") - G.globalData.InitVartypes() - - NewMkLine(NewLine("fname", "1", ".if !empty(PKGSRC_COMPILER:Mmycc)", nil)).checkIf() - - c.Check(s.Stdout(), equals, "WARN: fname:1: Invalid :M value \"mycc\". "+ - "Only { ccache ccc clang distcc f2c gcc hp icc ido gcc mipspro "+ - "mipspro-ucode pcc sunpro xlc } are allowed.\n") - - NewMkLine(NewLine("fname", "1", ".elif ${A} != ${B}", nil)).checkIf() - - c.Check(s.Stdout(), equals, "") // Unknown condition types are silently ignored - - NewMkLine(NewLine("fname", "1", ".if ${HOMEPAGE} == \"mailto:someone@example.org\"", nil)).checkIf() - - c.Check(s.Output(), equals, "WARN: fname:1: \"mailto:someone@example.org\" is not a valid URL.\n") -} diff --git a/pkgtools/pkglint/files/mkcontext.go b/pkgtools/pkglint/files/mkcontext.go deleted file mode 100644 index 5385e7f6151..00000000000 --- a/pkgtools/pkglint/files/mkcontext.go +++ /dev/null @@ -1,62 +0,0 @@ -package main - -// MkContext contains data for the Makefile (or *.mk) that is currently checked. -type MkContext struct { - forVars map[string]bool // The variables currently used in .for loops - indentation []int // Indentation depth of preprocessing directives - target string // Current make(1) target - vardef map[string]*Line // varname => line; for all variables that are defined in the current file - varuse map[string]*Line // varname => line; for all variables that are used in the current file - buildDefs map[string]bool // Variables that are registered in BUILD_DEFS, to ensure that all user-defined variables are added to it. - plistVars map[string]bool // Variables that are registered in PLIST_VARS, to ensure that all user-defined variables are added to it. - tools map[string]bool // Set of tools that are declared to be used. -} - -func newMkContext() *MkContext { - forVars := make(map[string]bool) - indentation := make([]int, 1) - vardef := make(map[string]*Line) - varuse := make(map[string]*Line) - buildDefs := make(map[string]bool) - plistVars := make(map[string]bool) - tools := make(map[string]bool) - for tool := range G.globalData.predefinedTools { - tools[tool] = true - } - return &MkContext{forVars, indentation, "", vardef, varuse, buildDefs, plistVars, tools} -} - -func (ctx *MkContext) indentDepth() int { - return ctx.indentation[len(ctx.indentation)-1] -} -func (ctx *MkContext) popIndent() { - ctx.indentation = ctx.indentation[:len(ctx.indentation)-1] -} -func (ctx *MkContext) pushIndent(indent int) { - ctx.indentation = append(ctx.indentation, indent) -} - -func (ctx *MkContext) defineVar(line *Line, varname string) { - if line.extra["value"] == nil { - line.errorf("Internal pkglint error: novalue") - return - } - if ctx.vardef[varname] == nil { - ctx.vardef[varname] = line - } - varcanon := varnameCanon(varname) - if ctx.vardef[varcanon] == nil { - ctx.vardef[varcanon] = line - } -} - -func (ctx *MkContext) varValue(varname string) (value string, found bool) { - if line := ctx.vardef[varname]; line != nil { - if value := line.extra["value"]; value != nil { - return value.(string), true - } else { - line.errorf("Internal pkglint error: novalue") - } - } - return "", false -} diff --git a/pkgtools/pkglint/files/mkline.go b/pkgtools/pkglint/files/mkline.go index f12eb5d02e3..04a5d4d7d25 100644 --- a/pkgtools/pkglint/files/mkline.go +++ b/pkgtools/pkglint/files/mkline.go @@ -3,109 +3,274 @@ package main // Checks concerning single lines in Makefiles. import ( + "fmt" + "os" "strconv" "strings" ) type MkLine struct { - line *Line + Line *Line + + xtype uint8 + xmustexist bool + xop MkOperator + xs1 string + xs2 string + xs3 string + xvalue string + xcomment string } -func NewMkLine(line *Line) *MkLine { - parselineMk(line) - return &MkLine{line} +func (mkline *MkLine) Error1(format, arg1 string) { mkline.Line.Error1(format, arg1) } +func (mkline *MkLine) Warn0(format string) { mkline.Line.Warn0(format) } +func (mkline *MkLine) Warn1(format, arg1 string) { mkline.Line.Warn1(format, arg1) } +func (mkline *MkLine) Warn2(format, arg1, arg2 string) { mkline.Line.Warn2(format, arg1, arg2) } +func (mkline *MkLine) Note0(format string) { mkline.Line.Note0(format) } +func (mkline *MkLine) Note2(format, arg1, arg2 string) { mkline.Line.Note2(format, arg1, arg2) } +func (mkline *MkLine) Debug1(format, arg1 string) { mkline.Line.Debug1(format, arg1) } +func (mkline *MkLine) Debug2(format, arg1, arg2 string) { mkline.Line.Debug2(format, arg1, arg2) } + +func NewMkLine(line *Line) (mkline *MkLine) { + mkline = &MkLine{Line: line} + + text := line.Text + + if hasPrefix(text, " ") { + mkline.Warn0("Makefile lines should not start with space characters.") + Explain3( + "If you want this line to contain a shell program, use a tab", + "character for indentation. Otherwise please remove the leading", + "white-space.") + } + + if m, varname, op, value, comment := MatchVarassign(text); m { + value = strings.Replace(value, "\\#", "#", -1) + varparam := varnameParam(varname) + + mkline.xtype = 1 + mkline.xs1 = varname + mkline.xs2 = varnameCanon(varname) + mkline.xs3 = varparam + mkline.xop = NewMkOperator(op) + mkline.xvalue = value + mkline.xcomment = comment + mkline.Tokenize(value) + return + } + + if hasPrefix(text, "\t") { + mkline.xtype = 2 + mkline.xs1 = text[1:] + mkline.Tokenize(mkline.xs1) + return + } + + if index := strings.IndexByte(text, '#'); index != -1 && strings.TrimSpace(text[:index]) == "" { + mkline.xtype = 3 + mkline.xcomment = text[index+1:] + return + } + + if strings.TrimSpace(text) == "" { + mkline.xtype = 4 + return + } + + if m, indent, directive, args := matchMkCond(text); m { + mkline.xtype = 5 + mkline.xs1 = indent + mkline.xs2 = directive + mkline.xs3 = args + return + } + + if m, directive, includefile := match2(text, reMkInclude); m { + mkline.xtype = 6 + mkline.xmustexist = directive == "include" + mkline.xs1 = includefile + return + } + + if m, directive, includefile := match2(text, `^\.\s*(s?include)\s+<([^>]+)>\s*(?:#.*)?$`); m { + mkline.xtype = 7 + mkline.xmustexist = directive == "include" + mkline.xs1 = includefile + return + } + + if m, targets, whitespace, sources := match3(text, `^([^\s:]+(?:\s*[^\s:]+)*)(\s*):\s*([^#]*?)(?:\s*#.*)?$`); m { + mkline.xtype = 8 + mkline.xs1 = targets + mkline.xs2 = sources + if whitespace != "" { + line.Warn0("Space before colon in dependency line.") + } + return + } + + if matches(text, `^(<<<<<<<|=======|>>>>>>>)`) { + return + } + + line.Error0("Unknown Makefile line format.") + return mkline +} + +func (mkline *MkLine) IsVarassign() bool { return mkline.xtype == 1 } +func (mkline *MkLine) Varname() string { return mkline.xs1 } +func (mkline *MkLine) Varcanon() string { return mkline.xs2 } +func (mkline *MkLine) Varparam() string { return mkline.xs3 } +func (mkline *MkLine) Op() MkOperator { return mkline.xop } +func (mkline *MkLine) Value() string { return mkline.xvalue } +func (mkline *MkLine) Comment() string { return mkline.xcomment } +func (mkline *MkLine) IsShellcmd() bool { return mkline.xtype == 2 } +func (mkline *MkLine) Shellcmd() string { return mkline.xs1 } +func (mkline *MkLine) IsComment() bool { return mkline.xtype == 3 } +func (mkline *MkLine) IsEmpty() bool { return mkline.xtype == 4 } +func (mkline *MkLine) IsCond() bool { return mkline.xtype == 5 } +func (mkline *MkLine) Indent() string { return mkline.xs1 } +func (mkline *MkLine) Directive() string { return mkline.xs2 } +func (mkline *MkLine) Args() string { return mkline.xs3 } +func (mkline *MkLine) IsInclude() bool { return mkline.xtype == 6 } +func (mkline *MkLine) MustExist() bool { return mkline.xmustexist } +func (mkline *MkLine) Includefile() string { return mkline.xs1 } +func (mkline *MkLine) IsSysinclude() bool { return mkline.xtype == 7 } +func (mkline *MkLine) IsDependency() bool { return mkline.xtype == 8 } +func (mkline *MkLine) Targets() string { return mkline.xs1 } +func (mkline *MkLine) Sources() string { return mkline.xs2 } + +func (mkline *MkLine) Tokenize(s string) { + p := NewParser(s) + p.MkTokens() + if p.Rest() != "" { + mkline.Error1("Invalid Makefile syntax at %q.", p.Rest()) + } } -func (ml *MkLine) checkVardef(varname, op string) { - defer tracecall("MkLine.checkVardef", varname, op)() +func (mkline *MkLine) CheckVardef(varname string, op MkOperator) { + if G.opts.DebugTrace { + defer tracecall(varname, op)() + } - defineVar(ml.line, varname) - ml.checkVardefPermissions(varname, op) + defineVar(mkline, varname) + mkline.CheckVardefPermissions(varname, op) } -func (ml *MkLine) checkVardefPermissions(varname, op string) { +func (mkline *MkLine) CheckVardefPermissions(varname string, op MkOperator) { if !G.opts.WarnPerm { return } - line := ml.line - perms := getVariablePermissions(line, varname) - var needed string + vartype := mkline.getVariableType(varname) + if vartype == nil { + if G.opts.DebugMisc { + mkline.Debug1("No type definition found for %q.", varname) + } + return + } + + perms := vartype.EffectivePermissions(mkline.Line.Fname) + var needed AclPermissions switch op { - case "=", "!=", ":=": - needed = "s" - case "?=": - needed = "d" - case "+=": - needed = "a" - } - - if !contains(perms, needed) { - line.warnf("Permission %q requested for %s, but only { %s } are allowed.", - ReadableVartypePermissions(needed), varname, ReadableVartypePermissions(perms)) - line.explain( - "Pkglint restricts the allowed actions on variables based on the filename.", - "", - "The available permissions are:", - "\tappend append something using +=", - "\tdefault set a default value using ?=", - "\tpreprocess use a variable during preprocessing (e.g. .if, .for)", - "\truntime use a variable at runtime", - "\t (when the shell commands are run)", - "\tset set a variable using :=, =, !=", - "\t (which happens during preprocessing)", - "", - "A \"?\" means that pkglint doesn't know which permissions are allowed", - "and which are not.") + case opAssign, opAssignShell, opAssignEval: + needed = aclpSet + case opAssignDefault: + needed = aclpSetDefault + case opAssignAppend: + needed = aclpAppend + } + + switch { + case perms.Contains(needed): + break + case perms == aclpUnknown: + if G.opts.DebugUnchecked { + mkline.Line.Debug1("Unknown permissions for %q.", varname) + } + default: + alternativeActions := perms & aclpAllWrite + alternativeFiles := vartype.AllowedFiles(needed) + switch { + case alternativeActions != 0 && alternativeFiles != "": + mkline.Line.Warnf("The variable %s may not be %s (only %s) in this file; it would be ok in %s.", + varname, needed.HumanString(), alternativeActions.HumanString(), alternativeFiles) + case alternativeFiles != "": + mkline.Line.Warnf("The variable %s may not be %s in this file; it would be ok in %s.", + varname, needed.HumanString(), alternativeFiles) + case alternativeActions != 0: + mkline.Line.Warnf("The variable %s may not be %s (only %s) in this file.", + varname, needed.HumanString(), alternativeActions.HumanString()) + default: + mkline.Line.Warnf("The variable %s may not be %s by any package.", + varname, needed.HumanString()) + } + Explain4( + "The allowed actions for a variable are determined based on the file", + "name in which the variable is used or defined. The exact rules are", + "hard-coded into pkglint. If they seem to be incorrect, please ask", + "on the tech-pkg@NetBSD.org mailing list.") } } -func (ml *MkLine) checkVaruse(varname string, mod string, vuc *VarUseContext) { - defer tracecall("MkLine.checkVaruse", ml.line, varname, mod, *vuc)() +func (mkline *MkLine) CheckVaruse(varname string, mod string, vuc *VarUseContext) { + if G.opts.DebugTrace { + defer tracecall(mkline, varname, mod, *vuc)() + } - line := ml.line - vartype := getVariableType(line, varname) + vartype := mkline.getVariableType(varname) if G.opts.WarnExtra && - (vartype == nil || vartype.guessed == guGuessed) && + (vartype == nil || vartype.guessed) && !varIsUsed(varname) && - !(G.mkContext != nil && G.mkContext.forVars[varname]) { - line.warnf("%s is used but not defined. Spelling mistake?", varname) + !(G.Mk != nil && G.Mk.forVars[varname]) { + mkline.Warn1("%s is used but not defined. Spelling mistake?", varname) } - ml.checkVarusePermissions(varname, vuc) + mkline.CheckVarusePermissions(varname, vuc) - if varname == "LOCALBASE" && !G.isInfrastructure { - ml.warnVaruseLocalbase() + if varname == "LOCALBASE" && !G.Infrastructure { + mkline.WarnVaruseLocalbase() } - needsQuoting := variableNeedsQuoting(line, varname, vuc) + needsQuoting := mkline.variableNeedsQuoting(varname, vuc) - if vuc.shellword == vucQuotFor { - ml.checkVaruseFor(varname, vartype, needsQuoting) + if vuc.quoting == vucQuotFor { + mkline.checkVaruseFor(varname, vartype, needsQuoting) } - if G.opts.WarnQuoting && vuc.shellword != vucQuotUnknown && needsQuoting != nqDontKnow { - ml.checkVaruseShellword(varname, vartype, vuc, mod, needsQuoting) + if G.opts.WarnQuoting && vuc.quoting != vucQuotUnknown && needsQuoting != nqDontKnow { + mkline.CheckVaruseShellword(varname, vartype, vuc, mod, needsQuoting) } - if G.globalData.userDefinedVars[varname] != nil && !G.globalData.systemBuildDefs[varname] && !G.mkContext.buildDefs[varname] { - line.warnf("The user-defined variable %s is used but not added to BUILD_DEFS.", varname) - line.explain( + if G.globalData.UserDefinedVars[varname] != nil && !G.globalData.SystemBuildDefs[varname] && !G.Mk.buildDefs[varname] { + mkline.Warn1("The user-defined variable %s is used but not added to BUILD_DEFS.", varname) + Explain( "When a pkgsrc package is built, many things can be configured by the", - "pkgsrc user in the mk.conf file. All these configurations should be", - "recorded in the binary package, so the package can be reliably rebuilt.", - "The BUILD_DEFS variable contains a list of all these user-settable", - "variables, so please add your variable to it, too.") + "pkgsrc user in the mk.conf file. All these configurations should be", + "recorded in the binary package, so the package can be reliably", + "rebuilt. The BUILD_DEFS variable contains a list of all these", + "user-settable variables, so please add your variable to it, too.") } } -func (ml *MkLine) checkVarusePermissions(varname string, vuc *VarUseContext) { +func (mkline *MkLine) CheckVarusePermissions(varname string, vuc *VarUseContext) { if !G.opts.WarnPerm { return } - line := ml.line - perms := getVariablePermissions(line, varname) + // This is the type of the variable that is being used. Not to + // be confused with vuc.vartype, which is the type of the + // context in which the variable is used (often a ShellCommand + // or, in an assignment, the type of the left hand side variable). + vartype := mkline.getVariableType(varname) + if vartype == nil { + if G.opts.DebugMisc { + mkline.Debug1("No type definition found for %q.", varname) + } + return + } + + perms := vartype.EffectivePermissions(mkline.Line.Fname) isLoadTime := false // Will the variable be used at load time? @@ -114,49 +279,48 @@ func (ml *MkLine) checkVarusePermissions(varname string, vuc *VarUseContext) { isIndirect := false switch { - case vuc.vartype != nil && vuc.vartype.guessed == guGuessed: + case vuc.vartype != nil && vuc.vartype.guessed: // Don't warn about unknown variables. - case vuc.time == vucTimeParse && !contains(perms, "p"): + case vuc.time == vucTimeParse && !perms.Contains(aclpUseLoadtime): isLoadTime = true - case vuc.vartype != nil && contains(vuc.vartype.union(), "p") && !contains(perms, "p"): + case vuc.vartype != nil && vuc.vartype.Union().Contains(aclpUseLoadtime) && !perms.Contains(aclpUseLoadtime): isLoadTime = true isIndirect = true } if isLoadTime && !isIndirect { - line.warnf("%s should not be evaluated at load time.", varname) - line.explain( + mkline.Warn1("%s should not be evaluated at load time.", varname) + Explain( "Many variables, especially lists of something, get their values", - "incrementally. Therefore it is generally unsafe to rely on their value", - "until it is clear that it will never change again. This point is", - "reached when the whole package Makefile is loaded and execution of the", - "shell commands starts, in some cases earlier.", + "incrementally. Therefore it is generally unsafe to rely on their", + "value until it is clear that it will never change again. This", + "point is reached when the whole package Makefile is loaded and", + "execution of the shell commands starts, in some cases earlier.", "", "Additionally, when using the \":=\" operator, each $$ is replaced", - "with a single $, so variables that have references to shell variables", - "or regular expressions are modified in a subtle way.") + "with a single $, so variables that have references to shell", + "variables or regular expressions are modified in a subtle way.") } if isLoadTime && isIndirect { - line.warnf("%s should not be evaluated indirectly at load time.", varname) - line.explain( - "The variable on the left-hand side may be evaluated at load time, but", - "the variable on the right-hand side may not. Due to this assignment, it", - "might be used indirectly at load-time, when it is not guaranteed to be", - "properly defined.") + mkline.Warn1("%s should not be evaluated indirectly at load time.", varname) + Explain4( + "The variable on the left-hand side may be evaluated at load time,", + "but the variable on the right-hand side may not. Because of the", + "assignment in this line, the variable might be used indirectly", + "at load time, before it is guaranteed to be properly initialized.") } - if !contains(perms, "p") && !contains(perms, "u") { - line.warnf("%s may not be used in this file.", varname) + if !perms.Contains(aclpUseLoadtime) && !perms.Contains(aclpUse) { + mkline.Warn1("%s may not be used in this file.", varname) } } -func (ml *MkLine) warnVaruseLocalbase() { - line := ml.line - line.warnf("The LOCALBASE variable should not be used by packages.") - line.explain( +func (mkline *MkLine) WarnVaruseLocalbase() { + mkline.Warn0("The LOCALBASE variable should not be used by packages.") + Explain( // from jlam via private mail. "Currently, LOCALBASE is typically used in these cases:", "", @@ -178,35 +342,32 @@ func (ml *MkLine) warnVaruseLocalbase() { "", "In the second case, the example is:", "", - " CONFIGURE_ENV+= --with-datafiles=${LOCALBASE}/share/battalion", + " CONFIGURE_ENV+= --with-datafiles=${LOCALBASE}/share/pkgbase", "", "This should really be:", "", - " CONFIGURE_ENV+= --with-datafiles=${PREFIX}/share/battalion") + " CONFIGURE_ENV+= --with-datafiles=${PREFIX}/share/pkgbase") } -func (ml *MkLine) checkVaruseFor(varname string, vartype *Vartype, needsQuoting NeedsQuoting) { - switch { - case vartype == nil: - // Cannot check anything here. - - case vartype.kindOfList == lkSpace: - // Fine - - case needsQuoting == nqDoesntMatter || needsQuoting == nqNo: - // Fine, this variable is not supposed to contain special characters. +func (mkline *MkLine) checkVaruseFor(varname string, vartype *Vartype, needsQuoting NeedsQuoting) { + if G.opts.DebugTrace { + defer tracecall(varname, vartype, needsQuoting)() + } - default: - ml.line.warnf("The variable %s should not be used in .for loops.", varname) - ml.line.explain( + if false && // Too many false positives + vartype != nil && + vartype.kindOfList != lkSpace && + needsQuoting != nqDoesntMatter { + mkline.Warn1("The variable %s should not be used in .for loops.", varname) + Explain4( "The .for loop splits its argument at sequences of white-space, as", "opposed to all other places in make(1), which act like the shell.", - "Therefore only variables that are specifically designed to match this", - "requirement should be used here.") + "Therefore only variables that are split at whitespace or don't", + "contain any special characters should be used here.") } } -func (ml *MkLine) checkVaruseShellword(varname string, vartype *Vartype, vuc *VarUseContext, mod string, needsQuoting NeedsQuoting) { +func (mkline *MkLine) CheckVaruseShellword(varname string, vartype *Vartype, vuc *VarUseContext, mod string, needsQuoting NeedsQuoting) { // In GNU configure scripts, a few variables need to be // passed through the :M* operator before they reach the @@ -214,65 +375,71 @@ func (ml *MkLine) checkVaruseShellword(varname string, vartype *Vartype, vuc *Va // // When doing checks outside a package, the :M* operator is needed for safety. needMstar := matches(varname, `^(?:.*_)?(?:CFLAGS||CPPFLAGS|CXXFLAGS|FFLAGS|LDFLAGS|LIBS)$`) && - (G.pkgContext == nil || G.pkgContext.vardef["GNU_CONFIGURE"] != nil) + (G.Pkg == nil || G.Pkg.vardef["GNU_CONFIGURE"] != nil) strippedMod := mod if m, stripped := match1(mod, `(.*?)(?::M\*)?(?::Q)?$`); m { strippedMod = stripped } - correctMod := strippedMod + ifelseStr(needMstar, ":M*:Q", ":Q") - line := ml.line if mod == ":M*:Q" && !needMstar { - line.notef("The :M* modifier is not needed here.") - - } else if mod != correctMod && needsQuoting == nqYes { - if vuc.shellword == vucQuotPlain { - line.warnf("Please use ${%s%s} instead of ${%s%s}.", varname, correctMod, varname, mod) - } else { - line.warnf("Please use ${%s%s} instead of ${%s%s} and make sure the variable appears outside of any quoting characters.", varname, correctMod, varname, mod) + mkline.Line.Note0("The :M* modifier is not needed here.") + + } else if needsQuoting == nqYes { + correctMod := strippedMod + ifelseStr(needMstar, ":M*:Q", ":Q") + if mod != correctMod { + if vuc.quoting == vucQuotPlain { + if !mkline.Line.AutofixReplace("${"+varname+mod+"}", "${"+varname+correctMod+"}") { + mkline.Line.Warnf("Please use ${%s%s} instead of ${%s%s}.", varname, correctMod, varname, mod) + } + } else { + mkline.Line.Warnf("Please use ${%s%s} instead of ${%s%s} and make sure"+ + " the variable appears outside of any quoting characters.", varname, correctMod, varname, mod) + } + Explain1( + "See the pkgsrc guide, section \"quoting guideline\", for details.") } - line.explain( - "See the pkgsrc guide, section \"quoting guideline\", for details.") } - if hasSuffix(mod, ":Q") { - expl := []string{ - "Many variables in pkgsrc do not need the :Q operator, since they", - "are not expected to contain white-space or other special characters.", - "", - "Another case is when a variable of type ShellWord appears in a context", - "that expects a shell word, it does not need to have a :Q operator. Even", - "when it is concatenated with another variable, it still stays _one_ word.", - "", - "Example:", - "\tWORD1= Have\\ fun # 1 word", - "\tWORD2= \"with BSD Make\" # 1 word, too", - "", - "\tdemo:", - "\t\techo ${WORD1}${WORD2} # still 1 word", + if hasSuffix(mod, ":Q") && (needsQuoting == nqNo || needsQuoting == nqDoesntMatter) { + bad := "${" + varname + mod + "}" + good := "${" + varname + strings.TrimSuffix(mod, ":Q") + "}" + needExplain := false + if needsQuoting == nqNo && !mkline.Line.AutofixReplace(bad, good) { + needExplain = true + mkline.Warn1("The :Q operator should not be used for ${%s} here.", varname) } - - switch needsQuoting { - case nqNo: - line.warnf("The :Q operator should not be used for ${%s} here.", varname) - line.explain(expl...) - case nqDoesntMatter: - line.notef("The :Q operator isn't necessary for ${%s} here.", varname) - line.explain(expl...) + if needsQuoting == nqDoesntMatter && !mkline.Line.AutofixReplace(bad, good) { + needExplain = true + mkline.Line.Note1("The :Q operator isn't necessary for ${%s} here.", varname) + } + if needExplain { + Explain( + "Many variables in pkgsrc do not need the :Q operator, since they", + "are not expected to contain white-space or other special characters.", + "Examples for these \"safe\" variables are:", + "", + "\t* filenames", + "\t* directory names", + "\t* user and group names", + "\t* tool names and tool paths", + "\t* variable names", + "\t* PKGNAME") } } } -func (ml *MkLine) checkDecreasingOrder(varname, value string) { - defer tracecall("MkLine.checkDecreasingOrder", varname, value)() +func (mkline *MkLine) CheckDecreasingOrder(varname, value string) { + if G.opts.DebugTrace { + defer tracecall2(varname, value)() + } strversions := splitOnSpace(value) intversions := make([]int, len(strversions)) for i, strversion := range strversions { iver, err := strconv.Atoi(strversion) if err != nil || !(iver > 0) { - ml.line.errorf("All values for %s must be positive integers.", varname) + mkline.Error1("All values for %s must be positive integers.", varname) return } intversions[i] = iver @@ -280,86 +447,71 @@ func (ml *MkLine) checkDecreasingOrder(varname, value string) { for i, ver := range intversions { if i > 0 && ver >= intversions[i-1] { - ml.line.warnf("The values for %s should be in decreasing order.", varname) - ml.line.explain( - "If they aren't, it may be possible that needless versions of packages", - "are installed.") + mkline.Warn1("The values for %s should be in decreasing order.", varname) + Explain2( + "If they aren't, it may be possible that needless versions of", + "packages are installed.") } } } -func (ml *MkLine) checkVarassign() { - defer tracecall("MkLine.checkVarassign")() +func (mkline *MkLine) CheckVarassign() { + if G.opts.DebugTrace { + defer tracecall0()() + } - line := ml.line - varname := line.extra["varname"].(string) - op := line.extra["op"].(string) - value := line.extra["value"].(string) - comment := line.extra["comment"].(string) - varbase := varnameBase(varname) + varname := mkline.Varname() + op := mkline.Op() + value := mkline.Value() + comment := mkline.Comment() varcanon := varnameCanon(varname) - ml.checkVardef(varname, op) + mkline.CheckVardef(varname, op) + mkline.CheckVarassignBsdPrefs() - if G.opts.WarnExtra && op == "?=" && G.pkgContext != nil && !G.pkgContext.seenBsdPrefsMk { - switch varbase { - case "BUILDLINK_PKGSRCDIR", "BUILDLINK_DEPMETHOD", "BUILDLINK_ABI_DEPENDS": - // FIXME: What about these ones? They occur quite often. - - default: - line.warnf("Please include \"../../mk/bsd.prefs.mk\" before using \"?=\".") - line.explain( - "The ?= operator is used to provide a default value to a variable. In", - "pkgsrc, many variables can be set by the pkgsrc user in the mk.conf", - "file. This file must be included explicitly. If a ?= operator appears", - "before mk.conf has been included, it will not care about the user's", - "preferences, which can result in unexpected behavior. The easiest way", - "to include the mk.conf file is by including the bsd.prefs.mk file,", - "which will take care of everything.") - } - } - - ml.checkText(value) - ml.checkVartype(varname, op, value, comment) + mkline.CheckText(value) + mkline.CheckVartype(varname, op, value, comment) // If the variable is not used and is untyped, it may be a spelling mistake. - if op == ":=" && varname == strings.ToLower(varname) { - _ = G.opts.DebugUnchecked && line.debugf("%s might be unused unless it is an argument to a procedure file.", varname) + if op == opAssignEval && varname == strings.ToLower(varname) { + if G.opts.DebugUnchecked { + mkline.Debug1("%s might be unused unless it is an argument to a procedure file.", varname) + } } else if !varIsUsed(varname) { if vartypes := G.globalData.vartypes; vartypes[varname] != nil || vartypes[varcanon] != nil { // Ok - } else if deprecated := G.globalData.deprecated; deprecated[varname] != "" || deprecated[varcanon] != "" { + } else if deprecated := G.globalData.Deprecated; deprecated[varname] != "" || deprecated[varcanon] != "" { // Ok } else { - line.warnf("%s is defined but not used. Spelling mistake?", varname) + mkline.Warn1("%s is defined but not used. Spelling mistake?", varname) } } if matches(value, `/etc/rc\.d`) { - line.warnf("Please use the RCD_SCRIPTS mechanism to install rc.d scripts automatically to ${RCD_SCRIPTS_EXAMPLEDIR}.") + mkline.Warn0("Please use the RCD_SCRIPTS mechanism to install rc.d scripts automatically to ${RCD_SCRIPTS_EXAMPLEDIR}.") } - if hasPrefix(varname, "_") && !G.isInfrastructure { - line.warnf("Variable names starting with an underscore (%s) are reserved for internal pkgsrc use.", varname) + if hasPrefix(varname, "_") && !G.Infrastructure { + mkline.Warn1("Variable names starting with an underscore (%s) are reserved for internal pkgsrc use.", varname) } - if varname == "PERL5_PACKLIST" && G.pkgContext != nil { - if m, p5pkgname := match1(G.pkgContext.effectivePkgbase, `^p5-(.*)`); m { + if varname == "PERL5_PACKLIST" && G.Pkg != nil { + if m, p5pkgname := match1(G.Pkg.EffectivePkgbase, `^p5-(.*)`); m { guess := "auto/" + strings.Replace(p5pkgname, "-", "/", -1) + "/.packlist" ucvalue, ucguess := strings.ToUpper(value), strings.ToUpper(guess) if ucvalue != ucguess && ucvalue != "${PERL5_SITEARCH}/"+ucguess { - line.warnf("Unusual value for PERL5_PACKLIST -- %q expected.", guess) + mkline.Warn1("Unusual value for PERL5_PACKLIST -- %q expected.", guess) } } } if varname == "CONFIGURE_ARGS" && matches(value, `=\$\{PREFIX\}/share/kde`) { - line.notef("Please .include \"../../meta-pkgs/kde3/kde3.mk\" instead of this line.") - line.explain( - "That file probably does many things automatically and consistently that", - "this package also does. When using kde3.mk, you can probably also leave", + mkline.Note0("Please .include \"../../meta-pkgs/kde3/kde3.mk\" instead of this line.") + Explain3( + "That file does many things automatically and consistently that this", + "package also does. When using kde3.mk, you can probably also leave", "out some explicit dependencies.") } @@ -369,82 +521,105 @@ func (ml *MkLine) checkVarassign() { // The variables mentioned in EVAL_PREFIX will later be // defined by find-prefix.mk. Therefore, they are marked // as known in the current file. - G.mkContext.vardef[evalVarname] = line + G.Mk.vardef[evalVarname] = mkline } } if varname == "PYTHON_VERSIONS_ACCEPTED" { - ml.checkDecreasingOrder(varname, value) + mkline.CheckDecreasingOrder(varname, value) } - if comment == "# defined" && !matches(varname, `.*(?:_MK|_COMMON)$`) { - line.notef("Please use \"# empty\", \"# none\" or \"yes\" instead of \"# defined\".") - line.explain( - "The value #defined says something about the state of the variable, but", - "not what that _means_. In some cases a variable that is defined means", - "\"yes\", in other cases it is an empty list (which is also only the", - "state of the variable), whose meaning could be described with \"none\".", - "It is this meaning that should be described.") + if comment == "# defined" && !hasSuffix(varname, "_MK") && !hasSuffix(varname, "_COMMON") { + mkline.Note0("Please use \"# empty\", \"# none\" or \"yes\" instead of \"# defined\".") + Explain( + "The value #defined says something about the state of the variable,", + "but not what that _means_. In some cases a variable that is defined", + "means \"yes\", in other cases it is an empty list (which is also", + "only the state of the variable), whose meaning could be described", + "with \"none\". It is this meaning that should be described.") } if m, revvarname := match1(value, `\$\{(PKGNAME|PKGVERSION)[:\}]`); m { if varname == "DIST_SUBDIR" || varname == "WRKSRC" { - line.warnf("%s should not be used in %s, as it includes the PKGREVISION. Please use %s_NOREV instead.", revvarname, varname, revvarname) + mkline.Line.Warnf("%s should not be used in %s, as it includes the PKGREVISION. Please use %s_NOREV instead.", revvarname, varname, revvarname) } } - if fix := G.globalData.deprecated[varname]; fix != "" { - line.warnf("Definition of %s is deprecated. %s", varname, fix) - } else if fix := G.globalData.deprecated[varcanon]; fix != "" { - line.warnf("Definition of %s is deprecated. %s", varname, fix) + if fix := G.globalData.Deprecated[varname]; fix != "" { + mkline.Warn2("Definition of %s is deprecated. %s", varname, fix) + } else if fix := G.globalData.Deprecated[varcanon]; fix != "" { + mkline.Warn2("Definition of %s is deprecated. %s", varname, fix) } if hasPrefix(varname, "SITES_") { - line.warnf("SITES_* is deprecated. Please use SITES.* instead.") - } - - if matches(value, `^[^=]@comment`) { - line.warnf("Please don't use @comment in %s.", varname) - line.explain( - "Here you are defining a variable containing @comment. As this value", - "typically includes a space as the last character you probably also used", - "quotes around the variable. This can lead to confusion when adding this", - "variable to PLIST_SUBST, as all other variables are quoted using the :Q", - "operator when they are appended. As it is hard to check whether a", - "variable that is appended to PLIST_SUBST is already quoted or not, you", - "should not have pre-quoted variables at all.", - "", - "To solve this, you should directly use PLIST_SUBST+= ${varname}=${value}", - "or use any other variable for collecting the list of PLIST substitutions", - "and later append that variable with PLIST_SUBST+= ${MY_PLIST_SUBST}.") + mkline.Warn0("SITES_* is deprecated. Please use SITES.* instead.") } - // Mark the variable as PLIST condition. This is later used in checkfile_PLIST. - if G.pkgContext != nil && G.pkgContext.plistSubstCond != nil { - if m, plistVarname := match1(value, `(.+)=.*@comment.*`); m { - G.pkgContext.plistSubstCond[plistVarname] = true - } - } + mkline.CheckVarassignPlistComment(varname, value) time := vucTimeRun - switch op { - case ":=", "!=": + if op == opAssignEval || op == opAssignShell { time = vucTimeParse } - usedVars := extractUsedVariables(line, value) - vuc := &VarUseContext{ - time, - getVariableType(line, varname), - vucQuotUnknown, - vucExtentUnknown} + usedVars := mkline.extractUsedVariables(value) + vuc := &VarUseContext{mkline.getVariableType(varname), time, vucQuotUnknown, vucExtentUnknown} for _, usedVar := range usedVars { - ml.checkVaruse(usedVar, "", vuc) + mkline.CheckVaruse(usedVar, "", vuc) + } +} + +func (mkline *MkLine) CheckVarassignBsdPrefs() { + if G.opts.WarnExtra && mkline.Op() == opAssignDefault && G.Pkg != nil && !G.Pkg.SeenBsdPrefsMk { + switch mkline.Varcanon() { + case "BUILDLINK_PKGSRCDIR.*", "BUILDLINK_DEPMETHOD.*", "BUILDLINK_ABI_DEPENDS.*": + return + } + + mkline.Warn0("Please include \"../../mk/bsd.prefs.mk\" before using \"?=\".") + Explain( + "The ?= operator is used to provide a default value to a variable.", + "In pkgsrc, many variables can be set by the pkgsrc user in the", + "mk.conf file. This file must be included explicitly. If a ?=", + "operator appears before mk.conf has been included, it will not care", + "about the user's preferences, which can result in unexpected", + "behavior.", + "", + "The easiest way to include the mk.conf file is by including the", + "bsd.prefs.mk file, which will take care of everything.") + } +} + +func (mkline *MkLine) CheckVarassignPlistComment(varname, value string) { + if false && // This is currently neither correct nor helpful + contains(value, "@comment") && !matches(value, `="@comment "`) { + mkline.Warn1("Please don't use @comment in %s.", varname) + Explain( + "If you are defining a PLIST conditional here, use one of the", + "following patterns instead:", + "", + "1. The direct way, without intermediate variable", + "", + "\tPLIST_SUBST+=\tMY_VAR=\"@comment \"", + "", + "2. The indirect way, with a separate variable", + "", + "\tPLIST_VARS+=\tMY_VAR", + "\t.if ...", + "\tMY_VAR?=\tyes", + "\t.endif") + } + + // Mark the variable as PLIST condition. This is later used in checkfile_PLIST. + if G.Pkg != nil { + if m, plistVarname := match1(value, `(.+)=.*@comment.*`); m { + G.Pkg.plistSubstCond[plistVarname] = true + } } } const reVarnamePlural = "^(?:" + - ".*S" + + ".*[Ss]" + "|.*LIST" + "|.*_AWK" + "|.*_ENV" + @@ -498,75 +673,79 @@ const reVarnamePlural = "^(?:" + "|TOOLS_NOOP" + ")$" -func (ml *MkLine) checkVartype(varname, op, value, comment string) { - defer tracecall("MkLine.checkVartype", varname, op, value, comment)() +func (mkline *MkLine) CheckVartype(varname string, op MkOperator, value, comment string) { + if G.opts.DebugTrace { + defer tracecall(varname, op, value, comment)() + } if !G.opts.WarnTypes { return } - line := ml.line varbase := varnameBase(varname) - vartype := getVariableType(line, varname) + vartype := mkline.getVariableType(varname) - if op == "+=" { + if op == opAssignAppend { if vartype != nil { - if !vartype.mayBeAppendedTo() { - line.warnf("The \"+=\" operator should only be used with lists.") + if !vartype.MayBeAppendedTo() { + mkline.Warn0("The \"+=\" operator should only be used with lists.") } - } else if !matches(varbase, `^_`) && !matches(varbase, reVarnamePlural) { - line.warnf("As %s is modified using \"+=\", its name should indicate plural.", varname) + } else if !hasPrefix(varbase, "_") && !matches(varbase, reVarnamePlural) { + mkline.Warn1("As %s is modified using \"+=\", its name should indicate plural.", varname) } } switch { case vartype == nil: // Cannot check anything if the type is not known. - _ = G.opts.DebugUnchecked && line.debugf("Unchecked variable assignment for %s.", varname) + if G.opts.DebugUnchecked { + mkline.Debug1("Unchecked variable assignment for %s.", varname) + } - case op == "!=": - _ = G.opts.DebugMisc && line.debugf("Use of !=: %q", value) + case op == opAssignShell: + if G.opts.DebugMisc { + mkline.Debug1("Use of !=: %q", value) + } case vartype.kindOfList == lkNone: - ml.checkVartypePrimitive(varname, vartype.checker, op, value, comment, vartype.isConsideredList(), vartype.guessed) + mkline.CheckVartypePrimitive(varname, vartype.checker, op, value, comment, vartype.IsConsideredList(), vartype.guessed) - default: - var words []string - if vartype.kindOfList == lkSpace { - words = splitOnSpace(value) - } else { - words, _ = splitIntoShellwords(line, value) + case vartype.kindOfList == lkSpace: + for _, word := range splitOnSpace(value) { + mkline.CheckVartypePrimitive(varname, vartype.checker, op, word, comment, true, vartype.guessed) } + case vartype.kindOfList == lkShell: + shline := NewShellLine(mkline) + words, _ := splitIntoShellWords(mkline.Line, value) for _, word := range words { - ml.checkVartypePrimitive(varname, vartype.checker, op, word, comment, true, vartype.guessed) - if vartype.kindOfList != lkSpace { - NewMkShellLine(ml.line).checkShellword(word, true) - } + mkline.CheckVartypePrimitive(varname, vartype.checker, op, word, comment, true, vartype.guessed) + shline.CheckToken(word, true) } } } -// The `op` parameter is one of `=`, `+=`, `:=`, `!=`, `?=`, `use`, `pp-use`, ``. -// For some variables (like BuildlinkDepth), the operator influences the valid values. +// For some variables (like `BuildlinkDepth`), `op` influences the valid values. // The `comment` parameter comes from a variable assignment, when a part of the line is commented out. -func (ml *MkLine) checkVartypePrimitive(varname string, checker *VarChecker, op, value, comment string, isList bool, guessed Guessed) { - defer tracecall("MkLine.checkVartypePrimitive", varname, op, value, comment, isList, guessed)() +func (mkline *MkLine) CheckVartypePrimitive(varname string, checker *VarChecker, op MkOperator, value, comment string, isList bool, guessed bool) { + if G.opts.DebugTrace { + defer tracecall(varname, op, value, comment, isList, guessed)() + } - ctx := &VartypeCheck{ml.line, varname, op, value, "", comment, isList, guessed == guGuessed} - ctx.valueNovar = ml.withoutMakeVariables(value, isList) + ctx := &VartypeCheck{mkline, mkline.Line, varname, op, value, "", comment, isList, guessed} + ctx.valueNovar = mkline.withoutMakeVariables(value, isList) checker.checker(ctx) } -func (ml *MkLine) withoutMakeVariables(value string, qModifierAllowed bool) string { +func (mkline *MkLine) withoutMakeVariables(value string, qModifierAllowed bool) string { valueNovar := value for { var m []string if m, valueNovar = replaceFirst(valueNovar, `\$\{([^{}]*)\}`, ""); m != nil { varuse := m[1] if !qModifierAllowed && hasSuffix(varuse, ":Q") { - ml.line.warnf("The :Q operator should only be used in lists and shell commands.") + mkline.Warn0("The :Q operator should only be used in lists and shell commands.") } } else { return valueNovar @@ -574,49 +753,57 @@ func (ml *MkLine) withoutMakeVariables(value string, qModifierAllowed bool) stri } } -func (ml *MkLine) checkVaralign() { - text := ml.line.text - if m := regcomp(reVarassign).FindStringSubmatchIndex(text); m != nil { - varname := text[m[2]:m[3]] - space1 := text[m[3]:m[4]] - op := text[m[4]:m[5]] - align := text[m[5]:m[6]] +func (mkline *MkLine) CheckVaralign() { + if !G.opts.WarnSpace { + return + } - if G.opts.WarnSpace && align != " " && strings.Trim(align, "\t") != "" { - ml.line.notef("Alignment of variable values should be done with tabs, not spaces.") - prefix := varname + space1 + op + if m, prefix, align := match2(mkline.Line.Text, `^( *[-*+A-Z_a-z0-9.${}\[]+\s*[!:?]?=)(\s*)`); m { + if align != " " && strings.Trim(align, "\t") != "" { alignedWidth := tabLength(prefix + align) tabs := "" for tabLength(prefix+tabs) < alignedWidth { tabs += "\t" } - ml.line.replace(prefix+align, prefix+tabs) + if !mkline.Line.AutofixReplace(prefix+align, prefix+tabs) { + mkline.Note0("Alignment of variable values should be done with tabs, not spaces.") + } } } } -func (ml *MkLine) checkText(text string) { - defer tracecall("MkLine.checkText", text)() +func (mkline *MkLine) CheckText(text string) { + if G.opts.DebugTrace { + defer tracecall1(text)() + } - line := ml.line if m, varname := match1(text, `^(?:[^#]*[^\$])?\$(\w+)`); m { - line.warnf("$%s is ambiguous. Use ${%s} if you mean a Makefile variable or $$%s if you mean a shell variable.", varname, varname, varname) + mkline.Warn1("$%[1]s is ambiguous. Use ${%[1]s} if you mean a Makefile variable or $$%[1]s if you mean a shell variable.", varname) } - if line.lines == "1" { - checklineRcsid(line, `# `, "# ") + if mkline.Line.firstLine == 1 { + mkline.Line.CheckRcsid(`# `, "# ") } - if contains(text, "${WRKSRC}/../") { - line.warnf("Using \"${WRKSRC}/..\" is conceptually wrong. Please use a combination of WRKSRC, CONFIGURE_DIRS and BUILD_DIRS instead.") - line.explain( - "You should define WRKSRC such that all of CONFIGURE_DIRS, BUILD_DIRS", - "and INSTALL_DIRS are subdirectories of it.") + if contains(text, "${WRKSRC}/..") { + mkline.Warn0("Building the package should take place entirely inside ${WRKSRC}, not \"${WRKSRC}/..\".") + Explain( + "WRKSRC should be defined so that there is no need to do anything", + "outside of this directory.", + "", + "Example:", + "", + "\tWRKSRC=\t${WRKDIR}", + "\tCONFIGURE_DIRS=\t${WRKSRC}/lib ${WRKSRC}/src", + "\tBUILD_DIRS=\t${WRKSRC}/lib ${WRKSRC}/src ${WRKSRC}/cmd", + "", + "See the pkgsrc guide, section \"Directories used during the build", + "process\" for more information.") } // Note: A simple -R is not detected, as the rate of false positives is too high. if m, flag := match1(text, `\b(-Wl,--rpath,|-Wl,-rpath-link,|-Wl,-rpath,|-Wl,-R)\b`); m { - line.warnf("Please use ${COMPILER_RPATH_FLAG} instead of %q.", flag) + mkline.Warn1("Please use ${COMPILER_RPATH_FLAG} instead of %q.", flag) } rest := text @@ -630,30 +817,30 @@ func (ml *MkLine) checkText(text string) { varbase, varext := m[1], m[2] varname := varbase + varext varcanon := varnameCanon(varname) - instead := G.globalData.deprecated[varname] + instead := G.globalData.Deprecated[varname] if instead == "" { - instead = G.globalData.deprecated[varcanon] + instead = G.globalData.Deprecated[varcanon] } if instead != "" { - line.warnf("Use of %q is deprecated. %s", varname, instead) + mkline.Warn2("Use of %q is deprecated. %s", varname, instead) } } } -func (ml *MkLine) checkIf() { - defer tracecall("MkLine.checkIf")() +func (mkline *MkLine) CheckIf() { + if G.opts.DebugTrace { + defer tracecall0()() + } - line := ml.line - condition := line.extra["args"].(string) - tree := parseMkCond(line, condition) + tree := mkline.parseMkCond(mkline.Args()) { var pvarname, ppattern *string if tree.Match(NewTree("not", NewTree("empty", NewTree("match", &pvarname, &ppattern)))) { - vartype := getVariableType(line, *pvarname) + vartype := mkline.getVariableType(*pvarname) if vartype != nil && vartype.checker.IsEnum() { if !matches(*ppattern, `[\$\[*]`) && !vartype.checker.HasEnum(*ppattern) { - line.warnf("Invalid :M value %q. Only { %s } are allowed.", *ppattern, vartype.checker.AllowedEnums()) + mkline.Warn2("Invalid :M value %q. Only { %s } are allowed.", *ppattern, vartype.checker.AllowedEnums()) } } return @@ -663,7 +850,470 @@ func (ml *MkLine) checkIf() { { var pop, pvarname, pvalue *string if tree.Match(NewTree("compareVarStr", &pvarname, &pop, &pvalue)) { - NewMkLine(line).checkVartype(*pvarname, "use", *pvalue, "") + mkline.CheckVartype(*pvarname, opUse, *pvalue, "") + } + } +} + +func (mkline *MkLine) CheckValidCharactersInValue(reValid string) { + rest := regcomp(reValid).ReplaceAllString(mkline.Value(), "") + if rest != "" { + uni := "" + for _, c := range rest { + uni += fmt.Sprintf(" %U", c) + } + mkline.Warn2("%s contains invalid characters (%s).", mkline.Varname(), uni[1:]) + } +} + +func (mkline *MkLine) explainRelativeDirs() { + Explain3( + "Directories in the form \"../../category/package\" make it easier to", + "move a package around in pkgsrc, for example from pkgsrc-wip to the", + "main pkgsrc repository.") +} + +func (mkline *MkLine) CheckRelativePkgdir(pkgdir string) { + mkline.CheckRelativePath(pkgdir, true) + pkgdir = resolveVarsInRelativePath(pkgdir, false) + + if m, otherpkgpath := match1(pkgdir, `^(?:\./)?\.\./\.\./([^/]+/[^/]+)$`); m { + if !fileExists(G.globalData.Pkgsrcdir + "/" + otherpkgpath + "/Makefile") { + mkline.Error1("There is no package in %q.", otherpkgpath) + } + + } else if !containsVarRef(pkgdir) { + mkline.Warn1("%q is not a valid relative package directory.", pkgdir) + Explain3( + "A relative pathname always starts with \"../../\", followed", + "by a category, a slash and a the directory name of the package.", + "For example, \"../../misc/screen\" is a valid relative pathname.") + } +} + +func (mkline *MkLine) CheckRelativePath(path string, mustExist bool) { + if !G.Wip && contains(path, "/wip/") { + mkline.Line.Error0("A main pkgsrc package must not depend on a pkgsrc-wip package.") + } + + resolvedPath := resolveVarsInRelativePath(path, true) + if containsVarRef(resolvedPath) { + return + } + + abs := resolvedPath + if !hasPrefix(abs, "/") { + abs = G.CurrentDir + "/" + abs + } + if _, err := os.Stat(abs); err != nil { + if mustExist { + mkline.Error1("%q does not exist.", resolvedPath) } + return + } + + if hasPrefix(path, "../") && + !matches(path, `^\.\./\.\./[^/]+/[^/]`) && + !(G.CurPkgsrcdir == ".." && hasPrefix(path, "../mk/")) && // For category Makefiles. + !hasPrefix(path, "../../mk/") { + mkline.Warn1("Invalid relative path %q.", path) + } +} + +func matchMkCond(text string) (m bool, indent, directive, args string) { + i, n := 0, len(text) + if i < n && text[i] == '.' { + i++ + } else { + return + } + + indentStart := i + for i < n && (text[i] == ' ' || text[i] == '\t') { + i++ + } + indentEnd := i + + directiveStart := i + for i < n && 'a' <= text[i] && text[i] <= 'z' { + i++ + } + directiveEnd := i + directive = text[directiveStart:directiveEnd] + switch directive { + case "if", "ifdef", "ifndef", "else", "elif", "endif", "for", "endfor", "undef": + break + default: + return + } + + for i < n && (text[i] == ' ' || text[i] == '\t') { + i++ + } + + argsStart := i + for i < n && text[i] != '#' { + i++ + } + for i > argsStart && (text[i-1] == ' ' || text[i-1] == '\t') { + i-- + } + argsEnd := i + + m = true + indent = text[indentStart:indentEnd] + args = text[argsStart:argsEnd] + return +} + +func (mkline *MkLine) parseMkCond(cond string) *Tree { + if G.opts.DebugTrace { + defer tracecall1(cond)() + } + + const ( + repartVarname = `[A-Z_][A-Z0-9_]*(?:\.[\w_+\-]+)?` + reDefined = `^defined\((` + repartVarname + `)\)` + reEmpty = `^empty\((` + repartVarname + `)\)` + reEmptyMatch = `^empty\((` + repartVarname + `):M([^\$:{})]+)\)` + reCompare = `^\$\{(` + repartVarname + `)\}\s+(==|!=)\s+"([^"\$\\]*)"` + ) + + if m, rest := replaceFirst(cond, `^!`, ""); m != nil { + return NewTree("not", mkline.parseMkCond(rest)) + } + if m, rest := replaceFirst(cond, reDefined, ""); m != nil { + return NewTree("defined", mkline.parseMkCond(rest)) + } + if m, _ := replaceFirst(cond, reEmpty, ""); m != nil { + return NewTree("empty", m[1]) + } + if m, _ := replaceFirst(cond, reEmptyMatch, ""); m != nil { + return NewTree("empty", NewTree("match", m[1], m[2])) + } + if m, _ := replaceFirst(cond, reCompare, ""); m != nil { + return NewTree("compareVarStr", m[1], m[2], m[3]) + } + return NewTree("unknown", cond) +} + +type NeedsQuoting uint8 + +const ( + nqNo NeedsQuoting = iota + nqYes + nqDoesntMatter + nqDontKnow +) + +func (mkline *MkLine) variableNeedsQuoting(varname string, vuc *VarUseContext) (needsQuoting NeedsQuoting) { + if G.opts.DebugTrace { + defer tracecall(varname, *vuc, "=>", needsQuoting)() + } + + vartype := mkline.getVariableType(varname) + if vartype == nil || vuc.vartype == nil { + return nqDontKnow + } + + isPlainWord := vartype.checker.IsEnum() + switch vartype.checker { + case CheckvarDistSuffix, + CheckvarFileMode, + CheckvarFilename, + CheckvarIdentifier, + CheckvarOption, + CheckvarPathname, + CheckvarPkgName, + CheckvarPkgOptionsVar, + CheckvarPkgRevision, + CheckvarRelativePkgDir, + CheckvarRelativePkgPath, + CheckvarUserGroupName, + CheckvarVarname, + CheckvarVersion, + CheckvarWrkdirSubdirectory: + isPlainWord = true + } + if isPlainWord { + if vartype.kindOfList == lkNone { + return nqDoesntMatter + } + if vartype.kindOfList == lkShell && vuc.extent != vucExtentWordpart { + return nqNo + } + } + + // In .for loops, the :Q operator is always misplaced, since + // the items are broken up at white-space, not as shell words + // like in all other parts of make(1). + if vuc.quoting == vucQuotFor { + return nqNo + } + + // Determine whether the context expects a list of shell words or not. + wantList := vuc.vartype.IsConsideredList() && (vuc.quoting == vucQuotBackt || vuc.extent != vucExtentWordpart) + haveList := vartype.IsConsideredList() + + if G.opts.DebugQuoting { + mkline.Line.Debugf("variableNeedsQuoting: varname=%q, context=%v, type=%v, wantList=%v, haveList=%v", + varname, vuc, vartype, wantList, haveList) + } + + // A shell word may appear as part of a shell word, for example COMPILER_RPATH_FLAG. + if vuc.extent == vucExtentWordpart && vuc.quoting == vucQuotPlain { + if vartype.kindOfList == lkNone && vartype.checker == CheckvarShellWord { + return nqNo + } + } + + // Assuming the tool definitions don't include very special characters, + // so they can safely be used inside any quotes. + if G.globalData.VarnameToToolname[varname] != "" { + switch vuc.quoting { + case vucQuotPlain: + if vuc.extent != vucExtentWordpart { + return nqNo + } + case vucQuotBackt: + return nqNo + case vucQuotDquot, vucQuotSquot: + return nqDoesntMatter + } + } + + // Variables that appear as parts of shell words generally need + // to be quoted. An exception is in the case of backticks, + // because the whole backticks expression is parsed as a single + // shell word by pkglint. + if vuc.extent == vucExtentWordpart && vuc.quoting != vucQuotBackt { + return nqYes + } + + // Assigning lists to lists does not require any quoting, though + // there may be cases like "CONFIGURE_ARGS+= -libs ${LDFLAGS:Q}" + // where quoting is necessary. + if wantList && haveList { + return nqDoesntMatter + } + + if wantList != haveList { + return nqYes + } + + if G.opts.DebugQuoting { + mkline.Line.Debug1("Don't know whether :Q is needed for %q", varname) + } + return nqDontKnow +} + +// Returns the type of the variable (maybe guessed based on the variable name), +// or nil if the type cannot even be guessed. +func (mkline *MkLine) getVariableType(varname string) *Vartype { + if vartype := G.globalData.vartypes[varname]; vartype != nil { + return vartype + } + if vartype := G.globalData.vartypes[varnameCanon(varname)]; vartype != nil { + return vartype + } + + if G.globalData.VarnameToToolname[varname] != "" { + return &Vartype{lkNone, CheckvarShellCommand, []AclEntry{{"*", aclpUse}}, false} + } + + if m, toolvarname := match1(varname, `^TOOLS_(.*)`); m && G.globalData.VarnameToToolname[toolvarname] != "" { + return &Vartype{lkNone, CheckvarPathname, []AclEntry{{"*", aclpUse}}, false} + } + + allowAll := []AclEntry{{"*", aclpAll}} + allowRuntime := []AclEntry{{"*", aclpAllRuntime}} + + // Guess the datatype of the variable based on naming conventions. + varbase := varnameBase(varname) + var gtype *Vartype + switch { + case hasSuffix(varbase, "DIRS"): + gtype = &Vartype{lkShell, CheckvarPathmask, allowRuntime, true} + case hasSuffix(varbase, "DIR"), hasSuffix(varname, "_HOME"): + gtype = &Vartype{lkNone, CheckvarPathname, allowRuntime, true} + case hasSuffix(varbase, "FILES"): + gtype = &Vartype{lkShell, CheckvarPathmask, allowRuntime, true} + case hasSuffix(varbase, "FILE"): + gtype = &Vartype{lkNone, CheckvarPathname, allowRuntime, true} + case hasSuffix(varbase, "PATH"): + gtype = &Vartype{lkNone, CheckvarPathlist, allowRuntime, true} + case hasSuffix(varbase, "PATHS"): + gtype = &Vartype{lkShell, CheckvarPathname, allowRuntime, true} + case hasSuffix(varbase, "_USER"): + gtype = &Vartype{lkNone, CheckvarUserGroupName, allowAll, true} + case hasSuffix(varbase, "_GROUP"): + gtype = &Vartype{lkNone, CheckvarUserGroupName, allowAll, true} + case hasSuffix(varbase, "_ENV"): + gtype = &Vartype{lkShell, CheckvarShellWord, allowRuntime, true} + case hasSuffix(varbase, "_CMD"): + gtype = &Vartype{lkNone, CheckvarShellCommand, allowRuntime, true} + case hasSuffix(varbase, "_ARGS"): + gtype = &Vartype{lkShell, CheckvarShellWord, allowRuntime, true} + case hasSuffix(varbase, "_CFLAGS"), hasSuffix(varname, "_CPPFLAGS"), hasSuffix(varname, "_CXXFLAGS"): + gtype = &Vartype{lkShell, CheckvarCFlag, allowRuntime, true} + case hasSuffix(varname, "_LDFLAGS"): + gtype = &Vartype{lkShell, CheckvarLdFlag, allowRuntime, true} + case hasSuffix(varbase, "_MK"): + gtype = &Vartype{lkNone, CheckvarUnchecked, allowAll, true} + case hasPrefix(varbase, "PLIST."): + gtype = &Vartype{lkNone, CheckvarYes, allowAll, true} + } + + if G.opts.DebugVartypes { + if gtype != nil { + mkline.Line.Debug2("The guessed type of %q is %q.", varname, gtype.String()) + } else { + mkline.Line.Debug1("No type definition found for %q.", varname) + } + } + return gtype +} + +// TODO: merge with determineUsedVariables +func (mkline *MkLine) extractUsedVariables(text string) []string { + re := regcomp(`^(?:[^\$]+|\$[\$*<>?@]|\$\{([.0-9A-Z_a-z]+)(?::(?:[^\${}]|\$[^{])+)?\})`) + rest := text + var result []string + for { + m := re.FindStringSubmatchIndex(rest) + if m == nil { + break + } + varname := rest[negToZero(m[2]):negToZero(m[3])] + rest = rest[:m[0]] + rest[m[1]:] + if varname != "" { + result = append(result, varname) + } + } + + if rest != "" && G.opts.DebugMisc { + mkline.Debug1("extractUsedVariables: rest=%q", rest) + } + return result +} + +func (mkline *MkLine) determineUsedVariables() (varnames []string) { + rest := mkline.Line.Text + + if strings.HasPrefix(rest, "#") { + return + } + + for { + p1 := strings.Index(rest, "${") + p2 := strings.Index(rest, "$(") + p3 := strings.Index(rest, "defined(") + p4 := strings.Index(rest, "empty(") + if p1 == -1 && p2 == -1 && p3 == -1 && p4 == -1 { + return + } + min := -1 + if min == -1 || (p1 != -1 && p1 < min) { + min = p1 + } + if min == -1 || (p2 != -1 && p2 < min) { + min = p2 + } + if min == -1 || (p3 != -1 && p3 < min) { + min = p3 + } + if min == -1 || (p4 != -1 && p4 < min) { + min = p4 + } + rest = rest[min:] + + m := regcomp(`(?:\$\{|\$\(|defined\(|empty\()([0-9+.A-Z_a-z]+)[:})]`).FindStringSubmatchIndex(rest) + if m == nil { + return + } + varname := rest[m[2]:m[3]] + varnames = append(varnames, varname) + rest = rest[:m[0]] + rest[m[1]:] + } + return +} + +// VarUseContext defines the context in which a variable is defined +// or used. Whether that is allowed depends on: +// +// * The variable’s data type, as defined in vardefs.go. +// * When used on the right-hand side of an assigment, the variable can +// represent a list of words, a single word or even only part of a +// word. This distinction decides upon the correct use of the :Q +// operator. +// * When used in preprocessing statements like .if or .for, the other +// operands of that statement should fit to the variable and are +// checked against the variable type. For example, comparing OPSYS to +// x86_64 doesn’t make sense. +type VarUseContext struct { + vartype *Vartype + time vucTime + quoting vucQuoting + extent vucExtent +} + +type vucTime uint8 + +const ( + vucTimeUnknown vucTime = iota + + // When Makefiles are loaded, the operators := and != are evaluated, + // as well as the conditionals .if, .elif and .for. + // During loading, not all variables are available yet. + // Variable values are still subject to change, especially lists. + vucTimeParse + + // All files have been read, all variables can be referenced. + // Variable values don’t change anymore. + vucTimeRun +) + +func (t vucTime) String() string { return [...]string{"unknown", "parse", "run"}[t] } + +// The quoting context in which the variable is used. +// Depending on this context, the modifiers :Q or :M can be allowed or not. +type vucQuoting uint8 + +const ( + vucQuotUnknown vucQuoting = iota + vucQuotPlain // Example: echo LOCALBASE=${LOCALBASE} + vucQuotDquot // Example: echo "The version is ${PKGVERSION}." + vucQuotSquot // Example: echo 'The version is ${PKGVERSION}.' + vucQuotBackt // Example: echo \`sed 1q ${WRKSRC}/README\` + + // The .for loop in Makefiles. This is the only place where + // variables are split on whitespace. Everywhere else (:Q, :M) + // they are split like in the shell. + // + // Example: .for f in ${EXAMPLE_FILES} + vucQuotFor +) + +func (q vucQuoting) String() string { + return [...]string{"unknown", "plain", "dquot", "squot", "backt", "mk-for"}[q] +} + +type vucExtent uint8 + +const ( + vucExtentUnknown vucExtent = iota + vucExtentWord // Example: echo ${LOCALBASE} + vucExtentWordpart // Example: echo LOCALBASE=${LOCALBASE} +) + +func (e vucExtent) String() string { + return [...]string{"unknown", "word", "wordpart"}[e] +} + +func (vuc *VarUseContext) String() string { + typename := "no-type" + if vuc.vartype != nil { + typename = vuc.vartype.String() } + return fmt.Sprintf("(%s %s %s %s)", vuc.time, typename, vuc.quoting, vuc.extent) } diff --git a/pkgtools/pkglint/files/mkline_test.go b/pkgtools/pkglint/files/mkline_test.go index 011e4fd7f59..afe599571ae 100644 --- a/pkgtools/pkglint/files/mkline_test.go +++ b/pkgtools/pkglint/files/mkline_test.go @@ -7,29 +7,29 @@ import ( func (s *Suite) TestChecklineMkVartype_SimpleType(c *check.C) { s.UseCommandLine(c, "-Wtypes", "-Dunchecked") G.globalData.InitVartypes() - ml := NewMkLine(NewLine("fname", "1", "COMMENT=\tA nice package", nil)) + mkline := NewMkLine(NewLine("fname", 1, "COMMENT=\tA nice package", nil)) vartype1 := G.globalData.vartypes["COMMENT"] c.Assert(vartype1, check.NotNil) - c.Check(vartype1.guessed, equals, guNotGuessed) + c.Check(vartype1.guessed, equals, false) - vartype := getVariableType(ml.line, "COMMENT") + vartype := mkline.getVariableType("COMMENT") c.Assert(vartype, check.NotNil) c.Check(vartype.checker.name, equals, "Comment") - c.Check(vartype.guessed, equals, guNotGuessed) + c.Check(vartype.guessed, equals, false) c.Check(vartype.kindOfList, equals, lkNone) - ml.checkVartype("COMMENT", "=", "A nice package", "") + mkline.CheckVartype("COMMENT", opAssign, "A nice package", "") c.Check(s.Stdout(), equals, "WARN: fname:1: COMMENT should not begin with \"A\".\n") } func (s *Suite) TestChecklineMkVartype(c *check.C) { G.globalData.InitVartypes() - ml := NewMkLine(NewLine("fname", "1", "DISTNAME=gcc-${GCC_VERSION}", nil)) + mkline := NewMkLine(NewLine("fname", 1, "DISTNAME=gcc-${GCC_VERSION}", nil)) - ml.checkVartype("DISTNAME", "=", "gcc-${GCC_VERSION}", "") + mkline.CheckVartype("DISTNAME", opAssign, "gcc-${GCC_VERSION}", "") } func (s *Suite) TestChecklineMkVaralign(c *check.C) { @@ -44,7 +44,7 @@ func (s *Suite) TestChecklineMkVaralign(c *check.C) { "VAR=\tvalue") // Already aligned with tabs only, left unchanged. for _, line := range lines { - NewMkLine(line).checkVaralign() + NewMkLine(line).CheckVaralign() } c.Check(lines[0].changed, equals, true) @@ -63,16 +63,284 @@ func (s *Suite) TestChecklineMkVaralign(c *check.C) { c.Check(lines[6].rawLines()[0].String(), equals, "7:VAR=\tvalue\n") c.Check(s.Output(), equals, ""+ "NOTE: file.mk:1: Alignment of variable values should be done with tabs, not spaces.\n"+ - "NOTE: file.mk:1: Autofix: replacing \"VAR= \" with \"VAR=\\t\".\n"+ + "AUTOFIX: file.mk:1: Replacing \"VAR= \" with \"VAR=\\t\".\n"+ "NOTE: file.mk:2: Alignment of variable values should be done with tabs, not spaces.\n"+ - "NOTE: file.mk:2: Autofix: replacing \"VAR= \" with \"VAR=\\t\".\n"+ + "AUTOFIX: file.mk:2: Replacing \"VAR= \" with \"VAR=\\t\".\n"+ "NOTE: file.mk:3: Alignment of variable values should be done with tabs, not spaces.\n"+ - "NOTE: file.mk:3: Autofix: replacing \"VAR= \" with \"VAR=\\t\\t\".\n"+ + "AUTOFIX: file.mk:3: Replacing \"VAR= \" with \"VAR=\\t\\t\".\n"+ "NOTE: file.mk:4: Alignment of variable values should be done with tabs, not spaces.\n"+ - "NOTE: file.mk:4: Autofix: replacing \"VAR= \\t\" with \"VAR=\\t\".\n"+ + "AUTOFIX: file.mk:4: Replacing \"VAR= \\t\" with \"VAR=\\t\".\n"+ "NOTE: file.mk:5: Alignment of variable values should be done with tabs, not spaces.\n"+ - "NOTE: file.mk:5: Autofix: replacing \"VAR= \\t\" with \"VAR=\\t\".\n"+ + "AUTOFIX: file.mk:5: Replacing \"VAR= \\t\" with \"VAR=\\t\".\n"+ "NOTE: file.mk:6: Alignment of variable values should be done with tabs, not spaces.\n"+ - "NOTE: file.mk:6: Autofix: replacing \"VAR= \\t\" with \"VAR=\\t\\t\".\n") + "AUTOFIX: file.mk:6: Replacing \"VAR= \\t\" with \"VAR=\\t\\t\".\n") c.Check(tabLength("VAR= \t"), equals, 16) } + +func (s *Suite) TestMkLine_fields(c *check.C) { + mklines := NewMkLines(s.NewLines("test.mk", + "VARNAME.param?=value # varassign comment", + "\tshell command # shell comment", + "# whole line comment", + "", + ". if !empty(PKGNAME:M*-*) # cond comment", + ".include \"../../mk/bsd.prefs.mk\" # include comment", + ".include <subdir.mk> # sysinclude comment", + "target1 target2: source1 source2", + "target : source", + "VARNAME+=value")) + ln := mklines.mklines + + c.Check(ln[0].IsVarassign(), equals, true) + c.Check(ln[0].Varname(), equals, "VARNAME.param") + c.Check(ln[0].Varcanon(), equals, "VARNAME.*") + c.Check(ln[0].Varparam(), equals, "param") + c.Check(ln[0].Op(), equals, opAssignDefault) + c.Check(ln[0].Value(), equals, "value") + c.Check(ln[0].Comment(), equals, "# varassign comment") + + c.Check(ln[1].IsShellcmd(), equals, true) + c.Check(ln[1].Shellcmd(), equals, "shell command # shell comment") + + c.Check(ln[2].IsComment(), equals, true) + c.Check(ln[2].Comment(), equals, " whole line comment") + + c.Check(ln[3].IsEmpty(), equals, true) + + c.Check(ln[4].IsCond(), equals, true) + c.Check(ln[4].Indent(), equals, " ") + c.Check(ln[4].Directive(), equals, "if") + c.Check(ln[4].Args(), equals, "!empty(PKGNAME:M*-*)") + c.Check(ln[4].Comment(), equals, "") // Not needed + + c.Check(ln[5].IsInclude(), equals, true) + c.Check(ln[5].MustExist(), equals, true) + c.Check(ln[5].Includefile(), equals, "../../mk/bsd.prefs.mk") + c.Check(ln[5].Comment(), equals, "") // Not needed + + c.Check(ln[6].IsSysinclude(), equals, true) + c.Check(ln[6].MustExist(), equals, true) + c.Check(ln[6].Includefile(), equals, "subdir.mk") + c.Check(ln[6].Comment(), equals, "") // Not needed + + c.Check(ln[7].IsDependency(), equals, true) + c.Check(ln[7].Targets(), equals, "target1 target2") + c.Check(ln[7].Sources(), equals, "source1 source2") + c.Check(ln[7].Comment(), equals, "") // Not needed + + c.Check(ln[9].IsVarassign(), equals, true) + c.Check(ln[9].Varname(), equals, "VARNAME") + c.Check(ln[9].Varcanon(), equals, "VARNAME") + c.Check(ln[9].Varparam(), equals, "") + + c.Check(s.Output(), equals, "WARN: test.mk:9: Space before colon in dependency line.\n") +} + +func (s *Suite) TestMkLine_checkVarassign(c *check.C) { + G.Pkg = NewPackage("graphics/gimp-fix-ca") + G.globalData.InitVartypes() + mkline := NewMkLine(NewLine("fname", 10, "MASTER_SITES=http://registry.gimp.org/file/fix-ca.c?action=download&id=9884&file=", nil)) + + mkline.CheckVarassign() + + c.Check(s.Output(), equals, "") +} + +func (s *Suite) TestParseMkCond_NotEmptyMatch(c *check.C) { + mkline := NewMkLine(NewLine("fname", 1, ".if !empty(USE_LIBTOOL:M[Yy][Ee][Ss])", nil)) + + cond := mkline.parseMkCond(mkline.Args()) + + c.Check(cond, check.DeepEquals, NewTree("not", NewTree("empty", NewTree("match", "USE_LIBTOOL", "[Yy][Ee][Ss]")))) +} + +func (s *Suite) TestParseMkCond_Compare(c *check.C) { + mkline := NewMkLine(NewLine("fname", 1, ".if ${VARNAME} != \"Value\"", nil)) + + cond := mkline.parseMkCond(mkline.Args()) + + c.Check(cond, check.DeepEquals, NewTree("compareVarStr", "VARNAME", "!=", "Value")) +} + +func (s *Suite) TestChecklineMkCondition(c *check.C) { + s.UseCommandLine(c, "-Wtypes") + G.globalData.InitVartypes() + + NewMkLine(NewLine("fname", 1, ".if !empty(PKGSRC_COMPILER:Mmycc)", nil)).CheckIf() + + c.Check(s.Stdout(), equals, "WARN: fname:1: Invalid :M value \"mycc\". "+ + "Only { ccache ccc clang distcc f2c gcc hp icc ido gcc mipspro "+ + "mipspro-ucode pcc sunpro xlc } are allowed.\n") + + NewMkLine(NewLine("fname", 1, ".elif ${A} != ${B}", nil)).CheckIf() + + c.Check(s.Stdout(), equals, "") // Unknown condition types are silently ignored + + NewMkLine(NewLine("fname", 1, ".if ${HOMEPAGE} == \"mailto:someone@example.org\"", nil)).CheckIf() + + c.Check(s.Output(), equals, "WARN: fname:1: \"mailto:someone@example.org\" is not a valid URL.\n") +} + +func (s *Suite) TestMkLine_variableNeedsQuoting(c *check.C) { + mkline := NewMkLine(NewLine("fname", 1, "PKGNAME := ${UNKNOWN}", nil)) + G.globalData.InitVartypes() + pkgnameType := G.globalData.vartypes["PKGNAME"] + + vuc := &VarUseContext{pkgnameType, vucTimeParse, vucQuotUnknown, vucExtentUnknown} + nq := mkline.variableNeedsQuoting("UNKNOWN", vuc) + + c.Check(nq, equals, nqDontKnow) +} + +func (s *Suite) TestMkLine_variableNeedsQuoting_Varbase(c *check.C) { + mkline := NewMkLine(NewLine("fname", 1, "# dummy", nil)) + G.globalData.InitVartypes() + + t1 := mkline.getVariableType("FONT_DIRS") + + c.Assert(t1, check.NotNil) + c.Check(t1.String(), equals, "ShellList of Pathmask") + + t2 := mkline.getVariableType("FONT_DIRS.ttf") + + c.Assert(t2, check.NotNil) + c.Check(t2.String(), equals, "ShellList of Pathmask") +} + +func (s *Suite) TestVarUseContext_ToString(c *check.C) { + G.globalData.InitVartypes() + mkline := NewMkLine(NewLine("fname", 1, "# dummy", nil)) + vartype := mkline.getVariableType("PKGNAME") + vuc := &VarUseContext{vartype, vucTimeUnknown, vucQuotBackt, vucExtentWord} + + c.Check(vuc.String(), equals, "(unknown PkgName backt word)") +} + +func (s *Suite) TestMkLine_(c *check.C) { + G.globalData.InitVartypes() + + G.Mk = s.NewMkLines("Makefile", + "# $"+"NetBSD$", + "ac_cv_libpari_libs+=\t-L${BUILDLINK_PREFIX.pari}/lib", // From math/clisp-pari/Makefile, rev. 1.8 + "var+=value") + + G.Mk.mklines[1].CheckVarassign() + G.Mk.mklines[2].CheckVarassign() + + c.Check(s.Output(), equals, ""+ + "WARN: Makefile:2: ac_cv_libpari_libs is defined but not used. Spelling mistake?\n"+ + "WARN: Makefile:3: As var is modified using \"+=\", its name should indicate plural.\n"+ + "WARN: Makefile:3: var is defined but not used. Spelling mistake?\n") +} + +// In variable assignments, a plain '#' introduces a line comment, unless +// it is escaped by a backslash. In shell commands, on the other hand, it +// is interpreted literally. +func (s *Suite) TestParselineMk(c *check.C) { + line1 := NewMkLine(NewLine("fname", 1, "SED_CMD=\t's,\\#,hash,g'", nil)) + + c.Check(line1.Varname(), equals, "SED_CMD") + c.Check(line1.Value(), equals, "'s,#,hash,g'") + + line2 := NewMkLine(NewLine("fname", 1, "\tsed -e 's,\\#,hash,g'", nil)) + + c.Check(line2.Shellcmd(), equals, "sed -e 's,\\#,hash,g'") +} + +func (s *Suite) TestMkLine_LeadingSpace(c *check.C) { + _ = NewMkLine(NewLine("rubyversion.mk", 427, " _RUBYVER=\t2.15", nil)) + + c.Check(s.Output(), equals, "WARN: rubyversion.mk:427: Makefile lines should not start with space characters.\n") +} + +func (s *Suite) TestMkLine_CheckVardefPermissions(c *check.C) { + s.UseCommandLine(c, "-Wall") + G.globalData.InitVartypes() + mklines := s.NewMkLines("options.mk", + "# $"+"NetBSD$", + "PKG_DEVELOPER?=\tyes", + "COMMENT=\t${PKG_DEVELOPER}") + + mklines.Check() + + c.Check(s.Output(), equals, "WARN: options.mk:2: The variable PKG_DEVELOPER may not be given a default value by any package.\n") +} + +func (s *Suite) TestMkLine_CheckVarusePermissions(c *check.C) { + s.UseCommandLine(c, "-Wall") + G.globalData.InitVartypes() + mklines := s.NewMkLines("options.mk", + "# $"+"NetBSD$", + "COMMENT=\t${GAMES_USER}", + "COMMENT:=\t${PKGBASE}", + "PYPKGPREFIX=${PKGBASE}") + G.globalData.UserDefinedVars = map[string]*MkLine{ + "GAMES_USER": mklines.mklines[0], + } + + mklines.Check() + + c.Check(s.Output(), equals, ""+ + "WARN: options.mk:2: The user-defined variable GAMES_USER is used but not added to BUILD_DEFS.\n"+ + "WARN: options.mk:3: PKGBASE should not be evaluated at load time.\n"+ + "WARN: options.mk:4: The variable PYPKGPREFIX may not be set in this file; it would be ok in pyversion.mk.\n"+ + "WARN: options.mk:4: \"${PKGBASE}\" is not valid for PYPKGPREFIX. Use one of { py27 py33 py34 } instead.\n"+ + "WARN: options.mk:4: PKGBASE should not be evaluated indirectly at load time.\n") +} + +func (s *Suite) TestMkLine_WarnVaruseLocalbase(c *check.C) { + mkline := NewMkLine(NewLine("options.mk", 56, "PKGNAME=${LOCALBASE}", nil)) + + mkline.WarnVaruseLocalbase() + + c.Check(s.Output(), equals, "WARN: options.mk:56: The LOCALBASE variable should not be used by packages.\n") +} + +func (s *Suite) TestMkLine_Misc(c *check.C) { + s.UseCommandLine(c, "-Wextra") + G.globalData.InitVartypes() + G.Pkg = NewPackage("category/pkgbase") + G.Mk = s.NewMkLines("options.mk", + "# $"+"NetBSD$", + ".for word in ${PKG_FAIL_REASON}", + "PYTHON_VERSIONS_ACCEPTED=\t27 35 30", + "CONFIGURE_ARGS+=\t--sharedir=${PREFIX}/share/kde", + "COMMENT=\t# defined", + ".endfor", + "GAMES_USER?=pkggames", + "PLIST_SUBST+= CONDITIONAL=${CONDITIONAL}", + "CONDITIONAL=\"@comment\"", + "BUILD_DIRS=\t${WRKSRC}/../build") + + G.Mk.Check() + + c.Check(s.Output(), equals, ""+ + "WARN: options.mk:3: The values for PYTHON_VERSIONS_ACCEPTED should be in decreasing order.\n"+ + "NOTE: options.mk:4: Please .include \"../../meta-pkgs/kde3/kde3.mk\" instead of this line.\n"+ + "NOTE: options.mk:5: Please use \"# empty\", \"# none\" or \"yes\" instead of \"# defined\".\n"+ + "WARN: options.mk:7: Please include \"../../mk/bsd.prefs.mk\" before using \"?=\".\n"+ + "WARN: options.mk:10: Building the package should take place entirely inside ${WRKSRC}, not \"${WRKSRC}/..\".\n"+ + "NOTE: options.mk:10: You can use \"../build\" instead of \"${WRKSRC}/../build\".\n") +} + +func (s *Suite) TestMkLine_CheckRelativePkgdir(c *check.C) { + mkline := NewMkLine(NewLine("Makefile", 46, "# dummy", nil)) + + mkline.CheckRelativePkgdir("../pkgbase") + + c.Check(s.Output(), equals, ""+ + "ERROR: Makefile:46: \"../pkgbase\" does not exist.\n"+ + "WARN: Makefile:46: \"../pkgbase\" is not a valid relative package directory.\n") +} + +// PR pkg/46570, item 2 +func (s *Suite) TestMkLine_UnfinishedVaruse(c *check.C) { + s.UseCommandLine(c, "-Dunchecked") + mkline := NewMkLine(NewLine("Makefile", 93, "EGDIRS=${EGDIR/apparmor.d ${EGDIR/dbus-1/system.d ${EGDIR/pam.d", nil)) + + mkline.CheckVarassign() + + c.Check(s.Output(), equals, ""+ + "ERROR: Makefile:93: Invalid Makefile syntax at \"${EGDIR/apparmor.d ${EGDIR/dbus-1/system.d ${EGDIR/pam.d\".\n"+ + "WARN: Makefile:93: EGDIRS is defined but not used. Spelling mistake?\n") +} diff --git a/pkgtools/pkglint/files/mklines.go b/pkgtools/pkglint/files/mklines.go new file mode 100644 index 00000000000..4390b313c97 --- /dev/null +++ b/pkgtools/pkglint/files/mklines.go @@ -0,0 +1,381 @@ +package main + +import ( + "path" + "strings" +) + +// MkLines contains data for the Makefile (or *.mk) that is currently checked. +type MkLines struct { + mklines []*MkLine + lines []*Line + forVars map[string]bool // The variables currently used in .for loops + indentation []int // Indentation depth of preprocessing directives + target string // Current make(1) target + vardef map[string]*MkLine // varname => line; for all variables that are defined in the current file + varuse map[string]*MkLine // varname => line; for all variables that are used in the current file + buildDefs map[string]bool // Variables that are registered in BUILD_DEFS, to ensure that all user-defined variables are added to it. + plistVars map[string]bool // Variables that are registered in PLIST_VARS, to ensure that all user-defined variables are added to it. + tools map[string]bool // Set of tools that are declared to be used. +} + +func NewMkLines(lines []*Line) *MkLines { + mklines := make([]*MkLine, len(lines)) + for i, line := range lines { + mklines[i] = NewMkLine(line) + } + tools := make(map[string]bool) + for tool := range G.globalData.PredefinedTools { + tools[tool] = true + } + + return &MkLines{ + mklines, + lines, + make(map[string]bool), + make([]int, 1), + "", + make(map[string]*MkLine), + make(map[string]*MkLine), + make(map[string]bool), + make(map[string]bool), + tools} +} + +func (mklines *MkLines) IndentDepth() int { + return mklines.indentation[len(mklines.indentation)-1] +} +func (mklines *MkLines) PopIndent() { + mklines.indentation = mklines.indentation[:len(mklines.indentation)-1] +} +func (mklines *MkLines) PushIndent(indent int) { + mklines.indentation = append(mklines.indentation, indent) +} + +func (mklines *MkLines) DefineVar(mkline *MkLine, varname string) { + if mklines.vardef[varname] == nil { + mklines.vardef[varname] = mkline + } + varcanon := varnameCanon(varname) + if mklines.vardef[varcanon] == nil { + mklines.vardef[varcanon] = mkline + } +} + +func (mklines *MkLines) UseVar(mkline *MkLine, varname string) { + varcanon := varnameCanon(varname) + mklines.varuse[varname] = mkline + mklines.varuse[varcanon] = mkline + if G.Pkg != nil { + G.Pkg.varuse[varname] = mkline + G.Pkg.varuse[varcanon] = mkline + } +} + +func (mklines *MkLines) VarValue(varname string) (value string, found bool) { + if mkline := mklines.vardef[varname]; mkline != nil { + return mkline.Value(), true + } + return "", false +} + +func (mklines *MkLines) Check() { + if G.opts.DebugTrace { + defer tracecall1(mklines.lines[0].Fname)() + } + + allowedTargets := make(map[string]bool) + substcontext := new(SubstContext) + + G.Mk = mklines + defer func() { G.Mk = nil }() + + mklines.DetermineUsedVariables() + + prefixes := splitOnSpace("pre do post") + actions := splitOnSpace("fetch extract patch tools wrapper configure build test install package clean") + for _, prefix := range prefixes { + for _, action := range actions { + allowedTargets[prefix+"-"+action] = true + } + } + + // In the first pass, all additions to BUILD_DEFS and USE_TOOLS + // are collected to make the order of the definitions irrelevant. + mklines.determineDefinedVariables() + + // In the second pass, the actual checks are done. + + mklines.lines[0].CheckRcsid(`#\s+`, "# ") + + for _, mkline := range mklines.mklines { + mkline.Line.CheckTrailingWhitespace() + mkline.Line.CheckValidCharacters(`[\t -~]`) + + switch { + case mkline.IsEmpty(): + substcontext.Finish(mkline) + + case mkline.IsVarassign(): + mklines.target = "" + mkline.CheckVaralign() + mkline.CheckVarassign() + substcontext.Varassign(mkline) + + case mkline.IsShellcmd(): + shellcmd := mkline.Shellcmd() + mkline.CheckText(shellcmd) + NewShellLine(mkline).CheckShellCommandLine(shellcmd) + + case mkline.IsInclude(): + mklines.target = "" + mklines.checklineInclude(mkline) + + case mkline.IsCond(): + mklines.checklineCond(mkline) + + case mkline.IsDependency(): + mklines.checklineDependencyRule(mkline, mkline.Targets(), mkline.Sources(), allowedTargets) + } + } + lastMkline := mklines.mklines[len(mklines.mklines)-1] + substcontext.Finish(lastMkline) + + ChecklinesTrailingEmptyLines(mklines.lines) + + if len(mklines.indentation) != 1 && mklines.IndentDepth() != 0 { + lastMkline.Line.Errorf("Directive indentation is not 0, but %d.", mklines.IndentDepth()) + } + + SaveAutofixChanges(mklines.lines) +} + +func (mklines *MkLines) determineDefinedVariables() { + for _, mkline := range mklines.mklines { + if !mkline.IsVarassign() { + continue + } + + varcanon := mkline.Varcanon() + switch varcanon { + case "BUILD_DEFS", "PKG_GROUPS_VARS", "PKG_USERS_VARS": + for _, varname := range splitOnSpace(mkline.Value()) { + mklines.buildDefs[varname] = true + if G.opts.DebugMisc { + mkline.Debug1("%q is added to BUILD_DEFS.", varname) + } + } + + case "PLIST_VARS": + for _, id := range splitOnSpace(mkline.Value()) { + mklines.plistVars["PLIST."+id] = true + if G.opts.DebugMisc { + mkline.Debug1("PLIST.%s is added to PLIST_VARS.", id) + } + mklines.UseVar(mkline, "PLIST."+id) + } + + case "USE_TOOLS": + for _, tool := range splitOnSpace(mkline.Value()) { + tool = strings.Split(tool, ":")[0] + mklines.tools[tool] = true + if G.opts.DebugMisc { + mkline.Debug1("%s is added to USE_TOOLS.", tool) + } + } + + case "SUBST_VARS.*": + for _, svar := range splitOnSpace(mkline.Value()) { + mklines.UseVar(mkline, varnameCanon(svar)) + if G.opts.DebugMisc { + mkline.Debug1("varuse %s", svar) + } + } + + case "OPSYSVARS": + for _, osvar := range splitOnSpace(mkline.Value()) { + mklines.UseVar(mkline, osvar+".*") + defineVar(mkline, osvar) + } + } + } +} + +func (mklines *MkLines) DetermineUsedVariables() { + for _, mkline := range mklines.mklines { + varnames := mkline.determineUsedVariables() + for _, varname := range varnames { + mklines.UseVar(mkline, varname) + } + } +} + +func (mklines *MkLines) checklineCond(mkline *MkLine) { + indent, directive, args := mkline.Indent(), mkline.Directive(), mkline.Args() + + switch directive { + case "endif", "endfor", "elif", "else": + if len(mklines.indentation) > 1 { + mklines.PopIndent() + } else { + mkline.Error1("Unmatched .%s.", directive) + } + } + + // Check the indentation + if expected := strings.Repeat(" ", mklines.IndentDepth()); indent != expected { + if G.opts.WarnSpace && !mkline.Line.AutofixReplace("."+indent, "."+expected) { + mkline.Line.Notef("This directive should be indented by %d spaces.", mklines.IndentDepth()) + } + } + + if directive == "if" && matches(args, `^!defined\([\w]+_MK\)$`) { + mklines.PushIndent(mklines.IndentDepth()) + + } else if matches(directive, `^(?:if|ifdef|ifndef|for|elif|else)$`) { + mklines.PushIndent(mklines.IndentDepth() + 2) + } + + reDirectivesWithArgs := `^(?:if|ifdef|ifndef|elif|for|undef)$` + if matches(directive, reDirectivesWithArgs) && args == "" { + mkline.Error1("\".%s\" requires arguments.", directive) + + } else if !matches(directive, reDirectivesWithArgs) && args != "" { + mkline.Error1("\".%s\" does not take arguments.", directive) + + if directive == "else" { + mkline.Note0("If you meant \"else if\", use \".elif\".") + } + + } else if directive == "if" || directive == "elif" { + mkline.CheckIf() + + } else if directive == "ifdef" || directive == "ifndef" { + if matches(args, `\s`) { + mkline.Error1("The \".%s\" directive can only handle _one_ argument.", directive) + } else { + mkline.Line.Warnf("The \".%s\" directive is deprecated. Please use \".if %sdefined(%s)\" instead.", + directive, ifelseStr(directive == "ifdef", "", "!"), args) + } + + } else if directive == "for" { + if m, vars, values := match2(args, `^(\S+(?:\s*\S+)*?)\s+in\s+(.*)$`); m { + for _, forvar := range splitOnSpace(vars) { + if !G.Infrastructure && hasPrefix(forvar, "_") { + mkline.Warn1("Variable names starting with an underscore (%s) are reserved for internal pkgsrc use.", forvar) + } + + if matches(forvar, `^[_a-z][_a-z0-9]*$`) { + // Fine. + } else if matches(forvar, `[A-Z]`) { + mkline.Warn0(".for variable names should not contain uppercase letters.") + } else { + mkline.Error1("Invalid variable name %q.", forvar) + } + + mklines.forVars[forvar] = true + } + + // Check if any of the value's types is not guessed. + guessed := true + for _, value := range splitOnSpace(values) { + if m, vname := match1(value, `^\$\{(.*)\}`); m { + vartype := mkline.getVariableType(vname) + if vartype != nil && !vartype.guessed { + guessed = false + } + } + } + + forLoopType := &Vartype{lkSpace, CheckvarUnchecked, []AclEntry{{"*", aclpAllRead}}, guessed} + forLoopContext := &VarUseContext{forLoopType, vucTimeParse, vucQuotFor, vucExtentWord} + for _, forLoopVar := range mkline.extractUsedVariables(values) { + mkline.CheckVaruse(forLoopVar, "", forLoopContext) + } + } + + } else if directive == "undef" && args != "" { + for _, uvar := range splitOnSpace(args) { + if mklines.forVars[uvar] { + mkline.Note0("Using \".undef\" after a \".for\" loop is unnecessary.") + } + } + } +} + +func (mklines *MkLines) checklineDependencyRule(mkline *MkLine, targets, dependencies string, allowedTargets map[string]bool) { + if G.opts.DebugMisc { + mkline.Debug2("targets=%q, dependencies=%q", targets, dependencies) + } + mklines.target = targets + + for _, source := range splitOnSpace(dependencies) { + if source == ".PHONY" { + for _, target := range splitOnSpace(targets) { + allowedTargets[target] = true + } + } + } + + for _, target := range splitOnSpace(targets) { + if target == ".PHONY" { + for _, dep := range splitOnSpace(dependencies) { + allowedTargets[dep] = true + } + + } else if target == ".ORDER" { + // TODO: Check for spelling mistakes. + + } else if !allowedTargets[target] { + mkline.Warn1("Unusual target %q.", target) + Explain3( + "If you want to define your own targets, you can \"declare\"", + "them by inserting a \".PHONY: my-target\" line before this line. This", + "will tell make(1) to not interpret this target's name as a filename.") + } + } +} + +func (mklines *MkLines) checklineInclude(mkline *MkLine) { + includefile := mkline.Includefile() + mustExist := mkline.MustExist() + if G.opts.DebugInclude { + mkline.Debug1("includefile=%s", includefile) + } + mkline.CheckRelativePath(includefile, mustExist) + + if hasSuffix(includefile, "/Makefile") { + mkline.Line.Error0("Other Makefiles must not be included directly.") + Explain4( + "If you want to include portions of another Makefile, extract", + "the common parts and put them into a Makefile.common. After", + "that, both this one and the other package should include the", + "Makefile.common.") + } + + if includefile == "../../mk/bsd.prefs.mk" { + if path.Base(mkline.Line.Fname) == "buildlink3.mk" { + mkline.Note0("For efficiency reasons, please include bsd.fast.prefs.mk instead of bsd.prefs.mk.") + } + if G.Pkg != nil { + G.Pkg.SeenBsdPrefsMk = true + } + } else if includefile == "../../mk/bsd.fast.prefs.mk" { + if G.Pkg != nil { + G.Pkg.SeenBsdPrefsMk = true + } + } + + if matches(includefile, `/x11-links/buildlink3\.mk$`) { + mkline.Error1("%s must not be included directly. Include \"../../mk/x11.buildlink3.mk\" instead.", includefile) + } + if matches(includefile, `/jpeg/buildlink3\.mk$`) { + mkline.Error1("%s must not be included directly. Include \"../../mk/jpeg.buildlink3.mk\" instead.", includefile) + } + if matches(includefile, `/intltool/buildlink3\.mk$`) { + mkline.Warn0("Please write \"USE_TOOLS+= intltool\" instead of this line.") + } + if m, dir := match1(includefile, `(.*)/builtin\.mk$`); m { + mkline.Line.Error2("%s must not be included directly. Include \"%s/buildlink3.mk\" instead.", includefile, dir) + } +} diff --git a/pkgtools/pkglint/files/mklines_test.go b/pkgtools/pkglint/files/mklines_test.go new file mode 100644 index 00000000000..6061894ec20 --- /dev/null +++ b/pkgtools/pkglint/files/mklines_test.go @@ -0,0 +1,59 @@ +package main + +import ( + check "gopkg.in/check.v1" +) + +func (s *Suite) TestMkLines_AutofixConditionalIndentation(c *check.C) { + s.UseCommandLine(c, "--autofix", "-Wspace") + tmpfile := s.CreateTmpFile(c, "fname.mk", "") + mklines := s.NewMkLines(tmpfile, + "# $"+"NetBSD$", + ".if defined(A)", + ".for a in ${A}", + ".if defined(C)", + ".endif", + ".endfor", + ".endif") + + mklines.Check() + + c.Check(s.OutputCleanTmpdir(), equals, ""+ + "AUTOFIX: ~/fname.mk:3: Replacing \".\" with \". \".\n"+ + "AUTOFIX: ~/fname.mk:4: Replacing \".\" with \". \".\n"+ + "AUTOFIX: ~/fname.mk:5: Replacing \".\" with \". \".\n"+ + "AUTOFIX: ~/fname.mk:6: Replacing \".\" with \". \".\n"+ + "AUTOFIX: ~/fname.mk: Has been auto-fixed. Please re-run pkglint.\n") + c.Check(s.LoadTmpFile(c, "fname.mk"), equals, ""+ + "# $NetBSD: mklines_test.go,v 1.1 2016/01/12 01:02:49 rillig Exp $\n"+ + ".if defined(A)\n"+ + ". for a in ${A}\n"+ + ". if defined(C)\n"+ + ". endif\n"+ + ". endfor\n"+ + ".endif\n") +} + +func (s *Suite) TestMkLines_UnusualTarget(c *check.C) { + mklines := s.NewMkLines("Makefile", + "# $"+"NetBSD$", + "", + "echo: echo.c", + "\tcc -o ${.TARGET} ${.IMPSRC}") + + mklines.Check() + + c.Check(s.Output(), equals, "WARN: Makefile:3: Unusual target \"echo\".\n") +} + +func (s *Suite) TestMkLines_checklineInclude_Makefile(c *check.C) { + mklines := s.NewMkLines("Makefile", + "# $"+"NetBSD$", + ".include \"../../other/package/Makefile\"") + + mklines.Check() + + c.Check(s.Output(), equals, ""+ + "ERROR: Makefile:2: \"/other/package/Makefile\" does not exist.\n"+ + "ERROR: Makefile:2: Other Makefiles must not be included directly.\n") +} diff --git a/pkgtools/pkglint/files/package.go b/pkgtools/pkglint/files/package.go index 57bdd429e7d..75b0d972abd 100644 --- a/pkgtools/pkglint/files/package.go +++ b/pkgtools/pkglint/files/package.go @@ -1,87 +1,146 @@ package main import ( + "fmt" "path" "regexp" "strconv" "strings" ) -func checkpackagePossibleDowngrade() { - defer tracecall("checkpackagePossibleDowngrade")() +// Package contains data for the pkgsrc package that is currently checked. +type Package struct { + Pkgpath string // e.g. "category/pkgdir" + Pkgdir string // PKGDIR from the package Makefile + Filesdir string // FILESDIR from the package Makefile + Patchdir string // PATCHDIR from the package Makefile + DistinfoFile string // DISTINFO_FILE from the package Makefile + EffectivePkgname string // PKGNAME or DISTNAME from the package Makefile, including nb13 + EffectivePkgbase string // The effective PKGNAME without the version + EffectivePkgversion string // The version part of the effective PKGNAME, excluding nb13 + EffectivePkgnameLine *MkLine // The origin of the three effective_* values + SeenBsdPrefsMk bool // Has bsd.prefs.mk already been included? + + vardef map[string]*MkLine // (varname, varcanon) => line + varuse map[string]*MkLine // (varname, varcanon) => line + bl3 map[string]*Line // buildlink3.mk name => line; contains only buildlink3.mk files that are directly included. + plistSubstCond map[string]bool // varname => true; list of all variables that are used as conditionals (@comment or nothing) in PLISTs. + included map[string]*Line // fname => line + seenMakefileCommon bool // Does the package have any .includes? +} + +func NewPackage(pkgpath string) *Package { + pkg := &Package{ + Pkgpath: pkgpath, + vardef: make(map[string]*MkLine), + varuse: make(map[string]*MkLine), + bl3: make(map[string]*Line), + plistSubstCond: make(map[string]bool), + included: make(map[string]*Line), + } + for varname, line := range G.globalData.UserDefinedVars { + pkg.vardef[varname] = line + } + return pkg +} - m, _, pkgversion := match2(G.pkgContext.effectivePkgname, rePkgname) +func (pkg *Package) defineVar(mkline *MkLine, varname string) { + if pkg.vardef[varname] == nil { + pkg.vardef[varname] = mkline + } + varcanon := varnameCanon(varname) + if pkg.vardef[varcanon] == nil { + pkg.vardef[varcanon] = mkline + } +} + +func (pkg *Package) varValue(varname string) (string, bool) { + if mkline := pkg.vardef[varname]; mkline != nil { + return mkline.Value(), true + } + return "", false +} + +func (pkg *Package) checkPossibleDowngrade() { + if G.opts.DebugTrace { + defer tracecall0()() + } + + m, _, pkgversion := match2(pkg.EffectivePkgname, rePkgname) if !m { return } - line := G.pkgContext.effectivePkgnameLine + mkline := pkg.EffectivePkgnameLine - change := G.globalData.lastChange[G.pkgContext.pkgpath] + change := G.globalData.LastChange[pkg.Pkgpath] if change == nil { - _ = G.opts.DebugMisc && line.debugf("No change log for package %q", G.pkgContext.pkgpath) + if G.opts.DebugMisc { + mkline.Debug1("No change log for package %q", pkg.Pkgpath) + } return } - if change.action == "Updated" { - if pkgverCmp(pkgversion, change.version) < 0 { - line.warnf("The package is being downgraded from %s to %s", change.version, pkgversion) + if change.Action == "Updated" { + if pkgverCmp(pkgversion, change.Version) < 0 { + mkline.Warn2("The package is being downgraded from %s to %s", change.Version, pkgversion) } } } -func checklinesBuildlink3Inclusion(lines []*Line) { - defer tracecall("checklinesbuildlink3Inclusion")() - - if G.pkgContext == nil { - return +func (pkg *Package) checklinesBuildlink3Inclusion(mklines *MkLines) { + if G.opts.DebugTrace { + defer tracecall0()() } // Collect all the included buildlink3.mk files from the file. - includedFiles := make(map[string]*Line) - for _, line := range lines { - if m, _, file := match2(line.text, reMkInclude); m { + includedFiles := make(map[string]*MkLine) + for _, mkline := range mklines.mklines { + if mkline.IsInclude() { + file := mkline.Includefile() if m, bl3 := match1(file, `^\.\./\.\./(.*)/buildlink3\.mk`); m { - includedFiles[bl3] = line - if G.pkgContext.bl3[bl3] == nil { - line.warnf("%s/buildlink3.mk is included by this file but not by the package.", bl3) + includedFiles[bl3] = mkline + if pkg.bl3[bl3] == nil { + mkline.Warn1("%s/buildlink3.mk is included by this file but not by the package.", bl3) } } } } if G.opts.DebugMisc { - for packageBl3, line := range G.pkgContext.bl3 { + for packageBl3, line := range pkg.bl3 { if includedFiles[packageBl3] == nil { - line.debugf("%s/buildlink3.mk is included by the package but not by the buildlink3.mk file.", packageBl3) + line.Debug1("%s/buildlink3.mk is included by the package but not by the buildlink3.mk file.", packageBl3) } } } } func checkdirPackage(pkgpath string) { - defer tracecall("checkdirPackage", pkgpath)() - ctx := newPkgContext(pkgpath) - G.pkgContext = ctx - defer func() { G.pkgContext = nil }() + if G.opts.DebugTrace { + defer tracecall1(pkgpath)() + } + + G.Pkg = NewPackage(pkgpath) + defer func() { G.Pkg = nil }() + pkg := G.Pkg // we need to handle the Makefile first to get some variables - lines := loadPackageMakefile(G.currentDir + "/Makefile") + lines := pkg.loadPackageMakefile(G.CurrentDir + "/Makefile") if lines == nil { - errorf(G.currentDir+"/Makefile", noLines, "Cannot be read.") return } - files := dirglob(G.currentDir) - if ctx.pkgdir != "." { - files = append(files, dirglob(G.currentDir+"/"+ctx.pkgdir)...) + files := dirglob(G.CurrentDir) + if pkg.Pkgdir != "." { + files = append(files, dirglob(G.CurrentDir+"/"+pkg.Pkgdir)...) } if G.opts.CheckExtra { - files = append(files, dirglob(G.currentDir+"/"+ctx.filesdir)...) + files = append(files, dirglob(G.CurrentDir+"/"+pkg.Filesdir)...) } - files = append(files, dirglob(G.currentDir+"/"+ctx.patchdir)...) - if ctx.distinfoFile != "distinfo" && ctx.distinfoFile != "./distinfo" { - files = append(files, G.currentDir+"/"+ctx.distinfoFile) + files = append(files, dirglob(G.CurrentDir+"/"+pkg.Patchdir)...) + if pkg.DistinfoFile != "distinfo" && pkg.DistinfoFile != "./distinfo" { + files = append(files, G.CurrentDir+"/"+pkg.DistinfoFile) } haveDistinfo := false havePatches := false @@ -90,24 +149,23 @@ func checkdirPackage(pkgpath string) { for _, fname := range files { if (hasPrefix(path.Base(fname), "Makefile.") || hasSuffix(fname, ".mk")) && !matches(fname, `patch-`) && - !contains(fname, G.pkgContext.pkgdir+"/") && - !contains(fname, G.pkgContext.filesdir+"/") { + !contains(fname, pkg.Pkgdir+"/") && + !contains(fname, pkg.Filesdir+"/") { if lines, err := readLines(fname, true); err == nil && lines != nil { - ParselinesMk(lines) - determineUsedVariables(lines) + NewMkLines(lines).DetermineUsedVariables() } } } for _, fname := range files { - if fname == G.currentDir+"/Makefile" { + if fname == G.CurrentDir+"/Makefile" { if G.opts.CheckMakefile { - checkfilePackageMakefile(fname, lines) + pkg.checkfilePackageMakefile(fname, lines) } } else { - checkfile(fname) + Checkfile(fname) } - if matches(fname, `/patches/patch-*$`) { + if contains(fname, "/patches/patch-") { havePatches = true } else if hasSuffix(fname, "/distinfo") { haveDistinfo = true @@ -116,186 +174,345 @@ func checkdirPackage(pkgpath string) { if G.opts.CheckDistinfo && G.opts.CheckPatches { if havePatches && !haveDistinfo { - warnf(G.currentDir+"/"+ctx.distinfoFile, noLines, "File not found. Please run \"%s makepatchsum\".", confMake) + Warnf(G.CurrentDir+"/"+pkg.DistinfoFile, noLines, "File not found. Please run \"%s makepatchsum\".", confMake) } } - if !isEmptyDir(G.currentDir + "/scripts") { - warnf(G.currentDir+"/scripts", noLines, "This directory and its contents are deprecated! Please call the script(s) explicitly from the corresponding target(s) in the pkg's Makefile.") + if !isEmptyDir(G.CurrentDir + "/scripts") { + Warnf(G.CurrentDir+"/scripts", noLines, "This directory and its contents are deprecated! Please call the script(s) explicitly from the corresponding target(s) in the pkg's Makefile.") } } -func checkfilePackageMakefile(fname string, lines []*Line) { - defer tracecall("checkfilePackageMakefile", fname, len(lines))() +func (pkg *Package) loadPackageMakefile(fname string) *MkLines { + if G.opts.DebugTrace { + defer tracecall1(fname)() + } + + mainLines, allLines := NewMkLines(nil), NewMkLines(nil) + if !readMakefile(fname, mainLines, allLines, "") { + return nil + } - vardef := G.pkgContext.vardef - if vardef["PLIST_SRC"] == nil && - vardef["GENERATE_PLIST"] == nil && - vardef["META_PACKAGE"] == nil && - !fileExists(G.currentDir+"/"+G.pkgContext.pkgdir+"/PLIST") && - !fileExists(G.currentDir+"/"+G.pkgContext.pkgdir+"/PLIST.common") { - warnf(fname, noLines, "Neither PLIST nor PLIST.common exist, and PLIST_SRC is unset. Are you sure PLIST handling is ok?") + if G.opts.DumpMakefile { + Debugf(G.CurrentDir, noLines, "Whole Makefile (with all included files) follows:") + for _, line := range allLines.lines { + fmt.Printf("%s\n", line.String()) + } } - if (vardef["NO_CHECKSUM"] != nil || vardef["META_PACKAGE"] != nil) && isEmptyDir(G.currentDir+"/"+G.pkgContext.patchdir) { - if distinfoFile := G.currentDir + "/" + G.pkgContext.distinfoFile; fileExists(distinfoFile) { - warnf(distinfoFile, noLines, "This file should not exist if NO_CHECKSUM or META_PACKAGE is set.") + allLines.DetermineUsedVariables() + + pkg.Pkgdir = expandVariableWithDefault("PKGDIR", ".") + pkg.DistinfoFile = expandVariableWithDefault("DISTINFO_FILE", "distinfo") + pkg.Filesdir = expandVariableWithDefault("FILESDIR", "files") + pkg.Patchdir = expandVariableWithDefault("PATCHDIR", "patches") + + if varIsDefined("PHPEXT_MK") { + if !varIsDefined("USE_PHP_EXT_PATCHES") { + pkg.Patchdir = "patches" } - } else { - if distinfoFile := G.currentDir + "/" + G.pkgContext.distinfoFile; !containsVarRef(distinfoFile) && !fileExists(distinfoFile) { - warnf(distinfoFile, noLines, "File not found. Please run \"%s makesum\".", confMake) + if varIsDefined("PECL_VERSION") { + pkg.DistinfoFile = "distinfo" } } - if vardef["REPLACE_PERL"] != nil && vardef["NO_CONFIGURE"] != nil { - vardef["REPLACE_PERL"].warnf("REPLACE_PERL is ignored when ...") - vardef["NO_CONFIGURE"].warnf("... NO_CONFIGURE is set.") + if G.opts.DebugMisc { + dummyLine.Debug1("DISTINFO_FILE=%s", pkg.DistinfoFile) + dummyLine.Debug1("FILESDIR=%s", pkg.Filesdir) + dummyLine.Debug1("PATCHDIR=%s", pkg.Patchdir) + dummyLine.Debug1("PKGDIR=%s", pkg.Pkgdir) } - if vardef["LICENSE"] == nil { - errorf(fname, noLines, "Each package must define its LICENSE.") + return mainLines +} + +func readMakefile(fname string, mainLines *MkLines, allLines *MkLines, includingFnameForUsedCheck string) bool { + if G.opts.DebugTrace { + defer tracecall1(fname)() } - if vardef["GNU_CONFIGURE"] != nil && vardef["USE_LANGUAGES"] != nil { - languagesLine := vardef["USE_LANGUAGES"] - value := languagesLine.extra["value"].(string) + fileLines := LoadNonemptyLines(fname, true) + if fileLines == nil { + return false + } + fileMklines := NewMkLines(fileLines) - if languagesLine.extra["comment"] != nil && matches(languagesLine.extra["comment"].(string), `(?-i)\b(?:c|empty|none)\b`) { - // Don't emit a warning, since the comment - // probably contains a statement that C is - // really not needed. + isMainMakefile := len(mainLines.mklines) == 0 + + for _, mkline := range fileMklines.mklines { + line := mkline.Line - } else if !matches(value, `(?:^|\s+)(?:c|c99|objc)(?:\s+|$)`) { - vardef["GNU_CONFIGURE"].warnf("GNU_CONFIGURE almost always needs a C compiler, ...") - languagesLine.warnf("... but \"c\" is not added to USE_LANGUAGES.") + if isMainMakefile { + mainLines.mklines = append(mainLines.mklines, mkline) + mainLines.lines = append(mainLines.lines, line) + } + allLines.mklines = append(allLines.mklines, mkline) + allLines.lines = append(allLines.lines, line) + + var includeFile, incDir, incBase string + if mkline.IsInclude() { + inc := mkline.Includefile() + includeFile = resolveVariableRefs(resolveVarsInRelativePath(inc, true)) + if containsVarRef(includeFile) { + if !contains(fname, "/mk/") { + line.Note1("Skipping include file %q. This may result in false warnings.", includeFile) + } + includeFile = "" + } + incDir, incBase = path.Split(includeFile) } - } - distnameLine := vardef["DISTNAME"] - pkgnameLine := vardef["PKGNAME"] + if includeFile != "" { + if path.Base(fname) != "buildlink3.mk" { + if m, bl3File := match1(includeFile, `^\.\./\.\./(.*)/buildlink3\.mk$`); m { + G.Pkg.bl3[bl3File] = line + if G.opts.DebugMisc { + line.Debug1("Buildlink3 file in package: %q", bl3File) + } + } + } + } - distname := "" - if distnameLine != nil { - distname = distnameLine.extra["value"].(string) + if includeFile != "" && G.Pkg.included[includeFile] == nil { + G.Pkg.included[includeFile] = line + + if matches(includeFile, `^\.\./[^./][^/]*/[^/]+`) { + mkline.Warn0("References to other packages should look like \"../../category/package\", not \"../package\".") + mkline.explainRelativeDirs() + } + + if path.Base(fname) == "Makefile" && !hasPrefix(incDir, "../../mk/") && incBase != "buildlink3.mk" && incBase != "builtin.mk" && incBase != "options.mk" { + if G.opts.DebugInclude { + line.Debug1("Including %q sets seenMakefileCommon.", includeFile) + } + G.Pkg.seenMakefileCommon = true + } + + if !contains(incDir, "/mk/") || strings.HasSuffix(includeFile, "/mk/haskell.mk") { + dirname, _ := path.Split(fname) + dirname = cleanpath(dirname) + + // Only look in the directory relative to the + // current file and in the current working directory. + // Pkglint doesn’t have an include dir list, like make(1) does. + if !fileExists(dirname + "/" + includeFile) { + dirname = G.CurrentDir + } + if !fileExists(dirname + "/" + includeFile) { + line.Error1("Cannot read %q.", dirname+"/"+includeFile) + return false + } + + if G.opts.DebugInclude { + line.Debug1("Including %q.", dirname+"/"+includeFile) + } + includingFname := ifelseStr(incBase == "Makefile.common" && incDir != "", fname, "") + if !readMakefile(dirname+"/"+includeFile, mainLines, allLines, includingFname) { + return false + } + } + } + + if mkline.IsVarassign() { + varname, op, value := mkline.Varname(), mkline.Op(), mkline.Value() + + if op != opAssignDefault || G.Pkg.vardef[varname] == nil { + if G.opts.DebugMisc { + line.Debugf("varassign(%q, %q, %q)", varname, op, value) + } + G.Pkg.vardef[varname] = mkline + } + } } - pkgname := "" - if pkgnameLine != nil { - pkgname = pkgnameLine.extra["value"].(string) + + if includingFnameForUsedCheck != "" { + fileMklines.checkForUsedComment(relpath(G.globalData.Pkgsrcdir, includingFnameForUsedCheck)) } - if distname != "" && pkgname != "" { - pkgname = pkgnameFromDistname(pkgname, distname) + return true +} + +func (pkg *Package) checkfilePackageMakefile(fname string, mklines *MkLines) { + if G.opts.DebugTrace { + defer tracecall1(fname)() } - if pkgname != "" && pkgname == distname && pkgnameLine.extra["comment"].(string) == "" { - pkgnameLine.notef("PKGNAME is ${DISTNAME} by default. You probably don't need to define PKGNAME.") + vardef := pkg.vardef + if vardef["PLIST_SRC"] == nil && + vardef["GENERATE_PLIST"] == nil && + vardef["META_PACKAGE"] == nil && + !fileExists(G.CurrentDir+"/"+pkg.Pkgdir+"/PLIST") && + !fileExists(G.CurrentDir+"/"+pkg.Pkgdir+"/PLIST.common") { + Warnf(fname, noLines, "Neither PLIST nor PLIST.common exist, and PLIST_SRC is unset. Are you sure PLIST handling is ok?") } - if pkgname == "" && distname != "" && !containsVarRef(distname) && !matches(distname, rePkgname) { - distnameLine.warnf("As DISTNAME is not a valid package name, please define the PKGNAME explicitly.") + if (vardef["NO_CHECKSUM"] != nil || vardef["META_PACKAGE"] != nil) && isEmptyDir(G.CurrentDir+"/"+pkg.Patchdir) { + if distinfoFile := G.CurrentDir + "/" + pkg.DistinfoFile; fileExists(distinfoFile) { + Warnf(distinfoFile, noLines, "This file should not exist if NO_CHECKSUM or META_PACKAGE is set.") + } + } else { + if distinfoFile := G.CurrentDir + "/" + pkg.DistinfoFile; !containsVarRef(distinfoFile) && !fileExists(distinfoFile) { + Warnf(distinfoFile, noLines, "File not found. Please run \"%s makesum\".", confMake) + } } - G.pkgContext.effectivePkgname, - G.pkgContext.effectivePkgnameLine, - G.pkgContext.effectivePkgbase, - G.pkgContext.effectivePkgversion = determineEffectivePkgVars(pkgname, pkgnameLine, distname, distnameLine) + if vardef["REPLACE_PERL"] != nil && vardef["NO_CONFIGURE"] != nil { + vardef["REPLACE_PERL"].Warn0("REPLACE_PERL is ignored when ...") + vardef["NO_CONFIGURE"].Warn0("... NO_CONFIGURE is set.") + } - if G.pkgContext.effectivePkgnameLine != nil { - _ = G.opts.DebugMisc && G.pkgContext.effectivePkgnameLine.debugf("Effective name=%q base=%q version=%q", - G.pkgContext.effectivePkgname, G.pkgContext.effectivePkgbase, G.pkgContext.effectivePkgversion) + if vardef["LICENSE"] == nil { + Errorf(fname, noLines, "Each package must define its LICENSE.") } - checkpackagePossibleDowngrade() + if vardef["GNU_CONFIGURE"] != nil && vardef["USE_LANGUAGES"] != nil { + languagesLine := vardef["USE_LANGUAGES"] - if vardef["COMMENT"] == nil { - warnf(fname, noLines, "No COMMENT given.") - } + if matches(languagesLine.Comment(), `(?-i)\b(?:c|empty|none)\b`) { + // Don't emit a warning, since the comment + // probably contains a statement that C is + // really not needed. - if vardef["USE_IMAKE"] != nil && vardef["USE_X11"] != nil { - vardef["USE_IMAKE"].notef("USE_IMAKE makes ...") - vardef["USE_X11"].notef("... USE_X11 superfluous.") + } else if !matches(languagesLine.Value(), `(?:^|\s+)(?:c|c99|objc)(?:\s+|$)`) { + vardef["GNU_CONFIGURE"].Warn0("GNU_CONFIGURE almost always needs a C compiler, ...") + languagesLine.Warn0("... but \"c\" is not added to USE_LANGUAGES.") + } } - if G.pkgContext.effectivePkgbase != "" { - for _, sugg := range G.globalData.getSuggestedPackageUpdates() { - if G.pkgContext.effectivePkgbase != sugg.pkgname { - continue - } + pkg.determineEffectivePkgVars() + pkg.checkPossibleDowngrade() - suggver, comment := sugg.version, sugg.comment - if comment != "" { - comment = " (" + comment + ")" - } + if vardef["COMMENT"] == nil { + Warnf(fname, noLines, "No COMMENT given.") + } - pkgnameLine := G.pkgContext.effectivePkgnameLine - cmp := pkgverCmp(G.pkgContext.effectivePkgversion, suggver) - switch { - case cmp < 0: - pkgnameLine.warnf("This package should be updated to %s%s.", sugg.version, comment) - pkgnameLine.explain( - "The wishlist for package updates in doc/TODO mentions that a newer", - "version of this package is available.") - case cmp > 0: - pkgnameLine.notef("This package is newer than the update request to %s%s.", suggver, comment) - default: - pkgnameLine.notef("The update request to %s from doc/TODO%s has been done.", suggver, comment) - } - } + if vardef["USE_IMAKE"] != nil && vardef["USE_X11"] != nil { + vardef["USE_IMAKE"].Note0("USE_IMAKE makes ...") + vardef["USE_X11"].Note0("... USE_X11 superfluous.") } - ChecklinesMk(lines) - ChecklinesPackageMakefileVarorder(lines) - saveAutofixChanges(lines) + pkg.checkUpdate() + mklines.Check() + pkg.ChecklinesPackageMakefileVarorder(mklines) + SaveAutofixChanges(mklines.lines) } -func getNbpart() string { - line := G.pkgContext.vardef["PKGREVISION"] +func (pkg *Package) getNbpart() string { + line := pkg.vardef["PKGREVISION"] if line == nil { return "" } - pkgrevision := line.extra["value"].(string) + pkgrevision := line.Value() if rev, err := strconv.Atoi(pkgrevision); err == nil { - return sprintf("nb%d", rev) + return "nb" + strconv.Itoa(rev) } return "" } -func determineEffectivePkgVars(pkgname string, pkgnameLine *Line, distname string, distnameLine *Line) (string, *Line, string, string) { +func (pkg *Package) determineEffectivePkgVars() { + distnameLine := pkg.vardef["DISTNAME"] + pkgnameLine := pkg.vardef["PKGNAME"] + + distname := "" + if distnameLine != nil { + distname = distnameLine.Value() + } + pkgname := "" + if pkgnameLine != nil { + pkgname = pkgnameLine.Value() + } + + if distname != "" && pkgname != "" { + pkgname = pkg.pkgnameFromDistname(pkgname, distname) + } + + if pkgname != "" && pkgname == distname && pkgnameLine.Comment() == "" { + pkgnameLine.Note0("PKGNAME is ${DISTNAME} by default. You probably don't need to define PKGNAME.") + } + + if pkgname == "" && distname != "" && !containsVarRef(distname) && !matches(distname, rePkgname) { + distnameLine.Warn0("As DISTNAME is not a valid package name, please define the PKGNAME explicitly.") + } + if pkgname != "" && !containsVarRef(pkgname) { if m, m1, m2 := match2(pkgname, rePkgname); m { - return pkgname + getNbpart(), pkgnameLine, m1, m2 + pkg.EffectivePkgname = pkgname + pkg.getNbpart() + pkg.EffectivePkgnameLine = pkgnameLine + pkg.EffectivePkgbase = m1 + pkg.EffectivePkgversion = m2 } } - if distname != "" && !containsVarRef(distname) { + if pkg.EffectivePkgnameLine == nil && distname != "" && !containsVarRef(distname) { if m, m1, m2 := match2(distname, rePkgname); m { - return distname + getNbpart(), distnameLine, m1, m2 + pkg.EffectivePkgname = distname + pkg.getNbpart() + pkg.EffectivePkgnameLine = distnameLine + pkg.EffectivePkgbase = m1 + pkg.EffectivePkgversion = m2 + } + } + if pkg.EffectivePkgnameLine != nil { + if G.opts.DebugMisc { + pkg.EffectivePkgnameLine.Line.Debugf("Effective name=%q base=%q version=%q", + pkg.EffectivePkgname, pkg.EffectivePkgbase, pkg.EffectivePkgversion) } } - return "", nil, "", "" } -func pkgnameFromDistname(pkgname, distname string) string { +func (pkg *Package) pkgnameFromDistname(pkgname, distname string) string { pkgname = strings.Replace(pkgname, "${DISTNAME}", distname, -1) if m, before, sep, subst, after := match4(pkgname, `^(.*)\$\{DISTNAME:S(.)([^\\}:]+)\}(.*)$`); m { qsep := regexp.QuoteMeta(sep) if m, left, from, right, to, mod := match5(subst, `^(\^?)([^:]*)(\$?)`+qsep+`([^:]*)`+qsep+`(g?)$`); m { newPkgname := before + mkopSubst(distname, left != "", from, right != "", to, mod != "") + after - _ = G.opts.DebugMisc && G.pkgContext.vardef["PKGNAME"].debugf("pkgnameFromDistname %q => %q", pkgname, newPkgname) + if G.opts.DebugMisc { + pkg.vardef["PKGNAME"].Debug2("pkgnameFromDistname %q => %q", pkgname, newPkgname) + } pkgname = newPkgname } } return pkgname } -func ChecklinesPackageMakefileVarorder(lines []*Line) { - defer tracecall("ChecklinesPackageMakefileVarorder", len(lines)) +func (pkg *Package) checkUpdate() { + if pkg.EffectivePkgbase != "" { + for _, sugg := range G.globalData.GetSuggestedPackageUpdates() { + if pkg.EffectivePkgbase != sugg.Pkgname { + continue + } + + suggver, comment := sugg.Version, sugg.Comment + if comment != "" { + comment = " (" + comment + ")" + } - if !G.opts.WarnOrder { + pkgnameLine := pkg.EffectivePkgnameLine + cmp := pkgverCmp(pkg.EffectivePkgversion, suggver) + switch { + case cmp < 0: + pkgnameLine.Warn2("This package should be updated to %s%s.", sugg.Version, comment) + Explain2( + "The wishlist for package updates in doc/TODO mentions that a newer", + "version of this package is available.") + case cmp > 0: + pkgnameLine.Note2("This package is newer than the update request to %s%s.", suggver, comment) + default: + pkgnameLine.Note2("The update request to %s from doc/TODO%s has been done.", suggver, comment) + } + } + } +} + +func (pkg *Package) ChecklinesPackageMakefileVarorder(mklines *MkLines) { + if G.opts.DebugTrace { + defer tracecall0()() + } + + if !G.opts.WarnOrder || pkg.seenMakefileCommon { return } - type OccCount int + type OccCount uint8 const ( once OccCount = iota optional @@ -378,10 +595,6 @@ func ChecklinesPackageMakefileVarorder(lines []*Line) { }, } - if G.pkgContext == nil || G.pkgContext.seenMakefileCommon { - return - } - lineno := 0 sectindex := -1 varindex := 0 @@ -399,12 +612,15 @@ func ChecklinesPackageMakefileVarorder(lines []*Line) { // - new lineno > old lineno // - new sectindex > old sectindex // - new sectindex == old sectindex && new varindex > old varindex - // - new next_section == true && old next_section == false - for lineno <= len(lines) { - line := lines[lineno] - text := line.text - - _ = G.opts.DebugMisc && line.debugf("[varorder] section %d variable %d", sectindex, varindex) + // - new nextSection == true && old nextSection == false + for lineno < len(mklines.lines) { + mkline := mklines.mklines[lineno] + line := mklines.lines[lineno] + text := line.Text + + if G.opts.DebugMisc { + line.Debugf("[varorder] section %d variable %d vars %v", sectindex, varindex, vars) + } if nextSection { nextSection = false @@ -421,14 +637,14 @@ func ChecklinesPackageMakefileVarorder(lines []*Line) { case hasPrefix(text, "#"): lineno++ - case line.extra["varcanon"] != nil: - varcanon := line.extra["varcanon"].(string) + case mkline.IsVarassign(): + varcanon := mkline.Varcanon() if belowText, exists := below[varcanon]; exists { if belowText != "" { - line.warnf("%s appears too late. Please put it below %s.", varcanon, belowText) + line.Warn2("%s appears too late. Please put it below %s.", varcanon, belowText) } else { - line.warnf("%s appears too late. It should be the very first definition.", varcanon) + line.Warn1("%s appears too late. It should be the very first definition.", varcanon) } lineno++ continue @@ -444,12 +660,12 @@ func ChecklinesPackageMakefileVarorder(lines []*Line) { switch { case !(varindex < len(vars)): if sections[sectindex].count != optional { - line.warnf("Empty line expected.") + line.Warn0("Empty line expected.") } nextSection = true case varcanon != vars[varindex].varname: - line.warnf("Expected %s, but found %s.", vars[varindex].varname, varcanon) + line.Warn2("Expected %s, but found %s.", vars[varindex].varname, varcanon) lineno++ default: @@ -464,7 +680,7 @@ func ChecklinesPackageMakefileVarorder(lines []*Line) { default: for varindex < len(vars) { if vars[varindex].count == once && !maySkipSection { - line.warnf("%s should be set here.", vars[varindex].varname) + line.Warn1("%s should be set here.", vars[varindex].varname) } below[vars[varindex].varname] = belowWhat varindex++ @@ -477,3 +693,37 @@ func ChecklinesPackageMakefileVarorder(lines []*Line) { } } } + +func (mklines *MkLines) checkForUsedComment(relativeName string) { + lines := mklines.lines + if len(lines) < 3 { + return + } + + expected := "# used by " + relativeName + for _, line := range lines { + if line.Text == expected { + return + } + } + + i := 0 + for i < 2 && hasPrefix(lines[i].Text, "#") { + i++ + } + + insertLine := lines[i] + if !insertLine.AutofixInsertBefore(expected) { + insertLine.Warn1("Please add a line %q here.", expected) + Explain( + "Since Makefile.common files usually don't have any comments and", + "therefore not a clearly defined interface, they should at least", + "contain references to all files that include them, so that it is", + "easier to see what effects future changes may have.", + "", + "If there are more than five packages that use a Makefile.common,", + "you should think about giving it a proper name (maybe plugin.mk) and", + "documenting its interface.") + } + SaveAutofixChanges(lines) +} diff --git a/pkgtools/pkglint/files/package_test.go b/pkgtools/pkglint/files/package_test.go index 5d80781c5f6..98672229767 100644 --- a/pkgtools/pkglint/files/package_test.go +++ b/pkgtools/pkglint/files/package_test.go @@ -5,45 +5,116 @@ import ( ) func (s *Suite) TestPkgnameFromDistname(c *check.C) { - G.pkgContext = newPkgContext("dummy") - G.pkgContext.vardef["PKGNAME"] = NewLine("dummy", "dummy", "dummy", nil) - - c.Check(pkgnameFromDistname("pkgname-1.0", "whatever"), equals, "pkgname-1.0") - c.Check(pkgnameFromDistname("${DISTNAME}", "distname-1.0"), equals, "distname-1.0") - c.Check(pkgnameFromDistname("${DISTNAME:S/dist/pkg/}", "distname-1.0"), equals, "pkgname-1.0") - c.Check(pkgnameFromDistname("${DISTNAME:S|a|b|g}", "panama-0.13"), equals, "pbnbmb-0.13") - c.Check(pkgnameFromDistname("${DISTNAME:S|^lib||}", "libncurses"), equals, "ncurses") - c.Check(pkgnameFromDistname("${DISTNAME:S|^lib||}", "mylib"), equals, "mylib") + pkg := NewPackage("dummy") + pkg.vardef["PKGNAME"] = NewMkLine(NewLine("Makefile", 5, "PKGNAME=dummy", nil)) + + c.Check(pkg.pkgnameFromDistname("pkgname-1.0", "whatever"), equals, "pkgname-1.0") + c.Check(pkg.pkgnameFromDistname("${DISTNAME}", "distname-1.0"), equals, "distname-1.0") + c.Check(pkg.pkgnameFromDistname("${DISTNAME:S/dist/pkg/}", "distname-1.0"), equals, "pkgname-1.0") + c.Check(pkg.pkgnameFromDistname("${DISTNAME:S|a|b|g}", "panama-0.13"), equals, "pbnbmb-0.13") + c.Check(pkg.pkgnameFromDistname("${DISTNAME:S|^lib||}", "libncurses"), equals, "ncurses") + c.Check(pkg.pkgnameFromDistname("${DISTNAME:S|^lib||}", "mylib"), equals, "mylib") + + c.Check(s.Output(), equals, "") } func (s *Suite) TestChecklinesPackageMakefileVarorder(c *check.C) { s.UseCommandLine(c, "-Worder") - G.pkgContext = newPkgContext("x11/9term") - lines := s.NewLines("Makefile", + pkg := NewPackage("x11/9term") + + pkg.ChecklinesPackageMakefileVarorder(s.NewMkLines("Makefile", "# $"+"NetBSD$", "", "DISTNAME=9term", - "CATEGORIES=x11") + "CATEGORIES=x11")) - ChecklinesPackageMakefileVarorder(lines) + c.Check(s.Output(), equals, "") + + pkg.ChecklinesPackageMakefileVarorder(s.NewMkLines("Makefile", + "# $"+"NetBSD$", + "", + "DISTNAME=9term", + "CATEGORIES=x11", + "", + ".include \"../../mk/bsd.pkg.mk\"")) c.Check(s.Output(), equals, ""+ - "WARN: Makefile:3: CATEGORIES should be set here.\n"+ - "WARN: Makefile:3: COMMENT should be set here.\n"+ - "WARN: Makefile:3: LICENSE should be set here.\n") + "WARN: Makefile:6: COMMENT should be set here.\n"+ + "WARN: Makefile:6: LICENSE should be set here.\n") } func (s *Suite) TestGetNbpart(c *check.C) { - G.pkgContext = newPkgContext("category/pkgbase") - line := NewLine("Makefile", "1", "PKGREVISION=14", nil) - parselineMk(line) - G.pkgContext.vardef["PKGREVISION"] = line + pkg := NewPackage("category/pkgbase") + pkg.vardef["PKGREVISION"] = NewMkLine(NewLine("Makefile", 1, "PKGREVISION=14", nil)) + + c.Check(pkg.getNbpart(), equals, "nb14") + + pkg.vardef["PKGREVISION"] = NewMkLine(NewLine("Makefile", 1, "PKGREVISION=asdf", nil)) + + c.Check(pkg.getNbpart(), equals, "") +} + +func (s *Suite) TestMkLines_CheckForUsedComment(c *check.C) { + s.UseCommandLine(c, "--show-autofix") + s.NewMkLines("Makefile.common", + "# $"+"NetBSD$", + "", + "# used by sysutils/mc", + ).checkForUsedComment("sysutils/mc") + + c.Check(s.Output(), equals, "") + + s.NewMkLines("Makefile.common").checkForUsedComment("category/package") + + c.Check(s.Output(), equals, "") + + s.NewMkLines("Makefile.common", + "# $"+"NetBSD$", + ).checkForUsedComment("category/package") + + c.Check(s.Output(), equals, "") + + s.NewMkLines("Makefile.common", + "# $"+"NetBSD$", + "", + ).checkForUsedComment("category/package") + + c.Check(s.Output(), equals, "") + + s.NewMkLines("Makefile.common", + "# $"+"NetBSD$", + "", + "VARNAME=\tvalue", + ).checkForUsedComment("category/package") + + c.Check(s.Output(), equals, ""+ + "WARN: Makefile.common:2: Please add a line \"# used by category/package\" here.\n"+ + "AUTOFIX: Makefile.common:2: Inserting a line \"# used by category/package\" before this line.\n") + + s.NewMkLines("Makefile.common", + "# $"+"NetBSD$", + "#", + "#", + ).checkForUsedComment("category/package") + + c.Check(s.Output(), equals, ""+ + "WARN: Makefile.common:3: Please add a line \"# used by category/package\" here.\n"+ + "AUTOFIX: Makefile.common:3: Inserting a line \"# used by category/package\" before this line.\n") +} + +func (s *Suite) TestPackage_DetermineEffectivePkgVars_Precedence(c *check.C) { + pkg := NewPackage("category/pkgbase") + pkgnameLine := NewMkLine(NewLine("Makefile", 3, "PKGNAME=pkgname-1.0", nil)) + distnameLine := NewMkLine(NewLine("Makefile", 4, "DISTNAME=distname-1.0", nil)) + pkgrevisionLine := NewMkLine(NewLine("Makefile", 5, "PKGREVISION=13", nil)) - c.Check(getNbpart(), equals, "nb14") + pkg.defineVar(pkgnameLine, pkgnameLine.Varname()) + pkg.defineVar(distnameLine, distnameLine.Varname()) + pkg.defineVar(pkgrevisionLine, pkgrevisionLine.Varname()) - line = NewLine("Makefile", "1", "PKGREVISION=asdf", nil) - parselineMk(line) - G.pkgContext.vardef["PKGREVISION"] = line + pkg.determineEffectivePkgVars() - c.Check(getNbpart(), equals, "") + c.Check(pkg.EffectivePkgbase, equals, "pkgname") + c.Check(pkg.EffectivePkgname, equals, "pkgname-1.0nb13") + c.Check(pkg.EffectivePkgversion, equals, "1.0") } diff --git a/pkgtools/pkglint/files/parser.go b/pkgtools/pkglint/files/parser.go new file mode 100644 index 00000000000..0b7f19d054c --- /dev/null +++ b/pkgtools/pkglint/files/parser.go @@ -0,0 +1,267 @@ +package main + +import ( + "strings" +) + +type Parser struct { + repl *PrefixReplacer +} + +func NewParser(s string) *Parser { + return &Parser{NewPrefixReplacer(s)} +} + +func (p *Parser) EOF() bool { + return p.repl.rest == "" +} + +func (p *Parser) Rest() string { + return p.repl.rest +} + +func (p *Parser) PkgbasePattern() (pkgbase string) { + repl := p.repl + + for { + if repl.AdvanceRegexp(`^\$\{\w+\}`) || + repl.AdvanceRegexp(`^[\w.*+,{}]+`) || + repl.AdvanceRegexp(`^\[[\d-]+\]`) { + pkgbase += repl.m[0] + continue + } + + mark := repl.Mark() + if repl.AdvanceStr("-") { + if repl.AdvanceRegexp(`^\d`) || + repl.AdvanceRegexp(`^\$\{\w*VER\w*\}`) || + repl.AdvanceStr("[") { + repl.Reset(mark) + return + } + pkgbase += "-" + } else { + return + } + } +} + +func (p *Parser) Dependency() *DependencyPattern { + repl := p.repl + + var dp DependencyPattern + mark := repl.Mark() + dp.pkgbase = p.PkgbasePattern() + if dp.pkgbase == "" { + return nil + } + + mark2 := repl.Mark() + if repl.AdvanceStr(">=") || repl.AdvanceStr(">") { + op := repl.s + if repl.AdvanceRegexp(`^(?:(?:\$\{\w+\})+|\d[\w.]*)`) { + dp.lowerOp = op + dp.lower = repl.m[0] + } else { + repl.Reset(mark2) + } + } + if repl.AdvanceStr("<=") || repl.AdvanceStr("<") { + op := repl.s + if repl.AdvanceRegexp(`^(?:(?:\$\{\w+\})+|\d[\w.]*)`) { + dp.upperOp = op + dp.upper = repl.m[0] + } else { + repl.Reset(mark2) + } + } + if dp.lowerOp != "" || dp.upperOp != "" { + return &dp + } + if repl.AdvanceStr("-") && repl.rest != "" { + dp.wildcard = repl.AdvanceRest() + return &dp + } + if hasPrefix(dp.pkgbase, "${") && hasSuffix(dp.pkgbase, "}") { + return &dp + } + if hasSuffix(dp.pkgbase, "-*") { + dp.pkgbase = strings.TrimSuffix(dp.pkgbase, "-*") + dp.wildcard = "*" + return &dp + } + + repl.Reset(mark) + return nil +} + +type MkToken struct { + literal string + varuse MkVarUse +} +type MkVarUse struct { + varname string + modifiers []string +} + +func (p *Parser) MkTokens() []*MkToken { + repl := p.repl + + var tokens []*MkToken + for !p.EOF() { + if varuse := p.VarUse(); varuse != nil { + tokens = append(tokens, &MkToken{varuse: *varuse}) + continue + } + + mark := repl.Mark() + needsReplace := false + again: + dollar := strings.IndexByte(repl.rest, '$') + if dollar == -1 { + dollar = len(repl.rest) + } + repl.Skip(dollar) + if repl.AdvanceStr("$$") { + needsReplace = true + goto again + } + literal := repl.Since(mark) + if needsReplace { + literal = strings.Replace(literal, "$$", "$", -1) + } + if literal != "" { + tokens = append(tokens, &MkToken{literal: literal}) + continue + } + + break + } + return tokens +} + +func (p *Parser) Varname() string { + repl := p.repl + + mark := repl.Mark() + repl.AdvanceStr(".") + for p.VarUse() != nil || repl.AdvanceBytes(0x00000000, 0x03ff6800, 0x87fffffe, 0x07fffffe, `[\w+\-.]`) { + } + return repl.Since(mark) +} + +func (p *Parser) VarUse() *MkVarUse { + repl := p.repl + + mark := repl.Mark() + if repl.AdvanceStr("${") || repl.AdvanceStr("$(") { + closing := "}" + if repl.Since(mark) == "$(" { + closing = ")" + } + + varnameMark := repl.Mark() + varname := p.Varname() + if varname != "" { + modifiers := p.VarUseModifiers(closing) + if repl.AdvanceStr(closing) { + return &MkVarUse{varname, modifiers} + } + } + + for p.VarUse() != nil || repl.AdvanceRegexp(`^([^$:`+closing+`]|\$\$)+`) { + } + rest := p.Rest() + if hasPrefix(rest, ":L") || hasPrefix(rest, ":sh") || hasPrefix(rest, ":?") { + varexpr := repl.Since(varnameMark) + modifiers := p.VarUseModifiers(closing) + if repl.AdvanceStr(closing) { + return &MkVarUse{varexpr, modifiers} + } + } + repl.Reset(mark) + } + + return nil +} + +func (p *Parser) VarUseModifiers(closing string) []string { + repl := p.repl + + var modifiers []string + for repl.AdvanceStr(":") { + modifierMark := repl.Mark() + + switch repl.PeekByte() { + case 'E', 'H', 'L', 'O', 'Q', 'R', 'T', 's', 't', 'u': + if repl.AdvanceRegexp(`^(E|H|L|Ox?|Q|R|T|sh|tA|tW|tl|ts.|tu|tw|u)`) { + modifiers = append(modifiers, repl.Since(modifierMark)) + continue + } + + case '=', 'D', 'M', 'N', 'U': + if repl.AdvanceRegexp(`^[=DMNU]`) { + for p.VarUse() != nil || repl.AdvanceRegexp(`^([^$:`+closing+`]|\$\$)+`) { + } + modifiers = append(modifiers, repl.Since(modifierMark)) + continue + } + + case 'C', 'S': + if repl.AdvanceRegexp(`^[CS]([%,/:;@^|])`) { + separator := repl.m[1] + repl.AdvanceStr("^") + re := `^([^\` + separator + `$` + closing + `\\]|\$\$|\\.)+` + for p.VarUse() != nil || repl.AdvanceRegexp(re) { + } + repl.AdvanceStr("$") + if repl.AdvanceStr(separator) { + for p.VarUse() != nil || repl.AdvanceRegexp(re) { + } + if repl.AdvanceStr(separator) { + repl.AdvanceRegexp(`^[1gW]`) + modifiers = append(modifiers, repl.Since(modifierMark)) + continue + } + } + } + + case '@': + if repl.AdvanceRegexp(`^@([\w.]+)@`) { + for p.VarUse() != nil || repl.AdvanceRegexp(`^([^$:@`+closing+`\\]|\$\$|\\.)+`) { + } + if repl.AdvanceStr("@") { + modifiers = append(modifiers, repl.Since(modifierMark)) + continue + } + } + + case '[': + if repl.AdvanceRegexp(`^\[[-.\d]+\]`) { + modifiers = append(modifiers, repl.Since(modifierMark)) + continue + } + + case '?': + repl.AdvanceStr("?") + re := `^([^$:` + closing + `]|\$\$)+` + for p.VarUse() != nil || repl.AdvanceRegexp(re) { + } + if repl.AdvanceStr(":") { + for p.VarUse() != nil || repl.AdvanceRegexp(re) { + } + modifiers = append(modifiers, repl.Since(modifierMark)) + continue + } + } + + repl.Reset(modifierMark) + for p.VarUse() != nil || repl.AdvanceRegexp(`^([^:$`+closing+`]|\$\$)+`) { + } + if suffixSubst := repl.Since(modifierMark); contains(suffixSubst, "=") { + modifiers = append(modifiers, suffixSubst) + continue + } + } + return modifiers +} diff --git a/pkgtools/pkglint/files/parser_test.go b/pkgtools/pkglint/files/parser_test.go new file mode 100644 index 00000000000..4f66b6c2c68 --- /dev/null +++ b/pkgtools/pkglint/files/parser_test.go @@ -0,0 +1,136 @@ +package main + +import ( + check "gopkg.in/check.v1" +) + +func (s *Suite) TestParser_PkgbasePattern(c *check.C) { + test := func(pattern, expected, rest string) { + parser := NewParser(pattern) + actual := parser.PkgbasePattern() + c.Check(actual, equals, expected) + c.Check(parser.Rest(), equals, rest) + } + + test("fltk", "fltk", "") + test("fltk|", "fltk", "|") + test("boost-build-1.59.*", "boost-build", "-1.59.*") + test("${PHP_PKG_PREFIX}-pdo-5.*", "${PHP_PKG_PREFIX}-pdo", "-5.*") + test("${PYPKGPREFIX}-metakit-[0-9]*", "${PYPKGPREFIX}-metakit", "-[0-9]*") +} + +func (s *Suite) TestParser_Dependency(c *check.C) { + + testDependencyRest := func(pattern string, expected DependencyPattern, rest string) { + parser := NewParser(pattern) + dp := parser.Dependency() + if c.Check(dp, check.NotNil) { + c.Check(*dp, equals, expected) + c.Check(parser.Rest(), equals, rest) + } + } + testDependency := func(pattern string, expected DependencyPattern) { + testDependencyRest(pattern, expected, "") + } + + testDependency("fltk>=1.1.5rc1<1.3", DependencyPattern{"fltk", ">=", "1.1.5rc1", "<", "1.3", ""}) + testDependency("libwcalc-1.0*", DependencyPattern{"libwcalc", "", "", "", "", "1.0*"}) + testDependency("${PHP_PKG_PREFIX}-pdo-5.*", DependencyPattern{"${PHP_PKG_PREFIX}-pdo", "", "", "", "", "5.*"}) + testDependency("${PYPKGPREFIX}-metakit-[0-9]*", DependencyPattern{"${PYPKGPREFIX}-metakit", "", "", "", "", "[0-9]*"}) + testDependency("boost-build-1.59.*", DependencyPattern{"boost-build", "", "", "", "", "1.59.*"}) + testDependency("${_EMACS_REQD}", DependencyPattern{"${_EMACS_REQD}", "", "", "", "", ""}) + testDependency("{gcc46,gcc46-libs}>=4.6.0", DependencyPattern{"{gcc46,gcc46-libs}", ">=", "4.6.0", "", "", ""}) + testDependency("perl5-*", DependencyPattern{"perl5", "", "", "", "", "*"}) + testDependency("verilog{,-current}-[0-9]*", DependencyPattern{"verilog{,-current}", "", "", "", "", "[0-9]*"}) + testDependency("mpg123{,-esound,-nas}>=0.59.18", DependencyPattern{"mpg123{,-esound,-nas}", ">=", "0.59.18", "", "", ""}) + testDependency("mysql*-{client,server}-[0-9]*", DependencyPattern{"mysql*-{client,server}", "", "", "", "", "[0-9]*"}) + testDependency("postgresql8[0-35-9]-${module}-[0-9]*", DependencyPattern{"postgresql8[0-35-9]-${module}", "", "", "", "", "[0-9]*"}) + testDependency("ncurses-${NC_VERS}{,nb*}", DependencyPattern{"ncurses", "", "", "", "", "${NC_VERS}{,nb*}"}) + testDependency("xulrunner10>=${MOZ_BRANCH}${MOZ_BRANCH_MINOR}", DependencyPattern{"xulrunner10", ">=", "${MOZ_BRANCH}${MOZ_BRANCH_MINOR}", "", "", ""}) + testDependencyRest("gnome-control-center>=2.20.1{,nb*}", DependencyPattern{"gnome-control-center", ">=", "2.20.1", "", "", ""}, "{,nb*}") + // "{ssh{,6}-[0-9]*,openssh-[0-9]*}" is not representable using the current data structure +} + +func (s *Suite) TestParser_MkTokens(c *check.C) { + parse := func(input string, expectedTokens []*MkToken, expectedRest string) { + p := NewParser(input) + actualTokens := p.MkTokens() + c.Check(actualTokens, deepEquals, expectedTokens) + for i, expectedToken := range expectedTokens { + if i < len(actualTokens) { + c.Check(*actualTokens[i], deepEquals, *expectedToken) + } + } + c.Check(p.Rest(), equals, expectedRest) + } + token := func(input string, expectedToken MkToken) { + parse(input, []*MkToken{&expectedToken}, "") + } + literal := func(literal string) MkToken { + return MkToken{literal: literal} + } + varuse := func(varname string, modifiers ...string) MkToken { + return MkToken{varuse: MkVarUse{varname: varname, modifiers: modifiers}} + } + + token("literal", literal("literal")) + token("\\/share\\/ { print \"share directory\" }", literal("\\/share\\/ { print \"share directory\" }")) + token("find . -name \\*.orig -o -name \\*.pre", literal("find . -name \\*.orig -o -name \\*.pre")) + token("-e 's|\\$${EC2_HOME.*}|EC2_HOME}|g'", literal("-e 's|\\${EC2_HOME.*}|EC2_HOME}|g'")) + + token("${VARIABLE}", varuse("VARIABLE")) + token("${VARIABLE.param}", varuse("VARIABLE.param")) + token("${VARIABLE.${param}}", varuse("VARIABLE.${param}")) + token("${VARIABLE.hicolor-icon-theme}", varuse("VARIABLE.hicolor-icon-theme")) + token("${VARIABLE.gtk+extra}", varuse("VARIABLE.gtk+extra")) + token("${VARIABLE:S/old/new/}", varuse("VARIABLE", "S/old/new/")) + token("${GNUSTEP_LFLAGS:S/-L//g}", varuse("GNUSTEP_LFLAGS", "S/-L//g")) + token("${SUSE_VERSION:S/.//}", varuse("SUSE_VERSION", "S/.//")) + token("${MASTER_SITE_GNOME:=sources/alacarte/0.13/}", varuse("MASTER_SITE_GNOME", "=sources/alacarte/0.13/")) + token("${INCLUDE_DIRS:H:T}", varuse("INCLUDE_DIRS", "H", "T")) + token("${A.${B.${C.${D}}}}", varuse("A.${B.${C.${D}}}")) + token("${RUBY_VERSION:C/([0-9]+)\\.([0-9]+)\\.([0-9]+)/\\1/}", varuse("RUBY_VERSION", "C/([0-9]+)\\.([0-9]+)\\.([0-9]+)/\\1/")) + token("${PERL5_${_var_}:Q}", varuse("PERL5_${_var_}", "Q")) + token("${PKGNAME_REQD:C/(^.*-|^)py([0-9][0-9])-.*/\\2/}", varuse("PKGNAME_REQD", "C/(^.*-|^)py([0-9][0-9])-.*/\\2/")) + token("${PYLIB:S|/|\\\\/|g}", varuse("PYLIB", "S|/|\\\\/|g")) + token("${PKGNAME_REQD:C/ruby([0-9][0-9]+)-.*/\\1/}", varuse("PKGNAME_REQD", "C/ruby([0-9][0-9]+)-.*/\\1/")) + token("${RUBY_SHLIBALIAS:S/\\//\\\\\\//}", varuse("RUBY_SHLIBALIAS", "S/\\//\\\\\\//")) + token("${RUBY_VER_MAP.${RUBY_VER}:U${RUBY_VER}}", varuse("RUBY_VER_MAP.${RUBY_VER}", "U${RUBY_VER}")) + token("${RUBY_VER_MAP.${RUBY_VER}:U18}", varuse("RUBY_VER_MAP.${RUBY_VER}", "U18")) + token("${CONFIGURE_ARGS:S/ENABLE_OSS=no/ENABLE_OSS=yes/g}", varuse("CONFIGURE_ARGS", "S/ENABLE_OSS=no/ENABLE_OSS=yes/g")) + token("${PLIST_RUBY_DIRS:S,DIR=\"PREFIX/,DIR=\",}", varuse("PLIST_RUBY_DIRS", "S,DIR=\"PREFIX/,DIR=\",")) + token("${LDFLAGS:S/-Wl,//g:Q}", varuse("LDFLAGS", "S/-Wl,//g", "Q")) + token("${_PERL5_REAL_PACKLIST:S/^/${DESTDIR}/}", varuse("_PERL5_REAL_PACKLIST", "S/^/${DESTDIR}/")) + token("${_PYTHON_VERSION:C/^([0-9])/\\1./1}", varuse("_PYTHON_VERSION", "C/^([0-9])/\\1./1")) + token("${PKGNAME:S/py${_PYTHON_VERSION}/py${i}/}", varuse("PKGNAME", "S/py${_PYTHON_VERSION}/py${i}/")) + token("${PKGNAME:C/-[0-9].*$/-[0-9]*/}", varuse("PKGNAME", "C/-[0-9].*$/-[0-9]*/")) + token("${PKGNAME:S/py${_PYTHON_VERSION}/py${i}/:C/-[0-9].*$/-[0-9]*/}", varuse("PKGNAME", "S/py${_PYTHON_VERSION}/py${i}/", "C/-[0-9].*$/-[0-9]*/")) + token("${_PERL5_VARS:tl:S/^/-V:/}", varuse("_PERL5_VARS", "tl", "S/^/-V:/")) + token("${_PERL5_VARS_OUT:M${_var_:tl}=*:S/^${_var_:tl}=${_PERL5_PREFIX:=/}//}", varuse("_PERL5_VARS_OUT", "M${_var_:tl}=*", "S/^${_var_:tl}=${_PERL5_PREFIX:=/}//")) + token("${RUBY${RUBY_VER}_PATCHLEVEL}", varuse("RUBY${RUBY_VER}_PATCHLEVEL")) + token("${DISTFILES:M*.gem}", varuse("DISTFILES", "M*.gem")) + token("$(GNUSTEP_USER_ROOT)", varuse("GNUSTEP_USER_ROOT")) + token("${LOCALBASE:S^/^_^}", varuse("LOCALBASE", "S^/^_^")) + token("${SOURCES:%.c=%.o}", varuse("SOURCES", "%.c=%.o")) + token("${GIT_TEMPLATES:@.t.@ ${EGDIR}/${GIT_TEMPLATEDIR}/${.t.} ${PREFIX}/${GIT_CORE_TEMPLATEDIR}/${.t.} @:M*}", + varuse("GIT_TEMPLATES", "@.t.@ ${EGDIR}/${GIT_TEMPLATEDIR}/${.t.} ${PREFIX}/${GIT_CORE_TEMPLATEDIR}/${.t.} @", "M*")) + token("${DISTNAME:C:_:-:}", varuse("DISTNAME", "C:_:-:")) + token("${CF_FILES:H:O:u:S@^@${PKG_SYSCONFDIR}/@}", varuse("CF_FILES", "H", "O", "u", "S@^@${PKG_SYSCONFDIR}/@")) + token("${ALT_GCC_RTS:S%${LOCALBASE}%%:S%/%%}", varuse("ALT_GCC_RTS", "S%${LOCALBASE}%%", "S%/%%")) + token("${PREFIX:C;///*;/;g:C;/$;;}", varuse("PREFIX", "C;///*;/;g", "C;/$;;")) + token("${GZIP_CMD:[1]:Q}", varuse("GZIP_CMD", "[1]", "Q")) + token("${DISTNAME:C/-[0-9]+$$//:C/_/-/}", varuse("DISTNAME", "C/-[0-9]+$$//", "C/_/-/")) + token("${DISTNAME:slang%=slang2%}", varuse("DISTNAME", "slang%=slang2%")) + token("${OSMAP_SUBSTVARS:@v@-e 's,\\@${v}\\@,${${v}},g' @}", varuse("OSMAP_SUBSTVARS", "@v@-e 's,\\@${v}\\@,${${v}},g' @")) + + /* weird features */ + token("${${EMACS_VERSION_MAJOR}>22:?@comment :}", varuse("${EMACS_VERSION_MAJOR}>22", "?@comment :")) + token("${${XKBBASE}/xkbcomp:L:Q}", varuse("${XKBBASE}/xkbcomp", "L", "Q")) + token("${${PKGBASE} ${PKGVERSION}:L}", varuse("${PKGBASE} ${PKGVERSION}", "L")) + token("${empty(CFLAGS):?:-cflags ${CFLAGS:Q}}", varuse("empty(CFLAGS)", "?:-cflags ${CFLAGS:Q}")) + token("${${${PKG_INFO} -E ${d} || echo:L:sh}:L:C/[^[0-9]]*/ /g:[1..3]:ts.}", + varuse("${${PKG_INFO} -E ${d} || echo:L:sh}", "L", "C/[^[0-9]]*/ /g", "[1..3]", "ts.")) + + parse("${VAR)", nil, "${VAR)") + parse("$(VAR}", nil, "$(VAR}") +} diff --git a/pkgtools/pkglint/files/patches.go b/pkgtools/pkglint/files/patches.go index 12129b9578d..74a8e3edb4b 100644 --- a/pkgtools/pkglint/files/patches.go +++ b/pkgtools/pkglint/files/patches.go @@ -7,7 +7,257 @@ import ( "strings" ) -type FileType int +func ChecklinesPatch(lines []*Line) { + if G.opts.DebugTrace { + defer tracecall1(lines[0].Fname)() + } + + (&PatchChecker{lines, NewExpecter(lines), false, false}).Check() +} + +type PatchChecker struct { + lines []*Line + exp *Expecter + seenDocumentation bool + previousLineEmpty bool +} + +const ( + rePatchUniFileDel = `^---\s(\S+)(?:\s+(.*))?$` + rePatchUniFileAdd = `^\+\+\+\s(\S+)(?:\s+(.*))?$` + rePatchUniHunk = `^@@ -(\d+)(?:,(\d+))? \+(\d+)(?:,(\d+))? @@(.*)$` +) + +func (ck *PatchChecker) Check() { + if G.opts.DebugTrace { + defer tracecall0()() + } + + if ck.lines[0].CheckRcsid(``, "") { + ck.exp.Advance() + } + ck.previousLineEmpty = ck.exp.ExpectEmptyLine() + + patchedFiles := 0 + for !ck.exp.EOF() { + line := ck.exp.CurrentLine() + if ck.exp.AdvanceIfMatches(rePatchUniFileDel) { + if ck.exp.AdvanceIfMatches(rePatchUniFileAdd) { + ck.checkBeginDiff(line, patchedFiles) + ck.checkUnifiedDiff(ck.exp.m[1]) + patchedFiles++ + continue + } + + ck.exp.StepBack() + } + + if ck.exp.AdvanceIfMatches(rePatchUniFileAdd) { + patchedFile := ck.exp.m[1] + if ck.exp.AdvanceIfMatches(rePatchUniFileDel) { + ck.checkBeginDiff(line, patchedFiles) + ck.exp.PreviousLine().Warn0("Unified diff headers should be first ---, then +++.") + ck.checkUnifiedDiff(patchedFile) + patchedFiles++ + continue + } + + ck.exp.StepBack() + } + + if ck.exp.AdvanceIfMatches(`^\*\*\*\s(\S+)(.*)$`) { + if ck.exp.AdvanceIfMatches(`^---\s(\S+)(.*)$`) { + ck.checkBeginDiff(line, patchedFiles) + line.Warn0("Please use unified diffs (diff -u) for patches.") + return + } + + ck.exp.StepBack() + } + + ck.exp.Advance() + ck.previousLineEmpty = line.Text == "" || hasPrefix(line.Text, "diff ") || hasPrefix(line.Text, "=============") + if !ck.previousLineEmpty { + ck.seenDocumentation = true + } + } + + if patchedFiles > 1 { + Warnf(ck.lines[0].Fname, noLines, "Contains patches for %d files, should be only one.", patchedFiles) + } else if patchedFiles == 0 { + Errorf(ck.lines[0].Fname, noLines, "Contains no patch.") + } + + ChecklinesTrailingEmptyLines(ck.lines) + SaveAutofixChanges(ck.lines) +} + +// See http://www.gnu.org/software/diffutils/manual/html_node/Detailed-Unified.html +func (ck *PatchChecker) checkUnifiedDiff(patchedFile string) { + if G.opts.DebugTrace { + defer tracecall0()() + } + + patchedFileType := guessFileType(ck.exp.CurrentLine(), patchedFile) + if G.opts.DebugMisc { + ck.exp.CurrentLine().Debugf("guessFileType(%q) = %s", patchedFile, patchedFileType) + } + + hasHunks := false + for ck.exp.AdvanceIfMatches(rePatchUniHunk) { + hasHunks = true + linesToDel := toInt(ck.exp.m[2], 1) + linesToAdd := toInt(ck.exp.m[4], 1) + if G.opts.DebugMisc { + ck.exp.PreviousLine().Debugf("hunk -%d +%d", linesToDel, linesToAdd) + } + ck.checktextUniHunkCr() + + for linesToDel > 0 || linesToAdd > 0 || hasPrefix(ck.exp.CurrentLine().Text, "\\") { + line := ck.exp.CurrentLine() + ck.exp.Advance() + text := line.Text + switch { + case text == "": + linesToDel-- + linesToAdd-- + case hasPrefix(text, " "), hasPrefix(text, "\t"): + linesToDel-- + linesToAdd-- + ck.checklineContext(text[1:], patchedFileType) + case hasPrefix(text, "-"): + linesToDel-- + case hasPrefix(text, "+"): + linesToAdd-- + ck.checklineAdded(text[1:], patchedFileType) + case hasPrefix(text, "\\"): + // \ No newline at end of file + default: + line.Error0("Invalid line in unified patch hunk") + return + } + } + } + if !hasHunks { + ck.exp.CurrentLine().Error1("No patch hunks for %q.", patchedFile) + } + if !ck.exp.EOF() { + line := ck.exp.CurrentLine() + if line.Text != "" && !matches(line.Text, rePatchUniFileDel) && !hasPrefix(line.Text, "Index:") && !hasPrefix(line.Text, "diff ") { + line.Warn0("Empty line or end of file expected.") + Explain3( + "This empty line makes the end of the patch clearly visible.", + "Otherwise the reader would have to count lines to see where", + "the patch ends.") + } + } +} + +func (ck *PatchChecker) checkBeginDiff(line *Line, patchedFiles int) { + if G.opts.DebugTrace { + defer tracecall0()() + } + + if !ck.seenDocumentation && patchedFiles == 0 { + line.Error0("Each patch must be documented.") + Explain( + "Pkgsrc tries to have as few patches as possible. Therefore, each", + "patch must document why it is necessary. Typical reasons are", + "portability or security.", + "", + "Patches that are related to a security issue should mention the", + "corresponding CVE identifier.", + "", + "Each patch should be sent to the upstream maintainers of the", + "package, so that they can include it in future versions. After", + "submitting a patch upstream, the corresponding bug report should", + "be mentioned in this file, to prevent duplicate work.") + } + if G.opts.WarnSpace && !ck.previousLineEmpty { + if !line.AutofixInsertBefore("") { + line.Note0("Empty line expected.") + } + } +} + +func (ck *PatchChecker) checklineContext(text string, patchedFileType FileType) { + if G.opts.DebugTrace { + defer tracecall2(text, patchedFileType.String())() + } + + if G.opts.WarnExtra { + ck.checklineAdded(text, patchedFileType) + } else { + ck.checktextRcsid(text) + } +} + +func (ck *PatchChecker) checklineAdded(addedText string, patchedFileType FileType) { + if G.opts.DebugTrace { + defer tracecall2(addedText, patchedFileType.String())() + } + + ck.checktextRcsid(addedText) + + line := ck.exp.PreviousLine() + switch patchedFileType { + case ftShell: + break + case ftMakefile: + // This check is not as accurate as the similar one in MkLine.checkShelltext. + shellTokens, _ := splitIntoShellTokens(line, addedText) + for _, shellToken := range shellTokens { + if !hasPrefix(shellToken, "#") { + line.CheckAbsolutePathname(shellToken) + } + } + case ftSource: + checklineSourceAbsolutePathname(line, addedText) + case ftConfigure: + if hasSuffix(addedText, ": Avoid regenerating within pkgsrc") { + line.Error0("This code must not be included in patches.") + Explain4( + "It is generated automatically by pkgsrc after the patch phase.", + "", + "For more details, look for \"configure-scripts-override\" in", + "mk/configure/gnu-configure.mk.") + } + case ftIgnore: + break + default: + checklineOtherAbsolutePathname(line, addedText) + } +} + +func (ck *PatchChecker) checktextUniHunkCr() { + if G.opts.DebugTrace { + defer tracecall0()() + } + + line := ck.exp.PreviousLine() + if hasSuffix(line.Text, "\r") { + if !line.AutofixReplace("\r\n", "\n") { + line.Error0("The hunk header must not end with a CR character.") + Explain1( + "The MacOS X patch utility cannot handle these.") + } + } +} + +func (ck *PatchChecker) checktextRcsid(text string) { + if strings.IndexByte(text, '$') == -1 { + return + } + if m, tagname := match1(text, `\$(Author|Date|Header|Id|Locker|Log|Name|RCSfile|Revision|Source|State|NetBSD)(?::[^\$]*)?\$`); m { + if matches(text, rePatchUniHunk) { + ck.exp.PreviousLine().Warn1("Found RCS tag \"$%s$\". Please remove it.", tagname) + } else { + ck.exp.PreviousLine().Warn1("Found RCS tag \"$%s$\". Please remove it by reducing the number of context lines using pkgdiff or \"diff -U[210]\".", tagname) + } + } +} + +type FileType uint8 const ( ftSource FileType = iota @@ -19,6 +269,18 @@ const ( ftUnknown ) +func (ft FileType) String() string { + return [...]string{ + "source code", + "shell code", + "Makefile", + "text file", + "configure file", + "ignored", + "unknown", + }[ft] +} + // This is used to select the proper subroutine for detecting absolute pathnames. func guessFileType(line *Line, fname string) FileType { basename := path.Base(fname) @@ -43,41 +305,52 @@ func guessFileType(line *Line, fname string) FileType { return ftUnknown } - _ = G.opts.DebugMisc && line.debugf("Unknown file type for %q", fname) + if G.opts.DebugMisc { + line.Debug1("Unknown file type for %q", fname) + } return ftUnknown } func checkwordAbsolutePathname(line *Line, word string) { - defer tracecall("checkwordAbsolutePathname", word)() + if G.opts.DebugTrace { + defer tracecall1(word)() + } switch { case matches(word, `^/dev/(?:null|tty|zero)$`): // These are defined by POSIX. case word == "/bin/sh": // This is usually correct, although on Solaris, it's pretty feature-crippled. + case matches(word, `^/s\W`): + // Probably a sed(1) command case matches(word, `^/(?:[a-z]|\$[({])`): // Absolute paths probably start with a lowercase letter. - line.warnf("Found absolute pathname: %s", word) - line.explain( - "Absolute pathnames are often an indicator for unportable code. As", + line.Warn1("Found absolute pathname: %s", word) + Explain( + "Absolute pathnames are often an indicator for unportable code. As", "pkgsrc aims to be a portable system, absolute pathnames should be", "avoided whenever possible.", "", - "A special variable in this context is ${DESTDIR}, which is used in GNU", - "projects to specify a different directory for installation than what", - "the programs see later when they are executed. Usually it is empty, so", - "if anything after that variable starts with a slash, it is considered", - "an absolute pathname.") + "A special variable in this context is ${DESTDIR}, which is used in", + "GNU projects to specify a different directory for installation than", + "what the programs see later when they are executed. Usually it is", + "empty, so if anything after that variable starts with a slash, it is", + "considered an absolute pathname.") } } // Looks for strings like "/dev/cd0" appearing in source code func checklineSourceAbsolutePathname(line *Line, text string) { - if matched, before, _, str := match3(text, `(.*)(["'])(/\w[^"']*)["']`); matched { - _ = G.opts.DebugMisc && line.debugf("checklineSourceAbsolutePathname: before=%q, str=%q", before, str) + if !strings.ContainsAny(text, "\"'") { + return + } + if matched, before, _, str := match3(text, `^(.*)(["'])(/\w[^"']*)["']`); matched { + if G.opts.DebugMisc { + line.Debug2("checklineSourceAbsolutePathname: before=%q, str=%q", before, str) + } switch { - case matches(before, `[A-Z_]+\s*$`): + case matches(before, `[A-Z_]\s*$`): // ok; C example: const char *echo_cmd = PREFIX "/bin/echo"; case matches(before, `\+\s*$`): @@ -90,7 +363,9 @@ func checklineSourceAbsolutePathname(line *Line, text string) { } func checklineOtherAbsolutePathname(line *Line, text string) { - defer tracecall("checklineOtherAbsolutePathname", text)() + if G.opts.DebugTrace { + defer tracecall1(text)() + } if hasPrefix(text, "#") && !hasPrefix(text, "#!") { // Don't warn for absolute pathnames in comments, except for shell interpreters. @@ -104,512 +379,10 @@ func checklineOtherAbsolutePathname(line *Line, text string) { case hasSuffix(before, "."): // Example: ../dir // XXX new: case matches(before, `s.$`): // Example: sed -e s,/usr,@PREFIX@, default: - _ = G.opts.DebugMisc && line.debugf("before=%q", before) - checkwordAbsolutePathname(line, path) - } - } -} - -const ( - rePatchNonempty = `^(.+)$` - rePatchEmpty = `^$` - rePatchTextError = `\*\*\* Error code` - rePatchCtxFileDel = `^\*\*\*\s(\S+)(.*)$` - rePatchCtxFileAdd = `^---\s(\S+)(.*)$` - rePatchCtxHunk = `^\*{15}(.*)$` - rePatchCtxHunkDel = `^\*\*\*\s(\d+)(?:,(\d+))?\s\*\*\*\*$` - rePatchCtxHunkAdd = `^-{3}\s(\d+)(?:,(\d+))?\s----$` - rePatchCtxLineDel = `^(?:-\s(.*))?$` - rePatchCtxLineMod = `^(?:!\s(.*))?$` - rePatchCtxLineAdd = `^(?:\+\s(.*))?$` - rePatchCtxLineContext = `^(?:\s\s(.*))?$` - rePatchUniFileDel = `^---\s(\S+)(?:\s+(.*))?$` - rePatchUniFileAdd = `^\+\+\+\s(\S+)(?:\s+(.*))?$` - rePatchUniHunk = `^@@\s-(?:(\d+),)?(\d+)\s\+(?:(\d+),)?(\d+)\s@@(.*)$` - rePatchUniLineDel = `^-(.*)$` - rePatchUniLineAdd = `^\+(.*)$` - rePatchUniLineContext = `^\s(.*)$` - rePatchUniLineNoNewline = `^\\ No newline at end of file$` -) - -type PatchState string - -const ( - pstOutside PatchState = "pstOutside" // Outside of a diff - - pstCtxFileAdd PatchState = "pstCtxFileAdd" // After the DeleteFile line of a context diff - pstCtxHunk PatchState = "pstCtxHunk" // After the AddFile line of a context diff - pstCtxHunkDel PatchState = "pstCtxHunkDel" // - pstCtxLineDel0 PatchState = "pstCtxLineDel0" // - pstCtxLineDel PatchState = "pstCtxLineDel" // - pstCtxLineAdd0 PatchState = "pstCtxLineAdd0" // - pstCtxLineAdd PatchState = "pstCtxLineAdd" // - - pstUniFileDelErr PatchState = "pstUniFileDelErr" // Sometimes, the DeleteFile and AddFile are reversed - pstUniFileAdd PatchState = "pstUniFileAdd" // After the DeleteFile line of a unified diff - pstUniHunk PatchState = "pstUniHunk" // After the AddFile line of a unified diff - pstUniLine PatchState = "pstUniLine" // After reading the hunk header -) - -func ptNop(ctx *CheckPatchContext) {} -func ptUniFileAdd(ctx *CheckPatchContext) { - ctx.currentFilename = ctx.m[1] - ctx.currentFiletype = new(FileType) - *ctx.currentFiletype = guessFileType(ctx.line, ctx.currentFilename) - _ = G.opts.DebugPatches && ctx.line.debugf("filename=%q filetype=%v", ctx.currentFilename, *ctx.currentFiletype) - ctx.patchedFiles++ - ctx.hunks = 0 -} - -type transition struct { - re string - next PatchState - action func(*CheckPatchContext) -} - -func (ctx *CheckPatchContext) checkOutside() { - text := ctx.line.text - if G.opts.WarnSpace && text != "" && ctx.needEmptyLineNow { - ctx.line.notef("Empty line expected.") - ctx.line.insertBefore("\n") - } - ctx.needEmptyLineNow = false - if text != "" { - ctx.seenComment = true - } - ctx.prevLineWasEmpty = text == "" -} - -func (ctx *CheckPatchContext) checkBeginDiff() { - if G.opts.WarnSpace && !ctx.prevLineWasEmpty { - ctx.line.notef("Empty line expected.") - ctx.line.insertBefore("\n") - } - if !ctx.seenComment { - ctx.line.errorf("Each patch must be documented.") - ctx.line.explain( - "Each patch must document why it is necessary. If it has been applied", - "because of a security issue, a reference to the CVE should be mentioned", - "as well.", - "", - "Since it is our goal to have as few patches as possible, all patches", - "should be sent to the upstream maintainers of the package. After you", - "have done so, you should add a reference to the bug report containing", - "the patch.") - } - ctx.checkOutside() -} - -var patchTransitions = map[PatchState][]transition{ - pstOutside: { - {rePatchEmpty, pstOutside, (*CheckPatchContext).checkOutside}, - {rePatchTextError, pstOutside, (*CheckPatchContext).checkOutside}, - {rePatchCtxFileDel, pstCtxFileAdd, func(ctx *CheckPatchContext) { - ctx.checkBeginDiff() - ctx.line.warnf("Please use unified diffs (diff -u) for patches.") - }}, - {rePatchUniFileDel, pstUniFileAdd, (*CheckPatchContext).checkBeginDiff}, - {rePatchUniFileAdd, pstUniFileDelErr, ptUniFileAdd}, - {rePatchNonempty, pstOutside, (*CheckPatchContext).checkOutside}, - }, - - pstUniFileDelErr: { - {rePatchUniFileDel, pstUniHunk, func(ctx *CheckPatchContext) { - ctx.line.warnf("Unified diff headers should be first ---, then +++.") - }}, - {"", pstOutside, ptNop}, - }, - - pstCtxFileAdd: { - {rePatchCtxFileAdd, pstCtxHunk, func(ctx *CheckPatchContext) { - ctx.currentFilename = ctx.m[1] - ctx.currentFiletype = new(FileType) - *ctx.currentFiletype = guessFileType(ctx.line, ctx.currentFilename) - _ = G.opts.DebugPatches && ctx.line.debugf("filename=%q filetype=%v", ctx.currentFilename, *ctx.currentFiletype) - ctx.patchedFiles++ - ctx.hunks = 0 - }}, - }, - - pstCtxHunk: { - {rePatchCtxHunk, pstCtxHunkDel, func(ctx *CheckPatchContext) { - ctx.hunks++ - }}, - {"", pstOutside, ptNop}, - }, - - pstCtxHunkDel: { - {rePatchCtxHunkDel, pstCtxLineDel0, func(ctx *CheckPatchContext) { - if ctx.m[2] != "" { - ctx.dellines = 1 + toInt(ctx.m[2]) - toInt(ctx.m[1]) - } else { - ctx.dellines = toInt(ctx.m[1]) - } - }}, - }, - - pstCtxLineDel0: { - {rePatchCtxLineContext, pstCtxLineDel, func(ctx *CheckPatchContext) { - ctx.checkHunkLine(1, 0, pstCtxLineDel0) - }}, - {rePatchCtxLineDel, pstCtxLineDel, func(ctx *CheckPatchContext) { - ctx.checkHunkLine(1, 0, pstCtxLineDel0) - }}, - {rePatchCtxLineMod, pstCtxLineDel, func(ctx *CheckPatchContext) { - ctx.checkHunkLine(1, 0, pstCtxLineDel0) - }}, - {rePatchCtxHunkAdd, pstCtxLineAdd0, func(ctx *CheckPatchContext) { - ctx.dellines = 0 - if 2 < len(ctx.m) { - ctx.addlines = 1 + toInt(ctx.m[2]) - toInt(ctx.m[1]) - } else { - ctx.addlines = toInt(ctx.m[1]) - } - }}, - }, - - pstCtxLineDel: { - {rePatchCtxLineContext, pstCtxLineDel, func(ctx *CheckPatchContext) { - ctx.checkHunkLine(1, 0, pstCtxLineDel0) - }}, - {rePatchCtxLineDel, pstCtxLineDel, func(ctx *CheckPatchContext) { - ctx.checkHunkLine(1, 0, pstCtxLineDel0) - }}, - {rePatchCtxLineMod, pstCtxLineDel, func(ctx *CheckPatchContext) { - ctx.checkHunkLine(1, 0, pstCtxLineDel0) - }}, - {"", pstCtxLineDel0, func(ctx *CheckPatchContext) { - if ctx.dellines != 0 { - ctx.line.warnf("Invalid number of deleted lines (%d missing).", ctx.dellines) - } - }}, - }, - - pstCtxLineAdd0: { - {rePatchCtxLineContext, pstCtxLineAdd, func(ctx *CheckPatchContext) { - ctx.checkHunkLine(0, 1, pstCtxHunk) - }}, - {rePatchCtxLineMod, pstCtxLineAdd, func(ctx *CheckPatchContext) { - ctx.checkHunkLine(0, 1, pstCtxHunk) - }}, - {rePatchCtxLineAdd, pstCtxLineAdd, func(ctx *CheckPatchContext) { - ctx.checkHunkLine(0, 1, pstCtxHunk) - }}, - {"", pstCtxHunk, ptNop}, - }, - - pstCtxLineAdd: { - {rePatchCtxLineContext, pstCtxLineAdd, func(ctx *CheckPatchContext) { - ctx.checkHunkLine(0, 1, pstCtxHunk) - }}, - {rePatchCtxLineMod, pstCtxLineAdd, func(ctx *CheckPatchContext) { - ctx.checkHunkLine(0, 1, pstCtxHunk) - }}, - {rePatchCtxLineAdd, pstCtxLineAdd, func(ctx *CheckPatchContext) { - ctx.checkHunkLine(0, 1, pstCtxHunk) - }}, - {"", pstCtxLineAdd0, func(ctx *CheckPatchContext) { - if ctx.addlines != 0 { - ctx.line.warnf("Invalid number of added lines (%d missing).", ctx.addlines) - } - }}, - }, - - pstUniFileAdd: { - {rePatchUniFileAdd, pstUniHunk, ptUniFileAdd}, - }, - - pstUniHunk: { - {rePatchUniHunk, pstUniLine, func(ctx *CheckPatchContext) { - m := ctx.m - if m[1] != "" { - ctx.dellines = toInt(m[2]) - } else { - ctx.dellines = 1 - } - if m[3] != "" { - ctx.addlines = toInt(m[4]) - } else { - ctx.addlines = 1 - } - ctx.checkText(ctx.line.text) - if hasSuffix(ctx.line.text, "\r") { - ctx.line.errorf("The hunk header must not end with a CR character.") - ctx.line.explain( - "The MacOS X patch utility cannot handle these.") - ctx.line.replace("\r\n", "\n") - } - ctx.hunks++ - if m[1] != "" && m[1] != "1" { - ctx.contextScanningLeading = new(bool) - *ctx.contextScanningLeading = true - } else { - ctx.contextScanningLeading = nil - } - ctx.leadingContextLines = 0 - ctx.trailingContextLines = 0 - }}, - {"", pstOutside, func(ctx *CheckPatchContext) { - if ctx.hunks == 0 { - ctx.line.warnf("No hunks for file %q.", ctx.currentFilename) - } - }}, - }, - - pstUniLine: { - {rePatchUniLineDel, pstUniLine, func(ctx *CheckPatchContext) { - ctx.checkHunkLine(1, 0, pstUniHunk) - }}, - {rePatchUniLineAdd, pstUniLine, func(ctx *CheckPatchContext) { - ctx.checkHunkLine(0, 1, pstUniHunk) - }}, - {rePatchUniLineContext, pstUniLine, func(ctx *CheckPatchContext) { - ctx.checkHunkLine(1, 1, pstUniHunk) - }}, - {rePatchUniLineNoNewline, pstUniLine, func(ctx *CheckPatchContext) { - }}, - {rePatchEmpty, pstUniLine, func(ctx *CheckPatchContext) { - if G.opts.WarnSpace { - ctx.line.notef("Leading white-space missing in hunk.") - ctx.line.replaceRegex(`^`, " ") - } - ctx.checkHunkLine(1, 1, pstUniHunk) - }}, - {"", pstUniHunk, func(ctx *CheckPatchContext) { - if ctx.dellines != 0 || ctx.addlines != 0 { - ctx.line.warnf("Unexpected end of hunk (-%d,+%d expected).", ctx.dellines, ctx.addlines) - } - }}, - }, -} - -func checklinesPatch(lines []*Line) { - defer tracecall("checklinesPatch", lines[0].fname)() - - checklineRcsid(lines[0], ``, "") - - ctx := CheckPatchContext{state: pstOutside, needEmptyLineNow: true} - for lineno := 1; lineno < len(lines); { - line := lines[lineno] - text := line.text - ctx.line = line - - _ = G.opts.DebugPatches && - line.debugf("state=%s hunks=%d del=%d add=%d text=%s", - ctx.state, ctx.hunks, ctx.dellines, ctx.addlines, text) - - found := false - for _, t := range patchTransitions[ctx.state] { - if t.re == "" { - ctx.m = ctx.m[:0] - } else if ctx.m = match(text, t.re); ctx.m == nil { - continue - } - - ctx.redostate = nil - ctx.nextstate = t.next - t.action(&ctx) - if ctx.redostate != nil { - ctx.state = *ctx.redostate - } else { - ctx.state = ctx.nextstate - if t.re != "" { - lineno++ - } - } - found = true - break - } - - if !found { - ctx.line.errorf("Internal pkglint error: checklinesPatch state=%s", ctx.state) - ctx.state = pstOutside - lineno++ - } - } - - fname := lines[0].fname - for ctx.state != pstOutside { - _ = G.opts.DebugPatches && - debugf(fname, "EOF", "state=%s hunks=%d del=%d add=%d", - ctx.state, ctx.hunks, ctx.dellines, ctx.addlines) - - found := false - for _, t := range patchTransitions[ctx.state] { - if t.re == "" { - ctx.m = ctx.m[:0] - ctx.redostate = nil - ctx.nextstate = t.next - t.action(&ctx) - if ctx.redostate != nil { - ctx.state = *ctx.redostate - } else { - ctx.state = ctx.nextstate - } - found = true - } - } - - if !found { - ctx.line.errorf("Internal pkglint error: checklinesPatch state=%s", ctx.state) - break - } - } - - if ctx.patchedFiles > 1 { - warnf(fname, noLines, "Contains patches for %d files, should be only one.", ctx.patchedFiles) - } else if ctx.patchedFiles == 0 { - errorf(fname, noLines, "Contains no patch.") - } - - checklinesTrailingEmptyLines(lines) - saveAutofixChanges(lines) -} - -type CheckPatchContext struct { - state PatchState - redostate *PatchState - nextstate PatchState - dellines int - addlines int - hunks int - seenComment bool - needEmptyLineNow bool - prevLineWasEmpty bool - currentFilename string - currentFiletype *FileType - patchedFiles int - leadingContextLines int - trailingContextLines int - contextScanningLeading *bool - line *Line - m []string -} - -func (ctx *CheckPatchContext) expectEmptyLine() { - if G.opts.WarnSpace { - ctx.line.notef("Empty line expected.") - ctx.line.insertBefore("\n") - } -} - -func (ctx *CheckPatchContext) useUnifiedDiffs() { - ctx.line.warnf("Please use unified diffs (diff -u) for patches.") -} - -func (ctx *CheckPatchContext) checkText(text string) { - if m, tagname := match1(text, `\$(Author|Date|Header|Id|Locker|Log|Name|RCSfile|Revision|Source|State|NetBSD)(?::[^\$]*)?\$`); m { - if matches(text, rePatchUniHunk) { - ctx.line.warnf("Found RCS tag \"$%s$\". Please remove it.", tagname) - } else { - ctx.line.warnf("Found RCS tag \"$%s$\". Please remove it by reducing the number of context lines using pkgdiff or \"diff -U[210]\".", tagname) - } - } -} - -func (ctx *CheckPatchContext) checkContents() { - if 1 < len(ctx.m) { - ctx.checkText(ctx.m[1]) - } -} - -func (ctx *CheckPatchContext) checkAddedContents() { - if !(1 < len(ctx.m)) { - return - } - - line := ctx.line - addedText := ctx.m[1] - - switch *ctx.currentFiletype { - case ftShell: - case ftMakefile: - // This check is not as accurate as the similar one in MkLine.checkShelltext. - shellwords, _ := splitIntoShellwords(line, addedText) - for _, shellword := range shellwords { - if !hasPrefix(shellword, "#") { - line.checkAbsolutePathname(shellword) - } - } - case ftSource: - checklineSourceAbsolutePathname(line, addedText) - case ftConfigure: - if matches(addedText, `: Avoid regenerating within pkgsrc$`) { - line.errorf("This code must not be included in patches.") - line.explain( - "It is generated automatically by pkgsrc after the patch phase.", - "", - "For more details, look for \"configure-scripts-override\" in", - "mk/configure/gnu-configure.mk.") - } - case ftIgnore: - break - default: - checklineOtherAbsolutePathname(line, addedText) - } -} - -func (ctx *CheckPatchContext) checkHunkEnd(deldelta, adddelta int, newstate PatchState) { - if deldelta > 0 && ctx.dellines == 0 { - ctx.redostate = &newstate - if ctx.addlines > 0 { - ctx.line.errorf("Expected %d more lines to be added.", ctx.addlines) - } - return - } - - if adddelta > 0 && ctx.addlines == 0 { - ctx.redostate = &newstate - if ctx.dellines > 0 { - ctx.line.errorf("Expected %d more lines to be deleted.", ctx.dellines) - } - return - } - - if ctx.contextScanningLeading != nil { - if deldelta != 0 && adddelta != 0 { - if *ctx.contextScanningLeading { - ctx.leadingContextLines++ - } else { - ctx.trailingContextLines++ - } - } else { - if *ctx.contextScanningLeading { - *ctx.contextScanningLeading = false - } else { - ctx.trailingContextLines = 0 - } - } - } - - if deldelta > 0 { - ctx.dellines -= deldelta - } - if adddelta > 0 { - ctx.addlines -= adddelta - } - - if ctx.dellines == 0 && ctx.addlines == 0 { - if ctx.contextScanningLeading != nil { - if ctx.leadingContextLines != ctx.trailingContextLines { - _ = G.opts.DebugPatches && ctx.line.warnf( - "The hunk that ends here does not have as many leading (%d) as trailing (%d) lines of context.", - ctx.leadingContextLines, ctx.trailingContextLines) + if G.opts.DebugMisc { + line.Debug1("before=%q", before) } + checkwordAbsolutePathname(line, path) } - ctx.nextstate = newstate - } -} - -func (ctx *CheckPatchContext) checkHunkLine(deldelta, adddelta int, newstate PatchState) { - ctx.checkContents() - ctx.checkHunkEnd(deldelta, adddelta, newstate) - - // If -Wextra is given, the context lines are checked for - // absolute paths and similar things. If it is not given, - // only those lines that really add something to the patched - // file are checked. - if adddelta > 0 && (deldelta == 0 || G.opts.WarnExtra) { - ctx.checkAddedContents() } } diff --git a/pkgtools/pkglint/files/patches_test.go b/pkgtools/pkglint/files/patches_test.go index cb3f6364f8f..317eba72c3a 100644 --- a/pkgtools/pkglint/files/patches_test.go +++ b/pkgtools/pkglint/files/patches_test.go @@ -2,7 +2,7 @@ package main import ( check "gopkg.in/check.v1" - "path/filepath" + "io/ioutil" ) func (s *Suite) TestChecklinesPatch_WithComment(c *check.C) { @@ -21,14 +21,13 @@ func (s *Suite) TestChecklinesPatch_WithComment(c *check.C) { "+old line", " context after") - checklinesPatch(lines) + ChecklinesPatch(lines) c.Check(s.Output(), equals, "") } func (s *Suite) TestChecklinesPatch_WithoutEmptyLine(c *check.C) { - tmpdir := c.MkDir() - fname := filepath.ToSlash(tmpdir + "/patch-WithoutEmptyLines") + fname := s.CreateTmpFile(c, "patch-WithoutEmptyLines", "dummy") s.UseCommandLine(c, "-Wall", "--autofix") lines := s.NewLines(fname, "$"+"NetBSD$", @@ -41,14 +40,27 @@ func (s *Suite) TestChecklinesPatch_WithoutEmptyLine(c *check.C) { "+old line", " context after") - checklinesPatch(lines) + ChecklinesPatch(lines) - c.Check(s.Output(), equals, ""+ - "NOTE: "+fname+":2: Empty line expected.\n"+ - "NOTE: "+fname+":2: Autofix: inserting a line \"\\n\" before this line.\n"+ - "NOTE: "+fname+":3: Empty line expected.\n"+ - "NOTE: "+fname+":3: Autofix: inserting a line \"\\n\" before this line.\n"+ - "NOTE: "+fname+": Has been auto-fixed. Please re-run pkglint.\n") + c.Check(s.OutputCleanTmpdir(), equals, ""+ + "AUTOFIX: ~/patch-WithoutEmptyLines:2: Inserting a line \"\" before this line.\n"+ + "AUTOFIX: ~/patch-WithoutEmptyLines:3: Inserting a line \"\" before this line.\n"+ + "AUTOFIX: ~/patch-WithoutEmptyLines: Has been auto-fixed. Please re-run pkglint.\n") + + fixed, err := ioutil.ReadFile(fname) + c.Assert(err, check.IsNil) + c.Check(string(fixed), equals, ""+ + "$"+"NetBSD$\n"+ + "\n"+ + "Text\n"+ + "\n"+ + "--- file.orig\n"+ + "+++ file\n"+ + "@@ -5,3 +5,3 @@\n"+ + " context before\n"+ + "-old line\n"+ + "+old line\n"+ + " context after\n") } func (s *Suite) TestChecklinesPatch_WithoutComment(c *check.C) { @@ -64,15 +76,15 @@ func (s *Suite) TestChecklinesPatch_WithoutComment(c *check.C) { "+old line", " context after") - checklinesPatch(lines) + ChecklinesPatch(lines) c.Check(s.Output(), equals, "ERROR: patch-WithoutComment:3: Each patch must be documented.\n") } func (s *Suite) TestChecklineOtherAbsolutePathname(c *check.C) { - line := NewLine("patch-ag", "1", "+$install -s -c ./bin/rosegarden ${DESTDIR}$BINDIR", nil) + line := NewLine("patch-ag", 1, "+$install -s -c ./bin/rosegarden ${DESTDIR}$BINDIR", nil) - checklineOtherAbsolutePathname(line, line.text) + checklineOtherAbsolutePathname(line, line.Text) c.Check(s.Output(), equals, "") } @@ -92,7 +104,7 @@ func (s *Suite) TestChecklinesPatch_ErrorCode(c *check.C) { "+old line", " context after") - checklinesPatch(lines) + ChecklinesPatch(lines) c.Check(s.Output(), equals, "") } @@ -113,7 +125,212 @@ func (s *Suite) TestChecklinesPatch_WrongOrder(c *check.C) { "+old line", " context after") - checklinesPatch(lines) + ChecklinesPatch(lines) c.Check(s.Output(), equals, "WARN: patch-WrongOrder:7: Unified diff headers should be first ---, then +++.\n") } + +func (s *Suite) TestChecklinesPatch_ContextDiff(c *check.C) { + s.UseCommandLine(c, "-Wall") + lines := s.NewLines("patch-ctx", + "$"+"NetBSD$", + "", + "diff -cr history.c.orig history.c", + "*** history.c.orig", + "--- history.c") + + ChecklinesPatch(lines) + + c.Check(s.Output(), equals, ""+ + "ERROR: patch-ctx:4: Each patch must be documented.\n"+ + "WARN: patch-ctx:4: Please use unified diffs (diff -u) for patches.\n") +} + +func (s *Suite) TestChecklinesPatch_NoPatch(c *check.C) { + lines := s.NewLines("patch-aa", + "$"+"NetBSD$", + "", + "-- oldfile", + "++ newfile") + + ChecklinesPatch(lines) + + c.Check(s.Output(), equals, "ERROR: patch-aa: Contains no patch.\n") +} + +func (s *Suite) TestChecklinesPatch_TwoPatches(c *check.C) { + lines := s.NewLines("patch-aa", + "$"+"NetBSD$", + "", + "--- oldfile", + "+++ newfile", + "@@ -1 +1 @@", + "-old", + "+new", + "--- oldfile2", + "+++ newfile2", + "@@ -1 +1 @@", + "-old", + "+new") + + ChecklinesPatch(lines) + + c.Check(s.Output(), equals, ""+ + "ERROR: patch-aa:3: Each patch must be documented.\n"+ + "WARN: patch-aa: Contains patches for 2 files, should be only one.\n") +} + +func (s *Suite) TestChecklinesPatch_PatchlikeDocumentation(c *check.C) { + lines := s.NewLines("patch-aa", + "$"+"NetBSD$", + "", + "--- oldfile", + "", + "+++ newfile", + "", + "*** oldOrNewFile") + + ChecklinesPatch(lines) + + c.Check(s.Output(), equals, "ERROR: patch-aa: Contains no patch.\n") +} + +func (s *Suite) TestChecklinesPatch_OnlyUnifiedHeader(c *check.C) { + lines := s.NewLines("patch-unified", + "$"+"NetBSD$", + "", + "Documentation for the patch", + "", + "--- file.orig", + "+++ file") + + ChecklinesPatch(lines) + + c.Check(s.Output(), equals, "ERROR: patch-unified:EOF: No patch hunks for \"file\".\n") +} + +func (s *Suite) TestChecklinesPatch_OnlyContextHeader(c *check.C) { + lines := s.NewLines("patch-context", + "$"+"NetBSD$", + "", + "Documentation for the patch", + "", + "*** file.orig", + "--- file") + + ChecklinesPatch(lines) + + c.Check(s.Output(), equals, "WARN: patch-context:5: Please use unified diffs (diff -u) for patches.\n") +} + +func (s *Suite) TestChecklinesPatch_Makefile(c *check.C) { + lines := s.NewLines("patch-unified", + "$"+"NetBSD$", + "", + "Documentation for the patch", + "", + "--- Makefile.orig", + "+++ Makefile", + "@@ -1,3 +1,5 @@", + " \t/bin/cp context before", + "-\t/bin/cp deleted", + "+\t/bin/cp added", + "+#\t/bin/cp added comment", + "+# added comment", + " \t/bin/cp context after") + + ChecklinesPatch(lines) + + c.Check(s.Output(), equals, ""+ + "WARN: patch-unified:10: Found absolute pathname: /bin/cp\n") + + G.opts.WarnExtra = true + + ChecklinesPatch(lines) + + c.Check(s.Output(), equals, ""+ + "WARN: patch-unified:8: Found absolute pathname: /bin/cp\n"+ + "WARN: patch-unified:10: Found absolute pathname: /bin/cp\n"+ + "WARN: patch-unified:13: Found absolute pathname: /bin/cp\n") +} + +func (s *Suite) TestChecklinesPatch_NoNewline_withFollowingText(c *check.C) { + lines := s.NewLines("patch-aa", + "$"+"NetBSD$", + "", + "comment", + "", + "--- oldfile", + "+++ newfile", + "@@ -1 +1 @@", + "-old", + "\\ No newline at end of file", + "+new", + "\\ No newline at end of file", + "last line (a comment)") + + ChecklinesPatch(lines) + + c.Check(s.Output(), equals, "WARN: patch-aa:12: Empty line or end of file expected.\n") +} + +func (s *Suite) TestChecklinesPatch_NoNewline(c *check.C) { + lines := s.NewLines("patch-aa", + "$"+"NetBSD$", + "", + "comment", + "", + "--- oldfile", + "+++ newfile", + "@@ -1 +1 @@", + "-old", + "\\ No newline at end of file", + "+new", + "\\ No newline at end of file") + + ChecklinesPatch(lines) + + c.Check(s.Output(), equals, "") +} + +func (s *Suite) TestChecklinesPatch_ShortAtEof(c *check.C) { + lines := s.NewLines("patch-aa", + "$"+"NetBSD$", + "", + "comment", + "", + "--- oldfile", + "+++ newfile", + "@@ -1,7 +1,6 @@", + " 1", + " 2", + " 3", + "-4", + " 5", + " 6") // Line 7 was empty, therefore omitted + + ChecklinesPatch(lines) + + c.Check(s.Output(), equals, "") +} + +// In some context lines, the leading space character is missing. +// Since this is no problem for patch(1), pkglint also doesn’t complain. +func (s *Suite) TestChecklinesPatch_AddTab(c *check.C) { + lines := s.NewLines("patch-aa", + "$"+"NetBSD$", + "", + "comment", + "", + "--- oldfile", + "+++ newfile", + "@@ -1,3 +1,3 @@", + "\tcontext", + "-old", + "+new", + "\tcontext") + + ChecklinesPatch(lines) + + c.Check(s.Output(), equals, "") +} diff --git a/pkgtools/pkglint/files/pkgcontext.go b/pkgtools/pkglint/files/pkgcontext.go deleted file mode 100644 index 43b1d8b36e1..00000000000 --- a/pkgtools/pkglint/files/pkgcontext.go +++ /dev/null @@ -1,60 +0,0 @@ -package main - -// PkgContext contains data for the package that is currently checked. -type PkgContext struct { - pkgpath string // e.g. "category/pkgdir" - pkgdir string // PKGDIR from the package Makefile - filesdir string // FILESDIR from the package Makefile - patchdir string // PATCHDIR from the package Makefile - distinfoFile string // DISTINFO_FILE from the package Makefile - effectivePkgname string // PKGNAME or DISTNAME from the package Makefile - effectivePkgbase string // The effective PKGNAME without the version - effectivePkgversion string // The version part of the effective PKGNAME - effectivePkgnameLine *Line // The origin of the three effective_* values - seenBsdPrefsMk bool // Has bsd.prefs.mk already been included? - - vardef map[string]*Line // varname => line - varuse map[string]*Line // varname => line - bl3 map[string]*Line // buildlink3.mk name => line; contains only buildlink3.mk files that are directly included. - plistSubstCond map[string]bool // varname => true; list of all variables that are used as conditionals (@comment or nothing) in PLISTs. - included map[string]*Line // fname => line - seenMakefileCommon bool // Does the package have any .includes? -} - -func newPkgContext(pkgpath string) *PkgContext { - ctx := &PkgContext{} - ctx.pkgpath = pkgpath - ctx.vardef = make(map[string]*Line) - ctx.varuse = make(map[string]*Line) - ctx.bl3 = make(map[string]*Line) - ctx.plistSubstCond = make(map[string]bool) - ctx.included = make(map[string]*Line) - for varname, line := range G.globalData.userDefinedVars { - ctx.vardef[varname] = line - } - return ctx -} - -func (ctx *PkgContext) defineVar(line *Line, varname string) { - if line.extra["value"] == nil { - line.errorf("Internal pkglint error: novalue") - return - } - if ctx.vardef[varname] == nil { - ctx.vardef[varname] = line - } - varcanon := varnameCanon(varname) - if ctx.vardef[varcanon] == nil { - ctx.vardef[varcanon] = line - } -} -func (ctx *PkgContext) varValue(varname string) (string, bool) { - if line := ctx.vardef[varname]; line != nil { - if value := line.extra["value"]; value != nil { - return value.(string), true - } else { - line.errorf("Internal pkglint error: novalue") - } - } - return "", false -} diff --git a/pkgtools/pkglint/files/pkglint.0 b/pkgtools/pkglint/files/pkglint.0 index 80564b751a9..895da87f366 100644 --- a/pkgtools/pkglint/files/pkglint.0 +++ b/pkgtools/pkglint/files/pkglint.0 @@ -28,9 +28,7 @@ DDEESSCCRRIIPPTTIIOONN --WW{{[[nnoo--]]wwaarrnn,,......}} Enable or disable specific warnings. For a list of warnings, see below. - --ee|----eexxppllaaiinn Print further explanations for diagnostics. Some- - times the reasons for diagnostics are not obvious and - need further explanation. + --ee|----eexxppllaaiinn Print verbose explanations for diagnostics. --gg|----ggcccc--oouuttppuutt--ffoorrmmaatt Use a format for the diagnostics that is understood @@ -51,7 +49,8 @@ DDEESSCCRRIIPPTTIIOONN --ss|----ssoouurrccee For all diagnostics having file and line number information, show the source code along with the - diagnostics. + diagnostics. This is especially useful together with + the --ff|----sshhooww--aauuttooffiixx option. CChheecckkss aallll Enable all checks. @@ -127,27 +126,13 @@ DDEESSCCRRIIPPTTIIOONN variable (e.g. sed instead of ${SED}). [[nnoo--]]eexxttrraa Emit some additional warnings that are not enabled by - default, for whatever reason. + default. [[nnoo--]]oorrddeerr Warn if Makefile variables are not in the preferred order. - [[nnoo--]]ppeerrmm Warn if a variable is used or defined outside its - specified scope. The available permissions are: - append - append something using += - default - set a default value using ?= - preprocess - use a variable during preprocessing - runtime - use a variable at runtime - set set a variable using :=, =, != - unknown - permissions for this variable currently not - known - A `?' means that it is not yet clear which permis- - sions are allowed and which aren't. + [[nnoo--]]ppeerrmm Warn if a variable is used or modified outside its + specified scope. [[nnoo--]]pplliisstt--ddeepprr Warn if deprecated pathnames are used in _P_L_I_S_T files. This warning is disabled by default. @@ -184,10 +169,6 @@ EEXXAAMMPPLLEESS Checks the category Makefile and reports any warnings it can find. - ppkkgglliinntt --rr --RR ''NNeettBBSSDD||IIdd'' //uussrr//ppkkggssrrcc - Check the whole pkgsrc tree while allowing `NetBSD' or `Id' - as the RCS Id. - DDIIAAGGNNOOSSTTIICCSS Diagnostics are written to the standard output. @@ -204,4 +185,4 @@ BBUUGGSS If you don't understand the messages, feel free to ask on the <tech-pkg@NetBSD.org> mailing list. -NetBSD 6.1 November 25, 2015 NetBSD 6.1 +NetBSD 7.0 January 12, 2016 NetBSD 7.0 diff --git a/pkgtools/pkglint/files/pkglint.1 b/pkgtools/pkglint/files/pkglint.1 index a3a3d01b27e..e84a5a6d696 100644 --- a/pkgtools/pkglint/files/pkglint.1 +++ b/pkgtools/pkglint/files/pkglint.1 @@ -1,4 +1,4 @@ -.\" $NetBSD: pkglint.1,v 1.49 2015/11/25 13:29:07 rillig Exp $ +.\" $NetBSD: pkglint.1,v 1.50 2016/01/12 01:02:49 rillig Exp $ .\" From FreeBSD: portlint.1,v 1.8 1997/11/25 14:53:14 itojun Exp .\" .\" Copyright (c) 1997 by Jun-ichiro Itoh <itojun@itojun.org>. @@ -6,9 +6,9 @@ .\" .\" Roland Illig <roland.illig@gmx.de>, 2004, 2005. .\" Thomas Klausner <wiz@NetBSD.org>, 2012. -.\" Roland Illig <rillig@NetBSD.org>, 2015. +.\" Roland Illig <rillig@NetBSD.org>, 2015, 2016. .\" -.Dd November 25, 2015 +.Dd January 12, 2016 .Dt PKGLINT 1 .Os .Sh NAME @@ -50,9 +50,7 @@ version number and exit. Enable or disable specific warnings. For a list of warnings, see below. .It Fl e Ns | Ns Fl -explain -Print further explanations for diagnostics. -Sometimes the reasons for diagnostics are not obvious and need further -explanation. +Print verbose explanations for diagnostics. .It Fl g Ns | Ns Fl -gcc-output-format Use a format for the diagnostics that is understood by most programs, especially editors, so they can provide a point-and-goto interface. @@ -71,6 +69,9 @@ line. .It Fl s Ns | Ns Fl -source For all diagnostics having file and line number information, show the source code along with the diagnostics. +This is especially useful together with the +.Fl f Ns | Ns Fl -show-autofix +option. .El .\" ======================================================================= .Ss Checks @@ -148,30 +149,11 @@ Warn if a file contains an absolute pathname. Warn if a system command name is used instead of a variable (e.g. sed instead of ${SED}). .It Cm [no-]extra -Emit some additional warnings that are not enabled by default, -for whatever reason. +Emit some additional warnings that are not enabled by default. .It Cm [no-]order Warn if Makefile variables are not in the preferred order. .It Cm [no-]perm -Warn if a variable is used or defined outside its specified scope. -The available permissions are: -.Bl -tag -width 3n -compact -.It append -append something using += -.It default -set a default value using ?= -.It preprocess -use a variable during preprocessing -.It runtime -use a variable at runtime -.It set -set a variable using :=, =, != -.It unknown -permissions for this variable currently not known -.El -A -.Sq \&? -means that it is not yet clear which permissions are allowed and which aren't. +Warn if a variable is used or modified outside its specified scope. .It Cm [no-]plist-depr Warn if deprecated pathnames are used in .Pa PLIST @@ -215,12 +197,6 @@ Files from the pkgsrc infrastructure. Checks the patches of the package in the current directory. .It Ic pkglint \-Wall /usr/pkgsrc/devel Checks the category Makefile and reports any warnings it can find. -.It Ic pkglint -r \-R 'NetBSD|Id' /usr/pkgsrc -Check the whole pkgsrc tree while allowing -.Ql NetBSD -or -.Ql Id -as the RCS Id. .El .Sh DIAGNOSTICS Diagnostics are written to the standard output. diff --git a/pkgtools/pkglint/files/pkglint.go b/pkgtools/pkglint/files/pkglint.go index 11dcf5be567..12e6091d474 100644 --- a/pkgtools/pkglint/files/pkglint.go +++ b/pkgtools/pkglint/files/pkglint.go @@ -1,32 +1,16 @@ package main -// based on pkglint.pl 1.896 - import ( - "fmt" "os" "path" "strings" ) const ( - reDependencyCmp = `^((?:\$\{[\w_]+\}|[\w_\.+]|-[^\d])+)[<>]=?(\d[^-*?\[\]]*)$` - reDependencyWildcard = `^((?:\$\{[\w_]+\}|[\w_\.+]|-[^\d\[])+)-(?:\[0-9\]\*|\d[^-]*)$` - reMkCond = `^\.(\s*)(if|ifdef|ifndef|else|elif|endif|for|endfor|undef)(?:\s+([^\s#][^#]*?))?\s*(?:#.*)?$` - reMkInclude = `^\.\s*(s?include)\s+\"([^\"]+)\"\s*(?:#.*)?$` - reVarassign = `^ *([-*+A-Z_a-z0-9.${}\[]+?)\s*([!+:?]?=)\s*((?:\\#|[^#])*?)(?:\s*(#.*))?$` - rePkgname = `^([\w\-.+]+)-(\d(?:\w|\.\d)*)$` - rePkgbase = `(?:[+.\w]|-[A-Z_a-z])+` - rePkgversion = `\d(?:\w|\.\d)*` + reMkInclude = `^\.\s*(s?include)\s+\"([^\"]+)\"\s*(?:#.*)?$` + rePkgname = `^([\w\-.+]+)-(\d(?:\w|\.\d)*)$` ) -func explainRelativeDirs(line *Line) { - line.explain( - "Directories in the form \"../../category/package\" make it easier to", - "move a package around in pkgsrc, for example from pkgsrc-wip to the", - "main pkgsrc repository.") -} - // Returns the pkgsrc top-level directory, relative to the given file or directory. func findPkgsrcTopdir(fname string) string { for _, dir := range []string{".", "..", "../..", "../../.."} { @@ -37,150 +21,10 @@ func findPkgsrcTopdir(fname string) string { return "" } -func loadPackageMakefile(fname string) []*Line { - defer tracecall("loadPackageMakefile", fname)() - - var mainLines, allLines []*Line - if !readMakefile(fname, &mainLines, &allLines) { - errorf(fname, noLines, "Cannot be read.") - return nil - } - - if G.opts.DumpMakefile { - debugf(G.currentDir, noLines, "Whole Makefile (with all included files) follows:") - for _, line := range allLines { - fmt.Printf("%s\n", line.String()) - } - } - - determineUsedVariables(allLines) - - G.pkgContext.pkgdir = expandVariableWithDefault("PKGDIR", ".") - G.pkgContext.distinfoFile = expandVariableWithDefault("DISTINFO_FILE", "distinfo") - G.pkgContext.filesdir = expandVariableWithDefault("FILESDIR", "files") - G.pkgContext.patchdir = expandVariableWithDefault("PATCHDIR", "patches") - - if varIsDefined("PHPEXT_MK") { - if !varIsDefined("USE_PHP_EXT_PATCHES") { - G.pkgContext.patchdir = "patches" - } - if varIsDefined("PECL_VERSION") { - G.pkgContext.distinfoFile = "distinfo" - } - } - - _ = G.opts.DebugMisc && - dummyLine.debugf("DISTINFO_FILE=%s", G.pkgContext.distinfoFile) && - dummyLine.debugf("FILESDIR=%s", G.pkgContext.filesdir) && - dummyLine.debugf("PATCHDIR=%s", G.pkgContext.patchdir) && - dummyLine.debugf("PKGDIR=%s", G.pkgContext.pkgdir) - - return mainLines -} - -func determineUsedVariables(lines []*Line) { - re := regcomp(`(?:\$\{|\$\(|defined\(|empty\()([0-9+.A-Z_a-z]+)[:})]`) - for _, line := range lines { - rest := line.text - for { - m := re.FindStringSubmatchIndex(rest) - if m == nil { - break - } - varname := rest[m[2]:m[3]] - useVar(line, varname) - rest = rest[:m[0]] + rest[m[1]:] - } - } -} - -func extractUsedVariables(line *Line, text string) []string { - re := regcomp(`^(?:[^\$]+|\$[\$*<>?@]|\$\{([.0-9A-Z_a-z]+)(?::(?:[^\${}]|\$[^{])+)?\})`) - rest := text - var result []string - for { - m := re.FindStringSubmatchIndex(rest) - if m == nil { - break - } - varname := rest[negToZero(m[2]):negToZero(m[3])] - rest = rest[:m[0]] + rest[m[1]:] - if varname != "" { - result = append(result, varname) - } - } - - if rest != "" { - _ = G.opts.DebugMisc && line.debugf("extractUsedVariables: rest=%q", rest) - } - return result -} - -// Returns the type of the variable (maybe guessed based on the variable name), -// or nil if the type cannot even be guessed. -func getVariableType(line *Line, varname string) *Vartype { - - if vartype := G.globalData.vartypes[varname]; vartype != nil { - return vartype - } - if vartype := G.globalData.vartypes[varnameCanon(varname)]; vartype != nil { - return vartype - } - - if G.globalData.varnameToToolname[varname] != "" { - return &Vartype{lkNone, CheckvarShellCommand, []AclEntry{{"*", "u"}}, guNotGuessed} - } - - if m, toolvarname := match1(varname, `^TOOLS_(.*)`); m && G.globalData.varnameToToolname[toolvarname] != "" { - return &Vartype{lkNone, CheckvarPathname, []AclEntry{{"*", "u"}}, guNotGuessed} - } - - allowAll := []AclEntry{{"*", "adpsu"}} - allowRuntime := []AclEntry{{"*", "adsu"}} - - // Guess the datatype of the variable based on naming conventions. - var gtype *Vartype - switch { - case hasSuffix(varname, "DIRS"): - gtype = &Vartype{lkShell, CheckvarPathmask, allowRuntime, guGuessed} - case hasSuffix(varname, "DIR"), hasSuffix(varname, "_HOME"): - gtype = &Vartype{lkNone, CheckvarPathname, allowRuntime, guGuessed} - case hasSuffix(varname, "FILES"): - gtype = &Vartype{lkShell, CheckvarPathmask, allowRuntime, guGuessed} - case hasSuffix(varname, "FILE"): - gtype = &Vartype{lkNone, CheckvarPathname, allowRuntime, guGuessed} - case hasSuffix(varname, "PATH"): - gtype = &Vartype{lkNone, CheckvarPathlist, allowRuntime, guGuessed} - case hasSuffix(varname, "PATHS"): - gtype = &Vartype{lkShell, CheckvarPathname, allowRuntime, guGuessed} - case hasSuffix(varname, "_USER"): - gtype = &Vartype{lkNone, CheckvarUserGroupName, allowAll, guGuessed} - case hasSuffix(varname, "_GROUP"): - gtype = &Vartype{lkNone, CheckvarUserGroupName, allowAll, guGuessed} - case hasSuffix(varname, "_ENV"): - gtype = &Vartype{lkShell, CheckvarShellWord, allowRuntime, guGuessed} - case hasSuffix(varname, "_CMD"): - gtype = &Vartype{lkNone, CheckvarShellCommand, allowRuntime, guGuessed} - case hasSuffix(varname, "_ARGS"): - gtype = &Vartype{lkShell, CheckvarShellWord, allowRuntime, guGuessed} - case hasSuffix(varname, "_CFLAGS"), hasSuffix(varname, "_CPPFLAGS"), hasSuffix(varname, "_CXXFLAGS"), hasSuffix(varname, "_LDFLAGS"): - gtype = &Vartype{lkShell, CheckvarShellWord, allowRuntime, guGuessed} - case hasSuffix(varname, "_MK"): - gtype = &Vartype{lkNone, CheckvarUnchecked, allowAll, guGuessed} - case hasPrefix(varname, "PLIST."): - gtype = &Vartype{lkNone, CheckvarYes, allowAll, guGuessed} - } - - if gtype != nil { - _ = G.opts.DebugVartypes && line.debugf("The guessed type of %q is %v.", varname, gtype) - } else { - _ = G.opts.DebugVartypes && line.debugf("No type definition found for %q.", varname) - } - return gtype -} - func resolveVariableRefs(text string) string { - defer tracecall("resolveVariableRefs", text)() + if G.opts.DebugTrace { + defer tracecall1(text)() + } visited := make(map[string]bool) // To prevent endless loops @@ -190,18 +34,18 @@ func resolveVariableRefs(text string) string { varname := m[2 : len(m)-1] if !visited[varname] { visited[varname] = true - if ctx := G.pkgContext; ctx != nil { - if value, ok := ctx.varValue(varname); ok { + if G.Pkg != nil { + if value, ok := G.Pkg.varValue(varname); ok { return value } } - if ctx := G.mkContext; ctx != nil { - if value, ok := ctx.varValue(varname); ok { + if G.Mk != nil { + if value, ok := G.Mk.VarValue(varname); ok { return value } } } - return sprintf("${%s}", varname) + return "${" + varname + "}" }) if replaced == str { return replaced @@ -211,215 +55,139 @@ func resolveVariableRefs(text string) string { } func expandVariableWithDefault(varname, defaultValue string) string { - line := G.pkgContext.vardef[varname] - if line == nil { + mkline := G.Pkg.vardef[varname] + if mkline == nil { return defaultValue } - value := line.extra["value"].(string) + value := mkline.Value() value = resolveVarsInRelativePath(value, true) if containsVarRef(value) { value = resolveVariableRefs(value) } - _ = G.opts.DebugMisc && line.debugf("Expanded %q to %q", varname, value) + if G.opts.DebugMisc { + mkline.Debug2("Expanded %q to %q", varname, value) + } return value } -func getVariablePermissions(line *Line, varname string) string { - if vartype := getVariableType(line, varname); vartype != nil { - return vartype.effectivePermissions(line.fname) +func CheckfileExtra(fname string) { + if G.opts.DebugTrace { + defer tracecall1(fname)() } - _ = G.opts.DebugMisc && line.debugf("No type definition found for %q.", varname) - return "adpsu" -} - -func checklineLength(line *Line, maxlength int) { - if len(line.text) > maxlength { - line.warnf("Line too long (should be no more than %d characters).", maxlength) - line.explain( - "Back in the old time, terminals with 80x25 characters were common.", - "And this is still the default size of many terminal emulators.", - "Moderately short lines also make reading easier.") + if lines := LoadNonemptyLines(fname, false); lines != nil { + ChecklinesTrailingEmptyLines(lines) } } -func checklineValidCharacters(line *Line, reChar string) { - rest := regcomp(reChar).ReplaceAllString(line.text, "") - if rest != "" { - uni := "" - for _, c := range rest { - uni += sprintf(" %U", c) - } - line.warnf("Line contains invalid characters (%s).", uni[1:]) +func ChecklinesDescr(lines []*Line) { + if G.opts.DebugTrace { + defer tracecall1(lines[0].Fname)() } -} -func checklineValidCharactersInValue(line *Line, reValid string) { - varname := line.extra["varname"].(string) - value := line.extra["value"].(string) - rest := regcomp(reValid).ReplaceAllString(value, "") - if rest != "" { - uni := "" - for _, c := range rest { - uni += sprintf(" %U", c) + for _, line := range lines { + line.CheckLength(80) + line.CheckTrailingWhitespace() + line.CheckValidCharacters(`[\t -~]`) + if contains(line.Text, "${") { + line.Note0("Variables are not expanded in the DESCR file.") } - line.warnf("%s contains invalid characters (%s).", varname, uni[1:]) } -} + ChecklinesTrailingEmptyLines(lines) -func checklineTrailingWhitespace(line *Line) { - if hasSuffix(line.text, " ") || hasSuffix(line.text, "\t") { - line.notef("Trailing white-space.") - line.explain( - "When a line ends with some white-space, that space is in most cases", - "irrelevant and can be removed.") - line.replaceRegex(`\s+\n$`, "\n") - } -} + if maxlines := 24; len(lines) > maxlines { + line := lines[maxlines] -func checklineRcsid(line *Line, prefixRe, suggestedPrefix string) bool { - defer tracecall("checklineRcsid", prefixRe, suggestedPrefix)() - - if !matches(line.text, `^`+prefixRe+`[$]NetBSD(?::[^\$]+)?\$$`) { - line.errorf("Expected %q.", suggestedPrefix+"$"+"NetBSD$") - line.explain( - "Several files in pkgsrc must contain the CVS Id, so that their current", - "version can be traced back later from a binary package. This is to", - "ensure reproducible builds, for example for finding bugs.") - line.insertBefore(suggestedPrefix + "$" + "NetBSD$") - return false + line.Warnf("File too long (should be no more than %d lines).", maxlines) + Explain3( + "The DESCR file should fit on a traditional terminal of 80x25", + "characters. It is also intended to give a _brief_ summary about", + "the package's contents.") } - return true -} -func checklineRelativePath(line *Line, path string, mustExist bool) { - if !G.isWip && contains(path, "/wip/") { - line.errorf("A main pkgsrc package must not depend on a pkgsrc-wip package.") - } - - resolvedPath := resolveVarsInRelativePath(path, true) - if containsVarRef(resolvedPath) { - return - } - - abs := ifelseStr(hasPrefix(resolvedPath, "/"), "", G.currentDir+"/") + resolvedPath - if _, err := os.Stat(abs); err != nil { - if mustExist { - line.errorf("%q does not exist.", resolvedPath) - } - return - } - - switch { - case matches(path, `^\.\./\.\./[^/]+/[^/]`): - case hasPrefix(path, "../../mk/"): - // There need not be two directory levels for mk/ files. - case matches(path, `^\.\./mk/`) && G.curPkgsrcdir == "..": - // That's fine for category Makefiles. - case matches(path, `^\.\.`): - line.warnf("Invalid relative path %q.", path) - } + SaveAutofixChanges(lines) } -func checkfileExtra(fname string) { - defer tracecall("checkfileExtra", fname)() - - if lines := LoadNonemptyLines(fname, false); lines != nil { - checklinesTrailingEmptyLines(lines) +func ChecklinesMessage(lines []*Line) { + if G.opts.DebugTrace { + defer tracecall1(lines[0].Fname)() } -} - -func checklinesMessage(lines []*Line) { - defer tracecall("checklinesMessage", lines[0].fname)() - explanation := []string{ - "A MESSAGE file should consist of a header line, having 75 \"=\"", - "characters, followed by a line containing only the RCS Id, then an", - "empty line, your text and finally the footer line, which is the", - "same as the header line."} + explainMessage := func() { + Explain4( + "A MESSAGE file should consist of a header line, having 75 \"=\"", + "characters, followed by a line containing only the RCS Id, then an", + "empty line, your text and finally the footer line, which is the", + "same as the header line.") + } if len(lines) < 3 { lastLine := lines[len(lines)-1] - lastLine.warnf("File too short.") - lastLine.explain(explanation...) + lastLine.Warn0("File too short.") + explainMessage() return } hline := strings.Repeat("=", 75) - if line := lines[0]; line.text != hline { - line.warnf("Expected a line of exactly 75 \"=\" characters.") - line.explain(explanation...) + if line := lines[0]; line.Text != hline { + line.Warn0("Expected a line of exactly 75 \"=\" characters.") + explainMessage() } - checklineRcsid(lines[1], ``, "") + lines[1].CheckRcsid(``, "") for _, line := range lines { - checklineLength(line, 80) - checklineTrailingWhitespace(line) - checklineValidCharacters(line, `[\t -~]`) + line.CheckLength(80) + line.CheckTrailingWhitespace() + line.CheckValidCharacters(`[\t -~]`) } - if lastLine := lines[len(lines)-1]; lastLine.text != hline { - lastLine.warnf("Expected a line of exactly 75 \"=\" characters.") - lastLine.explain(explanation...) + if lastLine := lines[len(lines)-1]; lastLine.Text != hline { + lastLine.Warn0("Expected a line of exactly 75 \"=\" characters.") + explainMessage() } - checklinesTrailingEmptyLines(lines) + ChecklinesTrailingEmptyLines(lines) } -func checklineRelativePkgdir(line *Line, pkgdir string) { - checklineRelativePath(line, pkgdir, true) - pkgdir = resolveVarsInRelativePath(pkgdir, false) - - if m, otherpkgpath := match1(pkgdir, `^(?:\./)?\.\./\.\./([^/]+/[^/]+)$`); m { - if !fileExists(G.globalData.pkgsrcdir + "/" + otherpkgpath + "/Makefile") { - line.errorf("There is no package in %q.", otherpkgpath) - } - - } else { - line.warnf("%q is not a valid relative package directory.", pkgdir) - line.explain( - "A relative pathname always starts with \"../../\", followed", - "by a category, a slash and a the directory name of the package.", - "For example, \"../../misc/screen\" is a valid relative pathname.") +func CheckfileMk(fname string) { + if G.opts.DebugTrace { + defer tracecall1(fname)() } -} - -func checkfileMk(fname string) { - defer tracecall("checkfileMk", fname)() lines := LoadNonemptyLines(fname, true) if lines == nil { return } - ParselinesMk(lines) - ChecklinesMk(lines) - saveAutofixChanges(lines) + NewMkLines(lines).Check() + SaveAutofixChanges(lines) } -func checkfile(fname string) { - defer tracecall("checkfile", fname)() +func Checkfile(fname string) { + if G.opts.DebugTrace { + defer tracecall1(fname)() + } basename := path.Base(fname) - if matches(basename, `^(?:work.*|.*~|.*\.orig|.*\.rej)$`) { + if hasPrefix(basename, "work") || hasSuffix(basename, "~") || hasSuffix(basename, ".orig") || hasSuffix(basename, ".rej") { if G.opts.Import { - errorf(fname, noLines, "Must be cleaned up before committing the package.") + Errorf(fname, noLines, "Must be cleaned up before committing the package.") } return } st, err := os.Lstat(fname) if err != nil { - errorf(fname, noLines, "%s", err) + Errorf(fname, noLines, "%s", err) return } if st.Mode().IsRegular() && st.Mode().Perm()&0111 != 0 && !isCommitted(fname) { - line := NewLine(fname, noLines, "", nil) - line.warnf("Should not be executable.") - line.explain( - "No package file should ever be executable. Even the INSTALL and", - "DEINSTALL scripts are usually not usable in the form they have in the", - "package, as the pathnames get adjusted during installation. So there is", - "no need to have any file executable.") + line := NewLine(fname, 0, "", nil) + line.Warn0("Should not be executable.") + Explain4( + "No package file should ever be executable. Even the INSTALL and", + "DEINSTALL scripts are usually not usable in the form they have in", + "the package, as the pathnames get adjusted during installation.", + "So there is no need to have any file executable.") } switch { @@ -430,79 +198,79 @@ func checkfile(fname string) { case matches(fname, `(?:^|/)files/[^/]*$`): // Ok case !isEmptyDir(fname): - warnf(fname, noLines, "Unknown directory name.") + Warnf(fname, noLines, "Unknown directory name.") } case st.Mode()&os.ModeSymlink != 0: if !matches(basename, `^work`) { - warnf(fname, noLines, "Unknown symlink name.") + Warnf(fname, noLines, "Unknown symlink name.") } case !st.Mode().IsRegular(): - errorf(fname, noLines, "Only files and directories are allowed in pkgsrc.") + Errorf(fname, noLines, "Only files and directories are allowed in pkgsrc.") case basename == "ALTERNATIVES": if G.opts.CheckAlternatives { - checkfileExtra(fname) + CheckfileExtra(fname) } case basename == "buildlink3.mk": if G.opts.CheckBuildlink3 { if lines := LoadNonemptyLines(fname, true); lines != nil { - checklinesBuildlink3Mk(lines) + ChecklinesBuildlink3Mk(NewMkLines(lines)) } } case hasPrefix(basename, "DESCR"): if G.opts.CheckDescr { if lines := LoadNonemptyLines(fname, false); lines != nil { - checklinesDescr(lines) + ChecklinesDescr(lines) } } - case hasPrefix(basename, "distinfo"): + case basename == "distinfo": if G.opts.CheckDistinfo { if lines := LoadNonemptyLines(fname, false); lines != nil { - checklinesDistinfo(lines) + ChecklinesDistinfo(lines) } } case basename == "DEINSTALL" || basename == "INSTALL": if G.opts.CheckInstall { - checkfileExtra(fname) + CheckfileExtra(fname) } case hasPrefix(basename, "MESSAGE"): if G.opts.CheckMessage { if lines := LoadNonemptyLines(fname, false); lines != nil { - checklinesMessage(lines) + ChecklinesMessage(lines) } } case matches(basename, `^patch-[-A-Za-z0-9_.~+]*[A-Za-z0-9_]$`): if G.opts.CheckPatches { if lines := LoadNonemptyLines(fname, false); lines != nil { - checklinesPatch(lines) + ChecklinesPatch(lines) } } case matches(fname, `(?:^|/)patches/manual[^/]*$`): if G.opts.DebugUnchecked { - debugf(fname, noLines, "Unchecked file %q.", fname) + Debugf(fname, noLines, "Unchecked file %q.", fname) } case matches(fname, `(?:^|/)patches/[^/]*$`): - warnf(fname, noLines, "Patch files should be named \"patch-\", followed by letters, '-', '_', '.', and digits only.") + Warnf(fname, noLines, "Patch files should be named \"patch-\", followed by letters, '-', '_', '.', and digits only.") case matches(basename, `^(?:.*\.mk|Makefile.*)$`) && !matches(fname, `files/`) && !matches(fname, `patches/`): if G.opts.CheckMk { - checkfileMk(fname) + CheckfileMk(fname) } case hasPrefix(basename, "PLIST"): if G.opts.CheckPlist { if lines := LoadNonemptyLines(fname, false); lines != nil { - checklinesPlist(lines) + ChecklinesPlist(lines) } } @@ -517,27 +285,135 @@ func checkfile(fname string) { // Skip default: - warnf(fname, noLines, "Unexpected file found.") + Warnf(fname, noLines, "Unexpected file found.") if G.opts.CheckExtra { - checkfileExtra(fname) + CheckfileExtra(fname) } } } -func checklinesTrailingEmptyLines(lines []*Line) { +func ChecklinesTrailingEmptyLines(lines []*Line) { max := len(lines) last := max - for last > 1 && lines[last-1].text == "" { + for last > 1 && lines[last-1].Text == "" { last-- } if last != max { - lines[last].notef("Trailing empty lines.") + lines[last].Note0("Trailing empty lines.") } } -func matchVarassign(text string) (m bool, varname, op, value, comment string) { - if contains(text, "=") { - m, varname, op, value, comment = match4(text, reVarassign) +func MatchVarassign(text string) (m bool, varname, op, value, comment string) { + i, n := 0, len(text) + + for i < n && text[i] == ' ' { + i++ + } + + varnameStart := i + for ; i < n; i++ { + b := text[i] + mask := uint(0) + switch b & 0xE0 { + case 0x20: + mask = 0x03ff6c10 + case 0x40: + mask = 0x8ffffffe + case 0x60: + mask = 0x2ffffffe + } + if (mask>>(b&0x1F))&1 == 0 { + break + } + } + varnameEnd := i + + if varnameEnd == varnameStart { + return + } + + for i < n && (text[i] == ' ' || text[i] == '\t') { + i++ + } + + opStart := i + if i < n { + if b := text[i]; b&0xE0 == 0x20 && (uint(0x84000802)>>(b&0x1F))&1 != 0 { + i++ + } + } + if i < n && text[i] == '=' { + i++ + } else { + return + } + opEnd := i + + if text[varnameEnd-1] == '+' && varnameEnd == opStart && text[opStart] == '=' { + varnameEnd-- + opStart-- + } + + for i < n && (text[i] == ' ' || text[i] == '\t') { + i++ + } + + valueStart := i + valuebuf := make([]byte, n-valueStart) + j := 0 + for ; i < n; i++ { + b := text[i] + if b == '#' && (i == valueStart || text[i-1] != '\\') { + break + } else if b != '\\' || i+1 >= n || text[i+1] != '#' { + valuebuf[j] = b + j++ + } } + + commentStart := i + commentEnd := n + + m = true + varname = text[varnameStart:varnameEnd] + op = text[opStart:opEnd] + value = strings.TrimSpace(string(valuebuf[:j])) + comment = text[commentStart:commentEnd] return } + +type DependencyPattern struct { + pkgbase string // "freeciv-client", "{gcc48,gcc48-libs}", "${EMACS_REQD}" + lowerOp string // ">=", ">" + lower string // "2.5.0", "${PYVER}" + upperOp string // "<", "<=" + upper string // "3.0", "${PYVER}" + wildcard string // "[0-9]*", "1.5.*", "${PYVER}" +} + +func resolveVarsInRelativePath(relpath string, adjustDepth bool) string { + + tmp := relpath + tmp = strings.Replace(tmp, "${PKGSRCDIR}", G.CurPkgsrcdir, -1) + tmp = strings.Replace(tmp, "${.CURDIR}", ".", -1) + tmp = strings.Replace(tmp, "${.PARSEDIR}", ".", -1) + tmp = strings.Replace(tmp, "${LUA_PKGSRCDIR}", "../../lang/lua52", -1) + tmp = strings.Replace(tmp, "${PHPPKGSRCDIR}", "../../lang/php55", -1) + tmp = strings.Replace(tmp, "${SUSE_DIR_PREFIX}", "suse100", -1) + tmp = strings.Replace(tmp, "${PYPKGSRCDIR}", "../../lang/python27", -1) + if G.Pkg != nil { + tmp = strings.Replace(tmp, "${FILESDIR}", G.Pkg.Filesdir, -1) + tmp = strings.Replace(tmp, "${PKGDIR}", G.Pkg.Pkgdir, -1) + } + + if adjustDepth { + if m, pkgpath := match1(tmp, `^\.\./\.\./([^.].*)$`); m { + tmp = G.CurPkgsrcdir + "/" + pkgpath + } + } + + if G.opts.DebugMisc { + dummyLine.Debug2("resolveVarsInRelativePath: %q => %q", relpath, tmp) + } + return tmp +} diff --git a/pkgtools/pkglint/files/pkglint_test.go b/pkgtools/pkglint/files/pkglint_test.go index 8c111e6053e..675b4e49666 100644 --- a/pkgtools/pkglint/files/pkglint_test.go +++ b/pkgtools/pkglint/files/pkglint_test.go @@ -1,54 +1,41 @@ package main import ( + "strings" + check "gopkg.in/check.v1" ) func (s *Suite) TestDetermineUsedVariables_simple(c *check.C) { - G.mkContext = newMkContext() - line := NewLine("fname", "1", "${VAR}", nil) - lines := []*Line{line} + mklines := s.NewMkLines("fname", + "\t${VAR}") + mkline := mklines.mklines[0] + G.Mk = mklines - determineUsedVariables(lines) + mklines.DetermineUsedVariables() - c.Check(len(G.mkContext.varuse), equals, 1) - c.Check(G.mkContext.varuse["VAR"], equals, line) + c.Check(len(mklines.varuse), equals, 1) + c.Check(mklines.varuse["VAR"], equals, mkline) } func (s *Suite) TestDetermineUsedVariables_nested(c *check.C) { - G.mkContext = newMkContext() - line := NewLine("fname", "2", "${outer.${inner}}", nil) - lines := []*Line{line} - - determineUsedVariables(lines) + mklines := s.NewMkLines("fname", + "\t${outer.${inner}}") + mkline := mklines.mklines[0] + G.Mk = mklines - c.Check(len(G.mkContext.varuse), equals, 3) - c.Check(G.mkContext.varuse["inner"], equals, line) - c.Check(G.mkContext.varuse["outer."], equals, line) - c.Check(G.mkContext.varuse["outer.*"], equals, line) -} + mklines.DetermineUsedVariables() -func (s *Suite) TestReShellword(c *check.C) { - re := `^(?:` + reShellword + `)$` - matches := check.NotNil - doesntMatch := check.IsNil - - c.Check(match("", re), doesntMatch) - c.Check(match("$var", re), matches) - c.Check(match("$var$var", re), matches) - c.Check(match("$var;;", re), doesntMatch) // More than one shellword - c.Check(match("'single-quoted'", re), matches) - c.Check(match("\"", re), doesntMatch) // Incomplete string - c.Check(match("'...'\"...\"", re), matches) // Mixed strings - c.Check(match("\"...\"", re), matches) - c.Check(match("`cat file`", re), matches) + c.Check(len(mklines.varuse), equals, 3) + c.Check(mklines.varuse["inner"], equals, mkline) + c.Check(mklines.varuse["outer."], equals, mkline) + c.Check(mklines.varuse["outer.*"], equals, mkline) } func (s *Suite) TestResolveVariableRefs_CircularReference(c *check.C) { - line := NewLine("fname", "1", "dummy", nil) - line.extra["value"] = "${GCC_VERSION}" - G.pkgContext = newPkgContext(".") - G.pkgContext.vardef["GCC_VERSION"] = line // circular reference + mkline := NewMkLine(NewLine("fname", 1, "GCC_VERSION=${GCC_VERSION}", nil)) + G.Pkg = NewPackage(".") + G.Pkg.vardef["GCC_VERSION"] = mkline resolved := resolveVariableRefs("gcc-${GCC_VERSION}") @@ -56,16 +43,13 @@ func (s *Suite) TestResolveVariableRefs_CircularReference(c *check.C) { } func (s *Suite) TestResolveVariableRefs_Multilevel(c *check.C) { - line1 := NewLine("fname", "dummy", "dummy", nil) - line1.extra["value"] = "${SECOND}" - line2 := NewLine("fname", "dummy", "dummy", nil) - line2.extra["value"] = "${THIRD}" - line3 := NewLine("fname", "dummy", "dummy", nil) - line3.extra["value"] = "got it" - G.pkgContext = newPkgContext(".") - G.pkgContext.vardef["FIRST"] = line1 - G.pkgContext.vardef["SECOND"] = line2 - G.pkgContext.vardef["THIRD"] = line3 + mkline1 := NewMkLine(NewLine("fname", 10, "_=${SECOND}", nil)) + mkline2 := NewMkLine(NewLine("fname", 11, "_=${THIRD}", nil)) + mkline3 := NewMkLine(NewLine("fname", 12, "_=got it", nil)) + G.Pkg = NewPackage(".") + defineVar(mkline1, "FIRST") + defineVar(mkline2, "SECOND") + defineVar(mkline3, "THIRD") resolved := resolveVariableRefs("you ${FIRST}") @@ -73,10 +57,9 @@ func (s *Suite) TestResolveVariableRefs_Multilevel(c *check.C) { } func (s *Suite) TestResolveVariableRefs_SpecialChars(c *check.C) { - line := NewLine("fname", "dummy", "dummy", nil) - line.extra["value"] = "x11" - G.pkgContext = newPkgContext("category/pkg") - G.pkgContext.vardef["GST_PLUGINS0.10_TYPE"] = line + mkline := NewMkLine(NewLine("fname", 10, "_=x11", nil)) + G.Pkg = NewPackage("category/pkg") + G.Pkg.vardef["GST_PLUGINS0.10_TYPE"] = mkline resolved := resolveVariableRefs("gst-plugins0.10-${GST_PLUGINS0.10_TYPE}/distinfo") @@ -92,7 +75,7 @@ func (s *Suite) TestChecklineRcsid(c *check.C) { "$"+"FreeBSD$") for _, line := range lines { - checklineRcsid(line, ``, "") + line.CheckRcsid(``, "") } c.Check(s.Output(), equals, ""+ @@ -102,11 +85,93 @@ func (s *Suite) TestChecklineRcsid(c *check.C) { } func (s *Suite) TestMatchVarassign(c *check.C) { - m, varname, op, value, comment := matchVarassign("C++=c11") + checkVarassign := func(text string, ck check.Checker, varname, op, value, comment string) { + type va struct { + varname, op, value, comment string + } + expected := va{varname, op, value, comment} + am, avarname, aop, avalue, acomment := MatchVarassign(text) + if !am { + c.Errorf("Text %q doesn’t match variable assignment", text) + return + } + actual := va{avarname, aop, avalue, acomment} + c.Check(actual, ck, expected) + } + checkNotVarassign := func(text string) { + m, _, _, _, _ := MatchVarassign(text) + if m { + c.Errorf("Text %q matches variable assignment, but shouldn’t.", text) + } + } + + checkVarassign("C++=c11", equals, "C+", "+=", "c11", "") + checkVarassign("V=v", equals, "V", "=", "v", "") + checkVarassign("VAR=#comment", equals, "VAR", "=", "", "#comment") + checkVarassign("VAR=\\#comment", equals, "VAR", "=", "#comment", "") + checkVarassign("VAR=\\\\\\##comment", equals, "VAR", "=", "\\\\#", "#comment") + checkVarassign("VAR=\\", equals, "VAR", "=", "\\", "") + checkVarassign("VAR += value", equals, "VAR", "+=", "value", "") + checkVarassign(" VAR=value", equals, "VAR", "=", "value", "") + checkNotVarassign("\tVAR=value") + checkNotVarassign("?=value") + checkNotVarassign("<=value") +} + +func (s *Suite) TestPackage_LoadPackageMakefile(c *check.C) { + makefile := s.CreateTmpFile(c, "category/package/Makefile", ""+ + "# $"+"NetBSD$\n"+ + "\n"+ + "PKGNAME=pkgname-1.67\n"+ + "DISTNAME=distfile_1_67\n"+ + ".include \"../../category/package/Makefile\"\n") + pkg := NewPackage("category/package") + G.CurrentDir = s.tmpdir + "/category/package" + G.CurPkgsrcdir = "../.." + G.Pkg = pkg + + pkg.loadPackageMakefile(makefile) + + c.Check(s.OutputCleanTmpdir(), equals, "") +} + +func (s *Suite) TestChecklinesDescr(c *check.C) { + lines := s.NewLines("DESCR", + strings.Repeat("X", 90), + "", "", "", "", "", "", "", "", "10", + "Try ${PREFIX}", + "", "", "", "", "", "", "", "", "20", + "", "", "", "", "", "", "", "", "", "30") + + ChecklinesDescr(lines) - c.Check(m, equals, true) - c.Check(varname, equals, "C+") - c.Check(op, equals, "+=") - c.Check(value, equals, "c11") - c.Check(comment, equals, "") + c.Check(s.Output(), equals, ""+ + "WARN: DESCR:1: Line too long (should be no more than 80 characters).\n"+ + "NOTE: DESCR:11: Variables are not expanded in the DESCR file.\n"+ + "WARN: DESCR:25: File too long (should be no more than 24 lines).\n") +} + +func (s *Suite) TestChecklinesMessage_short(c *check.C) { + lines := s.NewLines("MESSAGE", + "one line") + + ChecklinesMessage(lines) + + c.Check(s.Output(), equals, "WARN: MESSAGE:1: File too short.\n") +} + +func (s *Suite) TestChecklinesMessage_malformed(c *check.C) { + lines := s.NewLines("MESSAGE", + "1", + "2", + "3", + "4", + "5") + + ChecklinesMessage(lines) + + c.Check(s.Output(), equals, ""+ + "WARN: MESSAGE:1: Expected a line of exactly 75 \"=\" characters.\n"+ + "ERROR: MESSAGE:2: Expected \"$"+"NetBSD$\".\n"+ + "WARN: MESSAGE:5: Expected a line of exactly 75 \"=\" characters.\n") } diff --git a/pkgtools/pkglint/files/plist.go b/pkgtools/pkglint/files/plist.go index 0d5ea3b86d6..7373c05583b 100644 --- a/pkgtools/pkglint/files/plist.go +++ b/pkgtools/pkglint/files/plist.go @@ -2,402 +2,445 @@ package main import ( "path" + "sort" "strings" ) -type PlistContext struct { - allFiles map[string]*Line - allDirs map[string]*Line - lastFname string -} - -func checklinesPlist(lines []*Line) { - defer tracecall("checklinesPlist", lines[0].fname)() +func ChecklinesPlist(lines []*Line) { + if G.opts.DebugTrace { + defer tracecall1(lines[0].Fname)() + } - checklineRcsid(lines[0], `@comment `, "@comment ") + lines[0].CheckRcsid(`@comment `, "@comment ") if len(lines) == 1 { - lines[0].warnf("PLIST files shouldn't be empty.") - lines[0].explain( + lines[0].Warn0("PLIST files shouldn't be empty.") + Explain( "One reason for empty PLISTs is that this is a newly created package", "and that the author didn't run \"bmake print-PLIST\" after installing", "the files.", "", "Another reason, common for Perl packages, is that the final PLIST is", - "automatically generated. Since the source PLIST is not used at all,", + "automatically generated. Since the source PLIST is not used at all,", "you can remove it.", "", "Meta packages also don't need a PLIST file.") } - pctx := new(PlistContext) - pctx.allFiles = make(map[string]*Line) - pctx.allDirs = make(map[string]*Line) + ck := &PlistChecker{ + make(map[string]*PlistLine), + make(map[string]*PlistLine), + ""} + ck.Check(lines) +} + +type PlistChecker struct { + allFiles map[string]*PlistLine + allDirs map[string]*PlistLine + lastFname string +} + +type PlistLine struct { + line *Line + conditional string // e.g. PLIST.docs + text string // Like line.text, without the conditional +} - var extraLines []*Line - if fname := lines[0].fname; path.Base(fname) == "PLIST.common_end" { - commonLines, err := readLines(path.Dir(fname)+"/PLIST.common", false) +func (ck *PlistChecker) Check(plainLines []*Line) { + plines := ck.NewLines(plainLines) + ck.collectFilesAndDirs(plines) + + if fname := plines[0].line.Fname; path.Base(fname) == "PLIST.common_end" { + commonLines, err := readLines(strings.TrimSuffix(fname, "_end"), false) if err == nil { - extraLines = commonLines + ck.collectFilesAndDirs(ck.NewLines(commonLines)) } } - // Collect all files and directories that appear in the PLIST file. - for _, line := range append(append([]*Line(nil), extraLines...), lines...) { - text := line.text + for _, pline := range plines { + ck.checkline(pline) + pline.CheckTrailingWhitespace() + } - if hasPrefix(text, "${") { - if m, varname, rest := match2(text, `^\$\{([\w_]+)\}(.*)`); m { - if G.pkgContext != nil && G.pkgContext.plistSubstCond[varname] { - _ = G.opts.DebugMisc && line.debugf("Removed PLIST_SUBST conditional %q.", varname) - text = rest - } - } + ChecklinesTrailingEmptyLines(plainLines) + if G.opts.WarnPlistSort { + sorter := NewPlistLineSorter(plines) + sorter.Sort() + if !sorter.autofixed { + SaveAutofixChanges(plainLines) } + } else { + SaveAutofixChanges(plainLines) + } +} - if matches(text, `^[\w$]`) { - pctx.allFiles[text] = line - for dir := path.Dir(text); dir != "."; dir = path.Dir(dir) { - pctx.allDirs[dir] = line +func (ck *PlistChecker) NewLines(lines []*Line) []*PlistLine { + plines := make([]*PlistLine, len(lines)) + for i, line := range lines { + conditional, text := "", line.Text + if hasPrefix(text, "${PLIST.") { + if m, cond, rest := match2(text, `^\$\{(PLIST\.[\w-.]+)\}(.*)`); m { + conditional, text = cond, rest } } + plines[i] = &PlistLine{line, conditional, text} + } + return plines +} - if hasPrefix(text, "@") { - if m, dirname := match1(text, `^@exec \$\{MKDIR\} %D/(.*)$`); m { - for dir := dirname; dir != "."; dir = path.Dir(dir) { - pctx.allDirs[dir] = line +func (ck *PlistChecker) collectFilesAndDirs(plines []*PlistLine) { + for _, pline := range plines { + if text := pline.text; len(text) > 0 { + first := text[0] + switch { + case 'a' <= first && first <= 'z', + first == '$', + 'A' <= first && first <= 'Z', + '0' <= first && first <= '9': + if ck.allFiles[text] == nil { + ck.allFiles[text] = pline + } + for dir := path.Dir(text); dir != "."; dir = path.Dir(dir) { + ck.allDirs[dir] = pline + } + case first == '@': + if m, dirname := match1(text, `^@exec \$\{MKDIR\} %D/(.*)$`); m { + for dir := dirname; dir != "."; dir = path.Dir(dir) { + ck.allDirs[dir] = pline + } } } - } - } - for _, line := range lines { - pline := &PlistLine{line} - pline.check(pctx) - pline.checkTrailingWhitespace() + } } - - checklinesTrailingEmptyLines(lines) - saveAutofixChanges(lines) } -type PlistLine struct { - line *Line -} - -func (pline *PlistLine) check(pctx *PlistContext) { - text := pline.line.text +func (ck *PlistChecker) checkline(pline *PlistLine) { + text := pline.text if hasAlnumPrefix(text) { - pline.checkPathname(pctx, text) + ck.checkpath(pline) } else if m, cmd, arg := match2(text, `^(?:\$\{[\w.]+\})?@([a-z-]+)\s+(.*)`); m { - pline.checkDirective(cmd, arg) + pline.CheckDirective(cmd, arg) } else if hasPrefix(text, "$") { - pline.checkPathname(pctx, text) - } else if matches(text, `^\$\{[\w_]+\}$`) { - // A variable on its own line. + ck.checkpath(pline) } else { - pline.line.warnf("Unknown line type.") + pline.line.Warn0("Unknown line type.") } } -func (pline *PlistLine) checkTrailingWhitespace() { - line := pline.line +func (ck *PlistChecker) checkpath(pline *PlistLine) { + line, text := pline.line, pline.text + sdirname, basename := path.Split(text) + dirname := strings.TrimSuffix(sdirname, "/") - if hasSuffix(line.text, " ") || hasSuffix(line.text, "\t") { - line.errorf("pkgsrc does not support filenames ending in white-space.") - line.explain( - "Each character in the PLIST is relevant, even trailing white-space.") - } -} + ck.checkSorted(pline) -func (pline *PlistLine) checkDirective(cmd, arg string) { - line := pline.line + if contains(basename, "${IMAKE_MANNEWSUFFIX}") { + pline.warnImakeMannewsuffix() + } - if cmd == "unexec" { - if m, arg := match1(arg, `^(?:rmdir|\$\{RMDIR\} \%D/)(.*)`); m { - if !contains(arg, "true") && !contains(arg, "${TRUE}") { - line.warnf("Please remove this line. It is no longer necessary.") - } - } + topdir := "" + if firstSlash := strings.IndexByte(text, '/'); firstSlash != -1 { + topdir = text[:firstSlash] } - switch cmd { - case "exec", "unexec": - switch { - case contains(arg, "install-info"), - contains(arg, "${INSTALL_INFO}"): - line.warnf("@exec/unexec install-info is deprecated.") - case contains(arg, "ldconfig") && !contains(arg, "/usr/bin/true"): - line.errorf("ldconfig must be used with \"||/usr/bin/true\".") - } + switch topdir { + case "bin": + ck.checkpathBin(pline, dirname, basename) + case "doc": + line.Error0("Documentation must be installed under share/doc, not doc.") + case "etc": + ck.checkpathEtc(pline, dirname, basename) + case "info": + ck.checkpathInfo(pline, dirname, basename) + case "lib": + ck.checkpathLib(pline, dirname, basename) + case "man": + ck.checkpathMan(pline) + case "sbin": + ck.checkpathSbin(pline) + case "share": + ck.checkpathShare(pline) + } - case "comment": - // Nothing to do. + if contains(text, "${PKGLOCALEDIR}") && G.Pkg != nil && G.Pkg.vardef["USE_PKGLOCALEDIR"] == nil { + line.Warn0("PLIST contains ${PKGLOCALEDIR}, but USE_PKGLOCALEDIR was not found.") + } - case "dirrm": - line.warnf("@dirrm is obsolete. Please remove this line.") - line.explain( - "Directories are removed automatically when they are empty.", - "When a package needs an empty directory, it can use the @pkgdir", - "command in the PLIST") + if contains(text, "/CVS/") { + line.Warn0("CVS files should not be in the PLIST.") + } + if hasSuffix(text, ".orig") { + line.Warn0(".orig files should not be in the PLIST.") + } + if hasSuffix(text, "/perllocal.pod") { + line.Warn0("perllocal.pod files should not be in the PLIST.") + Explain2( + "This file is handled automatically by the INSTALL/DEINSTALL scripts,", + "since its contents changes frequently.") + } +} - case "imake-man": - args := splitOnSpace(arg) - if len(args) != 3 { - line.warnf("Invalid number of arguments for imake-man.") - } else { - if args[2] == "${IMAKE_MANNEWSUFFIX}" { - pline.warnAboutPlistImakeMannewsuffix() +func (ck *PlistChecker) checkSorted(pline *PlistLine) { + if text := pline.text; G.opts.WarnPlistSort && hasAlnumPrefix(text) && !containsVarRef(text) { + if ck.lastFname != "" { + if ck.lastFname > text && !G.opts.Autofix { + pline.line.Warn2("%q should be sorted before %q.", text, ck.lastFname) + Explain2( + "The files in the PLIST should be sorted alphabetically.", + "To fix this, run \"pkglint -F PLIST\".") + } + if prev := ck.allFiles[text]; prev != nil && prev != pline { + if !pline.line.AutofixDelete() { + pline.line.Errorf("Duplicate filename %q, already appeared in %s.", text, prev.line.ReferenceFrom(pline.line)) + } } } + ck.lastFname = text + } +} - case "pkgdir": - // Nothing to check. +func (ck *PlistChecker) checkpathBin(pline *PlistLine, dirname, basename string) { + if contains(dirname, "/") { + pline.line.Warn0("The bin/ directory should not have subdirectories.") + return + } - default: - line.warnf("Unknown PLIST directive \"@%s\".", cmd) + if G.opts.WarnExtra && + ck.allFiles["man/man1/"+basename+".1"] == nil && + ck.allFiles["man/man6/"+basename+".6"] == nil && + ck.allFiles["${IMAKE_MAN_DIR}/"+basename+".${IMAKE_MANNEWSUFFIX}"] == nil { + pline.line.Warn1("Manual page missing for bin/%s.", basename) + Explain( + "All programs that can be run directly by the user should have a", + "manual page for quick reference. The programs in the bin/ directory", + "should have corresponding manual pages in section 1 (filename", + "program.1), while the programs in the sbin/ directory have their", + "manual pages in section 8.") } } -func (pline *PlistLine) checkPathname(pctx *PlistContext, fullname string) { - line := pline.line - text := line.text - sdirname, basename := path.Split(fullname) - dirname := strings.TrimSuffix(sdirname, "/") +func (ck *PlistChecker) checkpathEtc(pline *PlistLine, dirname, basename string) { + if hasPrefix(pline.text, "etc/rc.d/") { + pline.line.Error0("RCD_SCRIPTS must not be registered in the PLIST. Please use the RCD_SCRIPTS framework.") + return + } - pline.checkSorted(pctx) + pline.line.Error0("Configuration files must not be registered in the PLIST. " + + "Please use the CONF_FILES framework, which is described in mk/pkginstall/bsd.pkginstall.mk.") +} - if contains(basename, "${IMAKE_MANNEWSUFFIX}") { - pline.warnAboutPlistImakeMannewsuffix() +func (ck *PlistChecker) checkpathInfo(pline *PlistLine, dirname, basename string) { + if pline.text == "info/dir" { + pline.line.Error0("\"info/dir\" must not be listed. Use install-info to add/remove an entry.") + return } - switch { - case hasPrefix(dirname, "bin/"): - line.warnf("The bin/ directory should not have subdirectories.") - - case dirname == "bin": - pline.checkpathBin(pctx, basename) + if G.Pkg != nil && G.Pkg.vardef["INFO_FILES"] == nil { + pline.line.Warn0("Packages that install info files should set INFO_FILES.") + } +} - case hasPrefix(text, "doc/"): - line.errorf("Documentation must be installed under share/doc, not doc.") +func (ck *PlistChecker) checkpathLib(pline *PlistLine, dirname, basename string) { + switch { + case G.Pkg != nil && G.Pkg.EffectivePkgbase != "" && hasPrefix(pline.text, "lib/"+G.Pkg.EffectivePkgbase+"/"): + return - case hasPrefix(text, "etc/rc.d/"): - line.errorf("RCD_SCRIPTS must not be registered in the PLIST. Please use the RCD_SCRIPTS framework.") + case hasPrefix(pline.text, "lib/locale/"): + pline.line.Error0("\"lib/locale\" must not be listed. Use ${PKGLOCALEDIR}/locale and set USE_PKGLOCALEDIR instead.") + return + } - case hasPrefix(text, "etc/"): - f := "mk/pkginstall/bsd.pkginstall.mk" - line.errorf("Configuration files must not be registered in the PLIST. "+ - "Please use the CONF_FILES framework, which is described in %s.", f) + switch ext := path.Ext(basename); ext { + case ".a", ".la", ".so": + if G.opts.WarnExtra && dirname == "lib" && !hasPrefix(basename, "lib") { + pline.line.Warn1("Library filename %q should start with \"lib\".", basename) + } + if ext == "la" { + if G.Pkg != nil && G.Pkg.vardef["USE_LIBTOOL"] == nil { + pline.line.Warn0("Packages that install libtool libraries should define USE_LIBTOOL.") + } + } + } - case hasPrefix(text, "include/") && matches(text, `^include/.*\.(?:h|hpp)$`): - // Fine. + if contains(basename, ".a") || contains(basename, ".so") { + if m, noext := match1(pline.text, `^(.*)(?:\.a|\.so[0-9.]*)$`); m { + if laLine := ck.allFiles[noext+".la"]; laLine != nil { + pline.line.Warn1("Redundant library found. The libtool library is in %s.", laLine.line.ReferenceFrom(pline.line)) + } + } + } +} - case text == "info/dir": - line.errorf("\"info/dir\" must not be listed. Use install-info to add/remove an entry.") +func (ck *PlistChecker) checkpathMan(pline *PlistLine) { + line := pline.line - case hasPrefix(text, "info/"): - if G.pkgContext != nil && G.pkgContext.vardef["INFO_FILES"] == nil { - line.warnf("Packages that install info files should set INFO_FILES.") - } + m, catOrMan, section, manpage, ext, gz := match5(pline.text, `^man/(cat|man)(\w+)/(.*?)\.(\w+)(\.gz)?$`) + if !m { + // maybe: line.warn1("Invalid filename %q for manual page.", text) + return + } - case G.pkgContext != nil && G.pkgContext.effectivePkgbase != "" && hasPrefix(text, "lib/"+G.pkgContext.effectivePkgbase+"/"): - // Fine. + if !matches(section, `^[\dln]$`) { + line.Warn1("Unknown section %q for manual page.", section) + } - case hasPrefix(text, "lib/locale/"): - line.errorf("\"lib/locale\" must not be listed. Use ${PKGLOCALEDIR}/locale and set USE_PKGLOCALEDIR instead.") + if catOrMan == "cat" && ck.allFiles["man/man"+section+"/"+manpage+"."+section] == nil { + line.Warn0("Preformatted manual page without unformatted one.") + } - case hasPrefix(text, "lib/"): - pline.checkpathLib(pctx, basename) + if catOrMan == "cat" { + if ext != "0" { + line.Warn0("Preformatted manual pages should end in \".0\".") + } + } else { + if !hasPrefix(ext, section) { + line.Warn2("Mismatch between the section (%s) and extension (%s) of the manual page.", section, ext) + } + } - case hasPrefix(text, "man/"): - pline.checkpathMan(pctx) + if gz != "" { + line.Note0("The .gz extension is unnecessary for manual pages.") + Explain4( + "Whether the manual pages are installed in compressed form or not is", + "configured by the pkgsrc user. Compression and decompression takes", + "place automatically, no matter if the .gz extension is mentioned in", + "the PLIST or not.") + } +} - case hasPrefix(text, "sbin/"): - pline.checkpathSbin(pctx) +func (ck *PlistChecker) checkpathSbin(pline *PlistLine) { + binname := pline.text[5:] + + if ck.allFiles["man/man8/"+binname+".8"] == nil && G.opts.WarnExtra { + pline.line.Warn1("Manual page missing for sbin/%s.", binname) + Explain( + "All programs that can be run directly by the user should have a", + "manual page for quick reference. The programs in the sbin/", + "directory should have corresponding manual pages in section 8", + "(filename program.8), while the programs in the bin/ directory", + "have their manual pages in section 1.") + } +} +func (ck *PlistChecker) checkpathShare(pline *PlistLine) { + line, text := pline.line, pline.text + switch { case hasPrefix(text, "share/applications/") && hasSuffix(text, ".desktop"): f := "../../sysutils/desktop-file-utils/desktopdb.mk" - if G.pkgContext != nil && G.pkgContext.included[f] == nil { - line.warnf("Packages that install a .desktop entry should .include %q.", f) - line.explain( - "If *.desktop files contain MimeType keys, the global MIME type registry", - "must be updated by desktop-file-utils. Otherwise, this warning is harmless.") + if G.opts.WarnExtra && G.Pkg != nil && G.Pkg.included[f] == nil { + line.Warn1("Packages that install a .desktop entry should .include %q.", f) + Explain3( + "If *.desktop files contain MimeType keys, the global MIME type", + "registry must be updated by desktop-file-utils. Otherwise, this", + "warning is harmless.") } - case hasPrefix(text, "share/icons/hicolor/") && G.pkgContext != nil && G.pkgContext.pkgpath != "graphics/hicolor-icon-theme": + case hasPrefix(text, "share/icons/hicolor/") && G.Pkg != nil && G.Pkg.Pkgpath != "graphics/hicolor-icon-theme": f := "../../graphics/hicolor-icon-theme/buildlink3.mk" - if G.pkgContext.included[f] == nil { - line.errorf("Packages that install hicolor icons must include %q in the Makefile.", f) + if G.Pkg.included[f] == nil { + line.Error1("Packages that install hicolor icons must include %q in the Makefile.", f) } - case hasPrefix(text, "share/icons/gnome") && G.pkgContext != nil && G.pkgContext.pkgpath != "graphics/gnome-icon-theme": + case hasPrefix(text, "share/icons/gnome") && G.Pkg != nil && G.Pkg.Pkgpath != "graphics/gnome-icon-theme": f := "../../graphics/gnome-icon-theme/buildlink3.mk" - if G.pkgContext.included[f] == nil { - line.errorf("The package Makefile must include %q.", f) - line.explain( - "Packages that install GNOME icons must maintain the icon theme cache.") + if G.Pkg.included[f] == nil { + line.Error1("The package Makefile must include %q.", f) + Explain2( + "Packages that install GNOME icons must maintain the icon theme", + "cache.") } - case dirname == "share/aclocal" && hasSuffix(basename, ".m4"): - // Fine. - case hasPrefix(text, "share/doc/html/"): - _ = G.opts.WarnPlistDepr && line.warnf("Use of \"share/doc/html\" is deprecated. Use \"share/doc/${PKGBASE}\" instead.") + if G.opts.WarnPlistDepr { + line.Warn0("Use of \"share/doc/html\" is deprecated. Use \"share/doc/${PKGBASE}\" instead.") + } - case G.pkgContext != nil && G.pkgContext.effectivePkgbase != "" && (hasPrefix(text, "share/doc/"+G.pkgContext.effectivePkgbase+"/") || - hasPrefix(text, "share/examples/"+G.pkgContext.effectivePkgbase+"/")): + case G.Pkg != nil && G.Pkg.EffectivePkgbase != "" && (hasPrefix(text, "share/doc/"+G.Pkg.EffectivePkgbase+"/") || + hasPrefix(text, "share/examples/"+G.Pkg.EffectivePkgbase+"/")): // Fine. - case text == "share/icons/hicolor/icon-theme.cache" && G.pkgContext != nil && G.pkgContext.pkgpath != "graphics/hicolor-icon-theme": - line.errorf("This file must not appear in any PLIST file.") - line.explain( + case text == "share/icons/hicolor/icon-theme.cache" && G.Pkg != nil && G.Pkg.Pkgpath != "graphics/hicolor-icon-theme": + line.Error0("This file must not appear in any PLIST file.") + Explain3( "Remove this line and add the following line to the package Makefile.", "", ".include \"../../graphics/hicolor-icon-theme/buildlink3.mk\"") case hasPrefix(text, "share/info/"): - line.warnf("Info pages should be installed into info/, not share/info/.") - line.explain( + line.Warn0("Info pages should be installed into info/, not share/info/.") + Explain1( "To fix this, you should add INFO_FILES=yes to the package Makefile.") case hasPrefix(text, "share/locale/") && hasSuffix(text, ".mo"): // Fine. case hasPrefix(text, "share/man/"): - line.warnf("Man pages should be installed into man/, not share/man/.") - - default: - _ = G.opts.DebugUnchecked && line.debugf("Unchecked pathname %q.", text) - } - - if contains(text, "${PKGLOCALEDIR}") && G.pkgContext != nil && G.pkgContext.vardef["USE_PKGLOCALEDIR"] == nil { - line.warnf("PLIST contains ${PKGLOCALEDIR}, but USE_PKGLOCALEDIR was not found.") - } - - if contains(text, "/CVS/") { - line.warnf("CVS files should not be in the PLIST.") - } - if hasSuffix(text, ".orig") { - line.warnf(".orig files should not be in the PLIST.") - } - if hasSuffix(text, "/perllocal.pod") { - line.warnf("perllocal.pod files should not be in the PLIST.") - line.explain( - "This file is handled automatically by the INSTALL/DEINSTALL scripts,", - "since its contents changes frequently.") + line.Warn0("Man pages should be installed into man/, not share/man/.") } } -func (pline *PlistLine) checkSorted(pctx *PlistContext) { - if text := pline.line.text; G.opts.WarnPlistSort && hasAlnumPrefix(text) && !containsVarRef(text) { - if pctx.lastFname != "" { - if pctx.lastFname > text { - pline.line.warnf("%q should be sorted before %q.", text, pctx.lastFname) - pline.line.explain( - "The files in the PLIST should be sorted alphabetically.") - } else if pctx.lastFname == text { - pline.line.errorf("Duplicate filename.") - } - } - pctx.lastFname = text +func (pline *PlistLine) CheckTrailingWhitespace() { + if hasSuffix(pline.text, " ") || hasSuffix(pline.text, "\t") { + pline.line.Error0("pkgsrc does not support filenames ending in white-space.") + Explain1( + "Each character in the PLIST is relevant, even trailing white-space.") } } -func (pline *PlistLine) checkpathBin(pctx *PlistContext, basename string) { - switch { - case pctx.allFiles["man/man1/"+basename+".1"] != nil: - case pctx.allFiles["man/man6/"+basename+".6"] != nil: - case pctx.allFiles["${IMAKE_MAN_DIR}/"+basename+".${IMAKE_MANNEWSUFFIX}"] != nil: - default: - if G.opts.WarnExtra { - pline.line.warnf("Manual page missing for bin/%s.", basename) - pline.line.explain( - "All programs that can be run directly by the user should have a manual", - "page for quick reference. The programs in the bin/ directory should have", - "corresponding manual pages in section 1 (filename program.1), not in", - "section 8.") - } - } -} +func (pline *PlistLine) CheckDirective(cmd, arg string) { + line := pline.line -func (pline *PlistLine) checkpathLib(pctx *PlistContext, basename string) { - if m, dir, lib, ext := match3(pline.line.text, `^(lib/(?:.*/)*)([^/]+)\.(so|a|la)$`); m { - if dir == "lib/" && !hasPrefix(lib, "lib") { - _ = G.opts.WarnExtra && pline.line.warnf("Library filename does not start with \"lib\".") - } - if ext == "la" { - if G.pkgContext != nil && G.pkgContext.vardef["USE_LIBTOOL"] == nil { - pline.line.warnf("Packages that install libtool libraries should define USE_LIBTOOL.") + if cmd == "unexec" { + if m, arg := match1(arg, `^(?:rmdir|\$\{RMDIR\} \%D/)(.*)`); m { + if !contains(arg, "true") && !contains(arg, "${TRUE}") { + pline.line.Warn0("Please remove this line. It is no longer necessary.") } } } - if contains(basename, ".a") || contains(basename, ".so") { - if m, noext := match1(pline.line.text, `^(.*)(?:\.a|\.so[0-9.]*)$`); m { - if laLine := pctx.allFiles[noext+".la"]; laLine != nil { - pline.line.warnf("Redundant library found. The libtool library is in line %s.", laLine) - } + switch cmd { + case "exec", "unexec": + switch { + case contains(arg, "install-info"), + contains(arg, "${INSTALL_INFO}"): + line.Warn0("@exec/unexec install-info is deprecated.") + case contains(arg, "ldconfig") && !contains(arg, "/usr/bin/true"): + pline.line.Error0("ldconfig must be used with \"||/usr/bin/true\".") } - } -} - -func (pline *PlistLine) checkpathMan(pctx *PlistContext) { - line := pline.line - - m, catOrMan, section, manpage, ext, gz := match5(pline.line.text, `^man/(cat|man)(\w+)/(.*?)\.(\w+)(\.gz)?$`) - if !m { - // maybe: line.warnf("Invalid filename %q for manual page.", text) - return - } - if !matches(section, `^[\dln]$`) { - line.warnf("Unknown section %q for manual page.", section) - } + case "comment": + // Nothing to do. - if catOrMan == "cat" && pctx.allFiles["man/man"+section+"/"+manpage+"."+section] == nil { - line.warnf("Preformatted manual page without unformatted one.") - } + case "dirrm": + line.Warn0("@dirrm is obsolete. Please remove this line.") + Explain3( + "Directories are removed automatically when they are empty.", + "When a package needs an empty directory, it can use the @pkgdir", + "command in the PLIST") - if catOrMan == "cat" { - if ext != "0" { - line.warnf("Preformatted manual pages should end in \".0\".") - } - } else { - if section != ext { - line.warnf("Mismatch between the section (%s) and extension (%s) of the manual page.", section, ext) + case "imake-man": + args := splitOnSpace(arg) + switch { + case len(args) != 3: + line.Warn0("Invalid number of arguments for imake-man.") + case args[2] == "${IMAKE_MANNEWSUFFIX}": + pline.warnImakeMannewsuffix() } - } - - if gz != "" { - line.notef("The .gz extension is unnecessary for manual pages.") - line.explain( - "Whether the manual pages are installed in compressed form or not is", - "configured by the pkgsrc user. Compression and decompression takes place", - "automatically, no matter if the .gz extension is mentioned in the PLIST", - "or not.") - } -} -func (pline *PlistLine) checkpathSbin(pctx *PlistContext) { - binname := pline.line.text[5:] + case "pkgdir": + // Nothing to check. - if pctx.allFiles["man/man8/"+binname+".8"] == nil && G.opts.WarnExtra { - pline.line.warnf("Manual page missing for sbin/%s.", binname) - pline.line.explain( - "All programs that can be run directly by the user should have a manual", - "page for quick reference. The programs in the sbin/ directory should have", - "corresponding manual pages in section 8 (filename program.8), not in", - "section 1.") + default: + line.Warn1("Unknown PLIST directive \"@%s\".", cmd) } } -func (pline *PlistLine) warnAboutPlistImakeMannewsuffix() { - line := pline.line - - line.warnf("IMAKE_MANNEWSUFFIX is not meant to appear in PLISTs.") - line.explain( +func (pline *PlistLine) warnImakeMannewsuffix() { + pline.line.Warn0("IMAKE_MANNEWSUFFIX is not meant to appear in PLISTs.") + Explain( "This is the result of a print-PLIST call that has not been edited", - "manually by the package maintainer. Please replace the", + "manually by the package maintainer. Please replace the", "IMAKE_MANNEWSUFFIX with:", "", "\tIMAKE_MAN_SUFFIX for programs,", @@ -406,3 +449,59 @@ func (pline *PlistLine) warnAboutPlistImakeMannewsuffix() { "\tIMAKE_GAMEMAN_SUFFIX for games,", "\tIMAKE_MISCMAN_SUFFIX for other man pages.") } + +type plistLineSorter struct { + first *PlistLine + plines []*PlistLine + lines []*Line + after map[*PlistLine][]*Line + swapped bool + autofixed bool +} + +func NewPlistLineSorter(plines []*PlistLine) *plistLineSorter { + s := &plistLineSorter{first: plines[0], after: make(map[*PlistLine][]*Line)} + prev := plines[0] + for _, pline := range plines[1:] { + if hasPrefix(pline.text, "@") || contains(pline.text, "$") { + s.after[prev] = append(s.after[prev], pline.line) + } else { + s.plines = append(s.plines, pline) + s.lines = append(s.lines, pline.line) + } + prev = pline + } + return s +} + +func (s *plistLineSorter) Len() int { + return len(s.plines) +} +func (s *plistLineSorter) Less(i, j int) bool { + return s.plines[i].text < s.plines[j].text +} +func (s *plistLineSorter) Swap(i, j int) { + s.swapped = true + s.lines[i], s.lines[j] = s.lines[j], s.lines[i] + s.plines[i], s.plines[j] = s.plines[j], s.plines[i] +} + +func (s *plistLineSorter) Sort() { + sort.Stable(s) + + if !s.swapped { + return + } + + firstLine := s.first.line + firstLine.RememberAutofix("Sorting the whole file.") + firstLine.logAutofix() + firstLine.changed = true // Otherwise the changes won’t be saved + lines := []*Line{firstLine} + lines = append(lines, s.after[s.first]...) + for _, pline := range s.plines { + lines = append(lines, pline.line) + lines = append(lines, s.after[pline]...) + } + s.autofixed = SaveAutofixChanges(lines) +} diff --git a/pkgtools/pkglint/files/plist_test.go b/pkgtools/pkglint/files/plist_test.go index a910a8fd1a9..04cafbb097c 100644 --- a/pkgtools/pkglint/files/plist_test.go +++ b/pkgtools/pkglint/files/plist_test.go @@ -5,17 +5,162 @@ import ( ) func (s *Suite) TestChecklinesPlist(c *check.C) { + s.UseCommandLine(c, "-Wall") + G.Pkg = NewPackage("category/pkgbase") lines := s.NewLines("PLIST", "bin/i386/6c", "bin/program", + "etc/my.cnf", + "etc/rc.d/service", "@exec ${MKDIR} include/pkgbase", + "info/dir", + "lib/c.so", + "lib/libc.so.6", + "lib/libc.la", "${PLIST.man}man/cat3/strcpy.4", - "${PLIST.obsolete}@unexec rmdir /tmp") + "man/man1/imake.${IMAKE_MANNEWSUFFIX}", + "${PLIST.obsolete}@unexec rmdir /tmp", + "sbin/clockctl", + "share/icons/gnome/delete-icon", + "share/tzinfo", + "share/tzinfo") - checklinesPlist(lines) + ChecklinesPlist(lines) c.Check(s.Output(), equals, ""+ "ERROR: PLIST:1: Expected \"@comment $"+"NetBSD$\".\n"+ "WARN: PLIST:1: The bin/ directory should not have subdirectories.\n"+ - "WARN: PLIST:5: Please remove this line. It is no longer necessary.\n") + "WARN: PLIST:2: Manual page missing for bin/program.\n"+ + "ERROR: PLIST:3: Configuration files must not be registered in the PLIST. Please use the CONF_FILES framework, which is described in mk/pkginstall/bsd.pkginstall.mk.\n"+ + "ERROR: PLIST:4: RCD_SCRIPTS must not be registered in the PLIST. Please use the RCD_SCRIPTS framework.\n"+ + "ERROR: PLIST:6: \"info/dir\" must not be listed. Use install-info to add/remove an entry.\n"+ + "WARN: PLIST:7: Library filename \"c.so\" should start with \"lib\".\n"+ + "WARN: PLIST:8: Redundant library found. The libtool library is in line 9.\n"+ + "WARN: PLIST:9: \"lib/libc.la\" should be sorted before \"lib/libc.so.6\".\n"+ + "WARN: PLIST:10: Preformatted manual page without unformatted one.\n"+ + "WARN: PLIST:10: Preformatted manual pages should end in \".0\".\n"+ + "WARN: PLIST:11: IMAKE_MANNEWSUFFIX is not meant to appear in PLISTs.\n"+ + "WARN: PLIST:12: Please remove this line. It is no longer necessary.\n"+ + "WARN: PLIST:13: Manual page missing for sbin/clockctl.\n"+ + "ERROR: PLIST:14: The package Makefile must include \"../../graphics/gnome-icon-theme/buildlink3.mk\".\n"+ + "ERROR: PLIST:16: Duplicate filename \"share/tzinfo\", already appeared in line 15.\n") +} + +func (s *Suite) TestChecklinesPlist_empty(c *check.C) { + lines := s.NewLines("PLIST", + "@comment $"+"NetBSD$") + + ChecklinesPlist(lines) + + c.Check(s.Output(), equals, "WARN: PLIST:1: PLIST files shouldn't be empty.\n") +} + +func (s *Suite) TestChecklinesPlist_commonEnd(c *check.C) { + s.CreateTmpFile(c, "PLIST.common", ""+ + "@comment $"+"NetBSD$\n"+ + "bin/common\n") + fname := s.CreateTmpFile(c, "PLIST.common_end", ""+ + "@comment $"+"NetBSD$\n"+ + "sbin/common_end\n") + + ChecklinesPlist(LoadExistingLines(fname, false)) + + c.Check(s.OutputCleanTmpdir(), equals, "") +} + +func (s *Suite) TestChecklinesPlist_conditional(c *check.C) { + G.Pkg = NewPackage("category/pkgbase") + G.Pkg.plistSubstCond["PLIST.bincmds"] = true + lines := s.NewLines("PLIST", + "@comment $"+"NetBSD$", + "${PLIST.bincmds}bin/subdir/command") + + ChecklinesPlist(lines) + + c.Check(s.Output(), equals, "WARN: PLIST:2: The bin/ directory should not have subdirectories.\n") +} + +func (s *Suite) TestChecklinesPlist_sorting(c *check.C) { + s.UseCommandLine(c, "-Wplist-sort") + lines := s.NewLines("PLIST", + "@comment $"+"NetBSD$", + "@comment Do not remove", + "sbin/i386/6c", + "sbin/program", + "bin/otherprogram", + "${PLIST.conditional}bin/cat") + + ChecklinesPlist(lines) + + c.Check(s.Output(), equals, ""+ + "WARN: PLIST:5: \"bin/otherprogram\" should be sorted before \"sbin/program\".\n"+ + "WARN: PLIST:6: \"bin/cat\" should be sorted before \"bin/otherprogram\".\n") +} + +func (s *Suite) TestPlistChecker_sort(c *check.C) { + s.UseCommandLine(c, "--autofix") + tmpfile := s.CreateTmpFile(c, "PLIST", "dummy\n") + ck := &PlistChecker{nil, nil, ""} + lines := s.NewLines(tmpfile, + "@comment $"+"NetBSD$", + "@comment Do not remove", + "A", + "b", + "CCC", + "lib/${UNKNOWN}.la", + "C", + "ddd", + "@exec echo \"after ddd\"", + "sbin/program", + "${PLIST.one}bin/program", + "${PKGMANDIR}/man1/program.1", + "${PLIST.two}bin/program2", + "lib/before.la", + "lib/after.la", + "@exec echo \"after lib/after.la\"") + plines := ck.NewLines(lines) + + NewPlistLineSorter(plines).Sort() + + c.Check(s.OutputCleanTmpdir(), equals, ""+ + "AUTOFIX: ~/PLIST:1: Sorting the whole file.\n"+ + "AUTOFIX: ~/PLIST: Has been auto-fixed. Please re-run pkglint.\n") + c.Check(s.LoadTmpFile(c, "PLIST"), equals, ""+ + "@comment $"+"NetBSD$\n"+ + "@comment Do not remove\n"+ + "A\n"+ + "C\n"+ + "CCC\n"+ + "lib/${UNKNOWN}.la\n"+ // Stays below the previous line + "b\n"+ + "${PLIST.one}bin/program\n"+ // Conditionals are ignored while sorting + "${PKGMANDIR}/man1/program.1\n"+ // Stays below the previous line + "${PLIST.two}bin/program2\n"+ + "ddd\n"+ + "@exec echo \"after ddd\"\n"+ // Stays below the previous line + "lib/after.la\n"+ + "@exec echo \"after lib/after.la\"\n"+ + "lib/before.la\n"+ + "sbin/program\n") +} + +func (s *Suite) TestPlistChecker_checkpathShare_Desktop(c *check.C) { + s.UseCommandLine(c, "-Wextra") + G.Pkg = NewPackage("category/pkgpath") + + ChecklinesPlist(s.NewLines("PLIST", + "@comment $"+"NetBSD$", + "share/applications/pkgbase.desktop")) + + c.Check(s.Output(), equals, "WARN: PLIST:2: Packages that install a .desktop entry should .include \"../../sysutils/desktop-file-utils/desktopdb.mk\".\n") +} + +func (s *Suite) TestPlistChecker_checkpathMan_gz(c *check.C) { + G.Pkg = NewPackage("category/pkgbase") + + ChecklinesPlist(s.NewLines("PLIST", + "@comment $"+"NetBSD$", + "man/man3/strerror.3.gz")) + + c.Check(s.Output(), equals, "NOTE: PLIST:2: The .gz extension is unnecessary for manual pages.\n") } diff --git a/pkgtools/pkglint/files/shell.go b/pkgtools/pkglint/files/shell.go index e8d83fc6ce3..ed79a958f10 100644 --- a/pkgtools/pkglint/files/shell.go +++ b/pkgtools/pkglint/files/shell.go @@ -10,12 +10,13 @@ import ( const ( reMkShellvaruse = `(?:^|[^\$])\$\$\{?(\w+)\}?` reVarnameDirect = `(?:[-*+.0-9A-Z_a-z{}\[]+)` - reShellword = `^\s*(` + + reShellToken = `^\s*(` + `#.*` + // shell comment `|(?:` + `'[^']*'` + // single quoted string - `|"(?:\\.|[^"\\])*"` + // double quoted string - "|`[^`]*`" + // backticks command execution + "|\"`[^`]+`\"" + // backticks command execution in double quotes + `|"(?:\\.|[^"])*"` + // double quoted string + "|`[^`]*`" + // backticks command execution, somewhat naive `|\\\$\$` + // a shell-escaped dollar sign `|\\[^\$]` + // other escaped characters `|\$[\w_]` + // one-character make(1) variable @@ -23,10 +24,10 @@ const ( `|\$\([^()]+\)` + // make(1) variable, $(...) `|\$[/@<^]` + // special make(1) variables `|\$\$[0-9A-Z_a-z]+` + // shell variable - `|\$\$[#?@]` + // special shell variables + `|\$\$[!#?@]` + // special shell variables `|\$\$[./]` + // unescaped dollar in shell, followed by punctuation `|\$\$\$\$` + // the special pid shell variable - `|\$\$\{[0-9A-Z_a-z]+\}` + // shell variable in braces + `|\$\$\{[0-9A-Z_a-z]+[#%:]?[^}]*\}` + // shell variable in braces `|\$\$\(` + // POSIX-style backticks replacement `|[^\(\)'\"\\\s;&\|<>` + "`" + `\$]` + // non-special character `|\$\{[^\s\"'` + "`" + `]+` + // HACK: nested make(1) variables @@ -44,80 +45,117 @@ const ( ) // ShellCommandState -type scState string +type scState uint8 const ( - scstStart scState = "start" - scstCont scState = "continuation" - scstInstall scState = "install" - scstInstallD scState = "install -d" - scstMkdir scState = "mkdir" - scstPax scState = "pax" - scstPaxS scState = "pax -s" - scstSed scState = "sed" - scstSedE scState = "sed -e" - scstSet scState = "set" - scstSetCont scState = "set-continuation" - scstCond scState = "cond" - scstCondCont scState = "cond-continuation" - scstCase scState = "case" - scstCaseIn scState = "case in" - scstCaseLabel scState = "case label" - scstCaseLabelCont scState = "case-label-continuation" - scstFor scState = "for" - scstForIn scState = "for-in" - scstForCont scState = "for-continuation" - scstEcho scState = "echo" - scstInstallDir scState = "install-dir" - scstInstallDir2 scState = "install-dir2" + scstStart scState = iota + scstCont + scstInstall + scstInstallD + scstMkdir + scstPax + scstPaxS + scstSed + scstSedE + scstSet + scstSetCont + scstCond + scstCondCont + scstCase + scstCaseIn + scstCaseLabel + scstCaseLabelCont + scstFor + scstForIn + scstForCont + scstEcho + scstInstallDir + scstInstallDir2 ) -type MkShellLine struct { - line *Line +func (st scState) String() string { + return [...]string{ + "start", + "continuation", + "install", + "install -d", + "mkdir", + "pax", + "pax -s", + "sed", + "sed -e", + "set", + "set-continuation", + "cond", + "cond-continuation", + "case", + "case in", + "case label", + "case-label-continuation", + "for", + "for-in", + "for-continuation", + "echo", + "install-dir", + "install-dir2", + }[st] +} + +type ShellLine struct { + line *Line + mkline *MkLine } -func NewMkShellLine(line *Line) *MkShellLine { - return &MkShellLine{line} +func NewShellLine(mkline *MkLine) *ShellLine { + return &ShellLine{mkline.Line, mkline} } -type ShellwordState string +type ShellwordState uint8 const ( - swstPlain ShellwordState = "plain" - swstSquot ShellwordState = "squot" - swstDquot ShellwordState = "dquot" - swstDquotBackt ShellwordState = "dquot+backt" - swstBackt ShellwordState = "backt" + swstPlain ShellwordState = iota + swstSquot + swstDquot + swstDquotBackt + swstBackt ) -func (msline *MkShellLine) checkShellword(shellword string, checkQuoting bool) { - defer tracecall("MkShellLine.checklineMkShellword", shellword, checkQuoting)() +func (st ShellwordState) String() string { + return [...]string{"plain", "squot", "dquot", "dquot+backt", "backt"}[st] +} + +func (shline *ShellLine) CheckToken(token string, checkQuoting bool) { + if G.opts.DebugTrace { + defer tracecall(token, checkQuoting)() + } - if shellword == "" || hasPrefix(shellword, "#") { + if token == "" || hasPrefix(token, "#") { return } - shellcommandContextType := &Vartype{lkNone, CheckvarShellCommand, []AclEntry{{"*", "adsu"}}, guNotGuessed} - shellwordVuc := &VarUseContext{vucTimeUnknown, shellcommandContextType, vucQuotPlain, vucExtentWord} + line := shline.line + shellcommandsContextType := &Vartype{lkNone, CheckvarShellCommands, []AclEntry{{"*", aclpAllRuntime}}, false} + shellwordVuc := &VarUseContext{shellcommandsContextType, vucTimeUnknown, vucQuotPlain, vucExtentWord} - line := msline.line - if m, varname, mod := match2(shellword, `^\$\{(`+reVarnameDirect+`)(:[^{}]+)?\}$`); m { - NewMkLine(line).checkVaruse(varname, mod, shellwordVuc) + if m, varname, mod := match2(token, `^\$\{(`+reVarnameDirect+`)(:[^{}]+)?\}$`); m { + shline.mkline.CheckVaruse(varname, mod, shellwordVuc) return } - if matches(shellword, `\$\{PREFIX\}/man(?:$|/)`) { - line.warnf("Please use ${PKGMANDIR} instead of \"man\".") + if matches(token, `\$\{PREFIX\}/man(?:$|/)`) { + line.Warn0("Please use ${PKGMANDIR} instead of \"man\".") } - if contains(shellword, "etc/rc.d") { - line.warnf("Please use the RCD_SCRIPTS mechanism to install rc.d scripts automatically to ${RCD_SCRIPTS_EXAMPLEDIR}.") + if contains(token, "etc/rc.d") { + line.Warn0("Please use the RCD_SCRIPTS mechanism to install rc.d scripts automatically to ${RCD_SCRIPTS_EXAMPLEDIR}.") } - repl := NewPrefixReplacer(shellword) + repl := NewPrefixReplacer(token) state := swstPlain outer: for repl.rest != "" { - _ = G.opts.DebugShell && line.debugf("shell state %s: %q", state, repl.rest) + if G.opts.DebugShell { + line.Debugf("shell state %s: %q", state, repl.rest) + } switch { // When parsing inside backticks, it is more @@ -126,20 +164,22 @@ outer: // make(1) variable. case state == swstBackt || state == swstDquotBackt: var backtCommand string - backtCommand, state = msline.unescapeBackticks(shellword, repl, state) - msline.checkShelltext(backtCommand) + backtCommand, state = shline.unescapeBackticks(token, repl, state) + setE := true + shline.CheckShellCommand(backtCommand, &setE) // Make(1) variables have the same syntax, no matter in which state we are currently. - case repl.startsWith(`^\$\{(` + reVarnameDirect + `|@)(:[^\{]+)?\}`), - repl.startsWith(`^\$\((` + reVarnameDirect + `|@])(:[^\)]+)?\)`), - repl.startsWith(`^\$([\w@])()`): + case repl.AdvanceRegexp(`^\$\{(` + reVarnameDirect + `|@)(:[^\{]+)?\}`), + repl.AdvanceRegexp(`^\$\((` + reVarnameDirect + `|@])(:[^\)]+)?\)`), + repl.AdvanceRegexp(`^\$([\w@<])()`): varname, mod := repl.m[1], repl.m[2] if varname == "@" { - line.warnf("Please use \"${.TARGET}\" instead of \"$@\".") - line.explain( - "The variable $@ can easily be confused with the shell variable of the", - "same name, which has a completely different meaning.") + line := shline.line + line.Warn0("Please use \"${.TARGET}\" instead of \"$@\".") + Explain2( + "The variable $@ can easily be confused with the shell variable of", + "the same name, which has a completely different meaning.") varname = ".TARGET" } @@ -151,10 +191,10 @@ outer: case (state == swstSquot || state == swstDquot) && matches(varname, `^(?:.*DIR|.*FILE|.*PATH|.*_VAR|PREFIX|.*BASE|PKGNAME)$`): // This is ok if we don't allow these variables to have embedded [\$\\\"\'\`]. case state == swstDquot && hasSuffix(mod, ":Q"): - line.warnf("Please don't use the :Q operator in double quotes.") - line.explain( - "Either remove the :Q or the double quotes. In most cases, it is more", - "appropriate to remove the double quotes.") + line.Warn0("Please don't use the :Q operator in double quotes.") + Explain2( + "Either remove the :Q or the double quotes. In most cases, it is", + "more appropriate to remove the double quotes.") } if varname != "@" { @@ -169,23 +209,23 @@ outer: case swstBackt: vucstate = vucQuotBackt } - vuc := &VarUseContext{vucTimeUnknown, shellcommandContextType, vucstate, vucExtentWordpart} - NewMkLine(line).checkVaruse(varname, mod, vuc) + vuc := &VarUseContext{shellcommandsContextType, vucTimeUnknown, vucstate, vucExtentWordpart} + shline.mkline.CheckVaruse(varname, mod, vuc) } // The syntax of the variable modifiers can get quite // hairy. In lack of motivation, we just skip anything // complicated, hoping that at least the braces are balanced. - case repl.startsWith(`^\$\{`): + case repl.AdvanceStr("${"): braces := 1 skip: for repl.rest != "" && braces > 0 { switch { - case repl.startsWith(`^\}`): + case repl.AdvanceStr("}"): braces-- - case repl.startsWith(`^\{`): + case repl.AdvanceStr("{"): braces++ - case repl.startsWith(`^[^{}]+`): + case repl.AdvanceRegexp(`^[^{}]+`): // skip default: break skip @@ -194,26 +234,27 @@ outer: case state == swstPlain: switch { - case repl.startsWith(`^[!#\%&\(\)*+,\-.\/0-9:;<=>?@A-Z\[\]^_a-z{|}~]+`), - repl.startsWith(`^\\(?:[ !"#'\(\)*;?\\^{|}]|\$\$)`): - case repl.startsWith(`^'`): + case repl.AdvanceRegexp(`^[!#\%&\(\)*+,\-.\/0-9:;<=>?@A-Z\[\]^_a-z{|}~]+`), + repl.AdvanceRegexp(`^\\(?:[ !"#'\(\)*./;?\\^{|}]|\$\$)`): + case repl.AdvanceStr("'"): state = swstSquot - case repl.startsWith(`^"`): + case repl.AdvanceStr("\""): state = swstDquot - case repl.startsWith("^`"): + case repl.AdvanceStr("`"): state = swstBackt - case repl.startsWith(`^\$\$([0-9A-Z_a-z]+|\#)`), - repl.startsWith(`^\$\$\{([0-9A-Z_a-z]+|\#)\}`), - repl.startsWith(`^\$\$(\$)\$`): + case repl.AdvanceRegexp(`^\$\$([0-9A-Z_a-z]+|#)`), + repl.AdvanceRegexp(`^\$\$\{([0-9A-Z_a-z]+|#)\}`), + repl.AdvanceRegexp(`^\$\$(\$)\$`): shvarname := repl.m[1] - if G.opts.WarnQuoting && checkQuoting && msline.variableNeedsQuoting(shvarname) { - line.warnf("Unquoted shell variable %q.", shvarname) - line.explain( - "When a shell variable contains white-space, it is expanded (split into", - "multiple words) when it is written as $variable in a shell script.", - "If that is not intended, you should add quotation marks around it,", - "like \"$variable\". Then, the variable will always expand to a single", - "word, preserving all white-space and other special characters.", + if G.opts.WarnQuoting && checkQuoting && shline.variableNeedsQuoting(shvarname) { + line.Warn1("Unquoted shell variable %q.", shvarname) + Explain( + "When a shell variable contains white-space, it is expanded (split", + "into multiple words) when it is written as $variable in a shell", + "script. If that is not intended, you should add quotation marks", + "around it, like \"$variable\". Then, the variable will always expand", + "to a single word, preserving all white-space and other special", + "characters.", "", "Example:", "\tfname=\"Curriculum vitae.doc\"", @@ -222,23 +263,26 @@ outer: "\tcp \"$fname\" /tmp", "\t# copies one file, as intended") } - case repl.startsWith(`^\$@`): - line.warnf("Please use %q instead of %q.", "${.TARGET}", "$@") - line.explain( + case repl.AdvanceStr("$@"): + line.Warn2("Please use %q instead of %q.", "${.TARGET}", "$@") + Explain2( "It is more readable and prevents confusion with the shell variable of", "the same name.") - case repl.startsWith(`^\$\$@`): - line.warnf("The $@ shell variable should only be used in double quotes.") + case repl.AdvanceStr("$$@"): + line.Warn0("The $@ shell variable should only be used in double quotes.") - case repl.startsWith(`^\$\$\?`): - line.warnf("The $? shell variable is often not available in \"set -e\" mode.") + case repl.AdvanceStr("$$?"): + line.Warn0("The $? shell variable is often not available in \"set -e\" mode.") - case repl.startsWith(`^\$\$\(`): - line.warnf("Invoking subshells via $(...) is not portable enough.") - line.explain( + case repl.AdvanceStr("$$("): + line.Warn0("Invoking subshells via $(...) is not portable enough.") + Explain2( "The Solaris /bin/sh does not know this way to execute a command in a", - "subshell. Please use backticks (`...`) as a replacement.") + "subshell. Please use backticks (`...`) as a replacement.") + + case repl.AdvanceStr("$$"): // Not part of a variable. + break default: break outer @@ -246,11 +290,11 @@ outer: case state == swstSquot: switch { - case repl.startsWith(`^'`): + case repl.AdvanceRegexp(`^'`): state = swstPlain - case repl.startsWith(`^[^\$\']+`): + case repl.AdvanceRegexp(`^[^\$\']+`): // just skip - case repl.startsWith(`^\$\$`): + case repl.AdvanceRegexp(`^\$\$`): // just skip default: break outer @@ -258,28 +302,21 @@ outer: case state == swstDquot: switch { - case repl.startsWith(`^"`): + case repl.AdvanceStr("\""): state = swstPlain - case repl.startsWith("^`"): + case repl.AdvanceStr("`"): state = swstDquotBackt - case repl.startsWith("^[^$\"\\\\`]+"): - // just skip - case repl.startsWith("^\\\\(?:[\\\\\"`]|\\$\\$)"): - // just skip - case repl.startsWith(`^\$\$\{([0-9A-Za-z_]+)\}`), - repl.startsWith(`^\$\$([0-9A-Z_a-z]+|[!#?@]|\$\$)`): - shvarname := repl.m[1] - _ = G.opts.DebugShell && line.debugf("checklineMkShellword: found double-quoted variable %q.", shvarname) - case repl.startsWith(`^\$\$`): - line.warnf("Unquoted $ or strange shell variable found.") - case repl.startsWith(`^\\(.)`): - char := repl.m[1] - line.warnf("Please use \"%s\" instead of \"%s\".", "\\\\"+char, "\\"+char) - line.explain( - "Although the current code may work, it is not good style to rely on", - "the shell passing this escape sequence exactly as is, and not", - "discarding the backslash. Alternatively you can use single quotes", - "instead of double quotes.") + case repl.AdvanceRegexp("^[^$\"\\\\`]+"): + break + case repl.AdvanceStr("\\$$"): + break + case repl.AdvanceRegexp(`^\\.`): // See http://pubs.opengroup.org/onlinepubs/009695399/utilities/xcu_chap02.html#tag_02_02_01 + break + case repl.AdvanceRegexp(`^\$\$\{\w+[#%+\-:]*[^{}]*\}`), + repl.AdvanceRegexp(`^\$\$(?:\w+|[!#?@]|\$\$)`): + break + case repl.AdvanceStr("$$"): + line.Warn0("Unescaped $ or strange shell variable found.") default: break outer } @@ -287,7 +324,7 @@ outer: } if strings.TrimSpace(repl.rest) != "" { - line.errorf("Internal pkglint error: checklineMkShellword state=%s, rest=%q, shellword=%q", state, repl.rest, shellword) + line.Errorf("Internal pkglint error: ShellLine.CheckToken state=%s, rest=%q, token=%q", state, repl.rest, token) } } @@ -296,11 +333,11 @@ outer: // before a dollar, a backslash or a backtick. // // See http://www.opengroup.org/onlinepubs/009695399/utilities/xcu_chap02.html#tag_02_06_03 -func (msline *MkShellLine) unescapeBackticks(shellword string, repl *PrefixReplacer, state ShellwordState) (unescaped string, newState ShellwordState) { - line := msline.line +func (shline *ShellLine) unescapeBackticks(shellword string, repl *PrefixReplacer, state ShellwordState) (unescaped string, newState ShellwordState) { + line := shline.line for repl.rest != "" { switch { - case repl.startsWith("^`"): + case repl.AdvanceStr("`"): if state == swstBackt { state = swstPlain } else { @@ -308,33 +345,33 @@ func (msline *MkShellLine) unescapeBackticks(shellword string, repl *PrefixRepla } return unescaped, state - case repl.startsWith("^\\\\([\\\\`$])"): + case repl.AdvanceRegexp("^\\\\([\\\\`$])"): unescaped += repl.m[1] - case repl.startsWith(`^(\\)`): - line.warnf("Backslashes should be doubled inside backticks.") - unescaped += repl.m[1] + case repl.AdvanceStr("\\"): + line.Warn0("Backslashes should be doubled inside backticks.") + unescaped += "\\" - case state == swstDquotBackt && repl.startsWith(`^"`): - line.warnf("Double quotes inside backticks inside double quotes are error prone.") - line.explain( + case state == swstDquotBackt && repl.AdvanceStr("\""): + line.Warn0("Double quotes inside backticks inside double quotes are error prone.") + Explain4( "According to the SUSv3, they produce undefined results.", "", "See the paragraph starting \"Within the backquoted ...\" in", "http://www.opengroup.org/onlinepubs/009695399/utilities/xcu_chap02.html") - case repl.startsWith("^([^\\\\`]+)"): + case repl.AdvanceRegexp("^([^\\\\`]+)"): unescaped += repl.m[1] default: - line.errorf("Internal pkglint error: checklineMkShellword shellword=%q rest=%q", shellword, repl.rest) + line.Errorf("Internal pkglint error: checklineMkShellword shellword=%q rest=%q", shellword, repl.rest) } } - line.errorf("Unfinished backquotes: rest=%q", repl.rest) + line.Error1("Unfinished backquotes: rest=%q", repl.rest) return unescaped, state } -func (msline *MkShellLine) variableNeedsQuoting(shvarname string) bool { +func (shline *ShellLine) variableNeedsQuoting(shvarname string) bool { switch shvarname { case "#", "?": return false // Definitely ok @@ -345,60 +382,82 @@ func (msline *MkShellLine) variableNeedsQuoting(shvarname string) bool { } type ShelltextContext struct { - line *Line + shline *ShellLine state scState shellword string } -func (msline *MkShellLine) checkShelltext(shelltext string) { - defer tracecall("MkShellLine.checklineMkShelltext", shelltext)() +func (shline *ShellLine) CheckShellCommandLine(shelltext string) { + if G.opts.DebugTrace { + defer tracecall1(shelltext)() + } - line := msline.line + line := shline.line if contains(shelltext, "${SED}") && contains(shelltext, "${MV}") { - line.notef("Please use the SUBST framework instead of ${SED} and ${MV}.") - line.explain( - "When converting things, pay attention to \"#\" characters. In shell", - "commands make(1) does not interpret them as comment character, but", - "in other lines it does. Therefore, instead of the shell command", - "", - "\tsed -e 's,#define foo,,'", - "", - "you need to write", + line.Note0("Please use the SUBST framework instead of ${SED} and ${MV}.") + Explain( + "Using the SUBST framework instead of explicit commands is easier", + "to understand, since all the complexity of using sed and mv is", + "hidden behind the scenes.", "", - "\tSUBST_SED.foo+=\t's,\\#define foo,,'") + "Run \"bmake help topic=subst\" for more information.") + if contains(shelltext, "#") { + Explain( + "When migrating to the SUBST framework, pay attention to \"#\"", + "characters. In shell commands, make(1) does not interpret them as", + "comment character, but in variable assignments it does. Therefore,", + "instead of the shell command", + "", + "\tsed -e 's,#define foo,,'", + "", + "you need to write", + "", + "\tSUBST_SED.foo+=\t's,\\#define foo,,'") + } } if m, cmd := match1(shelltext, `^@*-(.*(?:MKDIR|INSTALL.*-d|INSTALL_.*_DIR).*)`); m { - line.notef("You don't need to use \"-\" before %q.", cmd) + line.Note1("You don't need to use \"-\" before %q.", cmd) } - setE := false repl := NewPrefixReplacer(shelltext) - if repl.startsWith(`^\s*([-@]*)(\$\{_PKG_SILENT\}\$\{_PKG_DEBUG\}|\$\{RUN\}|)`) { - hidden, macro := repl.m[1], repl.m[2] - msline.checkLineStart(hidden, macro, repl.rest, &setE) + repl.AdvanceRegexp(`^\s+`) + if repl.AdvanceRegexp(`^[-@]+`) { + shline.checkHiddenAndSuppress(repl.m[0], repl.rest) + } + setE := false + if repl.AdvanceStr("${RUN}") { + setE = true + } else { + repl.AdvanceStr("${_PKG_SILENT}${_PKG_DEBUG}") } - state := scstStart - for repl.startsWith(reShellword) { - shellword := repl.m[1] + shline.CheckShellCommand(repl.rest, &setE) +} - _ = G.opts.DebugShell && line.debugf("checklineMkShelltext state=%v shellword=%q", state, shellword) +func (shline *ShellLine) CheckShellCommand(shellcmd string, pSetE *bool) { + state := scstStart + tokens, rest := splitIntoShellTokens(shline.line, shellcmd) + prevToken := "" + for _, token := range tokens { + if G.opts.DebugShell { + shline.line.Debugf("checkShellCommand state=%v token=%q", state, token) + } { quotingNecessary := state != scstCase && state != scstForCont && state != scstSetCont && - !(state == scstStart && matches(shellword, reShVarassign)) - msline.checkShellword(shellword, quotingNecessary) + !(state == scstStart && matches(token, reShVarassign)) + shline.CheckToken(token, quotingNecessary) } - st := &ShelltextContext{line, state, shellword} + st := &ShelltextContext{shline, state, token} st.checkCommandStart() st.checkConditionalCd() if state != scstPaxS && state != scstSedE && state != scstCaseLabel { - line.checkAbsolutePathname(shellword) + shline.line.CheckAbsolutePathname(token) } st.checkAutoMkdirs() st.checkInstallMulti() @@ -406,39 +465,46 @@ func (msline *MkShellLine) checkShelltext(shelltext string) { st.checkQuoteSubstitution() st.checkEchoN() st.checkPipeExitcode() - st.checkSetE(setE) + st.checkSetE(pSetE, prevToken) - if state == scstSet && matches(shellword, `^-.*e`) || state == scstStart && shellword == "${RUN}" { - setE = true + if state == scstSet && hasPrefix(token, "-") && contains(token, "e") || state == scstStart && token == "${RUN}" { + *pSetE = true } - state = nextState(line, state, shellword) + state = shline.nextState(state, token) + prevToken = token } - repl.startsWith(`^\s+`) - if repl.rest != "" { - line.errorf("Internal pkglint error: checklineMkShelltext state=%s rest=%q shellword=%q", state, repl.rest, shelltext) + if rest != "" { + shline.line.Errorf("Internal pkglint error: ShellLine.CheckShellCommand state=%s rest=%q shellcmd=%q", state, rest, shellcmd) } - } -func (msline *MkShellLine) checkLineStart(hidden, macro, rest string, eflag *bool) { - defer tracecall("MkShellLine.checkLineStart", hidden, macro, rest, eflag)() +func (shline *ShellLine) CheckShellCommands(shellcmds string) { + setE := true + shline.CheckShellCommand(shellcmds, &setE) + if !hasSuffix(shellcmds, ";") { + shline.line.Warn0("This shell command list should end with a semicolon.") + } +} - line := msline.line +func (shline *ShellLine) checkHiddenAndSuppress(hiddenAndSuppress, rest string) { + if G.opts.DebugTrace { + defer tracecall(hiddenAndSuppress, rest)() + } switch { - case !contains(hidden, "@"): + case !contains(hiddenAndSuppress, "@"): // Nothing is hidden at all. - case hasPrefix(G.mkContext.target, "show-") || hasSuffix(G.mkContext.target, "-message"): - // In these targets commands may be hidden. + case hasPrefix(G.Mk.target, "show-") || hasSuffix(G.Mk.target, "-message"): + // In these targets, all commands may be hidden. case hasPrefix(rest, "#"): // Shell comments may be hidden, since they cannot have side effects. default: - if m, cmd := match1(rest, reShellword); m { + if m, cmd := match1(rest, reShellToken); m { switch cmd { case "${DELAYED_ERROR_MSG}", "${DELAYED_WARNING_MSG}", "${DO_NADA}", @@ -447,33 +513,35 @@ func (msline *MkShellLine) checkLineStart(hidden, macro, rest string, eflag *boo "${PHASE_MSG}", "${PRINTF}", "${SHCOMMENT}", "${STEP_MSG}", "${WARNING_CAT}", "${WARNING_MSG}": + break default: - line.warnf("The shell command %q should not be hidden.", cmd) - line.explain( - "Hidden shell commands do not appear on the terminal or in the log file", - "when they are executed. When they fail, the error message cannot be", - "assigned to the command, which is very difficult to debug.") + shline.line.Warn1("The shell command %q should not be hidden.", cmd) + Explain( + "Hidden shell commands do not appear on the terminal or in the log", + "file when they are executed. When they fail, the error message", + "cannot be assigned to the command, which is very difficult to debug.", + "", + "It is better to insert ${RUN} at the beginning of the whole command", + "line. This will hide the command by default, but shows it when", + "PKG_DEBUG_LEVEL is set.") } } } - if contains(hidden, "-") { - line.warnf("The use of a leading \"-\" to suppress errors is deprecated.") - line.explain( - "If you really want to ignore any errors from this command (including", - "all errors you never thought of), append \"|| ${TRUE}\" to the", - "command.") - } - - if macro == "${RUN}" { - *eflag = true + if contains(hiddenAndSuppress, "-") { + shline.line.Warn0("Using a leading \"-\" to suppress errors is deprecated.") + Explain2( + "If you really want to ignore any errors from this command, append", + "\"|| ${TRUE}\" to the command.") } } func (ctx *ShelltextContext) checkCommandStart() { - defer tracecall("ShelltextContext.checkCommandStart", ctx.state, ctx.shellword)() + if G.opts.DebugTrace { + defer tracecall2(ctx.state.String(), ctx.shellword)() + } - line, state, shellword := ctx.line, ctx.state, ctx.shellword + state, shellword := ctx.state, ctx.shellword if state != scstStart && state != scstCond { return } @@ -489,8 +557,9 @@ func (ctx *ShelltextContext) checkCommandStart() { case ctx.handleComment(): default: if G.opts.WarnExtra { - line.warnf("Unknown shell command %q.", shellword) - line.explain( + line := ctx.shline.line + line.Warn1("Unknown shell command %q.", shellword) + Explain3( "If you want your package to be portable to all platforms that pkgsrc", "supports, you should only use shell commands that are covered by the", "tools framework.") @@ -499,22 +568,24 @@ func (ctx *ShelltextContext) checkCommandStart() { } func (ctx *ShelltextContext) handleTool() bool { - defer tracecall("ShelltextContext.handleTool", ctx.shellword)() + if G.opts.DebugTrace { + defer tracecall1(ctx.shellword)() + } shellword := ctx.shellword - if !G.globalData.tools[shellword] { + if !G.globalData.Tools[shellword] { return false } - if !G.mkContext.tools[shellword] && !G.mkContext.tools["g"+shellword] { - ctx.line.warnf("The %q tool is used but not added to USE_TOOLS.", shellword) + if !G.Mk.tools[shellword] && !G.Mk.tools["g"+shellword] { + ctx.shline.line.Warn1("The %q tool is used but not added to USE_TOOLS.", shellword) } if G.globalData.toolsVarRequired[shellword] { - ctx.line.warnf("Please use \"${%s}\" instead of %q.", G.globalData.vartools[shellword], shellword) + ctx.shline.line.Warn2("Please use \"${%s}\" instead of %q.", G.globalData.Vartools[shellword], shellword) } - NewMkShellLine(ctx.line).checkCommandUse(shellword) + ctx.shline.checkCommandUse(shellword) return true } @@ -525,8 +596,8 @@ func (ctx *ShelltextContext) handleForbiddenCommand() bool { return false } - ctx.line.errorf("%q must not be used in Makefiles.", ctx.shellword) - ctx.line.explain( + ctx.shline.line.Error1("%q must not be used in Makefiles.", ctx.shellword) + Explain3( "This command must appear in INSTALL scripts, not in the package", "Makefile, so that the package also works if it is installed as a binary", "package via pkg_add.") @@ -534,27 +605,29 @@ func (ctx *ShelltextContext) handleForbiddenCommand() bool { } func (ctx *ShelltextContext) handleCommandVariable() bool { - defer tracecall("ShelltextContext.handleCommandVariable", ctx.shellword)() + if G.opts.DebugTrace { + defer tracecall1(ctx.shellword)() + } shellword := ctx.shellword if m, varname := match1(shellword, `^\$\{([\w_]+)\}$`); m { - if toolname := G.globalData.varnameToToolname[varname]; toolname != "" { - if !G.mkContext.tools[toolname] { - ctx.line.warnf("The %q tool is used but not added to USE_TOOLS.", toolname) + if toolname := G.globalData.VarnameToToolname[varname]; toolname != "" { + if !G.Mk.tools[toolname] { + ctx.shline.line.Warn1("The %q tool is used but not added to USE_TOOLS.", toolname) } - NewMkShellLine(ctx.line).checkCommandUse(shellword) + ctx.shline.checkCommandUse(shellword) return true } - if vartype := getVariableType(ctx.line, varname); vartype != nil && vartype.checker.name == "ShellCommand" { - NewMkShellLine(ctx.line).checkCommandUse(shellword) + if vartype := ctx.shline.mkline.getVariableType(varname); vartype != nil && vartype.checker.name == "ShellCommand" { + ctx.shline.checkCommandUse(shellword) return true } // When the package author has explicitly defined a command // variable, assume it to be valid. - if G.pkgContext != nil && G.pkgContext.vardef[varname] != nil { + if G.Pkg != nil && G.Pkg.vardef[varname] != nil { return true } } @@ -562,44 +635,48 @@ func (ctx *ShelltextContext) handleCommandVariable() bool { } func (ctx *ShelltextContext) handleComment() bool { - defer tracecall("ShelltextContext.handleComment", ctx.shellword)() + if G.opts.DebugTrace { + defer tracecall1(ctx.shellword)() + } + line := ctx.shline.line shellword := ctx.shellword if !hasPrefix(shellword, "#") { return false } - line := ctx.line semicolon := contains(shellword, ";") - multiline := contains(line.lines, "--") + multiline := ctx.shline.line.IsMultiline() if semicolon { - line.warnf("A shell comment should not contain semicolons.") + line.Warn0("A shell comment should not contain semicolons.") } if multiline { - line.warnf("A shell comment does not stop at the end of line.") + line.Warn0("A shell comment does not stop at the end of line.") } if semicolon || multiline { - line.explain( - "When you split a shell command into multiple lines that are continued", - "with a backslash, they will nevertheless be converted to a single line", - "before the shell sees them. That means that even if it _looks_ like that", - "the comment only spans one line in the Makefile, in fact it spans until", - "the end of the whole shell command. To insert a comment into shell code,", - "you can pass it as an argument to the ${SHCOMMENT} macro, which expands", - "to a command doing nothing. Note that any special characters are", - "nevertheless interpreted by the shell.") + Explain( + "When you split a shell command into multiple lines that are", + "continued with a backslash, they will nevertheless be converted to", + "a single line before the shell sees them. That means that even if", + "it _looks_ like that the comment only spans one line in the", + "Makefile, in fact it spans until the end of the whole shell command.", + "", + "To insert a comment into shell code, you can write it like this:", + "", + "\t"+"${SHCOMMENT} \"The following command might fail; this is ok.\"", + "", + "Note that any special characters in the comment are still", + "interpreted by the shell.") } return true } func (ctx *ShelltextContext) checkConditionalCd() { - line, state, shellword := ctx.line, ctx.state, ctx.shellword - - if state == scstCond && shellword == "cd" { - line.errorf("The Solaris /bin/sh cannot handle \"cd\" inside conditionals.") - line.explain( + if ctx.state == scstCond && ctx.shellword == "cd" { + ctx.shline.line.Error0("The Solaris /bin/sh cannot handle \"cd\" inside conditionals.") + Explain3( "When the Solaris shell is in \"set -e\" mode and \"cd\" fails, the", "shell will exit, no matter if it is protected by an \"if\" or the", "\"||\" operator.") @@ -607,52 +684,52 @@ func (ctx *ShelltextContext) checkConditionalCd() { } func (ctx *ShelltextContext) checkAutoMkdirs() { - line, state, shellword := ctx.line, ctx.state, ctx.shellword + state, shellword := ctx.state, ctx.shellword + line := ctx.shline.line if (state == scstInstallD || state == scstMkdir) && matches(shellword, `^(?:\$\{DESTDIR\})?\$\{PREFIX(?:|:Q)\}/`) { - line.warnf("Please use AUTO_MKDIRS instead of %q.", + line.Warn1("Please use AUTO_MKDIRS instead of %q.", ifelseStr(state == scstMkdir, "${MKDIR}", "${INSTALL} -d")) - line.explain( - "Setting AUTO_MKDIRS=yes automatically creates all directories that are", - "mentioned in the PLIST. If you need additional directories, specify", - "them in INSTALLATION_DIRS, which is a list of directories relative to", - "${PREFIX}.") + Explain4( + "Setting AUTO_MKDIRS=yes automatically creates all directories that", + "are mentioned in the PLIST. If you need additional directories,", + "specify them in INSTALLATION_DIRS, which is a list of directories", + "relative to ${PREFIX}.") } if (state == scstInstallDir || state == scstInstallDir2) && !matches(shellword, reMkShellvaruse) { if m, dirname := match1(shellword, `^(?:\$\{DESTDIR\})?\$\{PREFIX(?:|:Q)\}/(.*)`); m { - line.notef("You can use AUTO_MKDIRS=yes or \"INSTALLATION_DIRS+= %s\" instead of this command.", dirname) - line.explain( - "This saves you some typing. You also don't have to think about which of", - "the many INSTALL_*_DIR macros is appropriate, since INSTALLATION_DIRS", - "takes care of that.", + line.Note1("You can use AUTO_MKDIRS=yes or \"INSTALLATION_DIRS+= %s\" instead of this command.", dirname) + Explain( + "Many packages include a list of all needed directories in their", + "PLIST file. In such a case, you can just set AUTO_MKDIRS=yes and", + "be done. The pkgsrc infrastructure will then create all directories", + "in advance.", "", - "Note that you should only do this if the package creates _all_", - "directories it needs before trying to install files into them.", - "", - "Many packages include a list of all needed directories in their PLIST", - "file. In that case, you can just set AUTO_MKDIRS=yes and be done.") + "To create directories that are not mentioned in the PLIST file, it", + "is easier to just list them in INSTALLATION_DIRS than to execute the", + "commands explicitly. That way, you don't have to think about which", + "of the many INSTALL_*_DIR variables is appropriate, since", + "INSTALLATION_DIRS takes care of that.") } } } func (ctx *ShelltextContext) checkInstallMulti() { - line, state, shellword := ctx.line, ctx.state, ctx.shellword - - if state == scstInstallDir2 && hasPrefix(shellword, "$") { - line.warnf("The INSTALL_*_DIR commands can only handle one directory at a time.") - line.explain( - "Many implementations of install(1) can handle more, but pkgsrc aims at", - "maximum portability.") + if ctx.state == scstInstallDir2 && hasPrefix(ctx.shellword, "$") { + line := ctx.shline.line + line.Warn0("The INSTALL_*_DIR commands can only handle one directory at a time.") + Explain2( + "Many implementations of install(1) can handle more, but pkgsrc aims", + "at maximum portability.") } } func (ctx *ShelltextContext) checkPaxPe() { - line, state, shellword := ctx.line, ctx.state, ctx.shellword - - if state == scstPax && shellword == "-pe" { - line.warnf("Please use the -pp option to pax(1) instead of -pe.") - line.explain( + if ctx.state == scstPax && ctx.shellword == "-pe" { + line := ctx.shline.line + line.Warn0("Please use the -pp option to pax(1) instead of -pe.") + Explain3( "The -pe option tells pax to preserve the ownership of the files, which", "means that the installed files will belong to the user that has built", "the package.") @@ -660,12 +737,11 @@ func (ctx *ShelltextContext) checkPaxPe() { } func (ctx *ShelltextContext) checkQuoteSubstitution() { - line, state, shellword := ctx.line, ctx.state, ctx.shellword - - if state == scstPaxS || state == scstSedE { - if false && !matches(shellword, `"^[\"\'].*[\"\']$`) { - line.warnf("Substitution commands like %q should always be quoted.", shellword) - line.explain( + if ctx.state == scstPaxS || ctx.state == scstSedE { + if false && !matches(ctx.shellword, `"^[\"\'].*[\"\']$`) { + line := ctx.shline.line + line.Warn1("Substitution commands like %q should always be quoted.", ctx.shellword) + Explain3( "Usually these substitution commands contain characters like '*' or", "other shell metacharacters that might lead to lookup of matching", "filenames and then expand to more than one word.") @@ -674,19 +750,16 @@ func (ctx *ShelltextContext) checkQuoteSubstitution() { } func (ctx *ShelltextContext) checkEchoN() { - line, state, shellword := ctx.line, ctx.state, ctx.shellword - - if state == scstEcho && shellword == "-n" { - line.warnf("Please use ${ECHO_N} instead of \"echo -n\".") + if ctx.state == scstEcho && ctx.shellword == "-n" { + ctx.shline.line.Warn0("Please use ${ECHO_N} instead of \"echo -n\".") } } func (ctx *ShelltextContext) checkPipeExitcode() { - line, state, shellword := ctx.line, ctx.state, ctx.shellword - - if G.opts.WarnExtra && state != scstCaseLabelCont && shellword == "|" { - line.warnf("The exitcode of the left-hand-side command of the pipe operator is ignored.") - line.explain( + if G.opts.WarnExtra && ctx.state != scstCaseLabelCont && ctx.shellword == "|" { + line := ctx.shline.line + line.Warn0("The exitcode of the left-hand-side command of the pipe operator is ignored.") + Explain( "In a shell command like \"cat *.txt | grep keyword\", if the command", "on the left side of the \"|\" fails, this failure is ignored.", "", @@ -695,32 +768,33 @@ func (ctx *ShelltextContext) checkPipeExitcode() { } } -func (ctx *ShelltextContext) checkSetE(eflag bool) { - line, state, shellword := ctx.line, ctx.state, ctx.shellword - - if G.opts.WarnExtra && shellword == ";" && state != scstCondCont && state != scstForCont && !eflag { - line.warnf("Please switch to \"set -e\" mode before using a semicolon to separate commands.") - line.explain( - "Older versions of the NetBSD make(1) had run the shell commands using", - "the \"-e\" option of /bin/sh. In 2004, this behavior has been changed to", - "follow the POSIX conventions, which is to not use the \"-e\" option.", - "The consequence of this change is that shell programs don't terminate", - "as soon as an error occurs, but try to continue with the next command.", - "Imagine what would happen for these commands:", - " cd \"HOME\"; cd /nonexistent; rm -rf *", - "To fix this warning, either insert \"set -e\" at the beginning of this", - "line or use the \"&&\" operator instead of the semicolon.") +func (ctx *ShelltextContext) checkSetE(eflag *bool, prevToken string) { + if G.opts.WarnExtra && ctx.shellword == ";" && ctx.state != scstCondCont && ctx.state != scstForCont && !*eflag { + *eflag = true + ctx.shline.line.Warn1("Please switch to \"set -e\" mode before using a semicolon (the one after %q) to separate commands.", prevToken) + Explain( + "Normally, when a shell command fails (returns non-zero), the", + "remaining commands are still executed. For example, the following", + "commands would remove all files from the HOME directory:", + "", + "\tcd \"$HOME\"; cd /nonexistent; rm -rf *", + "", + "To fix this warning, you can:", + "", + "* insert ${RUN} at the beginning of the line", + " (which among other things does \"set -e\")", + "* insert \"set -e\" explicitly at the beginning of the line", + "* use \"&&\" instead of \";\" to separate the commands") } } // Some shell commands should not be used in the install phase. -func (msline *MkShellLine) checkCommandUse(shellcmd string) { - line := msline.line - - if G.mkContext == nil || !matches(G.mkContext.target, `^(?:pre|do|post)-install$`) { +func (shline *ShellLine) checkCommandUse(shellcmd string) { + if G.Mk == nil || !matches(G.Mk.target, `^(?:pre|do|post)-install$`) { return } + line := shline.line switch shellcmd { case "${INSTALL}", "${INSTALL_DATA}", "${INSTALL_DATA_DIR}", @@ -735,17 +809,17 @@ func (msline *MkShellLine) checkCommandUse(shellcmd string) { case "sed", "${SED}", "tr", "${TR}": - line.warnf("The shell command %q should not be used in the install phase.", shellcmd) - line.explain( - "In the install phase, the only thing that should be done is to install", - "the prepared files to their final location. The file's contents should", - "not be changed anymore.") + line.Warn1("The shell command %q should not be used in the install phase.", shellcmd) + Explain3( + "In the install phase, the only thing that should be done is to", + "install the prepared files to their final location. The file's", + "contents should not be changed anymore.") case "cp", "${CP}": - line.warnf("${CP} should not be used to install files.") - line.explain( + line.Warn0("${CP} should not be used to install files.") + Explain( "The ${CP} command is highly platform dependent and cannot overwrite", - "files that don't have write permission. Please use ${PAX} instead.", + "read-only files. Please use ${PAX} instead.", "", "For example, instead of", "\t${CP} -R ${WRKSRC}/* ${PREFIX}/foodir", @@ -754,7 +828,7 @@ func (msline *MkShellLine) checkCommandUse(shellcmd string) { } } -func nextState(line *Line, state scState, shellword string) scState { +func (shline *ShellLine) nextState(state scState, shellword string) scState { switch { case shellword == ";;": return scstCaseLabel @@ -854,18 +928,37 @@ func nextState(line *Line, state scState, shellword string) scState { case state == scstEcho: return scstCont default: - _ = G.opts.DebugShell && line.errorf("Internal pkglint error: shellword.nextState state=%s shellword=%q", state, shellword) + if G.opts.DebugShell { + shline.line.Errorf("Internal pkglint error: shellword.nextState state=%s shellword=%q", state, shellword) + } return scstStart } } -func splitIntoShellwords(line *Line, text string) ([]string, string) { - var words []string +func splitIntoShellTokens(line *Line, text string) (words []string, rest string) { + repl := NewPrefixReplacer(text) + for repl.AdvanceRegexp(reShellToken) { + words = append(words, repl.m[1]) + } + repl.AdvanceRegexp(`^\s+`) + return words, repl.rest +} +// Compare devel/bmake/files/str.c, function brk_string. +func splitIntoShellWords(line *Line, text string) (words []string, rest string) { + reShellWord := `^\s*(` + + `#.*` + // shell comment + `|(?:` + + `'[^']*'` + // single quoted string + `|"(?:\\.|[^"\\])*"` + // double quoted string + "|`[^`]*`" + // backticks command execution + "|[^\\s\"'`\\\\]+" + // normal characters + `|\\.` + // escaped character + `)+)` // any of the above may be repeated repl := NewPrefixReplacer(text) - for repl.startsWith(reShellword) { + for repl.AdvanceRegexp(reShellWord) { words = append(words, repl.m[1]) } - repl.startsWith(`^\s+`) + repl.AdvanceRegexp(`^\s+`) return words, repl.rest } diff --git a/pkgtools/pkglint/files/shell_test.go b/pkgtools/pkglint/files/shell_test.go index 1d417cebd33..d1b37470353 100644 --- a/pkgtools/pkglint/files/shell_test.go +++ b/pkgtools/pkglint/files/shell_test.go @@ -4,80 +4,201 @@ import ( check "gopkg.in/check.v1" ) -func (s *Suite) TestSplitIntoShellwords_LineContinuation(c *check.C) { - line := NewLine("fname", "1", "dummy", nil) +func (s *Suite) TestReShellToken(c *check.C) { + re := `^(?:` + reShellToken + `)$` + matches := check.NotNil + doesntMatch := check.IsNil + + c.Check(match("", re), doesntMatch) + c.Check(match("$var", re), matches) + c.Check(match("$var$var", re), matches) + c.Check(match("$var;;", re), doesntMatch) // More than one token + c.Check(match("'single-quoted'", re), matches) + c.Check(match("\"", re), doesntMatch) // Incomplete string + c.Check(match("'...'\"...\"", re), matches) // Mixed strings + c.Check(match("\"...\"", re), matches) + c.Check(match("`cat file`", re), matches) + c.Check(match("${file%.c}.o", re), matches) +} + +func (s *Suite) TestSplitIntoShellTokens_LineContinuation(c *check.C) { + line := NewLine("fname", 10, "dummy", nil) - words, rest := splitIntoShellwords(line, "if true; then \\") + words, rest := splitIntoShellTokens(line, "if true; then \\") c.Check(words, check.DeepEquals, []string{"if", "true", ";", "then"}) c.Check(rest, equals, "\\") - words, rest = splitIntoShellwords(line, "pax -s /.*~$$//g") + words, rest = splitIntoShellTokens(line, "pax -s /.*~$$//g") c.Check(words, check.DeepEquals, []string{"pax", "-s", "/.*~$$//g"}) c.Check(rest, equals, "") } -func (s *Suite) TestChecklineMkShelltext(c *check.C) { +func (s *Suite) TestChecklineMkShellCommandLine(c *check.C) { s.UseCommandLine(c, "-Wall") - G.mkContext = newMkContext() - msline := NewMkShellLine(NewLine("fname", "1", "# dummy", nil)) + G.Mk = s.NewMkLines("fname", + "# dummy") + shline := NewShellLine(G.Mk.mklines[0]) - msline.checkShelltext("@# Comment") + shline.CheckShellCommandLine("@# Comment") c.Check(s.Output(), equals, "") - msline.checkShelltext("uname=`uname`; echo $$uname") + shline.CheckShellCommandLine("uname=`uname`; echo $$uname; echo") c.Check(s.Output(), equals, ""+ "WARN: fname:1: Unknown shell command \"uname\".\n"+ - "WARN: fname:1: Please switch to \"set -e\" mode before using a semicolon to separate commands.\n"+ + "WARN: fname:1: Please switch to \"set -e\" mode before using a semicolon (the one after \"uname=`uname`\") to separate commands.\n"+ "WARN: fname:1: Unknown shell command \"echo\".\n"+ - "WARN: fname:1: Unquoted shell variable \"uname\".\n") + "WARN: fname:1: Unquoted shell variable \"uname\".\n"+ + "WARN: fname:1: Unknown shell command \"echo\".\n") - G.globalData.tools = map[string]bool{"echo": true} - G.globalData.predefinedTools = map[string]bool{"echo": true} - G.mkContext = newMkContext() + G.globalData.Tools = map[string]bool{"echo": true} + G.globalData.PredefinedTools = map[string]bool{"echo": true} + G.Mk = s.NewMkLines("fname", + "# dummy") G.globalData.InitVartypes() - msline.checkShelltext("echo ${PKGNAME:Q}") // vucQuotPlain + shline.CheckShellCommandLine("echo ${PKGNAME:Q}") // vucQuotPlain c.Check(s.Output(), equals, ""+ "WARN: fname:1: PKGNAME may not be used in this file.\n"+ "NOTE: fname:1: The :Q operator isn't necessary for ${PKGNAME} here.\n") - msline.checkShelltext("echo \"${CFLAGS:Q}\"") // vucQuotDquot + shline.CheckShellCommandLine("echo \"${CFLAGS:Q}\"") // vucQuotDquot c.Check(s.Output(), equals, ""+ "WARN: fname:1: Please don't use the :Q operator in double quotes.\n"+ "WARN: fname:1: CFLAGS may not be used in this file.\n"+ "WARN: fname:1: Please use ${CFLAGS:M*:Q} instead of ${CFLAGS:Q} and make sure the variable appears outside of any quoting characters.\n") - msline.checkShelltext("echo '${COMMENT:Q}'") // vucQuotSquot + shline.CheckShellCommandLine("echo '${COMMENT:Q}'") // vucQuotSquot c.Check(s.Output(), equals, "WARN: fname:1: COMMENT may not be used in this file.\n") - msline.checkShelltext("echo $$@") + shline.CheckShellCommandLine("echo $$@") c.Check(s.Output(), equals, "WARN: fname:1: The $@ shell variable should only be used in double quotes.\n") - msline.checkShelltext("echo \"$$\"") // As seen by make(1); the shell sees: echo $ + shline.CheckShellCommandLine("echo \"$$\"") // As seen by make(1); the shell sees: echo $ + + c.Check(s.Output(), equals, "WARN: fname:1: Unescaped $ or strange shell variable found.\n") + + shline.CheckShellCommandLine("echo \"\\n\"") // As seen by make(1); the shell sees: echo "\n" + + c.Check(s.Output(), equals, "") + + shline.CheckShellCommandLine("${RUN} for f in *.c; do echo $${f%.c}; done") + + c.Check(s.Output(), equals, "") + + // Based on mail/thunderbird/Makefile, rev. 1.159 + shline.CheckShellCommandLine("${RUN} subdir=\"`unzip -c \"$$e\" install.rdf | awk '/re/ { print \"hello\" }'`\"") - c.Check(s.Output(), equals, "WARN: fname:1: Unquoted $ or strange shell variable found.\n") + c.Check(s.Output(), equals, ""+ + "WARN: fname:1: Unknown shell command \"unzip\".\n"+ + "WARN: fname:1: The exitcode of the left-hand-side command of the pipe operator is ignored.\n"+ + "WARN: fname:1: Unknown shell command \"awk\".\n") + + // From mail/thunderbird/Makefile, rev. 1.159 + shline.CheckShellCommandLine("" + + "${RUN} for e in ${XPI_FILES}; do " + + " subdir=\"`${UNZIP_CMD} -c \"$$e\" install.rdf | awk '/^ <em:id>/ {sub(\".*<em:id>\",\"\");sub(\"</em:id>.*\",\"\");print;exit;}'`\" && " + + " ${MKDIR} \"${WRKDIR}/extensions/$$subdir\" && " + + " cd \"${WRKDIR}/extensions/$$subdir\" && " + + " ${UNZIP_CMD} -aqo $$e; " + + "done") - msline.checkShelltext("echo \"\\n\"") // As seen by make(1); the shell sees: echo "\n" + c.Check(s.Output(), equals, ""+ + "WARN: fname:1: XPI_FILES is used but not defined. Spelling mistake?\n"+ + "WARN: fname:1: UNZIP_CMD is used but not defined. Spelling mistake?\n"+ + "WARN: fname:1: The exitcode of the left-hand-side command of the pipe operator is ignored.\n"+ + "WARN: fname:1: Unknown shell command \"awk\".\n"+ + "WARN: fname:1: MKDIR is used but not defined. Spelling mistake?\n"+ + "WARN: fname:1: Unknown shell command \"${MKDIR}\".\n"+ + "WARN: fname:1: UNZIP_CMD is used but not defined. Spelling mistake?\n"+ + "WARN: fname:1: Unquoted shell variable \"e\".\n") + + // From x11/wxGTK28/Makefile + shline.CheckShellCommandLine("" + + "set -e; cd ${WRKSRC}/locale; " + + "for lang in *.po; do " + + " [ \"$${lang}\" = \"wxstd.po\" ] && continue; " + + " ${TOOLS_PATH.msgfmt} -c -o \"$${lang%.po}.mo\" \"$${lang}\"; " + + "done") - c.Check(s.Output(), equals, "WARN: fname:1: Please use \"\\\\n\" instead of \"\\n\".\n") + c.Check(s.Output(), equals, ""+ + "WARN: fname:1: WRKSRC may not be used in this file.\n"+ + "WARN: fname:1: Unknown shell command \"[\".\n"+ + "WARN: fname:1: Unknown shell command \"${TOOLS_PATH.msgfmt}\".\n") + + shline.CheckShellCommandLine("@cp from to") + + c.Check(s.Output(), equals, ""+ + "WARN: fname:1: The shell command \"cp\" should not be hidden.\n"+ + "WARN: fname:1: Unknown shell command \"cp\".\n") + + shline.CheckShellCommandLine("${RUN} ${INSTALL_DATA_DIR} share/pkgbase ${PREFIX}/share/pkgbase") + + c.Check(s.Output(), equals, "NOTE: fname:1: You can use AUTO_MKDIRS=yes or \"INSTALLATION_DIRS+= share/pkgbase\" instead of this command.\n") } -func (s *Suite) TestMkShellLine_CheckShelltext_InternalError1(c *check.C) { +func (s *Suite) TestShellLine_CheckShelltext_nofix(c *check.C) { s.UseCommandLine(c, "-Wall") G.globalData.InitVartypes() - G.mkContext = newMkContext() - msline := NewMkShellLine(NewLine("fname", "1", "# dummy", nil)) + s.RegisterTool("echo", "ECHO", false) + G.Mk = s.NewMkLines("Makefile", + "\techo ${PKGNAME:Q}") + shline := NewShellLine(G.Mk.mklines[0]) + + c.Check(shline.line.raw[0].textnl, equals, "\techo ${PKGNAME:Q}\n") + c.Check(shline.line.raw[0].Lineno, equals, 1) + + shline.CheckShellCommandLine("echo ${PKGNAME:Q}") + + c.Check(s.Output(), equals, ""+ + "NOTE: Makefile:1: The :Q operator isn't necessary for ${PKGNAME} here.\n") +} + +func (s *Suite) TestShellLine_CheckShelltext_showAutofix(c *check.C) { + s.UseCommandLine(c, "-Wall", "--show-autofix") + G.globalData.InitVartypes() + s.RegisterTool("echo", "ECHO", false) + G.Mk = s.NewMkLines("Makefile", + "\techo ${PKGNAME:Q}") + shline := NewShellLine(G.Mk.mklines[0]) + + shline.CheckShellCommandLine("echo ${PKGNAME:Q}") + + c.Check(s.Output(), equals, ""+ + "NOTE: Makefile:1: The :Q operator isn't necessary for ${PKGNAME} here.\n"+ + "AUTOFIX: Makefile:1: Replacing \"${PKGNAME:Q}\" with \"${PKGNAME}\".\n") +} + +func (s *Suite) TestShellLine_CheckShelltext_autofix(c *check.C) { + s.UseCommandLine(c, "-Wall", "--autofix") + G.globalData.InitVartypes() + s.RegisterTool("echo", "ECHO", false) + G.Mk = s.NewMkLines("Makefile", + "\techo ${PKGNAME:Q}") + shline := NewShellLine(G.Mk.mklines[0]) + + shline.CheckShellCommandLine("echo ${PKGNAME:Q}") + + c.Check(s.Output(), equals, ""+ + "AUTOFIX: Makefile:1: Replacing \"${PKGNAME:Q}\" with \"${PKGNAME}\".\n") +} + +func (s *Suite) TestShellLine_CheckShelltext_InternalError1(c *check.C) { + s.UseCommandLine(c, "-Wall") + G.globalData.InitVartypes() + G.Mk = s.NewMkLines("fname", + "# dummy") + shline := NewShellLine(G.Mk.mklines[0]) // foobar="`echo \"foo bar\"`" - msline.checkShelltext("foobar=\"`echo \\\"foo bar\\\"`\"") + shline.CheckShellCommandLine("foobar=\"`echo \\\"foo bar\\\"`\"") c.Check(s.Output(), equals, ""+ "WARN: fname:1: Backslashes should be doubled inside backticks.\n"+ @@ -85,83 +206,111 @@ func (s *Suite) TestMkShellLine_CheckShelltext_InternalError1(c *check.C) { "WARN: fname:1: Backslashes should be doubled inside backticks.\n"+ "WARN: fname:1: Double quotes inside backticks inside double quotes are error prone.\n"+ "WARN: fname:1: Unknown shell command \"echo\".\n"+ - "ERROR: fname:1: Internal pkglint error: checklineMkShellword state=plain, rest=\"\\\\foo\", shellword=\"\\\\foo\"\n"+ - "ERROR: fname:1: Internal pkglint error: checklineMkShelltext state=continuation rest=\"\\\\\" shellword=\"echo \\\\foo bar\\\\\"\n") + "ERROR: fname:1: Internal pkglint error: ShellLine.CheckToken state=plain, rest=\"\\\\foo\", token=\"\\\\foo\"\n"+ + "ERROR: fname:1: Internal pkglint error: ShellLine.CheckShellCommand state=continuation rest=\"\\\\\" shellcmd=\"echo \\\\foo bar\\\\\"\n") } -func (s *Suite) TestMkShellLine_CheckShelltext_InternalError2(c *check.C) { +func (s *Suite) TestShellLine_CheckShelltext_DollarWithoutVariable(c *check.C) { G.globalData.InitVartypes() - msline := NewMkShellLine(NewLine("fname", "1", "# dummy", nil)) - G.mkContext = newMkContext() + G.Mk = s.NewMkLines("fname", + "# dummy") + shline := NewShellLine(G.Mk.mklines[0]) s.RegisterTool("pax", "PAX", false) - G.mkContext.tools["pax"] = true + G.Mk.tools["pax"] = true - msline.checkShelltext("pax -rwpp -s /.*~$$//g . ${DESTDIR}${PREFIX}") + shline.CheckShellCommandLine("pax -rwpp -s /.*~$$//g . ${DESTDIR}${PREFIX}") - c.Check(s.Output(), equals, "ERROR: fname:1: Internal pkglint error: checklineMkShellword state=plain, rest=\"$$//g\", shellword=\"/.*~$$//g\"\n") + c.Check(s.Output(), equals, "") } func (s *Suite) TestChecklineMkShellword(c *check.C) { s.UseCommandLine(c, "-Wall") G.globalData.InitVartypes() - msline := NewMkShellLine(NewLine("fname", "1", "# dummy", nil)) + shline := NewShellLine(NewMkLine(NewLine("fname", 1, "# dummy", nil))) c.Check(matches("${list}", `^`+reVarnameDirect+`$`), equals, false) - msline.checkShellword("${${list}}", false) + shline.CheckToken("${${list}}", false) c.Check(s.Output(), equals, "") - msline.checkShellword("\"$@\"", false) + shline.CheckToken("\"$@\"", false) c.Check(s.Output(), equals, "WARN: fname:1: Please use \"${.TARGET}\" instead of \"$@\".\n") + + shline.CheckToken("${COMMENT:Q}", true) + + c.Check(s.Output(), equals, "WARN: fname:1: COMMENT may not be used in this file.\n") + + shline.CheckToken("\"${DISTINFO_FILE:Q}\"", true) + + c.Check(s.Output(), equals, ""+ + "WARN: fname:1: DISTINFO_FILE may not be used in this file.\n"+ + "NOTE: fname:1: The :Q operator isn't necessary for ${DISTINFO_FILE} here.\n") + + shline.CheckToken("embed${DISTINFO_FILE:Q}ded", true) + + c.Check(s.Output(), equals, ""+ + "WARN: fname:1: DISTINFO_FILE may not be used in this file.\n"+ + "NOTE: fname:1: The :Q operator isn't necessary for ${DISTINFO_FILE} here.\n") + + shline.CheckToken("s,\\.,,", true) + + c.Check(s.Output(), equals, "") + + shline.CheckToken("\"s,\\.,,\"", true) + + c.Check(s.Output(), equals, "") } -func (s *Suite) TestMkShellLine_CheckShellword_InternalError(c *check.C) { - msline := NewMkShellLine(NewLine("fname", "1", "# dummy", nil)) +func (s *Suite) TestShellLine_CheckToken_DollarWithoutVariable(c *check.C) { + shline := NewShellLine(NewMkLine(NewLine("fname", 1, "# dummy", nil))) - msline.checkShellword("/.*~$$//g", false) + shline.CheckToken("/.*~$$//g", false) // Typical argument to pax(1). - c.Check(s.Output(), equals, "ERROR: fname:1: Internal pkglint error: checklineMkShellword state=plain, rest=\"$$//g\", shellword=\"/.*~$$//g\"\n") + c.Check(s.Output(), equals, "") } func (s *Suite) TestShelltextContext_CheckCommandStart(c *check.C) { s.UseCommandLine(c, "-Wall") s.RegisterTool("echo", "ECHO", true) - G.mkContext = newMkContext() - line := NewLine("fname", "3", "# dummy", nil) + G.Mk = s.NewMkLines("fname", + "# dummy") + mkline := NewMkLine(NewLine("fname", 3, "# dummy", nil)) + + mkline.CheckText("echo \"hello, world\"") - shellcmd := "echo \"hello, world\"" - NewMkLine(line).checkText(shellcmd) - NewMkShellLine(line).checkShelltext(shellcmd) + c.Check(s.Output(), equals, "") + + NewShellLine(mkline).CheckShellCommandLine("echo \"hello, world\"") c.Check(s.Output(), equals, ""+ - "WARN: fname:3: The \"echo\" tool is used but not added to USE_TOOLS.\n"+ "WARN: fname:3: Please use \"${ECHO}\" instead of \"echo\".\n") } -func (s *Suite) TestMkShellLine_checklineMkShelltext(c *check.C) { +func (s *Suite) TestShellLine_checklineMkShelltext(c *check.C) { - msline := NewMkShellLine(NewLine("Makefile", "3", "# dummy", nil)) + shline := NewShellLine(NewMkLine(NewLine("Makefile", 3, "# dummy", nil))) - msline.checkShelltext("for f in *.pl; do ${SED} s,@PREFIX@,${PREFIX}, < $f > $f.tmp && ${MV} $f.tmp $f; done") + shline.CheckShellCommandLine("for f in *.pl; do ${SED} s,@PREFIX@,${PREFIX}, < $f > $f.tmp && ${MV} $f.tmp $f; done") c.Check(s.Output(), equals, "NOTE: Makefile:3: Please use the SUBST framework instead of ${SED} and ${MV}.\n") - msline.checkShelltext("install -c manpage.1 ${PREFIX}/man/man1/manpage.1") + shline.CheckShellCommandLine("install -c manpage.1 ${PREFIX}/man/man1/manpage.1") c.Check(s.Output(), equals, "WARN: Makefile:3: Please use ${PKGMANDIR} instead of \"man\".\n") - msline.checkShelltext("cp init-script ${PREFIX}/etc/rc.d/service") + shline.CheckShellCommandLine("cp init-script ${PREFIX}/etc/rc.d/service") c.Check(s.Output(), equals, "WARN: Makefile:3: Please use the RCD_SCRIPTS mechanism to install rc.d scripts automatically to ${RCD_SCRIPTS_EXAMPLEDIR}.\n") } -func (s *Suite) TestMkShellLine_checkCommandUse(c *check.C) { - G.mkContext = newMkContext() - G.mkContext.target = "do-install" +func (s *Suite) TestShellLine_checkCommandUse(c *check.C) { + G.Mk = s.NewMkLines("fname", + "# dummy") + G.Mk.target = "do-install" - shline := NewMkShellLine(NewLine("fname", "1", "dummy", nil)) + shline := NewShellLine(NewMkLine(NewLine("fname", 1, "\tdummy", nil))) shline.checkCommandUse("sed") @@ -171,3 +320,72 @@ func (s *Suite) TestMkShellLine_checkCommandUse(c *check.C) { c.Check(s.Output(), equals, "WARN: fname:1: ${CP} should not be used to install files.\n") } + +func (s *Suite) TestSplitIntoShellWords(c *check.C) { + url := "http://registry.gimp.org/file/fix-ca.c?action=download&id=9884&file=" + + words, rest := splitIntoShellTokens(dummyLine, url) // Doesn’t really make sense + + c.Check(words, check.DeepEquals, []string{"http://registry.gimp.org/file/fix-ca.c?action=download", "&", "id=9884", "&", "file="}) + c.Check(rest, equals, "") + + words, rest = splitIntoShellWords(dummyLine, url) + + c.Check(words, check.DeepEquals, []string{"http://registry.gimp.org/file/fix-ca.c?action=download&id=9884&file="}) + c.Check(rest, equals, "") + + words, rest = splitIntoShellWords(dummyLine, "a b \"c c c\" d;;d;; \"e\"''`` 'rest") + + c.Check(words, check.DeepEquals, []string{"a", "b", "\"c c c\"", "d;;d;;", "\"e\"''``"}) + c.Check(rest, equals, "'rest") +} + +func (s *Suite) TestShellLine_CheckShellCommandLine_SedMv(c *check.C) { + shline := NewShellLine(NewMkLine(NewLine("Makefile", 85, "\t${RUN} ${SED} 's,#,// comment:,g' fname > fname.tmp; ${MV} fname.tmp fname", nil))) + + shline.CheckShellCommandLine(shline.mkline.Shellcmd()) + + c.Check(s.Output(), equals, "NOTE: Makefile:85: Please use the SUBST framework instead of ${SED} and ${MV}.\n") +} + +func (s *Suite) TestShellLine_CheckShellCommandLine_Subshell(c *check.C) { + shline := NewShellLine(NewMkLine(NewLine("Makefile", 85, "\t${RUN} uname=$$(uname)", nil))) + + shline.CheckShellCommandLine(shline.mkline.Shellcmd()) + + c.Check(s.Output(), equals, "WARN: Makefile:85: Invoking subshells via $(...) is not portable enough.\n") +} + +func (s *Suite) TestShellLine_CheckShellCommandLine_InstallDirs(c *check.C) { + shline := NewShellLine(NewMkLine(NewLine("Makefile", 85, "\t${RUN} ${INSTALL_DATA_DIR} ${DESTDIR}${PREFIX}/dir1 ${DESTDIR}${PREFIX}/dir2", nil))) + + shline.CheckShellCommandLine(shline.mkline.Shellcmd()) + + c.Check(s.Output(), equals, ""+ + "NOTE: Makefile:85: You can use AUTO_MKDIRS=yes or \"INSTALLATION_DIRS+= dir1\" instead of this command.\n"+ + "NOTE: Makefile:85: You can use AUTO_MKDIRS=yes or \"INSTALLATION_DIRS+= dir2\" instead of this command.\n"+ + "WARN: Makefile:85: The INSTALL_*_DIR commands can only handle one directory at a time.\n") +} + +func (s *Suite) TestShellLine_CheckShellCommandLine_InstallD(c *check.C) { + shline := NewShellLine(NewMkLine(NewLine("Makefile", 85, "\t${RUN} ${INSTALL} -d ${DESTDIR}${PREFIX}/dir1 ${DESTDIR}${PREFIX}/dir2", nil))) + + shline.CheckShellCommandLine(shline.mkline.Shellcmd()) + + c.Check(s.Output(), equals, ""+ + "WARN: Makefile:85: Please use AUTO_MKDIRS instead of \"${INSTALL} -d\".\n"+ + "WARN: Makefile:85: Please use AUTO_MKDIRS instead of \"${INSTALL} -d\".\n") +} + +func (s *Suite) TestShellLine_(c *check.C) { + tmpfile := s.CreateTmpFile(c, "Makefile", ""+ + "# $"+"NetBSD$\n"+ + "pre-install:\n"+ + "\t"+"# comment\\\n"+ + "\t"+"echo \"hello\"\n") + lines := LoadNonemptyLines(tmpfile, true) + + NewMkLines(lines).Check() + + c.Check(s.OutputCleanTmpdir(), equals, "WARN: ~/Makefile:3--4: A shell comment does not stop at the end of line.\n") +} diff --git a/pkgtools/pkglint/files/substcontext.go b/pkgtools/pkglint/files/substcontext.go index f29e5326db3..f70b930ee1a 100644 --- a/pkgtools/pkglint/files/substcontext.go +++ b/pkgtools/pkglint/files/substcontext.go @@ -17,17 +17,16 @@ func (ctx *SubstContext) Varassign(mkline *MkLine) { return } - line:=mkline.line - varname := line.extra["varname"].(string) - op := line.extra["op"].(string) - value := line.extra["value"].(string) + varname := mkline.Varname() + op := mkline.Op() + value := mkline.Value() if varname == "SUBST_CLASSES" { classes := splitOnSpace(value) if len(classes) > 1 { - line.warnf("Please add only one class at a time to SUBST_CLASSES.") + mkline.Warn0("Please add only one class at a time to SUBST_CLASSES.") } if ctx.id != "" { - line.warnf("SUBST_CLASSES should only appear once in a SUBST block.") + mkline.Warn0("SUBST_CLASSES should only appear once in a SUBST block.") } ctx.id = classes[0] return @@ -36,13 +35,13 @@ func (ctx *SubstContext) Varassign(mkline *MkLine) { m, varbase, varparam := match2(varname, `^(SUBST_(?:STAGE|MESSAGE|FILES|SED|VARS|FILTER_CMD))\.([\-\w_]+)$`) if !m { if ctx.id != "" { - line.warnf("Foreign variable %q in SUBST block.", varname) + mkline.Warn1("Foreign variable %q in SUBST block.", varname) } return } if ctx.id == "" { - line.warnf("SUBST_CLASSES should come before the definition of %q.", varname) + mkline.Warn1("SUBST_CLASSES should come before the definition of %q.", varname) ctx.id = varparam } @@ -56,26 +55,26 @@ func (ctx *SubstContext) Varassign(mkline *MkLine) { // but from a technically viewpoint, it is incorrect. ctx.id = varparam } else { - line.warnf("Variable %q does not match SUBST class %q.", varname, ctx.id) + mkline.Warn2("Variable %q does not match SUBST class %q.", varname, ctx.id) } return } switch varbase { case "SUBST_STAGE": - ctx.dup(line, &ctx.stage, varname, value) + ctx.dup(mkline, &ctx.stage, varname, value) case "SUBST_MESSAGE": - ctx.dup(line, &ctx.message, varname, value) + ctx.dup(mkline, &ctx.message, varname, value) case "SUBST_FILES": - ctx.duplist(line, &ctx.files, varname, op, value) + ctx.duplist(mkline, &ctx.files, varname, op, value) case "SUBST_SED": - ctx.duplist(line, &ctx.sed, varname, op, value) + ctx.duplist(mkline, &ctx.sed, varname, op, value) case "SUBST_FILTER_CMD": - ctx.dup(line, &ctx.filterCmd, varname, value) + ctx.dup(mkline, &ctx.filterCmd, varname, value) case "SUBST_VARS": - ctx.duplist(line, &ctx.vars, varname, op, value) + ctx.duplist(mkline, &ctx.vars, varname, op, value) default: - line.warnf("Foreign variable %q in SUBST block.", varname) + mkline.Warn1("Foreign variable %q in SUBST block.", varname) } } @@ -87,18 +86,17 @@ func (ctx *SubstContext) IsComplete() bool { } func (ctx *SubstContext) Finish(mkline *MkLine) { - line:=mkline.line if ctx.id == "" || !G.opts.WarnExtra { return } if ctx.stage == "" { - line.warnf("Incomplete SUBST block: %s missing.", ctx.varname("SUBST_STAGE")) + mkline.Warn1("Incomplete SUBST block: %s missing.", ctx.varname("SUBST_STAGE")) } if len(ctx.files) == 0 { - line.warnf("Incomplete SUBST block: %s missing.", ctx.varname("SUBST_FILES")) + mkline.Warn1("Incomplete SUBST block: %s missing.", ctx.varname("SUBST_FILES")) } if len(ctx.sed) == 0 && len(ctx.vars) == 0 && ctx.filterCmd == "" { - line.warnf("Incomplete SUBST block: %s, %s or %s missing.", + mkline.Line.Warnf("Incomplete SUBST block: %s, %s or %s missing.", ctx.varname("SUBST_SED"), ctx.varname("SUBST_VARS"), ctx.varname("SUBST_FILTER_CMD")) } ctx.id = "" @@ -111,6 +109,8 @@ func (ctx *SubstContext) Finish(mkline *MkLine) { } func (ctx *SubstContext) varname(varbase string) string { + switch { // prevent inlining + } if ctx.id != "" { return varbase + "." + ctx.id } else { @@ -118,16 +118,16 @@ func (ctx *SubstContext) varname(varbase string) string { } } -func (ctx *SubstContext) dup(line *Line, pstr *string, varname, value string) { +func (ctx *SubstContext) dup(mkline *MkLine, pstr *string, varname, value string) { if *pstr != "" { - line.warnf("Duplicate definition of %q.", varname) + mkline.Warn1("Duplicate definition of %q.", varname) } *pstr = value } -func (ctx *SubstContext) duplist(line *Line, plist *[]string, varname, op, value string) { - if len(*plist) > 0 && op != "+=" { - line.warnf("All but the first %q lines should use the \"+=\" operator.", varname) +func (ctx *SubstContext) duplist(mkline *MkLine, plist *[]string, varname string, op MkOperator, value string) { + if len(*plist) > 0 && op != opAssignAppend { + mkline.Warn1("All but the first %q lines should use the \"+=\" operator.", varname) } *plist = append(*plist, value) } diff --git a/pkgtools/pkglint/files/substcontext_test.go b/pkgtools/pkglint/files/substcontext_test.go index 03da1ecfc4e..380ccd855f7 100644 --- a/pkgtools/pkglint/files/substcontext_test.go +++ b/pkgtools/pkglint/files/substcontext_test.go @@ -8,23 +8,23 @@ func (s *Suite) TestSubstContext_Incomplete(c *check.C) { G.opts.WarnExtra = true ctx := new(SubstContext) - ctx.Varassign(newSubstLine("10", "PKGNAME=pkgname-1.0")) + ctx.Varassign(newSubstLine(10, "PKGNAME=pkgname-1.0")) c.Check(ctx.id, equals, "") - ctx.Varassign(newSubstLine("11", "SUBST_CLASSES+=interp")) + ctx.Varassign(newSubstLine(11, "SUBST_CLASSES+=interp")) c.Check(ctx.id, equals, "interp") - ctx.Varassign(newSubstLine("12", "SUBST_FILES.interp=Makefile")) + ctx.Varassign(newSubstLine(12, "SUBST_FILES.interp=Makefile")) c.Check(ctx.IsComplete(), equals, false) - ctx.Varassign(newSubstLine("13", "SUBST_SED.interp=s,@PREFIX@,${PREFIX},g")) + ctx.Varassign(newSubstLine(13, "SUBST_SED.interp=s,@PREFIX@,${PREFIX},g")) c.Check(ctx.IsComplete(), equals, false) - ctx.Finish(newSubstLine("14", "")) + ctx.Finish(newSubstLine(14, "")) c.Check(s.Output(), equals, "WARN: Makefile:14: Incomplete SUBST block: SUBST_STAGE.interp missing.\n") } @@ -33,18 +33,18 @@ func (s *Suite) TestSubstContext_Complete(c *check.C) { G.opts.WarnExtra = true ctx := new(SubstContext) - ctx.Varassign(newSubstLine("10", "PKGNAME=pkgname-1.0")) - ctx.Varassign(newSubstLine("11", "SUBST_CLASSES+=p")) - ctx.Varassign(newSubstLine("12", "SUBST_FILES.p=Makefile")) - ctx.Varassign(newSubstLine("13", "SUBST_SED.p=s,@PREFIX@,${PREFIX},g")) + ctx.Varassign(newSubstLine(10, "PKGNAME=pkgname-1.0")) + ctx.Varassign(newSubstLine(11, "SUBST_CLASSES+=p")) + ctx.Varassign(newSubstLine(12, "SUBST_FILES.p=Makefile")) + ctx.Varassign(newSubstLine(13, "SUBST_SED.p=s,@PREFIX@,${PREFIX},g")) c.Check(ctx.IsComplete(), equals, false) - ctx.Varassign(newSubstLine("14", "SUBST_STAGE.p=post-configure")) + ctx.Varassign(newSubstLine(14, "SUBST_STAGE.p=post-configure")) c.Check(ctx.IsComplete(), equals, true) - ctx.Finish(newSubstLine("15", "")) + ctx.Finish(newSubstLine(15, "")) c.Check(s.Output(), equals, "") } @@ -53,16 +53,16 @@ func (s *Suite) TestSubstContext_NoClass(c *check.C) { s.UseCommandLine(c, "-Wextra") ctx := new(SubstContext) - ctx.Varassign(newSubstLine("10", "UNRELATED=anything")) - ctx.Varassign(newSubstLine("11", "SUBST_FILES.repl+=Makefile.in")) - ctx.Varassign(newSubstLine("12", "SUBST_SED.repl+=-e s,from,to,g")) - ctx.Finish(newSubstLine("13","")) + ctx.Varassign(newSubstLine(10, "UNRELATED=anything")) + ctx.Varassign(newSubstLine(11, "SUBST_FILES.repl+=Makefile.in")) + ctx.Varassign(newSubstLine(12, "SUBST_SED.repl+=-e s,from,to,g")) + ctx.Finish(newSubstLine(13, "")) c.Check(s.Output(), equals, ""+ "WARN: Makefile:11: SUBST_CLASSES should come before the definition of \"SUBST_FILES.repl\".\n"+ "WARN: Makefile:13: Incomplete SUBST block: SUBST_STAGE.repl missing.\n") } -func newSubstLine(lineno, text string) *MkLine { +func newSubstLine(lineno int, text string) *MkLine { return NewMkLine(NewLine("Makefile", lineno, text, nil)) } diff --git a/pkgtools/pkglint/files/toplevel.go b/pkgtools/pkglint/files/toplevel.go index de34c1c76a3..3c8a0765985 100644 --- a/pkgtools/pkglint/files/toplevel.go +++ b/pkgtools/pkglint/files/toplevel.go @@ -5,46 +5,46 @@ type Toplevel struct { subdirs []string } -func checkdirToplevel() { - defer tracecall("checkdirToplevel", G.currentDir)() +func CheckdirToplevel() { + if G.opts.DebugTrace { + defer tracecall1(G.CurrentDir)() + } ctx := new(Toplevel) - - fname := G.currentDir + "/Makefile" + fname := G.CurrentDir + "/Makefile" lines := LoadNonemptyLines(fname, true) if lines == nil { return } - ParselinesMk(lines) - for _, line := range lines { - if m, commentedOut, indentation, subdir, comment := match4(line.text, `^(#?)SUBDIR\s*\+=(\s*)(\S+)\s*(?:#\s*(.*?)\s*|)$`); m { + if m, commentedOut, indentation, subdir, comment := match4(line.Text, `^(#?)SUBDIR\s*\+=(\s*)(\S+)\s*(?:#\s*(.*?)\s*|)$`); m { ctx.checkSubdir(line, commentedOut == "#", indentation, subdir, comment) } } - ChecklinesMk(lines) + NewMkLines(lines).Check() if G.opts.Recursive { if G.opts.CheckGlobal { - G.ipcUsedLicenses = make(map[string]bool) + G.UsedLicenses = make(map[string]bool) + G.Hash = make(map[string]*Hash) } - G.todo = append(G.todo, ctx.subdirs...) + G.Todo = append(G.Todo, ctx.subdirs...) } } func (ctx *Toplevel) checkSubdir(line *Line, commentedOut bool, indentation, subdir, comment string) { if commentedOut && comment == "" { - line.warnf("%q commented out without giving a reason.", subdir) + line.Warn1("%q commented out without giving a reason.", subdir) } if indentation != "\t" { - line.warnf("Indentation should be a single tab character.") + line.Warn0("Indentation should be a single tab character.") } - if contains(subdir, "$") || !fileExists(G.currentDir+"/"+subdir+"/Makefile") { + if contains(subdir, "$") || !fileExists(G.CurrentDir+"/"+subdir+"/Makefile") { return } @@ -53,15 +53,15 @@ func (ctx *Toplevel) checkSubdir(line *Line, commentedOut bool, indentation, sub case subdir > prev: // Correctly ordered case subdir == prev: - line.errorf("Each subdir must only appear once.") + line.Error0("Each subdir must only appear once.") case subdir == "archivers" && prev == "x11": // This exception is documented in the top-level Makefile. default: - line.warnf("%s should come before %s", subdir, prev) + line.Warn2("%s should come before %s", subdir, prev) } ctx.previousSubdir = subdir if !commentedOut { - ctx.subdirs = append(ctx.subdirs, G.currentDir+"/"+subdir) + ctx.subdirs = append(ctx.subdirs, G.CurrentDir+"/"+subdir) } } diff --git a/pkgtools/pkglint/files/toplevel_test.go b/pkgtools/pkglint/files/toplevel_test.go index f232369fcd2..4c98f9c0753 100644 --- a/pkgtools/pkglint/files/toplevel_test.go +++ b/pkgtools/pkglint/files/toplevel_test.go @@ -20,13 +20,13 @@ func (s *Suite) TestCheckdirToplevel(c *check.C) { s.CreateTmpFile(c, "ccc/Makefile", "") s.CreateTmpFile(c, "x11/Makefile", "") G.globalData.InitVartypes() - G.currentDir = s.tmpdir - checkdirToplevel() + G.CurrentDir = s.tmpdir + CheckdirToplevel() - c.Check(s.Output(), equals, ""+ - "WARN: "+s.tmpdir+"/Makefile:3: Indentation should be a single tab character.\n"+ - "ERROR: "+s.tmpdir+"/Makefile:6: Each subdir must only appear once.\n"+ - "WARN: "+s.tmpdir+"/Makefile:7: \"ignoreme\" commented out without giving a reason.\n"+ - "WARN: "+s.tmpdir+"/Makefile:9: bbb should come before ccc\n") + c.Check(s.OutputCleanTmpdir(), equals, ""+ + "WARN: ~/Makefile:3: Indentation should be a single tab character.\n"+ + "ERROR: ~/Makefile:6: Each subdir must only appear once.\n"+ + "WARN: ~/Makefile:7: \"ignoreme\" commented out without giving a reason.\n"+ + "WARN: ~/Makefile:9: bbb should come before ccc\n") } diff --git a/pkgtools/pkglint/files/tree.go b/pkgtools/pkglint/files/tree.go index a074662fbb8..b1377f88523 100644 --- a/pkgtools/pkglint/files/tree.go +++ b/pkgtools/pkglint/files/tree.go @@ -1,5 +1,9 @@ package main +import ( + "fmt" +) + type Tree struct { name string args []interface{} @@ -14,7 +18,9 @@ func NewTree(name string, args ...interface{}) *Tree { // If the match is partially successful, some or all of the variables // may have been copied or not. func (t *Tree) Match(pattern *Tree) bool { - defer tracecall("Tree.Match", t, pattern)() + if G.opts.DebugTrace { + defer tracecall(t, pattern)() + } if t.name != pattern.name || len(t.args) != len(pattern.args) { return false } @@ -55,10 +61,10 @@ func (t *Tree) String() string { continue } if arg, ok := arg.(string); ok { - s += sprintf(" %q", arg) + s += fmt.Sprintf(" %q", arg) continue } else { - s += sprintf(" %v", arg) + s += fmt.Sprintf(" %v", arg) } } return s + ")" diff --git a/pkgtools/pkglint/files/util.go b/pkgtools/pkglint/files/util.go index 637a79c2dc1..e323091c426 100644 --- a/pkgtools/pkglint/files/util.go +++ b/pkgtools/pkglint/files/util.go @@ -7,19 +7,19 @@ import ( "os" "path" "path/filepath" + "reflect" "regexp" + "runtime" "sort" "strconv" "strings" + "time" ) // Short names for commonly used functions. -var ( - sprintf = fmt.Sprintf - contains = strings.Contains - hasPrefix = strings.HasPrefix - hasSuffix = strings.HasSuffix -) +func contains(s, substr string) bool { return strings.Contains(s, substr) } +func hasPrefix(s, prefix string) bool { return strings.HasPrefix(s, prefix) } +func hasSuffix(s, suffix string) bool { return strings.HasSuffix(s, suffix) } func ifelseStr(cond bool, a, b string) string { if cond { @@ -28,11 +28,11 @@ func ifelseStr(cond bool, a, b string) string { return b } -func mustMatch(pattern string, s string) []string { - if m := regcomp(pattern).FindStringSubmatch(s); m != nil { +func mustMatch(s, re string) []string { + if m := match(s, re); m != nil { return m } - panic(sprintf("mustMatch %q %q", pattern, s)) + panic(fmt.Sprintf("mustMatch %q %q", s, re)) } func isEmptyDir(fname string) bool { @@ -56,7 +56,7 @@ func isEmptyDir(fname string) bool { func getSubdirs(fname string) []string { dirents, err := ioutil.ReadDir(fname) if err != nil { - fatalf(fname, noLines, "Cannot be read: %s", err) + Fatalf(fname, noLines, "Cannot be read: %s", err) } var subdirs []string @@ -77,7 +77,7 @@ func isCommitted(fname string) bool { return false } for _, line := range lines { - if hasPrefix(line.text, "/"+basename+"/") { + if hasPrefix(line.Text, "/"+basename+"/") { return true } } @@ -99,56 +99,52 @@ func tabLength(s string) int { } func varnameBase(varname string) string { - return strings.Split(varname, ".")[0] + dot := strings.IndexByte(varname, '.') + if dot != -1 { + return varname[:dot] + } + return varname } func varnameCanon(varname string) string { - parts := strings.SplitN(varname, ".", 2) - if len(parts) == 2 { - return parts[0] + ".*" + dot := strings.IndexByte(varname, '.') + if dot != -1 { + return varname[:dot] + ".*" } - return parts[0] + return varname } func varnameParam(varname string) string { - parts := strings.SplitN(varname, ".", 2) - return parts[len(parts)-1] + dot := strings.IndexByte(varname, '.') + if dot != -1 { + return varname[dot+1:] + } + return "" } -func defineVar(line *Line, varname string) { - if mk := G.mkContext; mk != nil { - mk.defineVar(line, varname) +func defineVar(mkline *MkLine, varname string) { + if G.Mk != nil { + G.Mk.DefineVar(mkline, varname) } - if pkg := G.pkgContext; pkg != nil { - pkg.defineVar(line, varname) + if G.Pkg != nil { + G.Pkg.defineVar(mkline, varname) } } func varIsDefined(varname string) bool { varcanon := varnameCanon(varname) - if mk := G.mkContext; mk != nil && (mk.vardef[varname] != nil || mk.vardef[varcanon] != nil) { + if G.Mk != nil && (G.Mk.vardef[varname] != nil || G.Mk.vardef[varcanon] != nil) { return true } - if pkg := G.pkgContext; pkg != nil && (pkg.vardef[varname] != nil || pkg.vardef[varcanon] != nil) { + if G.Pkg != nil && (G.Pkg.vardef[varname] != nil || G.Pkg.vardef[varcanon] != nil) { return true } return false } -func useVar(line *Line, varname string) { - varcanon := varnameCanon(varname) - if mk := G.mkContext; mk != nil { - mk.varuse[varname] = line - mk.varuse[varcanon] = line - } - if pkg := G.pkgContext; pkg != nil { - pkg.varuse[varname] = line - pkg.varuse[varcanon] = line - } -} func varIsUsed(varname string) bool { varcanon := varnameCanon(varname) - if mk := G.mkContext; mk != nil && (mk.varuse[varname] != nil || mk.varuse[varcanon] != nil) { + if G.Mk != nil && (G.Mk.varuse[varname] != nil || G.Mk.varuse[varcanon] != nil) { return true } - if pkg := G.pkgContext; pkg != nil && (pkg.varuse[varname] != nil || pkg.varuse[varcanon] != nil) { + if G.Pkg != nil && (G.Pkg.varuse[varname] != nil || G.Pkg.varuse[varcanon] != nil) { return true } return false @@ -180,13 +176,23 @@ func regcomp(re string) *regexp.Regexp { } func match(s, re string) []string { + if !G.opts.Profiling { + return regcomp(re).FindStringSubmatch(s) + } + + before := time.Now() + immediatelyBefore := time.Now() m := regcomp(re).FindStringSubmatch(s) - if G.opts.Profiling { - if m != nil { - G.rematch.add(re) - } else { - G.renomatch.add(re) - } + after := time.Now() + + delay := immediatelyBefore.UnixNano() - before.UnixNano() + timeTaken := after.UnixNano() - immediatelyBefore.UnixNano() - delay + + G.retime.Add(re, int(timeTaken)) + if m != nil { + G.rematch.Add(re, 1) + } else { + G.renomatch.Add(re, 1) } return m } @@ -195,9 +201,9 @@ func matches(s, re string) bool { matches := regcomp(re).MatchString(s) if G.opts.Profiling { if matches { - G.rematch.add(re) + G.rematch.Add(re, 1) } else { - G.renomatch.add(re) + G.renomatch.Add(re, 1) } } return matches @@ -206,7 +212,7 @@ func matches(s, re string) bool { func matchn(s, re string, n int) []string { if m := match(s, re); m != nil { if len(m) != 1+n { - panic(sprintf("expected match%d, got match%d for %q", len(m)-1, n, re)) + panic(fmt.Sprintf("expected match%d, got match%d for %q", len(m)-1, n, re)) } return m } @@ -245,7 +251,9 @@ func match5(s, re string) (matched bool, m1, m2, m3, m4, m5 string) { } func replaceFirst(s, re, replacement string) ([]string, string) { - defer tracecall("replaceFirst", s, re, replacement)() + if G.opts.DebugTrace { + defer tracecall(s, re, replacement)() + } if m := regcomp(re).FindStringSubmatchIndex(s); m != nil { replaced := s[:m[0]] + replacement + s[m[1]:] @@ -260,15 +268,71 @@ func replaceFirst(s, re, replacement string) ([]string, string) { type PrefixReplacer struct { rest string - m []string + s string // The last match from a prefix + m []string // The last match from a regular expression } func NewPrefixReplacer(s string) *PrefixReplacer { - return &PrefixReplacer{s, nil} + return &PrefixReplacer{s, "", nil} +} + +func (pr *PrefixReplacer) AdvanceBytes(bits0x00, bits0x20, bits0x40, bits0x60 uint32, re string) bool { + if G.TestingData != nil { + pr.verifyBits(bits0x00, bits0x20, bits0x40, bits0x60, re) + } + + rest := pr.rest + n := len(pr.rest) + i := 0 +loop: + for ; i < n; i++ { + b := rest[i] + var mask uint32 + switch b & 0xE0 { + case 0x00: + mask = bits0x00 + case 0x20: + mask = bits0x20 + case 0x40: + mask = bits0x40 + case 0x60: + mask = bits0x60 + } + if (mask>>(b&0x1F))&1 == 0 { + break loop + } + } + pr.s = rest[:i] + pr.rest = rest[i:] + return i != 0 } -func (pr *PrefixReplacer) startsWith(re string) bool { - if m := regcomp(re).FindStringSubmatch(pr.rest); m != nil { +func (pr *PrefixReplacer) AdvanceStr(prefix string) bool { + pr.m = nil + pr.s = "" + if hasPrefix(pr.rest, prefix) { + if G.opts.DebugTrace { + trace("", "PrefixReplacer.AdvanceStr", pr.rest, prefix) + } + pr.s = prefix + pr.rest = pr.rest[len(prefix):] + return true + } + return false +} +func (pr *PrefixReplacer) AdvanceRegexp(re string) bool { + pr.m = nil + pr.s = "" + if !hasPrefix(re, "^") { + panic(fmt.Sprintf("PrefixReplacer.AdvanceRegexp: regular expression %q must have prefix %q.", re, "^")) + } + if matches("", re) { + panic(fmt.Sprintf("PrefixReplacer.AdvanceRegexp: the empty string must not match the regular expression %q.", re)) + } + if m := match(pr.rest, re); m != nil { + if G.opts.DebugTrace { + trace("", "PrefixReplacer.AdvanceRegexp", pr.rest, re, m[0]) + } pr.rest = pr.rest[len(m[0]):] pr.m = m return true @@ -276,6 +340,72 @@ func (pr *PrefixReplacer) startsWith(re string) bool { return false } +func (pr *PrefixReplacer) PeekByte() int { + rest := pr.rest + if len(rest) == 0 { + return -1 + } + return int(rest[0]) +} + +func (pr *PrefixReplacer) Mark() string { + return pr.rest +} +func (pr *PrefixReplacer) Reset(mark string) { + pr.rest = mark +} +func (pr *PrefixReplacer) Skip(n int) { + pr.rest = pr.rest[n:] +} +func (pr *PrefixReplacer) Since(mark string) string { + return mark[:len(mark)-len(pr.rest)] +} +func (pr *PrefixReplacer) AdvanceRest() string { + rest := pr.rest + pr.rest = "" + return rest +} + +func (pr *PrefixReplacer) verifyBits(bits0x00, bits0x20, bits0x40, bits0x60 uint32, re string) { + if G.TestingData.VerifiedBits[re] { + return + } + G.TestingData.VerifiedBits[re] = true + rec := regcomp(re) + + var actual0x00, actual0x20, actual0x40, actual0x60 uint32 + mask := uint32(0) + for i := byte(0); i < 0x80; i++ { + if rec.Match([]byte{i}) { + mask |= uint32(1) << (i & 31) + } + switch i { + case 0x1F: + actual0x00 = mask + case 0x3F: + actual0x20 = mask + case 0x5F: + actual0x40 = mask + case 0x7F: + actual0x60 = mask + } + if i&0x1F == 0x1F { + mask = 0 + } + } + if actual0x00 == bits0x00 && actual0x20 == bits0x20 && actual0x40 == bits0x40 && actual0x60 == bits0x60 { + return + } + + print(fmt.Sprintf(""+ + "Expected bits(%#08x, %#08x, %#08x, %#08x), "+ + "not bits(%#08x, %#08x, %#08x, %#08x) "+ + "for pattern %q.\n", + actual0x00, actual0x20, actual0x40, actual0x60, + bits0x00, bits0x20, bits0x40, bits0x60, + re)) +} + // Useful in combination with regex.Find*Index func negToZero(i int) int { if i >= 0 { @@ -284,9 +414,11 @@ func negToZero(i int) int { return 0 } -func toInt(s string) int { - n, _ := strconv.Atoi(s) - return n +func toInt(s string, def int) int { + if n, err := strconv.Atoi(s); err == nil { + return n + } + return def } func dirglob(dirname string) []string { @@ -301,34 +433,71 @@ func dirglob(dirname string) []string { return fnames } +// http://stackoverflow.com/questions/13476349/check-for-nil-and-nil-interface-in-go +func isNil(a interface{}) bool { + defer func() { recover() }() + return a == nil || reflect.ValueOf(a).IsNil() +} + func argsStr(args ...interface{}) string { argsStr := "" for i, arg := range args { if i != 0 { argsStr += ", " } - argsStr += sprintf("%#v", arg) + if str, ok := arg.(fmt.Stringer); ok && !isNil(str) { + argsStr += str.String() + } else { + argsStr += fmt.Sprintf("%#v", arg) + } } return argsStr } func trace(action, funcname string, args ...interface{}) { if G.opts.DebugTrace { - io.WriteString(G.traceOut, sprintf("TRACE: %s%s%s(%s)\n", strings.Repeat("| ", G.traceDepth), action, funcname, argsStr(args...))) + io.WriteString(G.debugOut, fmt.Sprintf("TRACE: %s%s%s(%s)\n", strings.Repeat("| ", G.traceDepth), action, funcname, argsStr(args...))) } } -func tracecall(funcname string, args ...interface{}) func() { - if G.opts.DebugTrace { - trace("+ ", funcname, args...) - G.traceDepth++ - return func() { - G.traceDepth-- - trace("- ", funcname, args...) +func tracecallInternal(args ...interface{}) func() { + if !G.opts.DebugTrace { + panic("Internal pkglint error: calls to tracecall must only occur in tracing mode") + } + + funcname := "?" + if pc, _, _, ok := runtime.Caller(2); ok { + if fn := runtime.FuncForPC(pc); fn != nil { + funcname = strings.TrimSuffix(fn.Name(), "main.") } - } else { - return func() {} } + trace("+ ", funcname, args...) + G.traceDepth++ + + return func() { + G.traceDepth-- + trace("- ", funcname, args...) + } +} +func tracecall0() func() { + switch { // prevent inlining, for code size and performance + } + return tracecallInternal() +} +func tracecall1(arg1 string) func() { + switch { // prevent inlining, for code size and performance + } + return tracecallInternal(arg1) +} +func tracecall2(arg1, arg2 string) func() { + switch { // prevent inlining, for code size and performance + } + return tracecallInternal(arg1, arg2) +} +func tracecall(args ...interface{}) func() { + switch { // prevent inlining, for code size and performance + } + return tracecallInternal(args...) } // Emulates make(1)’s :S substitution operator. @@ -352,7 +521,9 @@ func relpath(from, to string) string { panic("relpath" + argsStr(from, to, err1, err2, err3)) } result := filepath.ToSlash(rel) - trace("", "relpath", from, to, "=>", result) + if G.opts.DebugTrace { + trace("", "relpath", from, to, "=>", result) + } return result } @@ -377,7 +548,7 @@ func stringStringMapKeys(m map[string]string) []string { func abspath(fname string) string { abs, err := filepath.Abs(fname) if err != nil { - fatalf(fname, noLines, "Cannot determine absolute path.") + Fatalf(fname, noLines, "Cannot determine absolute path.") } return filepath.ToSlash(abs) } @@ -386,8 +557,6 @@ func abspath(fname string) string { // Also, the initial directory is always kept. // This is to provide the package path as context in recursive invocations of pkglint. func cleanpath(fname string) string { - defer tracecall("cleanpath", fname)() - tmp := fname for len(tmp) > 2 && hasPrefix(tmp, "./") { tmp = tmp[2:] @@ -425,13 +594,13 @@ func NewHistogram() *Histogram { return h } -func (h *Histogram) add(s string) { +func (h *Histogram) Add(s string, n int) { if G.opts.Profiling { - h.histo[s]++ + h.histo[s] += n } } -func (h *Histogram) printStats(caption string, out io.Writer) { +func (h *Histogram) PrintStats(caption string, out io.Writer, limit int) { entries := make([]HistogramEntry, len(h.histo)) i := 0 @@ -444,7 +613,7 @@ func (h *Histogram) printStats(caption string, out io.Writer) { for i, entry := range entries { fmt.Fprintf(out, "%s %6d %s\n", caption, entry.count, entry.s) - if i >= 10 { + if limit > 0 && i >= limit { break } } diff --git a/pkgtools/pkglint/files/util_test.go b/pkgtools/pkglint/files/util_test.go index ec857f1e0f1..5c83e84cac9 100644 --- a/pkgtools/pkglint/files/util_test.go +++ b/pkgtools/pkglint/files/util_test.go @@ -73,8 +73,16 @@ func (s *Suite) TestIsEmptyDirAndGetSubdirs(c *check.C) { if nodir := s.tmpdir + "/nonexistent"; true { c.Check(isEmptyDir(nodir), equals, true) // Counts as empty. defer s.ExpectFatalError(func() { - c.Check(s.Output(), equals, "FATAL: "+nodir+": Cannot be read: open "+nodir+": The system cannot find the file specified.\n") + c.Check(s.Output(), check.Matches, `FATAL: (.+): Cannot be read: open (.+): (.+)\n`) }) c.Check(getSubdirs(nodir), check.DeepEquals, []string(nil)) + c.FailNow() } } + +func (s *Suite) TestPrefixReplacer_Since(c *check.C) { + repl := NewPrefixReplacer("hello, world") + mark := repl.Mark() + repl.AdvanceRegexp(`^\w+`) + c.Check(repl.Since(mark), equals, "hello") +} diff --git a/pkgtools/pkglint/files/vardefs.go b/pkgtools/pkglint/files/vardefs.go index 5ec76322763..b73307a7be6 100644 --- a/pkgtools/pkglint/files/vardefs.go +++ b/pkgtools/pkglint/files/vardefs.go @@ -1,5 +1,11 @@ package main +import ( + "fmt" + "path" + "strings" +) + // This file defines the specific type of some variables. // // There are two types of lists: @@ -88,87 +94,87 @@ func (gd *GlobalData) InitVartypes() { sys(".CURDIR", lkNone, CheckvarPathname) sys(".TARGET", lkNone, CheckvarPathname) - acl("ALL_ENV", lkShell, CheckvarShellWord) - acl("ALTERNATIVES_FILE", lkNone, CheckvarFilename) - acl("ALTERNATIVES_SRC", lkShell, CheckvarPathname) + acl("ALL_ENV", lkShell, CheckvarShellWord, "") + acl("ALTERNATIVES_FILE", lkNone, CheckvarFilename, "") + acl("ALTERNATIVES_SRC", lkShell, CheckvarPathname, "") pkg("APACHE_MODULE", lkNone, CheckvarYes) sys("AR", lkNone, CheckvarShellCommand) sys("AS", lkNone, CheckvarShellCommand) pkglist("AUTOCONF_REQD", lkShell, CheckvarVersion) - acl("AUTOMAKE_OVERRIDE", lkShell, CheckvarPathmask) + acl("AUTOMAKE_OVERRIDE", lkShell, CheckvarPathmask, "") pkglist("AUTOMAKE_REQD", lkShell, CheckvarVersion) pkg("AUTO_MKDIRS", lkNone, CheckvarYesNo) usr("BATCH", lkNone, CheckvarYes) - acl("BDB185_DEFAULT", lkNone, CheckvarUnchecked) + acl("BDB185_DEFAULT", lkNone, CheckvarUnchecked, "") sys("BDBBASE", lkNone, CheckvarPathname) pkg("BDB_ACCEPTED", lkShell, enum("db1 db2 db3 db4 db5")) - acl("BDB_DEFAULT", lkNone, enum("db1 db2 db3 db4 db5")) + acl("BDB_DEFAULT", lkNone, enum("db1 db2 db3 db4 db5"), "") sys("BDB_LIBS", lkShell, CheckvarLdFlag) sys("BDB_TYPE", lkNone, enum("db1 db2 db3 db4 db5")) sys("BINGRP", lkNone, CheckvarUserGroupName) sys("BINMODE", lkNone, CheckvarFileMode) sys("BINOWN", lkNone, CheckvarUserGroupName) - acl("BOOTSTRAP_DEPENDS", lkSpace, CheckvarDependencyWithPath, "Makefile.common:a", "Makefile:a", "options.mk:a", "*.mk:a") + acl("BOOTSTRAP_DEPENDS", lkSpace, CheckvarDependencyWithPath, "Makefile, Makefile.common, *.mk: append") pkg("BOOTSTRAP_PKG", lkNone, CheckvarYesNo) - acl("BROKEN", lkNone, CheckvarMessage) + acl("BROKEN", lkNone, CheckvarMessage, "") pkg("BROKEN_GETTEXT_DETECTION", lkNone, CheckvarYesNo) pkglist("BROKEN_EXCEPT_ON_PLATFORM", lkShell, CheckvarPlatformTriple) pkglist("BROKEN_ON_PLATFORM", lkSpace, CheckvarPlatformTriple) sys("BSD_MAKE_ENV", lkShell, CheckvarShellWord) - acl("BUILDLINK_ABI_DEPENDS.*", lkSpace, CheckvarDependency, "*:a") - acl("BUILDLINK_API_DEPENDS.*", lkSpace, CheckvarDependency, "*:a") - acl("BUILDLINK_CONTENTS_FILTER", lkShell, CheckvarShellWord) // Should better be ShellCommand + acl("BUILDLINK_ABI_DEPENDS.*", lkSpace, CheckvarDependency, "*: append") + acl("BUILDLINK_API_DEPENDS.*", lkSpace, CheckvarDependency, "*: append") + acl("BUILDLINK_CONTENTS_FILTER", lkNone, CheckvarShellCommand, "") sys("BUILDLINK_CFLAGS", lkShell, CheckvarCFlag) bl3list("BUILDLINK_CFLAGS.*", lkShell, CheckvarCFlag) sys("BUILDLINK_CPPFLAGS", lkShell, CheckvarCFlag) bl3list("BUILDLINK_CPPFLAGS.*", lkShell, CheckvarCFlag) - acl("BUILDLINK_CONTENTS_FILTER.*", lkNone, CheckvarShellCommand, "buildlink3.mk:s") - acl("BUILDLINK_DEPENDS", lkSpace, CheckvarIdentifier, "buildlink3.mk:a") - acl("BUILDLINK_DEPMETHOD.*", lkShell, CheckvarBuildlinkDepmethod, "buildlink3.mk:ad", "Makefile:as", "Makefile.common:a", "*.mk:a") // FIXME: buildlink3.mk:d may lead to unexpected behavior. + acl("BUILDLINK_CONTENTS_FILTER.*", lkNone, CheckvarShellCommand, "buildlink3.mk: set") + acl("BUILDLINK_DEPENDS", lkSpace, CheckvarIdentifier, "buildlink3.mk: append") + acl("BUILDLINK_DEPMETHOD.*", lkShell, CheckvarBuildlinkDepmethod, "buildlink3.mk: default, append; Makefile: set, append; Makefile.common, *.mk: append") sys("BUILDLINK_DIR", lkNone, CheckvarPathname) bl3list("BUILDLINK_FILES.*", lkShell, CheckvarPathmask) - acl("BUILDLINK_FILES_CMD.*", lkShell, CheckvarShellWord) // Should better be ShellCommand - acl("BUILDLINK_INCDIRS.*", lkShell, CheckvarPathname, "buildlink3.mk:ad") // Should [d]efault really be allowed in buildlink3.mk? - acl("BUILDLINK_JAVA_PREFIX.*", lkNone, CheckvarPathname, "buildlink3.mk:s") - acl("BUILDLINK_LDADD.*", lkShell, CheckvarLdFlag, "builtin.mk:adsu", "buildlink3.mk:", "Makefile:u", "Makefile.common:u", "*.mk:u") + acl("BUILDLINK_FILES_CMD.*", lkNone, CheckvarShellCommand, "") + acl("BUILDLINK_INCDIRS.*", lkShell, CheckvarPathname, "buildlink3.mk: default, append") + acl("BUILDLINK_JAVA_PREFIX.*", lkNone, CheckvarPathname, "buildlink3.mk: set") + acl("BUILDLINK_LDADD.*", lkShell, CheckvarLdFlag, "builtin.mk: set, default, append, use; buildlink3.mk: append; Makefile, Makefile.common, *.mk: use") sys("BUILDLINK_LDFLAGS", lkShell, CheckvarLdFlag) bl3list("BUILDLINK_LDFLAGS.*", lkShell, CheckvarLdFlag) bl3list("BUILDLINK_LIBDIRS.*", lkShell, CheckvarPathname) - acl("BUILDLINK_LIBS.*", lkShell, CheckvarLdFlag, "buildlink3.mk:a") - acl("BUILDLINK_PASSTHRU_DIRS", lkShell, CheckvarPathname, "Makefile:a", "Makefile.common:a", "buildlink3.mk:a", "hacks.mk:a") - acl("BUILDLINK_PASSTHRU_RPATHDIRS", lkShell, CheckvarPathname, "Makefile:a", "Makefile.common:a", "buildlink3.mk:a", "hacks.mk:a") - acl("BUILDLINK_PKGSRCDIR.*", lkNone, CheckvarRelativePkgDir, "buildlink3.mk:dp") - acl("BUILDLINK_PREFIX.*", lkNone, CheckvarPathname, "builtin.mk:su", "buildlink3.mk:", "Makefile:u", "Makefile.common:u", "*.mk:u") - acl("BUILDLINK_RPATHDIRS.*", lkShell, CheckvarPathname, "buildlink3.mk:a") - acl("BUILDLINK_TARGETS", lkShell, CheckvarIdentifier) - acl("BUILDLINK_FNAME_TRANSFORM.*", lkNone, CheckvarSedCommands, "Makefile:a", "builtin.mk:a", "hacks.mk:a", "buildlink3.mk:a") - acl("BUILDLINK_TRANSFORM", lkShell, CheckvarWrapperTransform, "*:a") - acl("BUILDLINK_TREE", lkShell, CheckvarIdentifier, "buildlink3.mk:a") - acl("BUILD_DEFS", lkShell, CheckvarVarname, "Makefile:a", "Makefile.common:a", "options.mk:a") - acl("BUILD_DEPENDS", lkSpace, CheckvarDependencyWithPath, "Makefile.common:a", "Makefile:a", "options.mk:a", "*.mk:a") + acl("BUILDLINK_LIBS.*", lkShell, CheckvarLdFlag, "buildlink3.mk: append") + acl("BUILDLINK_PASSTHRU_DIRS", lkShell, CheckvarPathname, "Makefile, Makefile.common, buildlink3.mk, hacks.mk: append") + acl("BUILDLINK_PASSTHRU_RPATHDIRS", lkShell, CheckvarPathname, "Makefile, Makefile.common, buildlink3.mk, hacks.mk: append") + acl("BUILDLINK_PKGSRCDIR.*", lkNone, CheckvarRelativePkgDir, "buildlink3.mk: default, use-loadtime") + acl("BUILDLINK_PREFIX.*", lkNone, CheckvarPathname, "builtin.mk: set, use; buildlink3.mk:; Makefile, Makefile.common, *.mk: use") + acl("BUILDLINK_RPATHDIRS.*", lkShell, CheckvarPathname, "buildlink3.mk: append") + acl("BUILDLINK_TARGETS", lkShell, CheckvarIdentifier, "") + acl("BUILDLINK_FNAME_TRANSFORM.*", lkNone, CheckvarSedCommands, "Makefile, buildlink3.mk, builtin.mk, hacks.mk: append") + acl("BUILDLINK_TRANSFORM", lkShell, CheckvarWrapperTransform, "*: append") + acl("BUILDLINK_TREE", lkShell, CheckvarIdentifier, "buildlink3.mk: append") + acl("BUILD_DEFS", lkShell, CheckvarVarname, "Makefile, Makefile.common, options.mk: append") + acl("BUILD_DEPENDS", lkSpace, CheckvarDependencyWithPath, "Makefile, Makefile.common, *.mk: append") pkglist("BUILD_DIRS", lkShell, CheckvarWrksrcSubdirectory) pkglist("BUILD_ENV", lkShell, CheckvarShellWord) sys("BUILD_MAKE_CMD", lkNone, CheckvarShellCommand) pkglist("BUILD_MAKE_FLAGS", lkShell, CheckvarShellWord) pkg("BUILD_TARGET", lkShell, CheckvarIdentifier) pkg("BUILD_USES_MSGFMT", lkNone, CheckvarYes) - acl("BUILTIN_PKG", lkNone, CheckvarIdentifier, "builtin.mk:psu") - acl("BUILTIN_PKG.*", lkNone, CheckvarPkgName, "builtin.mk:psu") - acl("BUILTIN_FIND_FILES_VAR", lkShell, CheckvarVarname, "builtin.mk:s") - acl("BUILTIN_FIND_FILES.*", lkShell, CheckvarPathname, "builtin.mk:s") - acl("BUILTIN_FIND_GREP.*", lkNone, CheckvarString, "builtin.mk:s") - acl("BUILTIN_FIND_LIBS", lkShell, CheckvarPathname, "builtin.mk:s") - acl("BUILTIN_IMAKE_CHECK", lkShell, CheckvarUnchecked, "builtin.mk:s") - acl("BUILTIN_IMAKE_CHECK.*", lkNone, CheckvarYesNo) + acl("BUILTIN_PKG", lkNone, CheckvarIdentifier, "builtin.mk: set, use-loadtime, use") + acl("BUILTIN_PKG.*", lkNone, CheckvarPkgName, "builtin.mk: set, use-loadtime, use") + acl("BUILTIN_FIND_FILES_VAR", lkShell, CheckvarVarname, "builtin.mk: set") + acl("BUILTIN_FIND_FILES.*", lkShell, CheckvarPathname, "builtin.mk: set") + acl("BUILTIN_FIND_GREP.*", lkNone, CheckvarString, "builtin.mk: set") + acl("BUILTIN_FIND_LIBS", lkShell, CheckvarPathname, "builtin.mk: set") + acl("BUILTIN_IMAKE_CHECK", lkShell, CheckvarUnchecked, "builtin.mk: set") + acl("BUILTIN_IMAKE_CHECK.*", lkNone, CheckvarYesNo, "") sys("BUILTIN_X11_TYPE", lkNone, CheckvarUnchecked) sys("BUILTIN_X11_VERSION", lkNone, CheckvarUnchecked) - acl("CATEGORIES", lkShell, CheckvarCategory, "Makefile:as", "Makefile.common:ads") + acl("CATEGORIES", lkShell, CheckvarCategory, "Makefile: set, append; Makefile.common: set, default, append") sys("CC_VERSION", lkNone, CheckvarMessage) sys("CC", lkNone, CheckvarShellCommand) pkglist("CFLAGS*", lkShell, CheckvarCFlag) // may also be changed by the user - acl("CHECK_BUILTIN", lkNone, CheckvarYesNo, "builtin.mk:d", "Makefile:s") - acl("CHECK_BUILTIN.*", lkNone, CheckvarYesNo, "*:p") - acl("CHECK_FILES_SKIP", lkShell, CheckvarBasicRegularExpression, "Makefile:a", "Makefile.common:a") + acl("CHECK_BUILTIN", lkNone, CheckvarYesNo, "builtin.mk: default; Makefile: set") + acl("CHECK_BUILTIN.*", lkNone, CheckvarYesNo, "buildlink3.mk: set; builtin.mk: default; *: use-loadtime") + acl("CHECK_FILES_SKIP", lkShell, CheckvarBasicRegularExpression, "Makefile, Makefile.common: append") pkg("CHECK_FILES_SUPPORTED", lkNone, CheckvarYesNo) usr("CHECK_HEADERS", lkNone, CheckvarYesNo) pkglist("CHECK_HEADERS_SKIP", lkShell, CheckvarPathmask) @@ -178,25 +184,25 @@ func (gd *GlobalData) InitVartypes() { pkglist("CHECK_PERMS_SKIP", lkShell, CheckvarPathmask) usr("CHECK_PORTABILITY", lkNone, CheckvarYesNo) pkglist("CHECK_PORTABILITY_SKIP", lkShell, CheckvarPathmask) - acl("CHECK_SHLIBS", lkNone, CheckvarYesNo, "Makefile:s") + acl("CHECK_SHLIBS", lkNone, CheckvarYesNo, "Makefile: set") pkglist("CHECK_SHLIBS_SKIP", lkShell, CheckvarPathmask) - acl("CHECK_SHLIBS_SUPPORTED", lkNone, CheckvarYesNo, "Makefile:s") + acl("CHECK_SHLIBS_SUPPORTED", lkNone, CheckvarYesNo, "Makefile: set") pkglist("CHECK_WRKREF_SKIP", lkShell, CheckvarPathmask) pkg("CMAKE_ARG_PATH", lkNone, CheckvarPathname) pkglist("CMAKE_ARGS", lkShell, CheckvarShellWord) - acl("COMMENT", lkNone, CheckvarComment, "Makefile:as", "Makefile.common:as") + acl("COMMENT", lkNone, CheckvarComment, "Makefile, Makefile.common: set, append") sys("COMPILER_RPATH_FLAG", lkNone, enum("-Wl,-rpath")) pkglist("CONFIGURE_ARGS", lkShell, CheckvarShellWord) pkglist("CONFIGURE_DIRS", lkShell, CheckvarWrksrcSubdirectory) - pkglist("CONFIGURE_ENV", lkShell, CheckvarShellWord) + acl("CONFIGURE_ENV", lkShell, CheckvarShellWord, "Makefile, Makefile.common: append, set, use; buildlink3.mk, builtin.mk: append; *.mk: append, use") pkg("CONFIGURE_HAS_INFODIR", lkNone, CheckvarYesNo) pkg("CONFIGURE_HAS_LIBDIR", lkNone, CheckvarYesNo) pkg("CONFIGURE_HAS_MANDIR", lkNone, CheckvarYesNo) pkg("CONFIGURE_SCRIPT", lkNone, CheckvarPathname) - acl("CONFIG_GUESS_OVERRIDE", lkShell, CheckvarPathmask, "Makefile:as", "Makefile.common:as") - acl("CONFIG_STATUS_OVERRIDE", lkShell, CheckvarPathmask, "Makefile:as", "Makefile.common:as") - acl("CONFIG_SHELL", lkNone, CheckvarPathname, "Makefile:s", "Makefile.common:s") - acl("CONFIG_SUB_OVERRIDE", lkShell, CheckvarPathmask, "Makefile:as", "Makefile.common:as") + acl("CONFIG_GUESS_OVERRIDE", lkShell, CheckvarPathmask, "Makefile, Makefile.common: set, append") + acl("CONFIG_STATUS_OVERRIDE", lkShell, CheckvarPathmask, "Makefile, Makefile.common: set, append") + acl("CONFIG_SHELL", lkNone, CheckvarPathname, "Makefile, Makefile.common: set") + acl("CONFIG_SUB_OVERRIDE", lkShell, CheckvarPathmask, "Makefile, Makefile.common: set, append") pkglist("CONFLICTS", lkSpace, CheckvarDependency) pkglist("CONF_FILES", lkShell, CheckvarShellWord) pkg("CONF_FILES_MODE", lkNone, enum("0644 0640 0600 0400")) @@ -204,38 +210,38 @@ func (gd *GlobalData) InitVartypes() { sys("COPY", lkNone, enum("-c")) // The flag that tells ${INSTALL} to copy a file sys("CPP", lkNone, CheckvarShellCommand) pkglist("CPPFLAGS*", lkShell, CheckvarCFlag) - acl("CRYPTO", lkNone, CheckvarYes, "Makefile:s") + acl("CRYPTO", lkNone, CheckvarYes, "Makefile: set") sys("CXX", lkNone, CheckvarShellCommand) pkglist("CXXFLAGS*", lkShell, CheckvarCFlag) - acl("DEINSTALL_FILE", lkNone, CheckvarPathname, "Makefile:s") - acl("DEINSTALL_SRC", lkShell, CheckvarPathname, "Makefile:s", "Makefile.common:ds") - acl("DEINSTALL_TEMPLATES", lkShell, CheckvarPathname, "Makefile:as", "Makefile.common:ads") + acl("DEINSTALL_FILE", lkNone, CheckvarPathname, "Makefile: set") + acl("DEINSTALL_SRC", lkShell, CheckvarPathname, "Makefile: set; Makefile.common: default, set") + acl("DEINSTALL_TEMPLATES", lkShell, CheckvarPathname, "Makefile: set, append; Makefile.common: set, default, append") sys("DELAYED_ERROR_MSG", lkNone, CheckvarShellCommand) sys("DELAYED_WARNING_MSG", lkNone, CheckvarShellCommand) pkglist("DEPENDS", lkSpace, CheckvarDependencyWithPath) usr("DEPENDS_TARGET", lkShell, CheckvarIdentifier) - acl("DESCR_SRC", lkShell, CheckvarPathname, "Makefile:s", "Makefile.common:ds") + acl("DESCR_SRC", lkShell, CheckvarPathname, "Makefile: set; Makefile.common: default, set") sys("DESTDIR", lkNone, CheckvarPathname) - acl("DESTDIR_VARNAME", lkNone, CheckvarVarname, "Makefile:s", "Makefile.common:s") + acl("DESTDIR_VARNAME", lkNone, CheckvarVarname, "Makefile, Makefile.common: set") sys("DEVOSSAUDIO", lkNone, CheckvarPathname) sys("DEVOSSSOUND", lkNone, CheckvarPathname) pkglist("DISTFILES", lkShell, CheckvarFilename) pkg("DISTINFO_FILE", lkNone, CheckvarRelativePkgPath) pkg("DISTNAME", lkNone, CheckvarFilename) pkg("DIST_SUBDIR", lkNone, CheckvarPathname) - acl("DJB_BUILD_ARGS", lkShell, CheckvarShellWord) - acl("DJB_BUILD_TARGETS", lkShell, CheckvarIdentifier) - acl("DJB_CONFIG_CMDS", lkShell, CheckvarShellWord, "options.mk:s") // ShellCommand, terminated by a semicolon - acl("DJB_CONFIG_DIRS", lkShell, CheckvarWrksrcSubdirectory) - acl("DJB_CONFIG_HOME", lkNone, CheckvarFilename) - acl("DJB_CONFIG_PREFIX", lkNone, CheckvarPathname) - acl("DJB_INSTALL_TARGETS", lkShell, CheckvarIdentifier) - acl("DJB_MAKE_TARGETS", lkNone, CheckvarYesNo) - acl("DJB_RESTRICTED", lkNone, CheckvarYesNo, "Makefile:s") - acl("DJB_SLASHPACKAGE", lkNone, CheckvarYesNo) - acl("DLOPEN_REQUIRE_PTHREADS", lkNone, CheckvarYesNo) - acl("DL_AUTO_VARS", lkNone, CheckvarYes, "Makefile:s", "Makefile.common:s", "options.mk:s") - acl("DL_LIBS", lkShell, CheckvarLdFlag) + acl("DJB_BUILD_ARGS", lkShell, CheckvarShellWord, "") + acl("DJB_BUILD_TARGETS", lkShell, CheckvarIdentifier, "") + acl("DJB_CONFIG_CMDS", lkNone, CheckvarShellCommands, "options.mk: set") + acl("DJB_CONFIG_DIRS", lkShell, CheckvarWrksrcSubdirectory, "") + acl("DJB_CONFIG_HOME", lkNone, CheckvarFilename, "") + acl("DJB_CONFIG_PREFIX", lkNone, CheckvarPathname, "") + acl("DJB_INSTALL_TARGETS", lkShell, CheckvarIdentifier, "") + acl("DJB_MAKE_TARGETS", lkNone, CheckvarYesNo, "") + acl("DJB_RESTRICTED", lkNone, CheckvarYesNo, "Makefile: set") + acl("DJB_SLASHPACKAGE", lkNone, CheckvarYesNo, "") + acl("DLOPEN_REQUIRE_PTHREADS", lkNone, CheckvarYesNo, "") + acl("DL_AUTO_VARS", lkNone, CheckvarYes, "Makefile, Makefile.common, options.mk: set") + acl("DL_LIBS", lkShell, CheckvarLdFlag, "") sys("DOCOWN", lkNone, CheckvarUserGroupName) sys("DOCGRP", lkNone, CheckvarUserGroupName) sys("DOCMODE", lkNone, CheckvarFileMode) @@ -252,19 +258,19 @@ func (gd *GlobalData) InitVartypes() { sys("EMACS_FLAVOR", lkNone, enum("emacs xemacs")) sys("EMACS_INFOPREFIX", lkNone, CheckvarPathname) sys("EMACS_LISPPREFIX", lkNone, CheckvarPathname) - acl("EMACS_MODULES", lkShell, CheckvarIdentifier, "Makefile:as", "Makefile.common:as") + acl("EMACS_MODULES", lkShell, CheckvarIdentifier, "Makefile, Makefile.common: set, append") sys("EMACS_PKGNAME_PREFIX", lkNone, CheckvarIdentifier) // Or the empty string. sys("EMACS_TYPE", lkNone, enum("emacs xemacs")) - acl("EMACS_USE_LEIM", lkNone, CheckvarYes) - acl("EMACS_VERSIONS_ACCEPTED", lkShell, enum("emacs25 emacs24 emacs24nox emacs23 emacs23nox emacs22 emacs22nox emacs21 emacs21nox emacs20 xemacs215 xemacs215nox xemacs214 xemacs214nox"), "Makefile:s") + acl("EMACS_USE_LEIM", lkNone, CheckvarYes, "") + acl("EMACS_VERSIONS_ACCEPTED", lkShell, enum("emacs25 emacs24 emacs24nox emacs23 emacs23nox emacs22 emacs22nox emacs21 emacs21nox emacs20 xemacs215 xemacs215nox xemacs214 xemacs214nox"), "Makefile: set") sys("EMACS_VERSION_MAJOR", lkNone, CheckvarInteger) sys("EMACS_VERSION_MINOR", lkNone, CheckvarInteger) - acl("EMACS_VERSION_REQD", lkShell, enum("emacs24 emacs24nox emacs23 emacs23nox emacs22 emacs22nox emacs21 emacs21nox emacs20 xemacs215 xemacs214"), "Makefile:as") + acl("EMACS_VERSION_REQD", lkShell, enum("emacs24 emacs24nox emacs23 emacs23nox emacs22 emacs22nox emacs21 emacs21nox emacs20 xemacs215 xemacs214"), "Makefile: set, append") sys("EMULDIR", lkNone, CheckvarPathname) sys("EMULSUBDIR", lkNone, CheckvarPathname) sys("OPSYS_EMULDIR", lkNone, CheckvarPathname) sys("EMULSUBDIRSLASH", lkNone, CheckvarPathname) - sys("EMUL_ARCH", lkNone, enum("i386 none")) + sys("EMUL_ARCH", lkNone, enum("i386 none x86_64")) sys("EMUL_DISTRO", lkNone, CheckvarIdentifier) sys("EMUL_IS_NATIVE", lkNone, CheckvarYes) pkg("EMUL_MODULES.*", lkShell, CheckvarIdentifier) @@ -277,21 +283,21 @@ func (gd *GlobalData) InitVartypes() { usr("EMUL_TYPE.*", lkNone, enum("native builtin suse suse-9.1 suse-9.x suse-10.0 suse-10.x")) sys("ERROR_CAT", lkNone, CheckvarShellCommand) sys("ERROR_MSG", lkNone, CheckvarShellCommand) - acl("EVAL_PREFIX", lkSpace, CheckvarShellWord, "Makefile:a", "Makefile.common:a") // XXX: Combining ShellWord with lkSpace looks weird. + acl("EVAL_PREFIX", lkSpace, CheckvarShellWord, "Makefile, Makefile.common: append") // XXX: Combining ShellWord with lkSpace looks weird. sys("EXPORT_SYMBOLS_LDFLAGS", lkShell, CheckvarLdFlag) sys("EXTRACT_CMD", lkNone, CheckvarShellCommand) pkg("EXTRACT_DIR", lkNone, CheckvarPathname) pkglist("EXTRACT_ELEMENTS", lkShell, CheckvarPathmask) pkglist("EXTRACT_ENV", lkShell, CheckvarShellWord) pkglist("EXTRACT_ONLY", lkShell, CheckvarPathname) - acl("EXTRACT_OPTS", lkShell, CheckvarShellWord, "Makefile:as", "Makefile.common:as") - acl("EXTRACT_OPTS_BIN", lkShell, CheckvarShellWord, "Makefile:as", "Makefile.common:as") - acl("EXTRACT_OPTS_LHA", lkShell, CheckvarShellWord, "Makefile:as", "Makefile.common:as") - acl("EXTRACT_OPTS_PAX", lkShell, CheckvarShellWord, "Makefile:as", "Makefile.common:as") - acl("EXTRACT_OPTS_RAR", lkShell, CheckvarShellWord, "Makefile:as", "Makefile.common:as") - acl("EXTRACT_OPTS_TAR", lkShell, CheckvarShellWord, "Makefile:as", "Makefile.common:as") - acl("EXTRACT_OPTS_ZIP", lkShell, CheckvarShellWord, "Makefile:as", "Makefile.common:as") - acl("EXTRACT_OPTS_ZOO", lkShell, CheckvarShellWord, "Makefile:as", "Makefile.common:as") + acl("EXTRACT_OPTS", lkShell, CheckvarShellWord, "Makefile, Makefile.common: set, append") + acl("EXTRACT_OPTS_BIN", lkShell, CheckvarShellWord, "Makefile, Makefile.common: set, append") + acl("EXTRACT_OPTS_LHA", lkShell, CheckvarShellWord, "Makefile, Makefile.common: set, append") + acl("EXTRACT_OPTS_PAX", lkShell, CheckvarShellWord, "Makefile, Makefile.common: set, append") + acl("EXTRACT_OPTS_RAR", lkShell, CheckvarShellWord, "Makefile, Makefile.common: set, append") + acl("EXTRACT_OPTS_TAR", lkShell, CheckvarShellWord, "Makefile, Makefile.common: set, append") + acl("EXTRACT_OPTS_ZIP", lkShell, CheckvarShellWord, "Makefile, Makefile.common: set, append") + acl("EXTRACT_OPTS_ZOO", lkShell, CheckvarShellWord, "Makefile, Makefile.common: set, append") pkg("EXTRACT_SUFX", lkNone, CheckvarDistSuffix) pkg("EXTRACT_USING", lkNone, enum("bsdtar gtar nbtar pax")) sys("FAIL_MSG", lkNone, CheckvarShellCommand) @@ -299,38 +305,38 @@ func (gd *GlobalData) InitVartypes() { pkg("FAM_ACCEPTED", lkShell, enum("fam gamin")) usr("FAM_DEFAULT", lkNone, enum("fam gamin")) sys("FAM_TYPE", lkNone, enum("fam gamin")) - acl("FETCH_BEFORE_ARGS", lkShell, CheckvarShellWord, "Makefile:as") + acl("FETCH_BEFORE_ARGS", lkShell, CheckvarShellWord, "Makefile: set, append") pkglist("FETCH_MESSAGE", lkShell, CheckvarShellWord) pkg("FILESDIR", lkNone, CheckvarRelativePkgPath) pkglist("FILES_SUBST", lkShell, CheckvarShellWord) - acl("FILES_SUBST_SED", lkShell, CheckvarShellWord) + acl("FILES_SUBST_SED", lkShell, CheckvarShellWord, "") pkglist("FIX_RPATH", lkShell, CheckvarVarname) pkglist("FLEX_REQD", lkShell, CheckvarVersion) - acl("FONTS_DIRS.*", lkShell, CheckvarPathname, "Makefile:as", "Makefile.common:a") + acl("FONTS_DIRS.*", lkShell, CheckvarPathname, "Makefile: set, append; Makefile.common: append") sys("GAMEDATAMODE", lkNone, CheckvarFileMode) sys("GAMES_GROUP", lkNone, CheckvarUserGroupName) sys("GAMEMODE", lkNone, CheckvarFileMode) sys("GAMES_USER", lkNone, CheckvarUserGroupName) pkglist("GCC_REQD", lkShell, CheckvarVersion) - pkglist("GENERATE_PLIST", lkShell, CheckvarShellWord) // List of Shellcommand, terminated with a semicolon + pkglist("GENERATE_PLIST", lkNone, CheckvarShellCommands) pkg("GITHUB_PROJECT", lkNone, CheckvarIdentifier) pkg("GITHUB_TAG", lkNone, CheckvarIdentifier) pkg("GITHUB_RELEASE", lkNone, CheckvarFilename) pkg("GITHUB_TYPE", lkNone, enum("tag release")) - acl("GNU_ARCH", lkNone, enum("mips")) - acl("GNU_CONFIGURE", lkNone, CheckvarYes, "Makefile.common:s", "Makefile:s") - acl("GNU_CONFIGURE_INFODIR", lkNone, CheckvarPathname, "Makefile:s", "Makefile.common:s") - acl("GNU_CONFIGURE_LIBDIR", lkNone, CheckvarPathname, "Makefile:s", "Makefile.common:s") + acl("GNU_ARCH", lkNone, enum("mips"), "") + acl("GNU_CONFIGURE", lkNone, CheckvarYes, "Makefile, Makefile.common: set") + acl("GNU_CONFIGURE_INFODIR", lkNone, CheckvarPathname, "Makefile, Makefile.common: set") + acl("GNU_CONFIGURE_LIBDIR", lkNone, CheckvarPathname, "Makefile, Makefile.common: set") pkg("GNU_CONFIGURE_LIBSUBDIR", lkNone, CheckvarPathname) - acl("GNU_CONFIGURE_MANDIR", lkNone, CheckvarPathname, "Makefile:s", "Makefile.common:s") - acl("GNU_CONFIGURE_PREFIX", lkNone, CheckvarPathname, "Makefile:s") - acl("HAS_CONFIGURE", lkNone, CheckvarYes, "Makefile:s", "Makefile.common:s") + acl("GNU_CONFIGURE_MANDIR", lkNone, CheckvarPathname, "Makefile, Makefile.common: set") + acl("GNU_CONFIGURE_PREFIX", lkNone, CheckvarPathname, "Makefile: set") + acl("HAS_CONFIGURE", lkNone, CheckvarYes, "Makefile, Makefile.common: set") pkglist("HEADER_TEMPLATES", lkShell, CheckvarPathname) pkg("HOMEPAGE", lkNone, CheckvarURL) - acl("IGNORE_PKG.*", lkNone, CheckvarYes, "*:sp") - acl("INCOMPAT_CURSES", lkSpace, CheckvarPlatformTriple, "Makefile:as") - acl("INCOMPAT_ICONV", lkSpace, CheckvarPlatformTriple) - acl("INFO_DIR", lkNone, CheckvarPathname) // relative to PREFIX + acl("IGNORE_PKG.*", lkNone, CheckvarYes, "*: set, use-loadtime") + acl("INCOMPAT_CURSES", lkSpace, CheckvarPlatformTriple, "Makefile: set, append") + acl("INCOMPAT_ICONV", lkSpace, CheckvarPlatformTriple, "") + acl("INFO_DIR", lkNone, CheckvarPathname, "") // relative to PREFIX pkg("INFO_FILES", lkNone, CheckvarYes) sys("INSTALL", lkNone, CheckvarShellCommand) pkglist("INSTALLATION_DIRS", lkShell, CheckvarPrefixPathname) @@ -339,7 +345,7 @@ func (gd *GlobalData) InitVartypes() { sys("INSTALL_DATA_DIR", lkNone, CheckvarShellCommand) pkglist("INSTALL_DIRS", lkShell, CheckvarWrksrcSubdirectory) pkglist("INSTALL_ENV", lkShell, CheckvarShellWord) - acl("INSTALL_FILE", lkNone, CheckvarPathname, "Makefile:s") + acl("INSTALL_FILE", lkNone, CheckvarPathname, "Makefile: set") sys("INSTALL_GAME", lkNone, CheckvarShellCommand) sys("INSTALL_GAME_DATA", lkNone, CheckvarShellCommand) sys("INSTALL_LIB", lkNone, CheckvarShellCommand) @@ -350,14 +356,14 @@ func (gd *GlobalData) InitVartypes() { sys("INSTALL_PROGRAM", lkNone, CheckvarShellCommand) sys("INSTALL_PROGRAM_DIR", lkNone, CheckvarShellCommand) sys("INSTALL_SCRIPT", lkNone, CheckvarShellCommand) - acl("INSTALL_SCRIPTS_ENV", lkShell, CheckvarShellWord) + acl("INSTALL_SCRIPTS_ENV", lkShell, CheckvarShellWord, "") sys("INSTALL_SCRIPT_DIR", lkNone, CheckvarShellCommand) - acl("INSTALL_SRC", lkShell, CheckvarPathname, "Makefile:s", "Makefile.common:ds") + acl("INSTALL_SRC", lkShell, CheckvarPathname, "Makefile: set; Makefile.common: default, set") pkg("INSTALL_TARGET", lkShell, CheckvarIdentifier) - acl("INSTALL_TEMPLATES", lkShell, CheckvarPathname, "Makefile:as", "Makefile.common:ads") - acl("INSTALL_UNSTRIPPED", lkNone, CheckvarYesNo, "Makefile:s", "Makefile.common:s") - pkg("INTERACTIVE_STAGE", lkShell, enum("fetch extract configure build install")) - acl("IS_BUILTIN.*", lkNone, CheckvarYesNoIndirectly, "builtin.mk:psu") + acl("INSTALL_TEMPLATES", lkShell, CheckvarPathname, "Makefile: set, append; Makefile.common: set, default, append") + acl("INSTALL_UNSTRIPPED", lkNone, CheckvarYesNo, "Makefile, Makefile.common: set") + pkg("INTERACTIVE_STAGE", lkShell, enum("fetch extract configure build test install")) + acl("IS_BUILTIN.*", lkNone, CheckvarYesNoIndirectly, "builtin.mk: set, use-loadtime, use") sys("JAVA_BINPREFIX", lkNone, CheckvarPathname) pkg("JAVA_CLASSPATH", lkNone, CheckvarShellWord) pkg("JAVA_HOME", lkNone, CheckvarPathname) @@ -366,7 +372,7 @@ func (gd *GlobalData) InitVartypes() { pkglist("JAVA_WRAPPERS", lkSpace, CheckvarFilename) pkg("JAVA_WRAPPER_BIN.*", lkNone, CheckvarPathname) sys("KRB5BASE", lkNone, CheckvarPathname) - acl("KRB5_ACCEPTED", lkShell, enum("heimdal mit-krb5")) + acl("KRB5_ACCEPTED", lkShell, enum("heimdal mit-krb5"), "") usr("KRB5_DEFAULT", lkNone, enum("heimdal mit-krb5")) sys("KRB5_TYPE", lkNone, CheckvarUnchecked) sys("LD", lkNone, CheckvarShellCommand) @@ -377,30 +383,30 @@ func (gd *GlobalData) InitVartypes() { sys("LIBOSSAUDIO", lkNone, CheckvarPathname) pkglist("LIBS*", lkShell, CheckvarLdFlag) sys("LIBTOOL", lkNone, CheckvarShellCommand) - acl("LIBTOOL_OVERRIDE", lkShell, CheckvarPathmask, "Makefile:as") + acl("LIBTOOL_OVERRIDE", lkShell, CheckvarPathmask, "Makefile: set, append") pkglist("LIBTOOL_REQD", lkShell, CheckvarVersion) - acl("LICENCE", lkNone, CheckvarLicense, "Makefile:s", "Makefile.common:s", "options.mk:s") - acl("LICENSE", lkNone, CheckvarLicense, "Makefile:s", "Makefile.common:s", "options.mk:s") + acl("LICENCE", lkNone, CheckvarLicense, "Makefile, Makefile.common: set; options.mk: set") + acl("LICENSE", lkNone, CheckvarLicense, "Makefile, Makefile.common: set; options.mk: set") pkg("LICENSE_FILE", lkNone, CheckvarPathname) sys("LINKER_RPATH_FLAG", lkNone, CheckvarShellWord) sys("LOWER_OPSYS", lkNone, CheckvarIdentifier) - acl("LTCONFIG_OVERRIDE", lkShell, CheckvarPathmask, "Makefile:as", "Makefile.common:a") + acl("LTCONFIG_OVERRIDE", lkShell, CheckvarPathmask, "Makefile: set, append; Makefile.common: append") sys("MACHINE_ARCH", lkNone, CheckvarIdentifier) sys("MACHINE_GNU_PLATFORM", lkNone, CheckvarPlatformTriple) - acl("MAINTAINER", lkNone, CheckvarMailAddress, "Makefile:s", "Makefile.common:d") + acl("MAINTAINER", lkNone, CheckvarMailAddress, "Makefile: set; Makefile.common: default") sys("MAKE", lkNone, CheckvarShellCommand) pkglist("MAKEFLAGS", lkShell, CheckvarShellWord) - acl("MAKEVARS", lkShell, CheckvarVarname, "builtin.mk:a", "buildlink3.mk:a", "hacks.mk:a") + acl("MAKEVARS", lkShell, CheckvarVarname, "builtin.mk: append; buildlink3.mk: append; hacks.mk: append") pkglist("MAKE_DIRS", lkShell, CheckvarPathname) pkglist("MAKE_DIRS_PERMS", lkShell, CheckvarShellWord) - pkglist("MAKE_ENV", lkShell, CheckvarShellWord) + acl("MAKE_ENV", lkShell, CheckvarShellWord, "Makefile: append, set, use; Makefile.common: append, set, use; buildlink3.mk: append; builtin.mk: append; *.mk: append, use") pkg("MAKE_FILE", lkNone, CheckvarPathname) pkglist("MAKE_FLAGS", lkShell, CheckvarShellWord) usr("MAKE_JOBS", lkNone, CheckvarInteger) pkg("MAKE_JOBS_SAFE", lkNone, CheckvarYesNo) pkg("MAKE_PROGRAM", lkNone, CheckvarShellCommand) - acl("MANCOMPRESSED", lkNone, CheckvarYesNo, "Makefile:s", "Makefile.common:ds") - acl("MANCOMPRESSED_IF_MANZ", lkNone, CheckvarYes, "Makefile:s", "Makefile.common:ds") + acl("MANCOMPRESSED", lkNone, CheckvarYesNo, "Makefile: set; Makefile.common: default, set") + acl("MANCOMPRESSED_IF_MANZ", lkNone, CheckvarYes, "Makefile: set; Makefile.common: default, set") sys("MANGRP", lkNone, CheckvarUserGroupName) sys("MANMODE", lkNone, CheckvarFileMode) sys("MANOWN", lkNone, CheckvarUserGroupName) @@ -437,54 +443,54 @@ func (gd *GlobalData) InitVartypes() { sys("MASTER_SITE_XCONTRIB", lkShell, CheckvarFetchURL) sys("MASTER_SITE_XEMACS", lkShell, CheckvarFetchURL) pkglist("MESSAGE_SRC", lkShell, CheckvarPathname) - acl("MESSAGE_SUBST", lkShell, CheckvarShellWord, "Makefile.common:a", "Makefile:a", "options.mk:a") + acl("MESSAGE_SUBST", lkShell, CheckvarShellWord, "Makefile.common: append; Makefile: append; options.mk: append") pkg("META_PACKAGE", lkNone, CheckvarYes) sys("MISSING_FEATURES", lkShell, CheckvarIdentifier) - acl("MYSQL_VERSIONS_ACCEPTED", lkShell, enum("51 55 56"), "Makefile:s") + acl("MYSQL_VERSIONS_ACCEPTED", lkShell, enum("51 55 56"), "Makefile: set") usr("MYSQL_VERSION_DEFAULT", lkNone, CheckvarVersion) sys("NM", lkNone, CheckvarShellCommand) sys("NONBINMODE", lkNone, CheckvarFileMode) pkg("NOT_FOR_COMPILER", lkShell, enum("ccache ccc clang distcc f2c gcc hp icc ido mipspro mipspro-ucode pcc sunpro xlc")) pkglist("NOT_FOR_PLATFORM", lkSpace, CheckvarPlatformTriple) pkg("NOT_FOR_UNPRIVILEGED", lkNone, CheckvarYesNo) - acl("NO_BIN_ON_CDROM", lkNone, CheckvarRestricted, "Makefile:s", "Makefile.common:s") - acl("NO_BIN_ON_FTP", lkNone, CheckvarRestricted, "Makefile:s", "Makefile.common:s") - acl("NO_BUILD", lkNone, CheckvarYes, "Makefile:s", "Makefile.common:s", "Makefile.*:ds") + acl("NO_BIN_ON_CDROM", lkNone, CheckvarRestricted, "Makefile, Makefile.common: set") + acl("NO_BIN_ON_FTP", lkNone, CheckvarRestricted, "Makefile, Makefile.common: set") + acl("NO_BUILD", lkNone, CheckvarYes, "Makefile, Makefile.common: set; Makefile.*: default, set") pkg("NO_CHECKSUM", lkNone, CheckvarYes) pkg("NO_CONFIGURE", lkNone, CheckvarYes) - acl("NO_EXPORT_CPP", lkNone, CheckvarYes, "Makefile:s") + acl("NO_EXPORT_CPP", lkNone, CheckvarYes, "Makefile: set") pkg("NO_EXTRACT", lkNone, CheckvarYes) pkg("NO_INSTALL_MANPAGES", lkNone, CheckvarYes) // only has an effect for Imake packages. - acl("NO_PKGTOOLS_REQD_CHECK", lkNone, CheckvarYes, "Makefile:s") - acl("NO_SRC_ON_CDROM", lkNone, CheckvarRestricted, "Makefile:s", "Makefile.common:s") - acl("NO_SRC_ON_FTP", lkNone, CheckvarRestricted, "Makefile:s", "Makefile.common:s") + acl("NO_PKGTOOLS_REQD_CHECK", lkNone, CheckvarYes, "Makefile: set") + acl("NO_SRC_ON_CDROM", lkNone, CheckvarRestricted, "Makefile, Makefile.common: set") + acl("NO_SRC_ON_FTP", lkNone, CheckvarRestricted, "Makefile, Makefile.common: set") pkglist("ONLY_FOR_COMPILER", lkShell, enum("ccc clang gcc hp icc ido mipspro mipspro-ucode pcc sunpro xlc")) pkglist("ONLY_FOR_PLATFORM", lkSpace, CheckvarPlatformTriple) pkg("ONLY_FOR_UNPRIVILEGED", lkNone, CheckvarYesNo) sys("OPSYS", lkNone, CheckvarIdentifier) - acl("OPSYSVARS", lkShell, CheckvarVarname, "Makefile:a", "Makefile.common:a") - acl("OSVERSION_SPECIFIC", lkNone, CheckvarYes, "Makefile:s", "Makefile.common:s") + acl("OPSYSVARS", lkShell, CheckvarVarname, "Makefile, Makefile.common: append") + acl("OSVERSION_SPECIFIC", lkNone, CheckvarYes, "Makefile, Makefile.common: set") sys("OS_VERSION", lkNone, CheckvarVersion) pkg("OVERRIDE_DIRDEPTH*", lkNone, CheckvarInteger) pkg("OVERRIDE_GNU_CONFIG_SCRIPTS", lkNone, CheckvarYes) - acl("OWNER", lkNone, CheckvarMailAddress, "Makefile:s", "Makefile.common:d") + acl("OWNER", lkNone, CheckvarMailAddress, "Makefile: set; Makefile.common: default") pkglist("OWN_DIRS", lkShell, CheckvarPathname) pkglist("OWN_DIRS_PERMS", lkShell, CheckvarShellWord) sys("PAMBASE", lkNone, CheckvarPathname) usr("PAM_DEFAULT", lkNone, enum("linux-pam openpam solaris-pam")) - acl("PATCHDIR", lkNone, CheckvarRelativePkgPath, "Makefile:s", "Makefile.common:ds") + acl("PATCHDIR", lkNone, CheckvarRelativePkgPath, "Makefile: set; Makefile.common: default, set") pkglist("PATCHFILES", lkShell, CheckvarFilename) - acl("PATCH_ARGS", lkShell, CheckvarShellWord) - acl("PATCH_DIST_ARGS", lkShell, CheckvarShellWord, "Makefile:as") - acl("PATCH_DIST_CAT", lkNone, CheckvarShellCommand) - acl("PATCH_DIST_STRIP*", lkNone, CheckvarShellWord, "Makefile:s", "Makefile.common:s", "buildlink3.mk:", "builtin.mk:", "*.mk:s") - acl("PATCH_SITES", lkShell, CheckvarURL, "Makefile:s", "options.mk:s", "Makefile.common:s") - acl("PATCH_STRIP", lkNone, CheckvarShellWord) + acl("PATCH_ARGS", lkShell, CheckvarShellWord, "") + acl("PATCH_DIST_ARGS", lkShell, CheckvarShellWord, "Makefile: set, append") + acl("PATCH_DIST_CAT", lkNone, CheckvarShellCommand, "") + acl("PATCH_DIST_STRIP*", lkNone, CheckvarShellWord, "Makefile, Makefile.common: set; buildlink3.mk:; builtin.mk:; *.mk: set") + acl("PATCH_SITES", lkShell, CheckvarURL, "Makefile: set; options.mk: set; Makefile.common: set") + acl("PATCH_STRIP", lkNone, CheckvarShellWord, "") pkg("PERL5_USE_PACKLIST", lkNone, CheckvarYesNo) - acl("PERL5_PACKLIST", lkShell, CheckvarPerl5Packlist, "Makefile:s", "options.mk:sa") - acl("PERL5_PACKLIST_DIR", lkNone, CheckvarPathname) + acl("PERL5_PACKLIST", lkShell, CheckvarPerl5Packlist, "Makefile: set; options.mk: set, append") + acl("PERL5_PACKLIST_DIR", lkNone, CheckvarPathname, "") sys("PGSQL_PREFIX", lkNone, CheckvarPathname) - acl("PGSQL_VERSIONS_ACCEPTED", lkShell, enum("91 92 93 94")) + acl("PGSQL_VERSIONS_ACCEPTED", lkShell, enum("91 92 93 94"), "") usr("PGSQL_VERSION_DEFAULT", lkNone, CheckvarVersion) sys("PG_LIB_EXT", lkNone, enum("dylib so")) sys("PGSQL_TYPE", lkNone, enum("postgresql81-client postgresql80-client")) @@ -492,7 +498,8 @@ func (gd *GlobalData) InitVartypes() { sys("PHASE_MSG", lkNone, CheckvarShellCommand) usr("PHP_VERSION_REQD", lkNone, CheckvarVersion) sys("PKGBASE", lkNone, CheckvarIdentifier) - acl("PKGCONFIG_OVERRIDE", lkShell, CheckvarPathmask, "Makefile:as", "Makefile.common:a") + acl("PKGCONFIG_FILE.*", lkShell, CheckvarPathname, "builtin.mk: set, append; pkgconfig-builtin.mk: use-loadtime") + acl("PKGCONFIG_OVERRIDE", lkShell, CheckvarPathmask, "Makefile: set, append; Makefile.common: append") pkg("PKGCONFIG_OVERRIDE_STAGE", lkNone, CheckvarStage) pkg("PKGDIR", lkNone, CheckvarRelativePkgDir) sys("PKGDIRMODE", lkNone, CheckvarFileMode) @@ -500,11 +507,11 @@ func (gd *GlobalData) InitVartypes() { pkg("PKGNAME", lkNone, CheckvarPkgName) sys("PKGNAME_NOREV", lkNone, CheckvarPkgName) sys("PKGPATH", lkNone, CheckvarPathname) - acl("PKGREPOSITORY", lkNone, CheckvarUnchecked) - acl("PKGREVISION", lkNone, CheckvarPkgRevision, "Makefile:s") + acl("PKGREPOSITORY", lkNone, CheckvarUnchecked, "") + acl("PKGREVISION", lkNone, CheckvarPkgRevision, "Makefile: set") sys("PKGSRCDIR", lkNone, CheckvarPathname) - acl("PKGSRCTOP", lkNone, CheckvarYes, "Makefile:s") - acl("PKGTOOLS_ENV", lkShell, CheckvarShellWord) + acl("PKGSRCTOP", lkNone, CheckvarYes, "Makefile: set") + acl("PKGTOOLS_ENV", lkShell, CheckvarShellWord, "") sys("PKGVERSION", lkNone, CheckvarVersion) sys("PKGWILDCARD", lkNone, CheckvarFilemask) sys("PKG_ADMIN", lkNone, CheckvarShellCommand) @@ -520,67 +527,67 @@ func (gd *GlobalData) InitVartypes() { cmdline("PKG_DEBUG_LEVEL", lkNone, CheckvarInteger) usr("PKG_DEFAULT_OPTIONS", lkShell, CheckvarOption) sys("PKG_DELETE", lkNone, CheckvarShellCommand) - acl("PKG_DESTDIR_SUPPORT", lkShell, enum("destdir user-destdir"), "Makefile:s", "Makefile.common:s") + acl("PKG_DESTDIR_SUPPORT", lkShell, enum("destdir user-destdir"), "Makefile, Makefile.common: set") pkglist("PKG_FAIL_REASON", lkShell, CheckvarShellWord) - acl("PKG_GECOS.*", lkNone, CheckvarMessage, "Makefile:s") - acl("PKG_GID.*", lkNone, CheckvarInteger, "Makefile:s") - acl("PKG_GROUPS", lkShell, CheckvarShellWord, "Makefile:as") + acl("PKG_GECOS.*", lkNone, CheckvarMessage, "Makefile: set") + acl("PKG_GID.*", lkNone, CheckvarInteger, "Makefile: set") + acl("PKG_GROUPS", lkShell, CheckvarShellWord, "Makefile: set, append") pkglist("PKG_GROUPS_VARS", lkShell, CheckvarVarname) - acl("PKG_HOME.*", lkNone, CheckvarPathname, "Makefile:s") - acl("PKG_HACKS", lkShell, CheckvarIdentifier, "hacks.mk:a") + acl("PKG_HOME.*", lkNone, CheckvarPathname, "Makefile: set") + acl("PKG_HACKS", lkShell, CheckvarIdentifier, "hacks.mk: append") sys("PKG_INFO", lkNone, CheckvarShellCommand) sys("PKG_JAVA_HOME", lkNone, CheckvarPathname) - jvms := enum("{ blackdown-jdk13 jdk jdk14 kaffe run-jdk13 sun-jdk14 sun-jdk15 sun-jdk6 openjdk7 openjdk7-bin sun-jdk7}") + jvms := enum("blackdown-jdk13 jdk jdk14 kaffe run-jdk13 sun-jdk14 sun-jdk15 sun-jdk6 openjdk7 openjdk7-bin sun-jdk7") sys("PKG_JVM", lkNone, jvms) - acl("PKG_JVMS_ACCEPTED", lkShell, jvms, "Makefile:s", "Makefile.common:ds") + acl("PKG_JVMS_ACCEPTED", lkShell, jvms, "Makefile: set; Makefile.common: default, set") usr("PKG_JVM_DEFAULT", lkNone, jvms) - acl("PKG_LEGACY_OPTIONS", lkShell, CheckvarOption) - acl("PKG_LIBTOOL", lkNone, CheckvarPathname, "Makefile:s") - acl("PKG_OPTIONS", lkSpace, CheckvarOption, "bsd.options.mk:s", "*:pu") + acl("PKG_LEGACY_OPTIONS", lkShell, CheckvarOption, "") + acl("PKG_LIBTOOL", lkNone, CheckvarPathname, "Makefile: set") + acl("PKG_OPTIONS", lkSpace, CheckvarOption, "bsd.options.mk: set; *: use-loadtime, use") usr("PKG_OPTIONS.*", lkSpace, CheckvarOption) - acl("PKG_OPTIONS_DEPRECATED_WARNINGS", lkShell, CheckvarShellWord) - acl("PKG_OPTIONS_GROUP.*", lkSpace, CheckvarOption, "options.mk:s", "Makefile:s") - acl("PKG_OPTIONS_LEGACY_OPTS", lkSpace, CheckvarUnchecked, "Makefile:a", "Makefile.common:a", "options.mk:a") - acl("PKG_OPTIONS_LEGACY_VARS", lkSpace, CheckvarUnchecked, "Makefile:a", "Makefile.common:a", "options.mk:a") - acl("PKG_OPTIONS_NONEMPTY_SETS", lkSpace, CheckvarIdentifier) - acl("PKG_OPTIONS_OPTIONAL_GROUPS", lkSpace, CheckvarIdentifier, "options.mk:as") - acl("PKG_OPTIONS_REQUIRED_GROUPS", lkSpace, CheckvarIdentifier, "options.mk:s", "Makefile:s") - acl("PKG_OPTIONS_SET.*", lkSpace, CheckvarOption) - acl("PKG_OPTIONS_VAR", lkNone, CheckvarPkgOptionsVar, "options.mk:s", "Makefile:s", "Makefile.common:s", "bsd.options.mk:p") - acl("PKG_PRESERVE", lkNone, CheckvarYes, "Makefile:s") - acl("PKG_SHELL", lkNone, CheckvarPathname, "Makefile:s", "Makefile.common:s") - acl("PKG_SHELL.*", lkNone, CheckvarPathname, "Makefile:s", "Makefile.common:s") - acl("PKG_SHLIBTOOL", lkNone, CheckvarPathname) + acl("PKG_OPTIONS_DEPRECATED_WARNINGS", lkShell, CheckvarShellWord, "") + acl("PKG_OPTIONS_GROUP.*", lkSpace, CheckvarOption, "options.mk: set; Makefile: set") + acl("PKG_OPTIONS_LEGACY_OPTS", lkSpace, CheckvarUnchecked, "Makefile, Makefile.common: append; options.mk: append") + acl("PKG_OPTIONS_LEGACY_VARS", lkSpace, CheckvarUnchecked, "Makefile, Makefile.common: append; options.mk: append") + acl("PKG_OPTIONS_NONEMPTY_SETS", lkSpace, CheckvarIdentifier, "") + acl("PKG_OPTIONS_OPTIONAL_GROUPS", lkSpace, CheckvarIdentifier, "options.mk: set, append") + acl("PKG_OPTIONS_REQUIRED_GROUPS", lkSpace, CheckvarIdentifier, "options.mk: set; Makefile: set") + acl("PKG_OPTIONS_SET.*", lkSpace, CheckvarOption, "") + acl("PKG_OPTIONS_VAR", lkNone, CheckvarPkgOptionsVar, "options.mk: set; Makefile, Makefile.common: set; bsd.options.mk: use-loadtime") + acl("PKG_PRESERVE", lkNone, CheckvarYes, "Makefile: set") + acl("PKG_SHELL", lkNone, CheckvarPathname, "Makefile, Makefile.common: set") + acl("PKG_SHELL.*", lkNone, CheckvarPathname, "Makefile, Makefile.common: set") + acl("PKG_SHLIBTOOL", lkNone, CheckvarPathname, "") pkglist("PKG_SKIP_REASON", lkShell, CheckvarShellWord) - acl("PKG_SUGGESTED_OPTIONS", lkShell, CheckvarOption, "options.mk:as", "Makefile:as", "Makefile.common:s") - acl("PKG_SUPPORTED_OPTIONS", lkShell, CheckvarOption, "options.mk:as", "Makefile:as", "Makefile.common:s") + acl("PKG_SUGGESTED_OPTIONS", lkShell, CheckvarOption, "options.mk: set, append; Makefile: set, append; Makefile.common: set") + acl("PKG_SUPPORTED_OPTIONS", lkShell, CheckvarOption, "options.mk: set, append; Makefile: set, append; Makefile.common: set") pkg("PKG_SYSCONFDIR*", lkNone, CheckvarPathname) pkglist("PKG_SYSCONFDIR_PERMS", lkShell, CheckvarShellWord) sys("PKG_SYSCONFBASEDIR", lkNone, CheckvarPathname) pkg("PKG_SYSCONFSUBDIR", lkNone, CheckvarPathname) - acl("PKG_SYSCONFVAR", lkNone, CheckvarIdentifier) // FIXME: name/type mismatch.") - acl("PKG_UID", lkNone, CheckvarInteger, "Makefile:s") - acl("PKG_USERS", lkShell, CheckvarShellWord, "Makefile:as") + acl("PKG_SYSCONFVAR", lkNone, CheckvarIdentifier, "") // FIXME: name/type mismatch. + acl("PKG_UID", lkNone, CheckvarInteger, "Makefile: set") + acl("PKG_USERS", lkShell, CheckvarShellWord, "Makefile: set, append") pkg("PKG_USERS_VARS", lkShell, CheckvarVarname) - acl("PKG_USE_KERBEROS", lkNone, CheckvarYes, "Makefile:s", "Makefile.common:s") + acl("PKG_USE_KERBEROS", lkNone, CheckvarYes, "Makefile, Makefile.common: set") // PLIST.* has special handling code pkglist("PLIST_VARS", lkShell, CheckvarIdentifier) pkglist("PLIST_SRC", lkShell, CheckvarRelativePkgPath) pkglist("PLIST_SUBST", lkShell, CheckvarShellWord) - acl("PLIST_TYPE", lkNone, enum("dynamic static")) - acl("PREPEND_PATH", lkShell, CheckvarPathname) - acl("PREFIX", lkNone, CheckvarPathname, "*:u") - acl("PREV_PKGPATH", lkNone, CheckvarPathname, "*:u") // doesn't exist any longer - acl("PRINT_PLIST_AWK", lkNone, CheckvarAwkCommand, "*:a") - acl("PRIVILEGED_STAGES", lkShell, enum("install package clean")) - acl("PTHREAD_AUTO_VARS", lkNone, CheckvarYesNo, "Makefile:s") + acl("PLIST_TYPE", lkNone, enum("dynamic static"), "") + acl("PREPEND_PATH", lkShell, CheckvarPathname, "") + acl("PREFIX", lkNone, CheckvarPathname, "*: use") + acl("PREV_PKGPATH", lkNone, CheckvarPathname, "*: use") // doesn't exist any longer + acl("PRINT_PLIST_AWK", lkNone, CheckvarAwkCommand, "*: append") + acl("PRIVILEGED_STAGES", lkShell, enum("install package clean"), "") + acl("PTHREAD_AUTO_VARS", lkNone, CheckvarYesNo, "Makefile: set") sys("PTHREAD_CFLAGS", lkShell, CheckvarCFlag) sys("PTHREAD_LDFLAGS", lkShell, CheckvarLdFlag) sys("PTHREAD_LIBS", lkShell, CheckvarLdFlag) - acl("PTHREAD_OPTS", lkShell, enum("native optional require"), "Makefile:as", "Makefile.common:a", "buildlink3.mk:a") + acl("PTHREAD_OPTS", lkShell, enum("native optional require"), "Makefile: set, append; Makefile.common: append; buildlink3.mk: append") sys("PTHREAD_TYPE", lkNone, CheckvarIdentifier) // Or "native" or "none". pkg("PY_PATCHPLIST", lkNone, CheckvarYes) - acl("PYPKGPREFIX", lkNone, enum("py27 py33 py34"), "*:pu", "pyversion.mk:s", "*:") + acl("PYPKGPREFIX", lkNone, enum("py27 py33 py34"), "pyversion.mk: set; *: use-loadtime, use") pkg("PYTHON_FOR_BUILD_ONLY", lkNone, CheckvarYes) pkglist("REPLACE_PYTHON", lkShell, CheckvarPathmask) pkg("PYTHON_VERSIONS_ACCEPTED", lkShell, CheckvarVersion) @@ -590,14 +597,14 @@ func (gd *GlobalData) InitVartypes() { pkglist("PYTHON_VERSIONED_DEPENDENCIES", lkShell, CheckvarPythonDependency) sys("RANLIB", lkNone, CheckvarShellCommand) pkglist("RCD_SCRIPTS", lkShell, CheckvarFilename) - acl("RCD_SCRIPT_SRC.*", lkShell, CheckvarPathname, "Makefile:s") - acl("REPLACE.*", lkNone, CheckvarString, "Makefile:s") + acl("RCD_SCRIPT_SRC.*", lkShell, CheckvarPathname, "Makefile: set") + acl("REPLACE.*", lkNone, CheckvarString, "Makefile: set") pkglist("REPLACE_AWK", lkShell, CheckvarPathmask) pkglist("REPLACE_BASH", lkShell, CheckvarPathmask) pkglist("REPLACE_CSH", lkShell, CheckvarPathmask) - acl("REPLACE_EMACS", lkShell, CheckvarPathmask) - acl("REPLACE_FILES.*", lkShell, CheckvarPathmask, "Makefile:as", "Makefile.common:as") - acl("REPLACE_INTERPRETER", lkShell, CheckvarIdentifier, "Makefile:a", "Makefile.common:a") + acl("REPLACE_EMACS", lkShell, CheckvarPathmask, "") + acl("REPLACE_FILES.*", lkShell, CheckvarPathmask, "Makefile, Makefile.common: set, append") + acl("REPLACE_INTERPRETER", lkShell, CheckvarIdentifier, "Makefile, Makefile.common: append") pkglist("REPLACE_KSH", lkShell, CheckvarPathmask) pkglist("REPLACE_LOCALEDIR_PATTERNS", lkShell, CheckvarFilemask) pkglist("REPLACE_LUA", lkShell, CheckvarPathmask) @@ -614,61 +621,63 @@ func (gd *GlobalData) InitVartypes() { usr("ROOT_GROUP", lkNone, CheckvarUserGroupName) usr("RUBY_VERSION_REQD", lkNone, CheckvarVersion) sys("RUN", lkNone, CheckvarShellCommand) - acl("SCRIPTS_ENV", lkShell, CheckvarShellWord, "Makefile:a", "Makefile.common:a") + sys("RUN_LDCONFIG", lkNone, CheckvarYesNo) + acl("SCRIPTS_ENV", lkShell, CheckvarShellWord, "Makefile, Makefile.common: append") usr("SETUID_ROOT_PERMS", lkShell, CheckvarShellWord) sys("SHAREGRP", lkNone, CheckvarUserGroupName) sys("SHAREMODE", lkNone, CheckvarFileMode) sys("SHAREOWN", lkNone, CheckvarUserGroupName) sys("SHCOMMENT", lkNone, CheckvarShellCommand) - acl("SHLIB_HANDLING", lkNone, enum("YES NO no")) - acl("SHLIBTOOL", lkNone, CheckvarShellCommand) - acl("SHLIBTOOL_OVERRIDE", lkShell, CheckvarPathmask, "Makefile:as", "Makefile.common:a") - acl("SITES.*", lkShell, CheckvarFetchURL, "Makefile:asu", "Makefile.common:asu", "options.mk:asu") + acl("SHLIB_HANDLING", lkNone, enum("YES NO no"), "") + acl("SHLIBTOOL", lkNone, CheckvarShellCommand, "") + acl("SHLIBTOOL_OVERRIDE", lkShell, CheckvarPathmask, "Makefile: set, append; Makefile.common: append") + acl("SITES.*", lkShell, CheckvarFetchURL, "Makefile, Makefile.common, options.mk: set, append, use") pkglist("SPECIAL_PERMS", lkShell, CheckvarShellWord) sys("STEP_MSG", lkNone, CheckvarShellCommand) - acl("SUBDIR", lkShell, CheckvarFilename, "Makefile:a", "*:") - acl("SUBST_CLASSES", lkShell, CheckvarIdentifier, "Makefile:a", "Makefile.common:a", "hacks.mk:a", "Makefile.*:a") - acl("SUBST_FILES.*", lkShell, CheckvarPathmask, "Makefile:as", "Makefile.common:as", "hacks.mk:as", "options.mk:as", "Makefile.*:as") - acl("SUBST_FILTER_CMD.*", lkNone, CheckvarShellCommand, "Makefile:s", "Makefile.common:s", "hacks.mk:s", "options.mk:s", "Makefile.*:s") - acl("SUBST_MESSAGE.*", lkNone, CheckvarMessage, "Makefile:s", "Makefile.common:s", "hacks.mk:s", "options.mk:s", "Makefile.*:s") - acl("SUBST_SED.*", lkNone, CheckvarSedCommands, "Makefile:as", "Makefile.common:as", "hacks.mk:as", "options.mk:as", "Makefile.*:as") + acl("SUBDIR", lkShell, CheckvarFilename, "Makefile: append; *:") + acl("SUBST_CLASSES", lkShell, CheckvarIdentifier, "Makefile: set, append; *: append") + acl("SUBST_FILES.*", lkShell, CheckvarPathmask, "Makefile: set, append; Makefile.*, *.mk: set, append") + acl("SUBST_FILTER_CMD.*", lkNone, CheckvarShellCommand, "Makefile, Makefile.*, *.mk: set") + acl("SUBST_MESSAGE.*", lkNone, CheckvarMessage, "Makefile, Makefile.*, *.mk: set") + acl("SUBST_SED.*", lkNone, CheckvarSedCommands, "Makefile, Makefile.*, *.mk: set, append") pkg("SUBST_STAGE.*", lkNone, CheckvarStage) pkglist("SUBST_VARS.*", lkShell, CheckvarVarname) pkglist("SUPERSEDES", lkSpace, CheckvarDependency) pkglist("TEST_DIRS", lkShell, CheckvarWrksrcSubdirectory) pkglist("TEST_ENV", lkShell, CheckvarShellWord) - acl("TEST_TARGET", lkShell, CheckvarIdentifier, "Makefile:s", "Makefile.common:ds", "options.mk:as") - acl("TEX_ACCEPTED", lkShell, enum("teTeX1 teTeX2 teTeX3"), "Makefile:s", "Makefile.common:s") - acl("TEX_DEPMETHOD", lkNone, enum("build run"), "Makefile:s", "Makefile.common:s") + acl("TEST_TARGET", lkShell, CheckvarIdentifier, "Makefile: set; Makefile.common: default, set; options.mk: set, append") + acl("TEX_ACCEPTED", lkShell, enum("teTeX1 teTeX2 teTeX3"), "Makefile, Makefile.common: set") + acl("TEX_DEPMETHOD", lkNone, enum("build run"), "Makefile, Makefile.common: set") pkglist("TEXINFO_REQD", lkShell, CheckvarVersion) - acl("TOOL_DEPENDS", lkSpace, CheckvarDependencyWithPath, "Makefile.common:a", "Makefile:a", "options.mk:a", "*.mk:a") + acl("TOOL_DEPENDS", lkSpace, CheckvarDependencyWithPath, "Makefile, Makefile.common, *.mk: append") sys("TOOLS_ALIASES", lkShell, CheckvarFilename) sys("TOOLS_BROKEN", lkShell, CheckvarTool) + sys("TOOLS_CMD.*", lkNone, CheckvarPathname) sys("TOOLS_CREATE", lkShell, CheckvarTool) - sys("TOOLS_DEPENDS.*", lkSpace, CheckvarDependencyWithPath) + acl("TOOLS_DEPENDS.*", lkSpace, CheckvarDependencyWithPath, "buildlink3.mk:; Makefile, Makefile.*: set, default; *: use") sys("TOOLS_GNU_MISSING", lkShell, CheckvarTool) sys("TOOLS_NOOP", lkShell, CheckvarTool) sys("TOOLS_PATH.*", lkNone, CheckvarPathname) sys("TOOLS_PLATFORM.*", lkNone, CheckvarShellCommand) sys("TOUCH_FLAGS", lkShell, CheckvarShellWord) pkglist("UAC_REQD_EXECS", lkShell, CheckvarPrefixPathname) - acl("UNLIMIT_RESOURCES", lkShell, enum("datasize stacksize memorysize"), "Makefile:as", "Makefile.common:a") + acl("UNLIMIT_RESOURCES", lkShell, enum("datasize stacksize memorysize"), "Makefile: set, append; Makefile.common: append") usr("UNPRIVILEGED_USER", lkNone, CheckvarUserGroupName) usr("UNPRIVILEGED_GROUP", lkNone, CheckvarUserGroupName) pkglist("UNWRAP_FILES", lkShell, CheckvarPathmask) usr("UPDATE_TARGET", lkShell, CheckvarIdentifier) pkg("USE_BSD_MAKEFILE", lkNone, CheckvarYes) - acl("USE_BUILTIN.*", lkNone, CheckvarYesNoIndirectly, "builtin.mk:s") + acl("USE_BUILTIN.*", lkNone, CheckvarYesNoIndirectly, "builtin.mk: set") pkg("USE_CMAKE", lkNone, CheckvarYes) - acl("USE_CROSSBASE", lkNone, CheckvarYes, "Makefile:s") + acl("USE_CROSSBASE", lkNone, CheckvarYes, "Makefile: set") pkg("USE_FEATURES", lkShell, CheckvarIdentifier) pkg("USE_GCC_RUNTIME", lkNone, CheckvarYesNo) pkg("USE_GNU_CONFIGURE_HOST", lkNone, CheckvarYesNo) - acl("USE_GNU_ICONV", lkNone, CheckvarYes, "Makefile:s", "Makefile.common:s", "options.mk:s") - acl("USE_IMAKE", lkNone, CheckvarYes, "Makefile:s") + acl("USE_GNU_ICONV", lkNone, CheckvarYes, "Makefile, Makefile.common: set; options.mk: set") + acl("USE_IMAKE", lkNone, CheckvarYes, "Makefile: set") pkg("USE_JAVA", lkNone, enum("run yes build")) pkg("USE_JAVA2", lkNone, enum("YES yes no 1.4 1.5 6 7 8")) - acl("USE_LANGUAGES", lkShell, enum("ada c c99 c++ fortran fortran77 java objc"), "Makefile:s", "Makefile.common:s", "options.mk:s") + acl("USE_LANGUAGES", lkShell, enum("ada c c99 c++ fortran fortran77 java objc"), "Makefile, Makefile.common, options.mk: set") pkg("USE_LIBTOOL", lkNone, CheckvarYes) pkg("USE_MAKEINFO", lkNone, CheckvarYes) pkg("USE_MSGFMT_PLURALS", lkNone, CheckvarYes) @@ -677,17 +686,17 @@ func (gd *GlobalData) InitVartypes() { pkg("USE_PKGINSTALL", lkNone, CheckvarYes) pkg("USE_PKGLOCALEDIR", lkNone, CheckvarYesNo) usr("USE_PKGSRC_GCC", lkNone, CheckvarYes) - acl("USE_TOOLS", lkShell, CheckvarTool, "*:a") + acl("USE_TOOLS", lkShell, CheckvarTool, "*: append") pkg("USE_X11", lkNone, CheckvarYes) sys("WARNING_MSG", lkNone, CheckvarShellCommand) sys("WARNING_CAT", lkNone, CheckvarShellCommand) - acl("WRAPPER_REORDER_CMDS", lkShell, CheckvarWrapperReorder, "buildlink3.mk:a", "Makefile.common:a", "Makefile:a") - acl("WRAPPER_TRANSFORM_CMDS", lkShell, CheckvarWrapperTransform, "buildlink3.mk:a", "Makefile.common:a", "Makefile:a") + acl("WRAPPER_REORDER_CMDS", lkShell, CheckvarWrapperReorder, "Makefile, Makefile.common, buildlink3.mk: append") + acl("WRAPPER_TRANSFORM_CMDS", lkShell, CheckvarWrapperTransform, "Makefile, Makefile.common, buildlink3.mk: append") sys("WRKDIR", lkNone, CheckvarPathname) pkg("WRKSRC", lkNone, CheckvarWrkdirSubdirectory) sys("X11_PKGSRCDIR.*", lkNone, CheckvarPathname) usr("XAW_TYPE", lkNone, enum("3d neXtaw standard xpm")) - acl("XMKMF_FLAGS", lkShell, CheckvarShellWord) + acl("XMKMF_FLAGS", lkShell, CheckvarShellWord, "") } func enum(values string) *VarChecker { @@ -698,16 +707,16 @@ func enum(values string) *VarChecker { name := "enum: " + values + " " // See IsEnum return &VarChecker{name, func(ctx *VartypeCheck) { if !vmap[ctx.value] { - ctx.line.warnf("%q is not valid for %s. Use one of { %s } instead.", ctx.value, ctx.varname, values) + ctx.line.Warnf("%q is not valid for %s. Use one of { %s } instead.", ctx.value, ctx.varname, values) } }} } -func acl(varname string, kindOfList KindOfList, checker *VarChecker, aclentries ...string) { - m := mustMatch(`^([A-Z_.][A-Z0-9_]*)(|\*|\.\*)$`, varname) +func acl(varname string, kindOfList KindOfList, checker *VarChecker, aclentries string) { + m := mustMatch(varname, `^([A-Z_.][A-Z0-9_]*)(|\*|\.\*)$`) varbase, varparam := m[1], m[2] - vtype := &Vartype{kindOfList, checker, parseAclEntries(aclentries), guNotGuessed} + vtype := &Vartype{kindOfList, checker, parseAclEntries(varname, aclentries), false} if G.globalData.vartypes == nil { G.globalData.vartypes = make(map[string]*Vartype) @@ -720,25 +729,67 @@ func acl(varname string, kindOfList KindOfList, checker *VarChecker, aclentries } } -func parseAclEntries(args []string) []AclEntry { +func parseAclEntries(varname string, aclentries string) []AclEntry { + if aclentries == "" { + return nil + } var result []AclEntry - for _, arg := range args { - m := mustMatch(`^([\w.*]+|_):([adpsu]*)$`, arg) - glob, perms := m[1], m[2] - result = append(result, AclEntry{glob, perms}) + for _, arg := range strings.Split(aclentries, "; ") { + var globs, perms string + if fields := strings.SplitN(arg, ": ", 2); len(fields) == 2 { + globs, perms = fields[0], fields[1] + } else { + globs = strings.TrimSuffix(arg, ":") + } + var permissions AclPermissions + for _, perm := range strings.Split(perms, ", ") { + switch perm { + case "append": + permissions |= aclpAppend + case "default": + permissions |= aclpSetDefault + case "set": + permissions |= aclpSet + case "use": + permissions |= aclpUse + case "use-loadtime": + permissions |= aclpUseLoadtime + case "": + break + default: + print(fmt.Sprintf("Invalid ACL permission %q for varname %q.\n", perm, varname)) + } + } + for _, glob := range strings.Split(globs, ", ") { + switch glob { + case "*", + "Makefile", "Makefile.common", "Makefile.*", + "buildlink3.mk", "builtin.mk", "options.mk", "hacks.mk", "*.mk", + "bsd.options.mk", "pkgconfig-builtin.mk", "pyversion.mk": + break + default: + print(fmt.Sprintf("Invalid ACL glob %q for varname %q.\n", glob, varname)) + } + for _, prev := range result { + if matched, err := path.Match(prev.glob, glob); err != nil || matched { + print(fmt.Sprintf("Ineffective ACL glob %q for varname %q.\n", glob, varname)) + } + } + result = append(result, AclEntry{glob, permissions}) + } } return result } // A package-defined variable may be set in all Makefiles except buildlink3.mk and builtin.mk. func pkg(varname string, kindOfList KindOfList, checker *VarChecker) { - acl(varname, kindOfList, checker, "Makefile:su", "Makefile.common:dsu", "buildlink3.mk:", "builtin.mk:", "*.mk:dsu") + acl(varname, kindOfList, checker, "Makefile: set, use; buildlink3.mk, builtin.mk:; Makefile.*, *.mk: default, set, use") } // A package-defined list may be appended to in all Makefiles except buildlink3.mk and builtin.mk. // Simple assignment (instead of appending) is only allowed in Makefile and Makefile.common. func pkglist(varname string, kindOfList KindOfList, checker *VarChecker) { - acl(varname, kindOfList, checker, "Makefile:asu", "Makefile.common:asu", "buildlink3.mk:", "builtin.mk:", "*.mk:au") + acl(varname, kindOfList, checker, "Makefile, Makefile.common: append, set, use; buildlink3.mk, builtin.mk:; *.mk: append, use") } // A user-defined or system-defined variable must not be set by any @@ -746,14 +797,14 @@ func pkglist(varname string, kindOfList KindOfList, checker *VarChecker) { // builtin.mk files or at load-time, since the system/user preferences // may not have been loaded when these files are included. func sys(varname string, kindOfList KindOfList, checker *VarChecker) { - acl(varname, kindOfList, checker, "buildlink3.mk:", "builtin.mk:u", "*:u") + acl(varname, kindOfList, checker, "buildlink3.mk:; *: use") } func usr(varname string, kindOfList KindOfList, checker *VarChecker) { - acl(varname, kindOfList, checker, "buildlink3.mk:", "builtin.mk:", "*:u") + acl(varname, kindOfList, checker, "buildlink3.mk:; *: use-loadtime, use") } func bl3list(varname string, kindOfList KindOfList, checker *VarChecker) { - acl(varname, kindOfList, checker, "buildlink3.mk:a", "builtin.mk:a") + acl(varname, kindOfList, checker, "buildlink3.mk, builtin.mk: append") } func cmdline(varname string, kindOfList KindOfList, checker *VarChecker) { - acl(varname, kindOfList, checker, "buildlink3.mk:", "builtin.mk:", "*:pu") + acl(varname, kindOfList, checker, "buildlink3.mk, builtin.mk:; *: use-loadtime, use") } diff --git a/pkgtools/pkglint/files/vars.go b/pkgtools/pkglint/files/vars.go deleted file mode 100644 index 089ab71b2fc..00000000000 --- a/pkgtools/pkglint/files/vars.go +++ /dev/null @@ -1,99 +0,0 @@ -package main - -type NeedsQuoting int - -const ( - nqNo NeedsQuoting = iota - nqYes - nqDoesntMatter - nqDontKnow -) - -func variableNeedsQuoting(line *Line, varname string, vuc *VarUseContext) NeedsQuoting { - defer tracecall("variableNeedsQuoting", varname, *vuc)() - - vartype := getVariableType(line, varname) - if vartype == nil || vuc.vartype == nil { - return nqDontKnow - } - - isPlainWord := vartype.checker.IsEnum() - switch vartype.checker.name { - case "DistSuffix", - "FileMode", "Filename", - "Identifier", - "Option", - "Pathname", "PkgName", "PkgOptionsVar", "PkgRevision", - "RelativePkgDir", "RelativePkgPath", - "UserGroupName", - "Varname", "Version", - "WrkdirSubdirectory": - isPlainWord = true - } - if isPlainWord { - if vartype.kindOfList == lkNone { - return nqDoesntMatter - } - if vartype.kindOfList == lkShell && vuc.extent != vucExtentWordpart { - return nqNo - } - } - - // In .for loops, the :Q operator is always misplaced, since - // the items are broken up at white-space, not as shell words - // like in all other parts of make(1). - if vuc.shellword == vucQuotFor { - return nqNo - } - - // Determine whether the context expects a list of shell words or not. - wantList := vuc.vartype.isConsideredList() && (vuc.shellword == vucQuotBackt || vuc.extent != vucExtentWordpart) - haveList := vartype.isConsideredList() - - _ = G.opts.DebugQuoting && line.debugf( - "variableNeedsQuoting: varname=%q, context=%v, type=%v, wantList=%v, haveList=%v", - varname, vuc, vartype, wantList, haveList) - - // A shell word may appear as part of a shell word, for example COMPILER_RPATH_FLAG. - if vuc.extent == vucExtentWordpart && vuc.shellword == vucQuotPlain { - if vartype.kindOfList == lkNone && vartype.checker.name == "ShellWord" { - return nqNo - } - } - - // Assuming the tool definitions don't include very special characters, - // so they can safely be used inside any quotes. - if G.globalData.varnameToToolname[varname] != "" { - shellword := vuc.shellword - switch { - case shellword == vucQuotPlain && vuc.extent != vucExtentWordpart: - return nqNo - case shellword == vucQuotBackt: - return nqNo - case shellword == vucQuotDquot || shellword == vucQuotSquot: - return nqDoesntMatter - } - } - - // Variables that appear as parts of shell words generally need - // to be quoted. An exception is in the case of backticks, - // because the whole backticks expression is parsed as a single - // shell word by pkglint. - if vuc.extent == vucExtentWordpart && vuc.shellword != vucQuotBackt { - return nqYes - } - - // Assigning lists to lists does not require any quoting, though - // there may be cases like "CONFIGURE_ARGS+= -libs ${LDFLAGS:Q}" - // where quoting is necessary. - if wantList && haveList { - return nqDoesntMatter - } - - if wantList != haveList { - return nqYes - } - - _ = G.opts.DebugQuoting && line.debugf("Don't know whether :Q is needed for %q", varname) - return nqDontKnow -} diff --git a/pkgtools/pkglint/files/vars_test.go b/pkgtools/pkglint/files/vars_test.go deleted file mode 100644 index 19c5b1c5910..00000000000 --- a/pkgtools/pkglint/files/vars_test.go +++ /dev/null @@ -1,17 +0,0 @@ -package main - -import ( - check "gopkg.in/check.v1" -) - -func (s *Suite) TestVariableNeedsQuoting(c *check.C) { - line := NewLine("fname", "1", "dummy", nil) - G.globalData.InitVartypes() - pkgnameType := G.globalData.vartypes["PKGNAME"] - - // In Makefile: PKGNAME := ${UNKNOWN} - vuc := &VarUseContext{vucTimeParse, pkgnameType, vucQuotUnknown, vucExtentUnknown} - nq := variableNeedsQuoting(line, "UNKNOWN", vuc) - - c.Check(nq, equals, nqDontKnow) -} diff --git a/pkgtools/pkglint/files/vartype.go b/pkgtools/pkglint/files/vartype.go index de1b723d39e..686565e8dda 100644 --- a/pkgtools/pkglint/files/vartype.go +++ b/pkgtools/pkglint/files/vartype.go @@ -11,10 +11,10 @@ type Vartype struct { kindOfList KindOfList checker *VarChecker aclEntries []AclEntry - guessed Guessed + guessed bool } -type KindOfList int +type KindOfList uint8 const ( lkNone KindOfList = iota // Plain data type @@ -24,64 +24,86 @@ const ( type AclEntry struct { glob string // Examples: "Makefile", "*.mk" - permissions string // Some of: "a"ppend, "d"efault, "s"et; "p"reprocessing, "u"se + permissions AclPermissions } -// Guessed says whether the type definition is guessed (based on the -// variable name) or explicitly defined (see vardefs.go). -type Guessed bool +type AclPermissions uint8 const ( - guNotGuessed Guessed = false - guGuessed Guessed = true + aclpSet AclPermissions = 1 << iota // VAR = value + aclpSetDefault // VAR ?= value + aclpAppend // VAR += value + aclpUseLoadtime // OTHER := ${VAR}, OTHER != ${VAR} + aclpUse // OTHER = ${VAR} + aclpUnknown + aclpAll AclPermissions = aclpAppend | aclpSetDefault | aclpSet | aclpUseLoadtime | aclpUse + aclpAllRuntime AclPermissions = aclpAppend | aclpSetDefault | aclpSet | aclpUse + aclpAllWrite AclPermissions = aclpSet | aclpSetDefault | aclpAppend + aclpAllRead AclPermissions = aclpUseLoadtime | aclpUse ) -// The allowed actions in this file, or "?" if unknown. -func (vt *Vartype) effectivePermissions(fname string) string { +func (perms AclPermissions) Contains(subset AclPermissions) bool { + return perms&subset == subset +} + +func (perms AclPermissions) String() string { + if perms == 0 { + return "none" + } + result := "" + + ifelseStr(perms.Contains(aclpSet), "set, ", "") + + ifelseStr(perms.Contains(aclpSetDefault), "set-default, ", "") + + ifelseStr(perms.Contains(aclpAppend), "append, ", "") + + ifelseStr(perms.Contains(aclpUseLoadtime), "use-loadtime, ", "") + + ifelseStr(perms.Contains(aclpUse), "use, ", "") + + ifelseStr(perms.Contains(aclpUnknown), "unknown, ", "") + return strings.TrimRight(result, ", ") +} + +func (perms AclPermissions) HumanString() string { + result := "" + + ifelseStr(perms.Contains(aclpSet), "set, ", "") + + ifelseStr(perms.Contains(aclpSetDefault), "given a default value, ", "") + + ifelseStr(perms.Contains(aclpAppend), "appended to, ", "") + + ifelseStr(perms.Contains(aclpUseLoadtime), "used at load time, ", "") + + ifelseStr(perms.Contains(aclpUse), "used, ", "") + return strings.TrimRight(result, ", ") +} + +func (vt *Vartype) EffectivePermissions(fname string) AclPermissions { for _, aclEntry := range vt.aclEntries { if m, _ := path.Match(aclEntry.glob, path.Base(fname)); m { return aclEntry.permissions } } - return "?" -} - -func ReadableVartypePermissions(perms string) string { - result := "" - for _, c := range perms { - switch c { - case 'a': - result += "append, " - case 'd': - result += "default, " - case 'p': - result += "preprocess, " - case 's': - result += "set, " - case 'u': - result += "runtime, " - case '?': - result += "unknown, " - } - } - return strings.TrimRight(result, ", ") + return aclpUnknown } // Returns the union of all possible permissions. This can be used to // check whether a variable may be defined or used at all, or if it is // read-only. -func (vt *Vartype) union() string { - var permissions string +func (vt *Vartype) Union() AclPermissions { + var permissions AclPermissions for _, aclEntry := range vt.aclEntries { - permissions += aclEntry.permissions + permissions |= aclEntry.permissions } return permissions } +func (vt *Vartype) AllowedFiles(perms AclPermissions) string { + files := make([]string, 0, len(vt.aclEntries)) + for _, aclEntry := range vt.aclEntries { + if aclEntry.permissions.Contains(perms) { + files = append(files, aclEntry.glob) + } + } + return strings.Join(files, ", ") +} + // Returns whether the type is considered a shell list. // This distinction between “real lists” and “considered a list” makes // the implementation of checklineMkVartype easier. -func (vt *Vartype) isConsideredList() bool { +func (vt *Vartype) IsConsideredList() bool { switch vt.kindOfList { case lkShell: return true @@ -89,16 +111,14 @@ func (vt *Vartype) isConsideredList() bool { return false } switch vt.checker { - case CheckvarSedCommands, CheckvarShellCommand: + case CheckvarAwkCommand, CheckvarSedCommands, CheckvarShellCommand, CheckvarShellCommands: return true } return false } -func (vt *Vartype) mayBeAppendedTo() bool { - return vt.kindOfList != lkNone || - vt.checker == CheckvarAwkCommand || - vt.checker == CheckvarSedCommands +func (vt *Vartype) MayBeAppendedTo() bool { + return vt.kindOfList != lkNone || vt.IsConsideredList() } func (vt *Vartype) String() string { @@ -123,7 +143,7 @@ func (vc *VarChecker) IsEnum() bool { return hasPrefix(vc.name, "enum: ") } func (vc *VarChecker) HasEnum(value string) bool { - return !matches(value, `\s`) && contains(vc.name, " "+value+" ") + return !contains(value, " ") && contains(vc.name, " "+value+" ") } func (vc *VarChecker) AllowedEnums() string { return vc.name[6 : len(vc.name)-1] @@ -168,6 +188,7 @@ var ( CheckvarSedCommand = &VarChecker{"SedCommand", (*VartypeCheck).SedCommand} CheckvarSedCommands = &VarChecker{"SedCommands", nil} CheckvarShellCommand = &VarChecker{"ShellCommand", nil} + CheckvarShellCommands = &VarChecker{"ShellCommands", nil} CheckvarShellWord = &VarChecker{"ShellWord", nil} CheckvarStage = &VarChecker{"Stage", (*VartypeCheck).Stage} CheckvarString = &VarChecker{"String", (*VartypeCheck).String} @@ -189,5 +210,6 @@ var ( func init() { // Necessary due to circular dependency CheckvarSedCommands.checker = (*VartypeCheck).SedCommands CheckvarShellCommand.checker = (*VartypeCheck).ShellCommand + CheckvarShellCommands.checker = (*VartypeCheck).ShellCommands CheckvarShellWord.checker = (*VartypeCheck).ShellWord } diff --git a/pkgtools/pkglint/files/vartype_test.go b/pkgtools/pkglint/files/vartype_test.go index e4b21558df6..c4fbf490c7e 100644 --- a/pkgtools/pkglint/files/vartype_test.go +++ b/pkgtools/pkglint/files/vartype_test.go @@ -11,17 +11,17 @@ func (s *Suite) TestVartypeEffectivePermissions(c *check.C) { t := G.globalData.vartypes["PREFIX"] c.Check(t.checker.name, equals, "Pathname") - c.Check(t.aclEntries, check.DeepEquals, []AclEntry{{glob: "*", permissions: "u"}}) - c.Check(t.effectivePermissions("Makefile"), equals, "u") + c.Check(t.aclEntries, check.DeepEquals, []AclEntry{{glob: "*", permissions: aclpUse}}) + c.Check(t.EffectivePermissions("Makefile"), equals, aclpUse) } { t := G.globalData.vartypes["EXTRACT_OPTS"] c.Check(t.checker.name, equals, "ShellWord") - c.Check(t.effectivePermissions("Makefile"), equals, "as") - c.Check(t.effectivePermissions("../Makefile"), equals, "as") - c.Check(t.effectivePermissions("options.mk"), equals, "?") + c.Check(t.EffectivePermissions("Makefile"), equals, aclpAppend|aclpSet) + c.Check(t.EffectivePermissions("../Makefile"), equals, aclpAppend|aclpSet) + c.Check(t.EffectivePermissions("options.mk"), equals, aclpUnknown) } } @@ -32,3 +32,17 @@ func (s *Suite) TestVarCheckerHasEnum(c *check.C) { c.Check(vc.HasEnum("middle"), equals, true) c.Check(vc.HasEnum("maninstall"), equals, true) } + +func (s *Suite) TestAclPermissions_contains(c *check.C) { + perms := aclpAllRuntime + + c.Check(perms.Contains(aclpAllRuntime), equals, true) + c.Check(perms.Contains(aclpUse), equals, true) + c.Check(perms.Contains(aclpUseLoadtime), equals, false) +} + +func (s *Suite) TestAclPermissions_String(c *check.C) { + c.Check(AclPermissions(0).String(), equals, "none") + c.Check(aclpAll.String(), equals, "set, set-default, append, use-loadtime, use") + c.Check(aclpUnknown.String(), equals, "unknown") +} diff --git a/pkgtools/pkglint/files/vartypecheck.go b/pkgtools/pkglint/files/vartypecheck.go index ad5cbec0acc..f37269ed01f 100644 --- a/pkgtools/pkglint/files/vartypecheck.go +++ b/pkgtools/pkglint/files/vartypecheck.go @@ -6,32 +6,69 @@ import ( ) type VartypeCheck struct { + mkline *MkLine line *Line varname string - op string + op MkOperator value string valueNovar string comment string listContext bool - guessed Guessed + guessed bool // Whether the type definition is guessed (based on the variable name) or explicitly defined (see vardefs.go). +} + +type MkOperator uint8 + +const ( + opAssign MkOperator = iota // = + opAssignShell // != + opAssignEval // := + opAssignAppend // += + opAssignDefault // ?= + opUseLoadtime + opUse +) + +func NewMkOperator(op string) MkOperator { + switch op { + case "=": + return opAssign + case "!=": + return opAssignShell + case ":=": + return opAssignEval + case "+=": + return opAssignAppend + case "?=": + return opAssignDefault + } + return opAssign +} + +func (op MkOperator) String() string { + return [...]string{"=", "!=", ":=", "+=", "?=", "use", "use-loadtime"}[op] } func (cv *VartypeCheck) AwkCommand() { - _ = G.opts.DebugUnchecked && cv.line.debugf("Unchecked AWK command: %q", cv.value) + if G.opts.DebugUnchecked { + cv.line.Debug1("Unchecked AWK command: %q", cv.value) + } } func (cv *VartypeCheck) BasicRegularExpression() { - _ = G.opts.DebugUnchecked && cv.line.debugf("Unchecked basic regular expression: %q", cv.value) + if G.opts.DebugUnchecked { + cv.line.Debug1("Unchecked basic regular expression: %q", cv.value) + } } func (cv *VartypeCheck) BuildlinkDepmethod() { if !containsVarRef(cv.value) && cv.value != "build" && cv.value != "full" { - cv.line.warnf("Invalid dependency method %q. Valid methods are \"build\" or \"full\".", cv.value) + cv.line.Warn1("Invalid dependency method %q. Valid methods are \"build\" or \"full\".", cv.value) } } func (cv *VartypeCheck) Category() { - if fileExists(G.currentDir + "/" + G.curPkgsrcdir + "/" + cv.value + "/Makefile") { + if fileExists(G.CurrentDir + "/" + G.CurPkgsrcdir + "/" + cv.value + "/Makefile") { return } switch cv.value { @@ -48,22 +85,27 @@ func (cv *VartypeCheck) Category() { "windowmaker", "xmms": default: - cv.line.errorf("Invalid category %q.", cv.value) + cv.line.Error1("Invalid category %q.", cv.value) } } // A single option to the C/C++ compiler. func (cv *VartypeCheck) CFlag() { - line, value := cv.line, cv.value - + cflag := cv.value switch { - case matches(value, `^-[DILOUWfgm]`), - hasPrefix(value, "-std="), - value == "-c99": - case hasPrefix(value, "-"): - line.warnf("Unknown compiler flag %q.", value) - case !containsVarRef(value): - line.warnf("Compiler flag %q should start with a hyphen.", value) + case matches(cflag, `^-[DILOUWfgm]`), + hasPrefix(cflag, "-std="), + cflag == "-c99", + cflag == "-c", + cflag == "-no-integrated-as", + cflag == "-pthread", + hasPrefix(cflag, "`") && hasSuffix(cflag, "`"), + containsVarRef(cflag): + return + case hasPrefix(cflag, "-"): + cv.line.Warn1("Unknown compiler flag %q.", cflag) + default: + cv.line.Warn1("Compiler flag %q should start with a hyphen.", cflag) } } @@ -71,78 +113,86 @@ func (cv *VartypeCheck) CFlag() { func (cv *VartypeCheck) Comment() { line, value := cv.line, cv.value - if value == "SHORT_DESCRIPTION_OF_THE_PACKAGE" { - line.errorf("COMMENT must be set.") + if value == "TODO: Short description of the package" { // See pkgtools/url2pkg/files/url2pkg.pl, keyword "COMMENT". + line.Error0("COMMENT must be set.") } if m, first := match1(value, `^(?i)(a|an)\s`); m { - line.warnf("COMMENT should not begin with %q.", first) + line.Warn1("COMMENT should not begin with %q.", first) } if matches(value, `^[a-z]`) { - line.warnf("COMMENT should start with a capital letter.") + line.Warn0("COMMENT should start with a capital letter.") } if hasSuffix(value, ".") { - line.warnf("COMMENT should not end with a period.") + line.Warn0("COMMENT should not end with a period.") } if len(value) > 70 { - line.warnf("COMMENT should not be longer than 70 characters.") + line.Warn0("COMMENT should not be longer than 70 characters.") } } func (cv *VartypeCheck) Dependency() { line, value := cv.line, cv.value - if m, depbase, depop, depversion := match3(value, `^(`+rePkgbase+`)(<|=|>|<=|>=|!=|-)(`+rePkgversion+`)$`); m { - _, _, _ = depbase, depop, depversion + parser := NewParser(value) + deppat := parser.Dependency() + if deppat != nil && deppat.wildcard == "" && (parser.Rest() == "{,nb*}" || parser.Rest() == "{,nb[0-9]*}") { + line.Warn0("Dependency patterns of the form pkgbase>=1.0 don't need the \"{,nb*}\" extension.") + Explain4( + "The \"{,nb*}\" extension is only necessary for dependencies of the", + "form \"pkgbase-1.2\", since the pattern \"pkgbase-1.2\" doesn't match", + "the version \"pkgbase-1.2nb5\". For dependency patterns using the", + "comparison operators, this is not necessary.") + + } else if deppat == nil || !parser.EOF() { + line.Warn1("Unknown dependency pattern %q.", value) + Explain( + "Typical dependencies have the following forms:", + "", + "\tpackage>=2.5", + "\tpackage-[0-9]*", + "\tpackage-3.141", + "\tpackage>=2.71828<=3.1415") return } - if m, depbase, bracket, version, versionWildcard, other := match5(value, `^(`+rePkgbase+`)-(?:\[(.*)\]\*|(\d+(?:\.\d+)*(?:\.\*)?)(\{,nb\*\}|\*|)|(.*))?$`); m { - switch { - case bracket != "": - if bracket != "0-9" { - line.warnf("Only [0-9]* is allowed in the numeric part of a dependency.") - } - - case version != "" && versionWildcard != "": - // Fine. - - case version != "": - line.warnf("Please append \"{,nb*}\" to the version number of this dependency.") - line.explain( - "Usually, a dependency should stay valid when the PKGREVISION is", - "increased, since those changes are most often editorial. In the", - "current form, the dependency only matches if the PKGREVISION is", - "undefined.") - - case other == "*": - line.warnf("Please use \"%s-[0-9]*\" instead of \"%s-*\".", depbase, depbase) - line.explain( - "If you use a * alone, the package specification may match other", - "packages that have the same prefix, but a longer name. For example,", - "foo-* matches foo-1.2, but also foo-client-1.2 and foo-server-1.2.") - - default: - line.errorf("Unknown dependency pattern %q.", value) + wildcard := deppat.wildcard + if m, inside := match1(wildcard, `^\[(.*)\]\*$`); m { + if inside != "0-9" { + line.Warn0("Only [0-9]* is allowed in the numeric part of a dependency.") } - return - } - switch { - case contains(value, "{"): - // No check yet for alternative dependency patterns. - _ = G.opts.DebugUnchecked && line.debugf("Unchecked alternative dependency pattern: %s", value) + } else if m, ver, suffix := match2(wildcard, `^(\d\w*(?:\.\w+)*)(\.\*|\{,nb\*\}|\{,nb\[0-9\]\*\}|\*|)$`); m { + if suffix == "" { + line.Warn2("Please use %q instead of %q as the version pattern.", ver+"{,nb*}", ver) + Explain3( + "Without the \"{,nb*}\" suffix, this version pattern only matches", + "package versions that don't have a PKGREVISION (which is the part", + "after the \"nb\").") + } + if suffix == "*" { + line.Warn2("Please use %q instead of %q as the version pattern.", ver+".*", ver+"*") + Explain2( + "For example, the version \"1*\" also matches \"10.0.0\", which is", + "probably not intended.") + } - case value != cv.valueNovar: - _ = G.opts.DebugUnchecked && line.debugf("Unchecked dependency: %s", value) + } else if wildcard == "*" { + line.Warn1("Please use \"%[1]s-[0-9]*\" instead of \"%[1]s-*\".", deppat.pkgbase) + Explain3( + "If you use a * alone, the package specification may match other", + "packages that have the same prefix, but a longer name. For example,", + "foo-* matches foo-1.2, but also foo-client-1.2 and foo-server-1.2.") + } - default: - line.warnf("Unknown dependency format: %s", value) - line.explain( - "Typical dependencies have the following forms:", + if nocclasses := regcomp(`\[[\d-]+\]`).ReplaceAllString(wildcard, ""); contains(nocclasses, "-") { + line.Warn1("The version pattern %q should not contain a hyphen.", wildcard) + Explain( + "Pkgsrc interprets package names with version numbers like this:", + "", + "\t\"foo-2.0-2.1.x\" => pkgbase \"foo\", version \"2.0-2.1.x\"", "", - "* package>=2.5", - "* package-[0-9]*", - "* package-3.141") + "To make the \"2.0\" above part of the package basename, the hyphen", + "must be omitted, so the full package name becomes \"foo2.0-2.1.x\".") } } @@ -152,33 +202,31 @@ func (cv *VartypeCheck) DependencyWithPath() { return // It's probably not worth checking this. } - if m, pattern, relpath, _, pkg := match4(value, `(.*):(\.\./\.\./([^/]+)/([^/]+))$`); m { - checklineRelativePkgdir(line, relpath) + if m, pattern, relpath, pkg := match3(value, `(.*):(\.\./\.\./[^/]+/([^/]+))$`); m { + cv.mkline.CheckRelativePkgdir(relpath) switch pkg { case "msgfmt", "gettext": - line.warnf("Please use USE_TOOLS+=msgfmt instead of this dependency.") + line.Warn0("Please use USE_TOOLS+=msgfmt instead of this dependency.") case "perl5": - line.warnf("Please use USE_TOOLS+=perl:run instead of this dependency.") + line.Warn0("Please use USE_TOOLS+=perl:run instead of this dependency.") case "gmake": - line.warnf("Please use USE_TOOLS+=gmake instead of this dependency.") + line.Warn0("Please use USE_TOOLS+=gmake instead of this dependency.") } - if !matches(pattern, reDependencyCmp) && !matches(pattern, reDependencyWildcard) { - line.errorf("Unknown dependency pattern %q.", pattern) - } + cv.mkline.CheckVartypePrimitive(cv.varname, CheckvarDependency, cv.op, pattern, cv.comment, cv.listContext, cv.guessed) return } if matches(value, `:\.\./[^/]+$`) { - line.warnf("Dependencies should have the form \"../../category/package\".") - explainRelativeDirs(line) + line.Warn0("Dependencies should have the form \"../../category/package\".") + cv.mkline.explainRelativeDirs() return } - line.warnf("Unknown dependency format.") - line.explain( - "Examples for valid dependencies are:", + line.Warn1("Unknown dependency pattern with path %q.", value) + Explain4( + "Examples for valid dependency patterns with path are:", " package-[0-9]*:../../category/package", " package>=3.41:../../category/package", " package-2.718:../../category/package") @@ -186,7 +234,7 @@ func (cv *VartypeCheck) DependencyWithPath() { func (cv *VartypeCheck) DistSuffix() { if cv.value == ".tar.gz" { - cv.line.notef("%s is \".tar.gz\" by default, so this definition may be redundant.", cv.varname) + cv.line.Note1("%s is \".tar.gz\" by default, so this definition may be redundant.", cv.varname) } } @@ -194,41 +242,49 @@ func (cv *VartypeCheck) EmulPlatform() { if m, opsys, arch := match2(cv.value, `^(\w+)-(\w+)$`); m { if !matches(opsys, `^(?:bitrig|bsdos|cygwin|darwin|dragonfly|freebsd|haiku|hpux|interix|irix|linux|mirbsd|netbsd|openbsd|osf1|sunos|solaris)$`) { - cv.line.warnf("Unknown operating system: %s", opsys) + cv.line.Warnf("Unknown operating system: %s", opsys) } // no check for os_version if !matches(arch, `^(?:i386|alpha|amd64|arc|arm|arm32|cobalt|convex|dreamcast|hpcmips|hpcsh|hppa|ia64|m68k|m88k|mips|mips64|mipsel|mipseb|mipsn32|ns32k|pc532|pmax|powerpc|rs6000|s390|sparc|sparc64|vax|x86_64)$`) { - cv.line.warnf("Unknown hardware architecture: %s", arch) + cv.line.Warn1("Unknown hardware architecture: %s", arch) } } else { - cv.line.warnf("%q is not a valid emulation platform.", cv.value) - cv.line.explain( + cv.line.Warn1("%q is not a valid emulation platform.", cv.value) + Explain( "An emulation platform has the form <OPSYS>-<MACHINE_ARCH>.", - "OPSYS is the lower-case name of the operating system, and MACHINE_ARCH", - "is the hardware architecture.", + "OPSYS is the lower-case name of the operating system, and", + "MACHINE_ARCH is the hardware architecture.", "", "Examples: linux-i386, irix-mipsel.") } } func (cv *VartypeCheck) FetchURL() { - NewMkLine(cv.line).checkVartypePrimitive(cv.varname, CheckvarURL, cv.op, cv.value, cv.comment, cv.listContext, cv.guessed) + cv.mkline.CheckVartypePrimitive(cv.varname, CheckvarURL, cv.op, cv.value, cv.comment, cv.listContext, cv.guessed) - for siteUrl, siteName := range G.globalData.masterSiteUrls { - if hasPrefix(cv.value, siteUrl) { - subdir := cv.value[len(siteUrl):] - isGithub := hasPrefix(cv.value, "https://github.com/") - if isGithub { + for siteURL, siteName := range G.globalData.MasterSiteUrls { + if hasPrefix(cv.value, siteURL) { + subdir := cv.value[len(siteURL):] + if hasPrefix(cv.value, "https://github.com/") { subdir = strings.SplitAfter(subdir, "/")[0] - } - cv.line.warnf("Please use ${%s:=%s} instead of %q.", siteName, subdir, cv.value) - if isGithub { - cv.line.warnf("Run \"%s help topic=github\" for further tips.", confMake) + cv.line.Warnf("Please use ${%s:=%s} instead of %q and run \"%s help topic=github\" for further tips.", + siteName, subdir, cv.value, confMake) + } else { + cv.line.Warnf("Please use ${%s:=%s} instead of %q.", siteName, subdir, cv.value) } return } } + + if m, name, subdir := match2(cv.value, `\$\{(MASTER_SITE_[^:]*).*:=(.*)\}$`); m { + if !G.globalData.MasterSiteVars[name] { + cv.line.Error1("%s does not exist.", name) + } + if !hasSuffix(subdir, "/") { + cv.line.Error1("The subdirectory in %s must end with a slash.", name) + } + } } // See Pathname @@ -236,15 +292,15 @@ func (cv *VartypeCheck) FetchURL() { func (cv *VartypeCheck) Filename() { switch { case contains(cv.valueNovar, "/"): - cv.line.warnf("A filename should not contain a slash.") + cv.line.Warn0("A filename should not contain a slash.") case !matches(cv.valueNovar, `^[-0-9@A-Za-z.,_~+%]*$`): - cv.line.warnf("%q is not a valid filename.", cv.value) + cv.line.Warn1("%q is not a valid filename.", cv.value) } } func (cv *VartypeCheck) Filemask() { if !matches(cv.valueNovar, `^[-0-9A-Za-z._~+%*?]*$`) { - cv.line.warnf("%q is not a valid filename mask.", cv.value) + cv.line.Warn1("%q is not a valid filename mask.", cv.value) } } @@ -255,7 +311,7 @@ func (cv *VartypeCheck) FileMode() { case matches(cv.value, `^[0-7]{3,4}`): // Fine. default: - cv.line.warnf("Invalid file mode %q.", cv.value) + cv.line.Warn1("Invalid file mode %q.", cv.value) } } @@ -269,32 +325,42 @@ func (cv *VartypeCheck) Identifier() { case cv.value != "" && cv.valueNovar == "": // Don't warn here. default: - cv.line.warnf("Invalid identifier %q.", cv.value) + cv.line.Warn1("Invalid identifier %q.", cv.value) } } func (cv *VartypeCheck) Integer() { if !matches(cv.value, `^\d+$`) { - cv.line.warnf("Invalid integer %q.", cv.value) + cv.line.Warn1("Invalid integer %q.", cv.value) } } func (cv *VartypeCheck) LdFlag() { - if matches(cv.value, `^-[Ll]`) || cv.value == "-static" { + ldflag := cv.value + if m, rpathFlag := match1(ldflag, `^(-Wl,(?:-R|-rpath|--rpath))`); m { + cv.line.Warn1("Please use \"${COMPILER_RPATH_FLAG}\" instead of %q.", rpathFlag) return - } else if m, rpathFlag := match1(cv.value, `^(-Wl,(?:-R|-rpath|--rpath))`); m { - cv.line.warnf("Please use ${COMPILER_RPATH_FLAG} instead of %s.", rpathFlag) - - } else if hasPrefix(cv.value, "-") { - cv.line.warnf("Unknown linker flag %q.", cv.value) + } - } else if cv.value == cv.valueNovar { - cv.line.warnf("Linker flag %q does not start with a dash.", cv.value) + switch { + case hasPrefix(ldflag, "-L"), + hasPrefix(ldflag, "-l"), + ldflag == "-pthread", + ldflag == "-static", + hasPrefix(ldflag, "-static-"), + hasPrefix(ldflag, "-Wl,-"), + hasPrefix(ldflag, "`") && hasSuffix(ldflag, "`"), + ldflag != cv.valueNovar: + return + case hasPrefix(ldflag, "-"): + cv.line.Warn1("Unknown linker flag %q.", cv.value) + default: + cv.line.Warn1("Linker flag %q should start with a hypen.", cv.value) } } func (cv *VartypeCheck) License() { - checklineLicense(cv.line, cv.value) + checklineLicense(cv.mkline, cv.value) } func (cv *VartypeCheck) MailAddress() { @@ -302,14 +368,14 @@ func (cv *VartypeCheck) MailAddress() { if m, _, domain := match2(value, `^([+\-.0-9A-Z_a-z]+)@([-\w\d.]+)$`); m { if strings.EqualFold(domain, "NetBSD.org") && domain != "NetBSD.org" { - line.warnf("Please write \"NetBSD.org\" instead of %q.", domain) + line.Warn1("Please write \"NetBSD.org\" instead of %q.", domain) } if matches(value, `(?i)^(tech-pkg|packages)@NetBSD\.org$`) { - line.errorf("This mailing list address is obsolete. Use pkgsrc-users@NetBSD.org instead.") + line.Error0("This mailing list address is obsolete. Use pkgsrc-users@NetBSD.org instead.") } } else { - line.warnf("\"%s\" is not a valid mail address.", value) + line.Warn1("\"%s\" is not a valid mail address.", value) } } @@ -318,15 +384,15 @@ func (cv *VartypeCheck) Message() { line, varname, value := cv.line, cv.varname, cv.value if matches(value, `^[\"'].*[\"']$`) { - line.warnf("%s should not be quoted.", varname) - line.explain( + line.Warn1("%s should not be quoted.", varname) + Explain( "The quoting is only needed for variables which are interpreted as", - "multiple words (or, generally speaking, a list of something). A single", - "text message does not belong to this class, since it is only printed", - "as a whole.", + "multiple words (or, generally speaking, a list of something). A", + "single text message does not belong to this class, since it is only", + "printed as a whole.", "", - "On the other hand, PKG_FAIL_REASON is a _list_ of text messages, so in", - "that case, the quoting has to be done.`") + "On the other hand, PKG_FAIL_REASON is a _list_ of text messages, so", + "in that case, the quoting has to be done.") } } @@ -335,34 +401,36 @@ func (cv *VartypeCheck) Option() { line, value, valueNovar := cv.line, cv.value, cv.valueNovar if value != valueNovar { - _ = G.opts.DebugUnchecked && line.debugf("Unchecked option name: %q", value) + if G.opts.DebugUnchecked { + line.Debug1("Unchecked option name: %q", value) + } return } - if m, optname := match1(value, `^-?([a-z][-0-9a-z\+]*)$`); m { - if _, found := G.globalData.pkgOptions[optname]; !found { // There’s a difference between empty and absent here. - line.warnf("Unknown option \"%s\".", optname) - line.explain( + if m, optname := match1(value, `^-?([a-z][-0-9a-z+]*)$`); m { + if _, found := G.globalData.PkgOptions[optname]; !found { // There’s a difference between empty and absent here. + line.Warn1("Unknown option \"%s\".", optname) + Explain4( "This option is not documented in the mk/defaults/options.description", - "file. If this is not a typo, please think of a brief but precise", - "description and either update that file yourself or ask on the", - "tech-pkg@NetBSD.org mailing list.") + "file. Please think of a brief but precise description and either", + "update that file yourself or suggest a description for this option", + "on the tech-pkg@NetBSD.org mailing list.") } return } if matches(value, `^-?([a-z][-0-9a-z_\+]*)$`) { - line.warnf("Use of the underscore character in option names is deprecated.") + line.Warn0("Use of the underscore character in option names is deprecated.") return } - line.errorf("Invalid option name.") + line.Error1("Invalid option name %q. Option names must start with a lowercase letter and be all-lowercase.", value) } // The PATH environment variable func (cv *VartypeCheck) Pathlist() { - if !contains(cv.value, ":") && cv.guessed == guGuessed { - NewMkLine(cv.line).checkVartypePrimitive(cv.varname, CheckvarPathname, cv.op, cv.value, cv.comment, cv.listContext, cv.guessed) + if !contains(cv.value, ":") && cv.guessed { + cv.mkline.CheckVartypePrimitive(cv.varname, CheckvarPathname, cv.op, cv.value, cv.comment, cv.listContext, cv.guessed) return } @@ -372,11 +440,11 @@ func (cv *VartypeCheck) Pathlist() { } if !matches(path, `^[-0-9A-Za-z._~+%/]*$`) { - cv.line.warnf("%q is not a valid pathname.", path) + cv.line.Warn1("%q is not a valid pathname.", path) } if !hasPrefix(path, "/") { - cv.line.warnf("All components of %s (in this case %q) should be absolute paths.", cv.varname, path) + cv.line.Warn2("All components of %s (in this case %q) should be absolute paths.", cv.varname, path) } } } @@ -385,37 +453,37 @@ func (cv *VartypeCheck) Pathlist() { // See Filemask func (cv *VartypeCheck) Pathmask() { if !matches(cv.valueNovar, `^[#\-0-9A-Za-z._~+%*?/\[\]]*`) { - cv.line.warnf("%q is not a valid pathname mask.", cv.value) + cv.line.Warn1("%q is not a valid pathname mask.", cv.value) } - cv.line.checkAbsolutePathname(cv.value) + cv.line.CheckAbsolutePathname(cv.value) } // Like Filename, but including slashes // See http://www.opengroup.org/onlinepubs/009695399/basedefs/xbd_chap03.html#tag_03_266 func (cv *VartypeCheck) Pathname() { if !matches(cv.valueNovar, `^[#\-0-9A-Za-z._~+%/]*$`) { - cv.line.warnf("%q is not a valid pathname.", cv.value) + cv.line.Warn1("%q is not a valid pathname.", cv.value) } - cv.line.checkAbsolutePathname(cv.value) + cv.line.CheckAbsolutePathname(cv.value) } func (cv *VartypeCheck) Perl5Packlist() { if cv.value != cv.valueNovar { - cv.line.warnf("%s should not depend on other variables.", cv.varname) + cv.line.Warn1("%s should not depend on other variables.", cv.varname) } } func (cv *VartypeCheck) PkgName() { if cv.value == cv.valueNovar && !matches(cv.value, rePkgname) { - cv.line.warnf("%q is not a valid package name. A valid package name has the form packagename-version, where version consists only of digits, letters and dots.", cv.value) + cv.line.Warn1("%q is not a valid package name. A valid package name has the form packagename-version, where version consists only of digits, letters and dots.", cv.value) } } func (cv *VartypeCheck) PkgOptionsVar() { - NewMkLine(cv.line).checkVartypePrimitive(cv.varname, CheckvarVarname, cv.op, cv.value, cv.comment, false, cv.guessed) + cv.mkline.CheckVartypePrimitive(cv.varname, CheckvarVarname, cv.op, cv.value, cv.comment, false, cv.guessed) if matches(cv.value, `\$\{PKGBASE[:\}]`) { - cv.line.errorf("PKGBASE must not be used in PKG_OPTIONS_VAR.") - cv.line.explain( + cv.line.Error0("PKGBASE must not be used in PKG_OPTIONS_VAR.") + Explain3( "PKGBASE is defined in bsd.pkg.mk, which is included as the", "very last file, but PKG_OPTIONS_VAR is evaluated earlier.", "Use ${PKGNAME:C/-[0-9].*//} instead.") @@ -425,21 +493,21 @@ func (cv *VartypeCheck) PkgOptionsVar() { // A directory name relative to the top-level pkgsrc directory. // Despite its name, it is more similar to RelativePkgDir than to RelativePkgPath. func (cv *VartypeCheck) PkgPath() { - checklineRelativePkgdir(cv.line, G.curPkgsrcdir+"/"+cv.value) + cv.mkline.CheckRelativePkgdir(G.CurPkgsrcdir + "/" + cv.value) } func (cv *VartypeCheck) PkgRevision() { if !matches(cv.value, `^[1-9]\d*$`) { - cv.line.warnf("%s must be a positive integer number.", cv.varname) + cv.line.Warn1("%s must be a positive integer number.", cv.varname) } - if path.Base(cv.line.fname) != "Makefile" { - cv.line.errorf("%s only makes sense directly in the package Makefile.", cv.varname) - cv.line.explain( + if path.Base(cv.line.Fname) != "Makefile" { + cv.line.Error1("%s only makes sense directly in the package Makefile.", cv.varname) + Explain( "Usually, different packages using the same Makefile.common have", - "different dependencies and will be bumped at different times (e.g. for", - "shlib major bumps) and thus the PKGREVISIONs must be in the separate", - "Makefiles. There is no practical way of having this information in a", - "commonly used Makefile.") + "different dependencies and will be bumped at different times (e.g.", + "for shlib major bumps) and thus the PKGREVISIONs must be in the", + "separate Makefiles. There is no practical way of having this", + "information in a commonly used Makefile.") } } @@ -452,16 +520,16 @@ func (cv *VartypeCheck) PlatformTriple() { reTriple := `^(` + rePart + `)-(` + rePart + `)-(` + rePart + `)$` if m, opsys, _, arch := match3(cv.value, reTriple); m { if !matches(opsys, `^(?:\*|Bitrig|BSDOS|Cygwin|Darwin|DragonFly|FreeBSD|Haiku|HPUX|Interix|IRIX|Linux|MirBSD|NetBSD|OpenBSD|OSF1|QNX|SunOS)$`) { - cv.line.warnf("Unknown operating system: %s", opsys) + cv.line.Warnf("Unknown operating system: %s", opsys) } // no check for os_version if !matches(arch, `^(?:\*|i386|alpha|amd64|arc|arm|arm32|cobalt|convex|dreamcast|hpcmips|hpcsh|hppa|ia64|m68k|m88k|mips|mips64|mipsel|mipseb|mipsn32|ns32k|pc532|pmax|powerpc|rs6000|s390|sparc|sparc64|vax|x86_64)$`) { - cv.line.warnf("Unknown hardware architecture: %s", arch) + cv.line.Warn1("Unknown hardware architecture: %s", arch) } } else { - cv.line.warnf("%q is not a valid platform triple.", cv.value) - cv.line.explain( + cv.line.Warn1("%q is not a valid platform triple.", cv.value) + Explain3( "A platform triple has the form <OPSYS>-<OS_VERSION>-<MACHINE_ARCH>.", "Each of these components may be a shell globbing expression.", "Examples: NetBSD-*-i386, *-*-*, Linux-*-*.") @@ -470,41 +538,40 @@ func (cv *VartypeCheck) PlatformTriple() { func (cv *VartypeCheck) PrefixPathname() { if m, mansubdir := match1(cv.value, `^man/(.+)`); m { - cv.line.warnf("Please use \"${PKGMANDIR}/%s\" instead of %q.", mansubdir, cv.value) + cv.line.Warn2("Please use \"${PKGMANDIR}/%s\" instead of %q.", mansubdir, cv.value) } } func (cv *VartypeCheck) PythonDependency() { if cv.value != cv.valueNovar { - cv.line.warnf("Python dependencies should not contain variables.") - } - if !matches(cv.valueNovar, `^[+\-.0-9A-Z_a-z]+(?:|:link|:build)$`) { - cv.line.warnf("Invalid Python dependency %q.", cv.value) - cv.line.explain( - "Python dependencies must be an identifier for a package, as specified", - "in lang/python/versioned_dependencies.mk. This identifier may be", - "followed by :build for a build-time only dependency, or by :link for", - "a run-time only dependency.") + cv.line.Warn0("Python dependencies should not contain variables.") + } else if !matches(cv.valueNovar, `^[+\-.0-9A-Z_a-z]+(?:|:link|:build)$`) { + cv.line.Warn1("Invalid Python dependency %q.", cv.value) + Explain4( + "Python dependencies must be an identifier for a package, as", + "specified in lang/python/versioned_dependencies.mk. This", + "identifier may be followed by :build for a build-time only", + "dependency, or by :link for a run-time only dependency.") } } // Refers to a package directory. func (cv *VartypeCheck) RelativePkgDir() { - checklineRelativePkgdir(cv.line, cv.value) + cv.mkline.CheckRelativePkgdir(cv.value) } // Refers to a file or directory. func (cv *VartypeCheck) RelativePkgPath() { - checklineRelativePath(cv.line, cv.value, true) + cv.mkline.CheckRelativePath(cv.value, true) } func (cv *VartypeCheck) Restricted() { if cv.value != "${RESTRICTED}" { - cv.line.warnf("The only valid value for %s is ${RESTRICTED}.", cv.varname) - cv.line.explain( - "These variables are used to control which files may be mirrored on FTP", - "servers or CD-ROM collections. They are not intended to mark packages", - "whose only MASTER_SITES are on ftp.NetBSD.org.") + cv.line.Warn1("The only valid value for %s is ${RESTRICTED}.", cv.varname) + Explain3( + "These variables are used to control which files may be mirrored on", + "FTP servers or CD-ROM collections. They are not intended to mark", + "packages whose only MASTER_SITES are on ftp.NetBSD.org.") } } @@ -513,12 +580,14 @@ func (cv *VartypeCheck) SedCommand() { func (cv *VartypeCheck) SedCommands() { line := cv.line + mkline := cv.mkline + shline := NewShellLine(mkline) - words, rest := splitIntoShellwords(line, cv.value) + tokens, rest := splitIntoShellTokens(line, cv.value) if rest != "" { - if contains(cv.value, "#") { - line.errorf("Invalid shell words in sed commands.") - line.explain( + if strings.Contains(line.Text, "#") { + line.Error1("Invalid shell words %q in sed commands.", rest) + Explain4( "When sed commands have embedded \"#\" characters, they need to be", "escaped with a backslash, otherwise make(1) will interpret them as a", "comment, no matter if they occur in single or double quotes or", @@ -527,22 +596,22 @@ func (cv *VartypeCheck) SedCommands() { return } - nwords := len(words) + ntokens := len(tokens) ncommands := 0 - for i := 0; i < nwords; i++ { - word := words[i] - NewMkShellLine(cv.line).checkShellword(word, true) + for i := 0; i < ntokens; i++ { + token := tokens[i] + shline.CheckToken(token, true) switch { - case word == "-e": - if i+1 < nwords { + case token == "-e": + if i+1 < ntokens { // Check the real sed command here. i++ ncommands++ if ncommands > 1 { - line.notef("Each sed command should appear in an assignment of its own.") - line.explain( + line.Note0("Each sed command should appear in an assignment of its own.") + Explain( "For example, instead of", " SUBST_SED.foo+= -e s,command1,, -e s,command2,,", "use", @@ -551,40 +620,46 @@ func (cv *VartypeCheck) SedCommands() { "", "This way, short sed commands cannot be hidden at the end of a line.") } - NewMkShellLine(line).checkShellword(words[i-1], true) - NewMkShellLine(line).checkShellword(words[i], true) - NewMkLine(line).checkVartypePrimitive(cv.varname, CheckvarSedCommand, cv.op, words[i], cv.comment, cv.listContext, cv.guessed) + shline.CheckToken(tokens[i-1], true) + shline.CheckToken(tokens[i], true) + mkline.CheckVartypePrimitive(cv.varname, CheckvarSedCommand, cv.op, tokens[i], cv.comment, cv.listContext, cv.guessed) } else { - line.errorf("The -e option to sed requires an argument.") + line.Error0("The -e option to sed requires an argument.") } - case word == "-E": + case token == "-E": // Switch to extended regular expressions mode. - case word == "-n": + case token == "-n": // Don't print lines per default. - case i == 0 && matches(word, `^(["']?)(?:\d*|/.*/)s.+["']?$`): - line.notef("Please always use \"-e\" in sed commands, even if there is only one substitution.") + case i == 0 && matches(token, `^(["']?)(?:\d*|/.*/)s.+["']?$`): + line.Note0("Please always use \"-e\" in sed commands, even if there is only one substitution.") default: - line.warnf("Unknown sed command %q.", word) + line.Warn1("Unknown sed command %q.", token) } } } func (cv *VartypeCheck) ShellCommand() { - NewMkShellLine(cv.line).checkShelltext(cv.value) + setE := true + NewShellLine(cv.mkline).CheckShellCommand(cv.value, &setE) +} + +// Zero or more shell commands, each terminated with a semicolon. +func (cv *VartypeCheck) ShellCommands() { + NewShellLine(cv.mkline).CheckShellCommands(cv.value) } func (cv *VartypeCheck) ShellWord() { if !cv.listContext { - NewMkShellLine(cv.line).checkShellword(cv.value, true) + NewShellLine(cv.mkline).CheckToken(cv.value, true) } } func (cv *VartypeCheck) Stage() { if !matches(cv.value, `^(?:pre|do|post)-(?:extract|patch|configure|build|test|install)`) { - cv.line.warnf("Invalid stage name %q. Use one of {pre,do,post}-{extract,patch,configure,build,test,install}.", cv.value) + cv.line.Warn1("Invalid stage name %q. Use one of {pre,do,post}-{extract,patch,configure,build,test,install}.", cv.value) } } @@ -593,20 +668,20 @@ func (cv *VartypeCheck) String() { } func (cv *VartypeCheck) Tool() { - if cv.varname == "TOOLS_NOOP" && cv.op == "+=" { + if cv.varname == "TOOLS_NOOP" && cv.op == opAssignAppend { // no warning for package-defined tool definitions } else if m, toolname, tooldep := match2(cv.value, `^([-\w]+|\[)(?::(\w+))?$`); m { - if !G.globalData.tools[toolname] { - cv.line.errorf("Unknown tool %q.", toolname) + if !G.globalData.Tools[toolname] { + cv.line.Error1("Unknown tool %q.", toolname) } switch tooldep { case "", "bootstrap", "build", "pkgsrc", "run": default: - cv.line.errorf("Unknown tool dependency %q. Use one of \"build\", \"pkgsrc\" or \"run\".", tooldep) + cv.line.Error1("Unknown tool dependency %q. Use one of \"build\", \"pkgsrc\" or \"run\".", tooldep) } } else { - cv.line.errorf("Invalid tool syntax: %q.", cv.value) + cv.line.Error1("Invalid tool syntax: %q.", cv.value) } } @@ -620,49 +695,41 @@ func (cv *VartypeCheck) URL() { if value == "" && hasPrefix(cv.comment, "#") { // Ok - } else if m, name, subdir := match2(value, `\$\{(MASTER_SITE_[^:]*).*:=(.*)\}$`); m { - if !G.globalData.masterSiteVars[name] { - line.errorf("%s does not exist.", name) - } - if !hasSuffix(subdir, "/") { - line.errorf("The subdirectory in %s must end with a slash.", name) - } - } else if containsVarRef(value) { // No further checks - } else if m, _, host, _, _ := match4(value, `^(https?|ftp|gopher)://([-0-9A-Za-z.]+)(?::(\d+))?/([-%&+,./0-9:=?@A-Z_a-z~]|#)*$`); m { + } else if m, _, host, _, _ := match4(value, `^(https?|ftp|gopher)://([-0-9A-Za-z.]+)(?::(\d+))?/([-%&+,./0-9:;=?@A-Z_a-z~]|#)*$`); m { if matches(host, `(?i)\.NetBSD\.org$`) && !matches(host, `\.NetBSD\.org$`) { - line.warnf("Please write NetBSD.org instead of %s.", host) + line.Warn1("Please write NetBSD.org instead of %s.", host) } } else if m, scheme, _, absPath := match3(value, `^([0-9A-Za-z]+)://([^/]+)(.*)$`); m { switch { case scheme != "ftp" && scheme != "http" && scheme != "https" && scheme != "gopher": - line.warnf("%q is not a valid URL. Only ftp, gopher, http, and https URLs are allowed here.", value) + line.Warn1("%q is not a valid URL. Only ftp, gopher, http, and https URLs are allowed here.", value) case absPath == "": - line.notef("For consistency, please add a trailing slash to %q.", value) + line.Note1("For consistency, please add a trailing slash to %q.", value) default: - line.warnf("%q is not a valid URL.", value) + line.Warn1("%q is not a valid URL.", value) } } else { - line.warnf("%q is not a valid URL.", value) + line.Warn1("%q is not a valid URL.", value) } } func (cv *VartypeCheck) UserGroupName() { if cv.value == cv.valueNovar && !matches(cv.value, `^[0-9_a-z]+$`) { - cv.line.warnf("Invalid user or group name %q.", cv.value) + cv.line.Warn1("Invalid user or group name %q.", cv.value) } } func (cv *VartypeCheck) Varname() { if cv.value == cv.valueNovar && !matches(cv.value, `^[A-Z_][0-9A-Z_]*(?:[.].*)?$`) { - cv.line.warnf("%q is not a valid variable name.", cv.value) - cv.line.explain( + cv.line.Warn1("%q is not a valid variable name.", cv.value) + Explain( "Variable names are restricted to only uppercase letters and the", "underscore in the basename, and arbitrary characters in the", "parameterized part, following the dot.", @@ -675,30 +742,30 @@ func (cv *VartypeCheck) Varname() { func (cv *VartypeCheck) Version() { if !matches(cv.value, `^([\d.])+$`) { - cv.line.warnf("Invalid version number %q.", cv.value) + cv.line.Warn1("Invalid version number %q.", cv.value) } } func (cv *VartypeCheck) WrapperReorder() { if !matches(cv.value, `^reorder:l:([\w\-]+):([\w\-]+)$`) { - cv.line.warnf("Unknown wrapper reorder command %q.", cv.value) + cv.line.Warn1("Unknown wrapper reorder command %q.", cv.value) } } func (cv *VartypeCheck) WrapperTransform() { - switch { - case matches(cv.value, `^rm:(?:-[DILOUWflm].*|-std=.*)$`): - case matches(cv.value, `^l:([^:]+):(.+)$`): - case matches(cv.value, `^'?(?:opt|rename|rm-optarg|rmdir):.*$`): - case cv.value == "-e": - case matches(cv.value, `^\"?'?s[|:,]`): - default: - cv.line.warnf("Unknown wrapper transform command %q.", cv.value) + cmd := cv.value + if hasPrefix(cmd, "rm:-") || + matches(cmd, `^(R|l|rpath):([^:]+):(.+)$`) || + matches(cmd, `^'?(opt|rename|rm-optarg|rmdir):.*$`) || + cmd == "-e" || + matches(cmd, `^["']?s[|:,]`) { + return } + cv.line.Warn1("Unknown wrapper transform command %q.", cmd) } func (cv *VartypeCheck) WrkdirSubdirectory() { - NewMkLine(cv.line).checkVartypePrimitive(cv.varname, CheckvarPathname, cv.op, cv.value, cv.comment, cv.listContext, cv.guessed) + cv.mkline.CheckVartypePrimitive(cv.varname, CheckvarPathname, cv.op, cv.value, cv.comment, cv.listContext, cv.guessed) } // A directory relative to ${WRKSRC}, for use in CONFIGURE_DIRS and similar variables. @@ -707,23 +774,25 @@ func (cv *VartypeCheck) WrksrcSubdirectory() { if rest == "" { rest = "." } - cv.line.notef("You can use %q instead of %q.", rest, cv.value) + cv.line.Note2("You can use %q instead of %q.", rest, cv.value) + Explain1( + "These directories are interpreted relative to ${WRKSRC}.") } else if cv.value != "" && cv.valueNovar == "" { // The value of another variable } else if !matches(cv.valueNovar, `^(?:\.|[0-9A-Za-z_@][-0-9A-Za-z_@./+]*)$`) { - cv.line.warnf("%q is not a valid subdirectory of ${WRKSRC}.", cv.value) + cv.line.Warn1("%q is not a valid subdirectory of ${WRKSRC}.", cv.value) } } // Used for variables that are checked using `.if defined(VAR)`. func (cv *VartypeCheck) Yes() { if !matches(cv.value, `^(?:YES|yes)(?:\s+#.*)?$`) { - cv.line.warnf("%s should be set to YES or yes.", cv.varname) - cv.line.explain( + cv.line.Warn1("%s should be set to YES or yes.", cv.varname) + Explain4( "This variable means \"yes\" if it is defined, and \"no\" if it is", - "undefined. Even when it has the value \"no\", this means \"yes\".", + "undefined. Even when it has the value \"no\", this means \"yes\".", "Therefore when it is defined, its value should correspond to its", "meaning.") } @@ -734,7 +803,7 @@ func (cv *VartypeCheck) Yes() { // func (cv *VartypeCheck) YesNo() { if !matches(cv.value, `^(?:YES|yes|NO|no)(?:\s+#.*)?$`) { - cv.line.warnf("%s should be set to YES, yes, NO, or no.", cv.varname) + cv.line.Warn1("%s should be set to YES, yes, NO, or no.", cv.varname) } } @@ -742,6 +811,6 @@ func (cv *VartypeCheck) YesNo() { // != operator. func (cv *VartypeCheck) YesNoIndirectly() { if cv.valueNovar != "" && !matches(cv.value, `^(?:YES|yes|NO|no)(?:\s+#.*)?$`) { - cv.line.warnf("%s should be set to YES, yes, NO, or no.", cv.varname) + cv.line.Warn1("%s should be set to YES, yes, NO, or no.", cv.varname) } } diff --git a/pkgtools/pkglint/files/vartypecheck_test.go b/pkgtools/pkglint/files/vartypecheck_test.go index c0023ff6606..a9b866a0091 100644 --- a/pkgtools/pkglint/files/vartypecheck_test.go +++ b/pkgtools/pkglint/files/vartypecheck_test.go @@ -2,267 +2,373 @@ package main import ( check "gopkg.in/check.v1" - "io/ioutil" - "os" ) func (s *Suite) TestVartypeCheck_AwkCommand(c *check.C) { - newVartypeCheck("PLIST_AWK", "+=", "{print $0}").AwkCommand() + s.UseCommandLine(c, "-Dunchecked") + runVartypeChecks("PLIST_AWK", opAssignAppend, (*VartypeCheck).AwkCommand, + "{print $0}", + "{print $$0}") + + c.Check(s.Output(), equals, ""+ + "ERROR: fname:1: Invalid Makefile syntax at \"$0}\".\n"+ + "DEBUG: fname:1: Unchecked AWK command: \"{print $0}\"\n"+ + "DEBUG: fname:2: Unchecked AWK command: \"{print $$0}\"\n") } func (s *Suite) TestVartypeCheck_BasicRegularExpression(c *check.C) { - newVartypeCheck("REPLACE_FILES.pl", "=", ".*\\.pl$").BasicRegularExpression() + runVartypeChecks("REPLACE_FILES.pl", opAssign, (*VartypeCheck).BasicRegularExpression, + ".*\\.pl$", + ".*\\.pl$$") + + c.Check(s.Output(), equals, "ERROR: fname:1: Invalid Makefile syntax at \"$\".\n") } func (s *Suite) TestVartypeCheck_BuildlinkDepmethod(c *check.C) { - newVartypeCheck("BUILDLINK_DEPMETHOD.libc", "?=", "full").BuildlinkDepmethod() - newVartypeCheck("BUILDLINK_DEPMETHOD.libc", "?=", "unknown").BuildlinkDepmethod() + runVartypeChecks("BUILDLINK_DEPMETHOD.libc", opAssignDefault, (*VartypeCheck).BuildlinkDepmethod, + "full", + "unknown") - c.Check(s.Output(), equals, "WARN: fname:1: Invalid dependency method \"unknown\". Valid methods are \"build\" or \"full\".\n") + c.Check(s.Output(), equals, "WARN: fname:2: Invalid dependency method \"unknown\". Valid methods are \"build\" or \"full\".\n") } func (s *Suite) TestVartypeCheck_Category(c *check.C) { - tmpdir := c.MkDir() - categorydir := tmpdir + "/filesyscategory" - categoryMakefile := categorydir + "/Makefile" - os.Mkdir(categorydir, 0777) - ioutil.WriteFile(categoryMakefile, []byte("# Nothing\n"), 0777) - G.currentDir = tmpdir - G.curPkgsrcdir = "." - - newVartypeCheck("CATEGORIES", "=", "chinese").Category() - newVartypeCheck("CATEGORIES", "=", "arabic").Category() - newVartypeCheck("CATEGORIES", "=", "filesyscategory").Category() - - c.Check(s.Output(), equals, "ERROR: fname:1: Invalid category \"arabic\".\n") + s.CreateTmpFile(c, "filesyscategory/Makefile", "# empty\n") + G.CurrentDir = s.tmpdir + G.CurPkgsrcdir = "." + + runVartypeChecks("CATEGORIES", opAssign, (*VartypeCheck).Category, + "chinese", + "arabic", + "filesyscategory") + + c.Check(s.Output(), equals, "ERROR: fname:2: Invalid category \"arabic\".\n") } func (s *Suite) TestVartypeCheck_CFlag(c *check.C) { - newVartypeCheck("CFLAGS", "+=", "-Wall").CFlag() - newVartypeCheck("CFLAGS", "+=", "/W3").CFlag() - newVartypeCheck("CFLAGS", "+=", "target:sparc64").CFlag() - newVartypeCheck("CFLAGS", "+=", "-std=c99").CFlag() - newVartypeCheck("CFLAGS", "+=", "-XX:+PrintClassHistogramAfterFullGC").CFlag() + runVartypeChecks("CFLAGS", opAssignAppend, (*VartypeCheck).CFlag, + "-Wall", + "/W3", + "target:sparc64", + "-std=c99", + "-XX:+PrintClassHistogramAfterFullGC", + "`pkg-config pidgin --cflags`") c.Check(s.Output(), equals, ""+ - "WARN: fname:1: Compiler flag \"/W3\" should start with a hyphen.\n"+ - "WARN: fname:1: Compiler flag \"target:sparc64\" should start with a hyphen.\n"+ - "WARN: fname:1: Unknown compiler flag \"-XX:+PrintClassHistogramAfterFullGC\".\n") + "WARN: fname:2: Compiler flag \"/W3\" should start with a hyphen.\n"+ + "WARN: fname:3: Compiler flag \"target:sparc64\" should start with a hyphen.\n"+ + "WARN: fname:5: Unknown compiler flag \"-XX:+PrintClassHistogramAfterFullGC\".\n") } func (s *Suite) TestVartypeCheck_Comment(c *check.C) { - newVartypeCheck("COMMENT", "=", "Versatile Programming Language").Comment() - newVartypeCheck("COMMENT", "=", "SHORT_DESCRIPTION_OF_THE_PACKAGE").Comment() - newVartypeCheck("COMMENT", "=", "A great package.").Comment() - newVartypeCheck("COMMENT", "=", "some packages need a very very long comment to explain their basic usefulness").Comment() + runVartypeChecks("COMMENT", opAssign, (*VartypeCheck).Comment, + "Versatile Programming Language", + "TODO: Short description of the package", + "A great package.", + "some packages need a very very long comment to explain their basic usefulness") c.Check(s.Output(), equals, ""+ - "ERROR: fname:1: COMMENT must be set.\n"+ - "WARN: fname:1: COMMENT should not begin with \"A\".\n"+ - "WARN: fname:1: COMMENT should not end with a period.\n"+ - "WARN: fname:1: COMMENT should start with a capital letter.\n"+ - "WARN: fname:1: COMMENT should not be longer than 70 characters.\n") + "ERROR: fname:2: COMMENT must be set.\n"+ + "WARN: fname:3: COMMENT should not begin with \"A\".\n"+ + "WARN: fname:3: COMMENT should not end with a period.\n"+ + "WARN: fname:4: COMMENT should start with a capital letter.\n"+ + "WARN: fname:4: COMMENT should not be longer than 70 characters.\n") } func (s *Suite) TestVartypeCheck_Dependency(c *check.C) { - newVartypeCheck("CONFLICTS", "+=", "Perl").Dependency() - - c.Check(s.Output(), equals, "WARN: fname:1: Unknown dependency format: Perl\n") - - newVartypeCheck("CONFLICTS", "+=", "perl5>=5.22").Dependency() - - c.Check(s.Output(), equals, "") - - newVartypeCheck("CONFLICTS", "+=", "perl5-*").Dependency() - - c.Check(s.Output(), equals, "WARN: fname:1: Please use \"perl5-[0-9]*\" instead of \"perl5-*\".\n") - - newVartypeCheck("CONFLICTS", "+=", "perl5-5.22.*").Dependency() - - c.Check(s.Output(), equals, "WARN: fname:1: Please append \"{,nb*}\" to the version number of this dependency.\n") - - newVartypeCheck("CONFLICTS", "+=", "perl5-[5.10-5.22]*").Dependency() - - c.Check(s.Output(), equals, "WARN: fname:1: Only [0-9]* is allowed in the numeric part of a dependency.\n") - - newVartypeCheck("CONFLICTS", "+=", "py-docs").Dependency() - - c.Check(s.Output(), equals, "ERROR: fname:1: Unknown dependency pattern \"py-docs\".\n") + runVartypeChecks("CONFLICTS", opAssignAppend, (*VartypeCheck).Dependency, + "Perl", + "perl5>=5.22", + "perl5-*", + "perl5-5.22.*", + "perl5-[5.10-5.22]*", + "py-docs", + "perl5-5.22.*{,nb*}", + "libkipi>=0.1.5<4.0", + "gtk2+>=2.16", + "perl-5.22", + "perl-5*", + "gtksourceview-sharp-2.0-[0-9]*", + "perl-5.22{,nb*}", + "perl-5.22{,nb[0-9]*}", + "mbrola-301h{,nb[0-9]*}", + "mpg123{,-esound,-nas}>=0.59.18", + "mysql*-{client,server}-[0-9]*", + "postgresql8[0-35-9]-${module}-[0-9]*", + "ncurses-${NC_VERS}{,nb*}", + "{ssh{,6}-[0-9]*,openssh-[0-9]*}", + "gnome-control-center>=2.20.1{,nb*}") - newVartypeCheck("CONFLICTS", "+=", "perl5-5.22.*{,nb*}").Dependency() - - c.Check(s.Output(), equals, "") + c.Check(s.Output(), equals, ""+ + "WARN: fname:1: Unknown dependency pattern \"Perl\".\n"+ + "WARN: fname:3: Please use \"perl5-[0-9]*\" instead of \"perl5-*\".\n"+ + "WARN: fname:5: Only [0-9]* is allowed in the numeric part of a dependency.\n"+ + "WARN: fname:5: The version pattern \"[5.10-5.22]*\" should not contain a hyphen.\n"+ + "WARN: fname:6: Unknown dependency pattern \"py-docs\".\n"+ + "WARN: fname:10: Please use \"5.22{,nb*}\" instead of \"5.22\" as the version pattern.\n"+ + "WARN: fname:11: Please use \"5.*\" instead of \"5*\" as the version pattern.\n"+ + "WARN: fname:12: The version pattern \"2.0-[0-9]*\" should not contain a hyphen.\n"+ + "WARN: fname:20: The version pattern \"[0-9]*,openssh-[0-9]*}\" should not contain a hyphen.\n"+ // XXX + "WARN: fname:21: Dependency patterns of the form pkgbase>=1.0 don't need the \"{,nb*}\" extension.\n") } -func (s *Suite) TestVartypeCheck_DependencyWithPatch(c *check.C) { - G.curPkgsrcdir = "../.." - - newVartypeCheck("DEPENDS", "+=", "Perl").DependencyWithPath() - - c.Check(s.Output(), equals, "WARN: fname:1: Unknown dependency format.\n") - - newVartypeCheck("DEPENDS", "+=", "perl5>=5.22:../perl5").DependencyWithPath() - - c.Check(s.Output(), equals, "WARN: fname:1: Dependencies should have the form \"../../category/package\".\n") - - newVartypeCheck("DEPENDS", "+=", "perl5>=5.24:../../lang/perl5").DependencyWithPath() +func (s *Suite) TestVartypeCheck_DependencyWithPath(c *check.C) { + s.CreateTmpFile(c, "x11/alacarte/Makefile", "# empty\n") + s.CreateTmpFile(c, "category/package/Makefile", "# empty\n") + G.globalData.Pkgsrcdir = s.tmpdir + G.CurrentDir = s.tmpdir + "/category/package" + G.CurPkgsrcdir = "../.." + + runVartypeChecks("DEPENDS", opAssignAppend, (*VartypeCheck).DependencyWithPath, + "Perl", + "perl5>=5.22:../perl5", + "perl5>=5.24:../../lang/perl5", + "broken0.12.1:../../x11/alacarte", + "broken[0-9]*:../../x11/alacarte", + "broken[0-9]*../../x11/alacarte", + "broken>=:../../x11/alacarte", + "broken=0:../../x11/alacarte", + "broken=:../../x11/alacarte", + "broken-:../../x11/alacarte", + "broken>:../../x11/alacarte", + "gtk2+>=2.16:../../x11/alacarte") c.Check(s.Output(), equals, ""+ - "ERROR: fname:1: \"../../lang/perl5\" does not exist.\n"+ - "ERROR: fname:1: There is no package in \"lang/perl5\".\n"+ - "WARN: fname:1: Please use USE_TOOLS+=perl:run instead of this dependency.\n") + "WARN: fname:1: Unknown dependency pattern with path \"Perl\".\n"+ + "WARN: fname:2: Dependencies should have the form \"../../category/package\".\n"+ + "ERROR: fname:3: \"../../lang/perl5\" does not exist.\n"+ + "ERROR: fname:3: There is no package in \"lang/perl5\".\n"+ + "WARN: fname:3: Please use USE_TOOLS+=perl:run instead of this dependency.\n"+ + "WARN: fname:4: Unknown dependency pattern \"broken0.12.1\".\n"+ + "WARN: fname:5: Unknown dependency pattern \"broken[0-9]*\".\n"+ + "WARN: fname:6: Unknown dependency pattern with path \"broken[0-9]*../../x11/alacarte\".\n"+ + "WARN: fname:7: Unknown dependency pattern \"broken>=\".\n"+ + "WARN: fname:8: Unknown dependency pattern \"broken=0\".\n"+ + "WARN: fname:9: Unknown dependency pattern \"broken=\".\n"+ + "WARN: fname:10: Unknown dependency pattern \"broken-\".\n"+ + "WARN: fname:11: Unknown dependency pattern \"broken>\".\n") } func (s *Suite) TestVartypeCheck_DistSuffix(c *check.C) { - newVartypeCheck("EXTRACT_SUFX", "=", ".tar.gz").DistSuffix() - newVartypeCheck("EXTRACT_SUFX", "=", ".tar.bz2").DistSuffix() + runVartypeChecks("EXTRACT_SUFX", opAssign, (*VartypeCheck).DistSuffix, + ".tar.gz", + ".tar.bz2") c.Check(s.Output(), equals, "NOTE: fname:1: EXTRACT_SUFX is \".tar.gz\" by default, so this definition may be redundant.\n") } func (s *Suite) TestVartypeCheck_EmulPlatform(c *check.C) { - newVartypeCheck("EMUL_PLATFORM", "=", "linux-i386").EmulPlatform() - newVartypeCheck("EMUL_PLATFORM", "=", "nextbsd-8087").EmulPlatform() - newVartypeCheck("EMUL_PLATFORM", "=", "${LINUX}").EmulPlatform() + runVartypeChecks("EMUL_PLATFORM", opAssign, (*VartypeCheck).EmulPlatform, + "linux-i386", + "nextbsd-8087", + "${LINUX}") c.Check(s.Output(), equals, ""+ - "WARN: fname:1: Unknown operating system: nextbsd\n"+ - "WARN: fname:1: Unknown hardware architecture: 8087\n"+ - "WARN: fname:1: \"${LINUX}\" is not a valid emulation platform.\n") + "WARN: fname:2: Unknown operating system: nextbsd\n"+ + "WARN: fname:2: Unknown hardware architecture: 8087\n"+ + "WARN: fname:3: \"${LINUX}\" is not a valid emulation platform.\n") } func (s *Suite) TestVartypeCheck_FetchURL(c *check.C) { - G.globalData.masterSiteUrls = map[string]string{ + G.globalData.MasterSiteUrls = map[string]string{ "https://github.com/": "MASTER_SITE_GITHUB", "http://ftp.gnu.org/pub/gnu/": "MASTER_SITE_GNU", } - G.globalData.masterSiteVars = map[string]bool{ + G.globalData.MasterSiteVars = map[string]bool{ "MASTER_SITE_GITHUB": true, "MASTER_SITE_GNU": true, } - newVartypeCheck("MASTER_SITES", "=", "https://github.com/example/project/").FetchURL() + runVartypeChecks("MASTER_SITES", opAssign, (*VartypeCheck).FetchURL, + "https://github.com/example/project/", + "http://ftp.gnu.org/pub/gnu/bison", // Missing a slash at the end + "${MASTER_SITE_GNU:=bison}", + "${MASTER_SITE_INVALID:=subdir/}") c.Check(s.Output(), equals, ""+ - "WARN: fname:1: Please use ${MASTER_SITE_GITHUB:=example/} instead of \"https://github.com/example/project/\".\n"+ - "WARN: fname:1: Run \""+confMake+" help topic=github\" for further tips.\n") + "WARN: fname:1: Please use ${MASTER_SITE_GITHUB:=example/} instead of \"https://github.com/example/project/\" and run \""+confMake+" help topic=github\" for further tips.\n"+ + "WARN: fname:2: Please use ${MASTER_SITE_GNU:=bison} instead of \"http://ftp.gnu.org/pub/gnu/bison\".\n"+ + "ERROR: fname:3: The subdirectory in MASTER_SITE_GNU must end with a slash.\n"+ + "ERROR: fname:4: MASTER_SITE_INVALID does not exist.\n") - newVartypeCheck("MASTER_SITES", "=", "http://ftp.gnu.org/pub/gnu/bison").FetchURL() // Missing a slash at the end + // PR 46570, keyword gimp-fix-ca + runVartypeChecks("MASTER_SITES", opAssign, (*VartypeCheck).FetchURL, + "https://example.org/download.cgi?fname=fname&sha1=12341234") - c.Check(s.Output(), equals, "WARN: fname:1: Please use ${MASTER_SITE_GNU:=bison} instead of \"http://ftp.gnu.org/pub/gnu/bison\".\n") - - newVartypeCheck("MASTER_SITES", "=", "${MASTER_SITE_GNU:=bison}").FetchURL() - - c.Check(s.Output(), equals, "ERROR: fname:1: The subdirectory in MASTER_SITE_GNU must end with a slash.\n") - - newVartypeCheck("MASTER_SITES", "=", "${MASTER_SITE_INVALID:=subdir/}").FetchURL() - - c.Check(s.Output(), equals, "ERROR: fname:1: MASTER_SITE_INVALID does not exist.\n") + c.Check(s.Output(), equals, "") } func (s *Suite) TestVartypeCheck_Filename(c *check.C) { - newVartypeCheck("FNAME", "=", "Filename with spaces.docx").Filename() + runVartypeChecks("FNAME", opAssign, (*VartypeCheck).Filename, + "Filename with spaces.docx", + "OS/2-manual.txt") - c.Check(s.Output(), equals, "WARN: fname:1: \"Filename with spaces.docx\" is not a valid filename.\n") + c.Check(s.Output(), equals, ""+ + "WARN: fname:1: \"Filename with spaces.docx\" is not a valid filename.\n"+ + "WARN: fname:2: A filename should not contain a slash.\n") +} - newVartypeCheck("FNAME", "=", "OS/2-manual.txt").Filename() +func (s *Suite) TestVartypeCheck_LdFlag(c *check.C) { + runVartypeChecks("LDFLAGS", opAssignAppend, (*VartypeCheck).LdFlag, + "-lc", + "-L/usr/lib64", + "`pkg-config pidgin --ldflags`", + "-unknown") - c.Check(s.Output(), equals, "WARN: fname:1: A filename should not contain a slash.\n") + c.Check(s.Output(), equals, "WARN: fname:4: Unknown linker flag \"-unknown\".\n") } func (s *Suite) TestVartypeCheck_MailAddress(c *check.C) { - newVartypeCheck("MAINTAINER", "=", "pkgsrc-users@netbsd.org").MailAddress() + runVartypeChecks("MAINTAINER", opAssign, (*VartypeCheck).MailAddress, + "pkgsrc-users@netbsd.org") c.Check(s.Output(), equals, "WARN: fname:1: Please write \"NetBSD.org\" instead of \"netbsd.org\".\n") } func (s *Suite) TestVartypeCheck_Message(c *check.C) { - - newVartypeCheck("SUBST_MESSAGE.id", "=", "\"Correct paths\"").Message() + runVartypeChecks("SUBST_MESSAGE.id", opAssign, (*VartypeCheck).Message, + "\"Correct paths\"", + "Correct paths") c.Check(s.Output(), equals, "WARN: fname:1: SUBST_MESSAGE.id should not be quoted.\n") +} + +func (s *Suite) TestVartypeCheck_Option(c *check.C) { + G.globalData.PkgOptions = map[string]string{ + "documented": "Option description", + "undocumented": "", + } - newVartypeCheck("SUBST_MESSAGE.id", "=", "Correct paths").Message() + runVartypeChecks("PKG_OPTIONS.pkgbase", opAssign, (*VartypeCheck).Option, + "documented", + "undocumented", + "unknown") - c.Check(s.Output(), equals, "") + c.Check(s.Output(), equals, "WARN: fname:3: Unknown option \"unknown\".\n") } func (s *Suite) TestVartypeCheck_Pathlist(c *check.C) { - - newVartypeCheck("PATH", "=", "/usr/bin:/usr/sbin:.:${LOCALBASE}/bin").Pathlist() + runVartypeChecks("PATH", opAssign, (*VartypeCheck).Pathlist, + "/usr/bin:/usr/sbin:.:${LOCALBASE}/bin") c.Check(s.Output(), equals, "WARN: fname:1: All components of PATH (in this case \".\") should be absolute paths.\n") } -func (s *Suite) TestVartypeCheck_PkgRevision(c *check.C) { +func (s *Suite) TestVartypeCheck_PkgOptionsVar(c *check.C) { + runVartypeChecks("PKG_OPTIONS_VAR.screen", opAssign, (*VartypeCheck).PkgOptionsVar, + "PKG_OPTIONS.${PKGBASE}") + + c.Check(s.Output(), equals, "ERROR: fname:1: PKGBASE must not be used in PKG_OPTIONS_VAR.\n") +} - newVartypeCheck("PKGREVISION", "=", "3a").PkgRevision() +func (s *Suite) TestVartypeCheck_PkgRevision(c *check.C) { + runVartypeChecks("PKGREVISION", opAssign, (*VartypeCheck).PkgRevision, + "3a") c.Check(s.Output(), equals, ""+ "WARN: fname:1: PKGREVISION must be a positive integer number.\n"+ "ERROR: fname:1: PKGREVISION only makes sense directly in the package Makefile.\n") - vc := newVartypeCheck("PKGREVISION", "=", "3") - vc.line.fname = "Makefile" - vc.PkgRevision() + runVartypeChecksFname("Makefile", "PKGREVISION", opAssign, (*VartypeCheck).PkgRevision, + "3") c.Check(s.Output(), equals, "") } func (s *Suite) TestVartypeCheck_PlatformTriple(c *check.C) { - newVartypeCheck("ONLY_FOR_PLATFORM", "=", "linux-i386").PlatformTriple() - newVartypeCheck("ONLY_FOR_PLATFORM", "=", "nextbsd-5.0-8087").PlatformTriple() - newVartypeCheck("ONLY_FOR_PLATFORM", "=", "${LINUX}").PlatformTriple() + runVartypeChecks("ONLY_FOR_PLATFORM", opAssign, (*VartypeCheck).PlatformTriple, + "linux-i386", + "nextbsd-5.0-8087", + "${LINUX}") c.Check(s.Output(), equals, ""+ "WARN: fname:1: \"linux-i386\" is not a valid platform triple.\n"+ - "WARN: fname:1: Unknown operating system: nextbsd\n"+ - "WARN: fname:1: Unknown hardware architecture: 8087\n") + "WARN: fname:2: Unknown operating system: nextbsd\n"+ + "WARN: fname:2: Unknown hardware architecture: 8087\n") } -func (s *Suite) TestVartypeCheck_SedCommands(c *check.C) { - - newVartypeCheck("SUBST_SED.dummy", "=", "s,@COMPILER@,gcc,g").SedCommands() - - c.Check(s.Output(), equals, "NOTE: fname:1: Please always use \"-e\" in sed commands, even if there is only one substitution.\n") +func (s *Suite) TestVartypeCheck_PythonDependency(c *check.C) { + runVartypeChecks("PYTHON_VERSIONED_DEPENDENCIES", opAssign, (*VartypeCheck).PythonDependency, + "cairo", + "${PYDEP}", + "cairo,X") - newVartypeCheck("SUBST_SED.dummy", "=", "-e s,a,b, -e a,b,c,").SedCommands() - - c.Check(s.Output(), equals, "NOTE: fname:1: Each sed command should appear in an assignment of its own.\n") + c.Check(s.Output(), equals, ""+ + "WARN: fname:2: Python dependencies should not contain variables.\n"+ + "WARN: fname:3: Invalid Python dependency \"cairo,X\".\n") } -func (s *Suite) TestVartypeCheck_Stage(c *check.C) { - - newVartypeCheck("SUBST_STAGE.dummy", "=", "post-patch").Stage() +func (s *Suite) TestVartypeCheck_Restricted(c *check.C) { + runVartypeChecks("NO_BIN_ON_CDROM", opAssign, (*VartypeCheck).Restricted, + "May only be distributed free of charge") - c.Check(s.Output(), equals, "") + c.Check(s.Output(), equals, "WARN: fname:1: The only valid value for NO_BIN_ON_CDROM is ${RESTRICTED}.\n") +} - newVartypeCheck("SUBST_STAGE.dummy", "=", "post-modern").Stage() +func (s *Suite) TestVartypeCheck_SedCommands(c *check.C) { + runVartypeChecks("SUBST_SED.dummy", opAssign, (*VartypeCheck).SedCommands, + "s,@COMPILER@,gcc,g", + "-e s,a,b, -e a,b,c,", + "-e \"s,#,comment ,\"", + "-e \"s,\\#,comment ,\"") - c.Check(s.Output(), equals, "WARN: fname:1: Invalid stage name \"post-modern\". Use one of {pre,do,post}-{extract,patch,configure,build,test,install}.\n") + c.Check(s.Output(), equals, ""+ + "NOTE: fname:1: Please always use \"-e\" in sed commands, even if there is only one substitution.\n"+ + "NOTE: fname:2: Each sed command should appear in an assignment of its own.\n"+ + "ERROR: fname:3: Invalid shell words \"\\\"s,\" in sed commands.\n") +} - newVartypeCheck("SUBST_STAGE.dummy", "=", "pre-test").Stage() +func (s *Suite) TestVartypeCheck_ShellCommands(c *check.C) { + runVartypeChecks("GENERATE_PLIST", opAssign, (*VartypeCheck).ShellCommands, + "echo bin/program", + "echo bin/program;") - c.Check(s.Output(), equals, "") + c.Check(s.Output(), equals, "WARN: fname:1: This shell command list should end with a semicolon.\n") } -func (s *Suite) TestVartypeCheck_Yes(c *check.C) { +func (s *Suite) TestVartypeCheck_Stage(c *check.C) { + runVartypeChecks("SUBST_STAGE.dummy", opAssign, (*VartypeCheck).Stage, + "post-patch", + "post-modern", + "pre-test") - newVartypeCheck("APACHE_MODULE", "=", "yes").Yes() + c.Check(s.Output(), equals, "WARN: fname:2: Invalid stage name \"post-modern\". Use one of {pre,do,post}-{extract,patch,configure,build,test,install}.\n") +} - c.Check(s.Output(), equals, "") +func (s *Suite) TestVartypeCheck_URL(c *check.C) { + runVartypeChecks("MASTER_SITES", opAssign, (*VartypeCheck).URL, + "http://example.org/distfiles/", + "http://example.org/download?fname=distfile;version=1.0", + "http://example.org/download?fname=<distfile>;version=<version>") - newVartypeCheck("APACHE_MODULE", "=", "no").Yes() + c.Check(s.Output(), equals, "WARN: fname:3: \"http://example.org/download?fname=<distfile>;version=<version>\" is not a valid URL.\n") +} - c.Check(s.Output(), equals, "WARN: fname:1: APACHE_MODULE should be set to YES or yes.\n") +func (s *Suite) TestVartypeCheck_Yes(c *check.C) { + runVartypeChecks("APACHE_MODULE", opAssign, (*VartypeCheck).Yes, + "yes", + "no", + "${YESVAR}") - newVartypeCheck("APACHE_MODULE", "=", "${YESVAR}").Yes() + c.Check(s.Output(), equals, ""+ + "WARN: fname:2: APACHE_MODULE should be set to YES or yes.\n"+ + "WARN: fname:3: APACHE_MODULE should be set to YES or yes.\n") +} - c.Check(s.Output(), equals, "WARN: fname:1: APACHE_MODULE should be set to YES or yes.\n") +func runVartypeChecks(varname string, op MkOperator, checker func(*VartypeCheck), values ...string) { + for i, value := range values { + mkline := NewMkLine(NewLine("fname", i+1, varname+op.String()+value, nil)) + valueNovar := mkline.withoutMakeVariables(mkline.Value(), true) + vc := &VartypeCheck{mkline, mkline.Line, mkline.Varname(), mkline.Op(), mkline.Value(), valueNovar, "", true, false} + checker(vc) + } } -func newVartypeCheck(varname, op, value string) *VartypeCheck { - line := NewLine("fname", "1", varname+op+value, nil) - valueNovar := NewMkLine(line).withoutMakeVariables(value, true) - return &VartypeCheck{line, varname, op, value, valueNovar, "", true, guNotGuessed} +func runVartypeChecksFname(fname, varname string, op MkOperator, checker func(*VartypeCheck), values ...string) { + for i, value := range values { + mkline := NewMkLine(NewLine(fname, i+1, varname+op.String()+value, nil)) + valueNovar := mkline.withoutMakeVariables(value, true) + vc := &VartypeCheck{mkline, mkline.Line, varname, op, value, valueNovar, "", true, false} + checker(vc) + } } diff --git a/pkgtools/pkglint/files/varusecontext.go b/pkgtools/pkglint/files/varusecontext.go deleted file mode 100644 index cf0e4e61668..00000000000 --- a/pkgtools/pkglint/files/varusecontext.go +++ /dev/null @@ -1,75 +0,0 @@ -package main - -// VarUseContext defines the context in which a variable is defined -// or used. Whether that is allowed depends on: -// -// * The variable’s data type, as defined in vardefs.go. -// * When used on the right-hand side of an assigment, the variable can -// represent a list of words, a single word or even only part of a -// word. This distinction decides upon the correct use of the :Q -// operator. -// * When used in preprocessing statements like .if or .for, the other -// operands of that statement should fit to the variable and are -// checked against the variable type. For example, comparing OPSYS to -// x86_64 doesn’t make sense. -type VarUseContext struct { - time vucTime - vartype *Vartype - shellword vucQuoting - extent vucExtent -} - -type vucTime int - -const ( - vucTimeUnknown vucTime = iota - - // When Makefiles are loaded, the operators := and != are evaluated, - // as well as the conditionals .if, .elif and .for. - // During loading, not all variables are available yet. - // Variable values are still subject to change, especially lists. - vucTimeParse - - // All files have been read, all variables can be referenced. - // Variable values don’t change anymore. - vucTimeRun -) - -// The quoting context in which the variable is used. -// Depending on this context, the modifiers :Q or :M can be allowed or not. -type vucQuoting int - -const ( - vucQuotUnknown vucQuoting = iota - vucQuotPlain // Example: echo LOCALBASE=${LOCALBASE} - vucQuotDquot // Example: echo "The version is ${PKGVERSION}." - vucQuotSquot // Example: echo 'The version is ${PKGVERSION}.' - vucQuotBackt // Example: echo \`sed 1q ${WRKSRC}/README\` - - // The .for loop in Makefiles. This is the only place where - // variables are split on whitespace. Everywhere else (:Q, :M) - // they are split like in the shell. - // - // Example: .for f in ${EXAMPLE_FILES} - vucQuotFor -) - -type vucExtent int - -const ( - vucExtentUnknown vucExtent = iota - vucExtentWord // Example: echo ${LOCALBASE} - vucExtentWordpart // Example: echo LOCALBASE=${LOCALBASE} -) - -func (vuc *VarUseContext) String() string { - typename := "no-type" - if vuc.vartype != nil { - typename = vuc.vartype.String() - } - return sprintf("(%s %s %s %s)", - []string{"unknown", "load-time", "run-time"}[vuc.time], - typename, - []string{"unknown", "plain", "dquot", "squot", "backt", "for"}[vuc.shellword], - []string{"unknown", "word", "word-part"}[vuc.extent]) -} diff --git a/pkgtools/pkglint/files/varusecontext_test.go b/pkgtools/pkglint/files/varusecontext_test.go deleted file mode 100644 index 931bdd45455..00000000000 --- a/pkgtools/pkglint/files/varusecontext_test.go +++ /dev/null @@ -1,13 +0,0 @@ -package main - -import ( - check "gopkg.in/check.v1" -) - -func (s *Suite) TestVarUseContext_ToString(c *check.C) { - G.globalData.InitVartypes() - vartype := getVariableType(NewLine("fname", "1", "dummy", nil), "PKGNAME") - vuc := &VarUseContext{vucTimeUnknown, vartype, vucQuotBackt, vucExtentWord} - - c.Check(vuc.String(), equals, "(unknown PkgName backt word)") -} diff --git a/pkgtools/pkglint/files/vercmp.go b/pkgtools/pkglint/files/vercmp.go index f440bedc3b3..fd43fa8626d 100644 --- a/pkgtools/pkglint/files/vercmp.go +++ b/pkgtools/pkglint/files/vercmp.go @@ -23,12 +23,12 @@ func icmp(a, b int) int { } func pkgverCmp(left, right string) int { - lv := mkversion(left) - rv := mkversion(right) + lv := newVersion(left) + rv := newVersion(right) m := imax(len(lv.v), len(rv.v)) for i := 0; i < m; i++ { - if c := icmp(lv.place(i), rv.place(i)); c != 0 { + if c := icmp(lv.Place(i), rv.Place(i)); c != 0 { return c } } @@ -40,7 +40,7 @@ type version struct { nb int } -func mkversion(vstr string) *version { +func newVersion(vstr string) *version { v := new(version) rest := strings.ToLower(vstr) for rest != "" { @@ -49,40 +49,40 @@ func mkversion(vstr string) *version { n := 0 i := 0 for i < len(rest) && isdigit(rest[i]) { - n = 10*n + (int(rest[i]) - '0') + n = 10*n + int(rest[i]-'0') i++ } rest = rest[i:] - v.add(n) + v.Add(n) + case rest[0] == '_' || rest[0] == '.': + v.Add(0) + rest = rest[1:] case hasPrefix(rest, "alpha"): - v.add(-3) + v.Add(-3) rest = rest[5:] case hasPrefix(rest, "beta"): - v.add(-2) + v.Add(-2) rest = rest[4:] case hasPrefix(rest, "pre"): - v.add(-1) + v.Add(-1) rest = rest[3:] case hasPrefix(rest, "rc"): - v.add(-1) + v.Add(-1) rest = rest[2:] case hasPrefix(rest, "pl"): - v.add(0) + v.Add(0) rest = rest[2:] - case hasPrefix(rest, "_") || hasPrefix(rest, "."): - v.add(0) - rest = rest[1:] case hasPrefix(rest, "nb"): i := 2 n := 0 for i < len(rest) && isdigit(rest[i]) { - n = 10*n + (int(rest[i]) - '0') + n = 10*n + int(rest[i]-'0') i++ } v.nb = n rest = rest[i:] - case 'a' <= rest[0] && rest[0] <= 'z': - v.add(int(rest[0]) - 'a' + 1) + case rest[0]-'a' <= 'z'-'a': + v.Add(int(rest[0] - 'a' + 1)) rest = rest[1:] default: rest = rest[1:] @@ -91,13 +91,13 @@ func mkversion(vstr string) *version { return v } -func (v *version) add(i int) { +func (v *version) Add(i int) { v.v = append(v.v, i) } func isdigit(b byte) bool { - return '0' <= b && b <= '9' + return b-'0' <= 9 } -func (v *version) place(i int) int { +func (v *version) Place(i int) int { if i < len(v.v) { return v.v[i] } diff --git a/pkgtools/pkglint/files/vercmp_test.go b/pkgtools/pkglint/files/vercmp_test.go index 1ff3fa98f74..190f5779559 100644 --- a/pkgtools/pkglint/files/vercmp_test.go +++ b/pkgtools/pkglint/files/vercmp_test.go @@ -5,14 +5,16 @@ import ( ) func (s *Suite) TestMkversion(c *check.C) { - c.Check(mkversion("5.0"), check.DeepEquals, &version{[]int{5, 0, 0}, 0}) - c.Check(mkversion("5.0nb5"), check.DeepEquals, &version{[]int{5, 0, 0}, 5}) - c.Check(mkversion("0.0.1-SNAPSHOT"), check.DeepEquals, &version{[]int{0, 0, 0, 0, 1, 19, 14, 1, 16, 19, 8, 15, 20}, 0}) - c.Check(mkversion("1.0alpha3"), check.DeepEquals, &version{[]int{1, 0, 0, -3, 3}, 0}) - c.Check(mkversion("2.5beta"), check.DeepEquals, &version{[]int{2, 0, 5, -2}, 0}) - c.Check(mkversion("20151110"), check.DeepEquals, &version{[]int{20151110}, 0}) - c.Check(mkversion("0"), check.DeepEquals, &version{[]int{0}, 0}) - c.Check(mkversion("nb1"), check.DeepEquals, &version{nil, 1}) + c.Check(newVersion("5.0"), check.DeepEquals, &version{[]int{5, 0, 0}, 0}) + c.Check(newVersion("5.0nb5"), check.DeepEquals, &version{[]int{5, 0, 0}, 5}) + c.Check(newVersion("0.0.1-SNAPSHOT"), check.DeepEquals, &version{[]int{0, 0, 0, 0, 1, 19, 14, 1, 16, 19, 8, 15, 20}, 0}) + c.Check(newVersion("1.0alpha3"), check.DeepEquals, &version{[]int{1, 0, 0, -3, 3}, 0}) + c.Check(newVersion("2.5beta"), check.DeepEquals, &version{[]int{2, 0, 5, -2}, 0}) + c.Check(newVersion("20151110"), check.DeepEquals, &version{[]int{20151110}, 0}) + c.Check(newVersion("0"), check.DeepEquals, &version{[]int{0}, 0}) + c.Check(newVersion("nb1"), check.DeepEquals, &version{nil, 1}) + c.Check(newVersion("1.0.1a"), deepEquals, &version{[]int{1, 0, 0, 0, 1, 1}, 0}) + c.Check(newVersion("1.0.1z"), deepEquals, &version{[]int{1, 0, 0, 0, 1, 26}, 0}) } func (s *Suite) TestPkgverCmp(c *check.C) { |