diff options
author | rillig <rillig@pkgsrc.org> | 2021-08-08 22:04:15 +0000 |
---|---|---|
committer | rillig <rillig@pkgsrc.org> | 2021-08-08 22:04:15 +0000 |
commit | 1f3004789b8ebae920ce18d8e800dfd1797caef8 (patch) | |
tree | 8981bfe85a3978106edf7a378128b1e1d6378246 /pkgtools | |
parent | c2cdbc9ec4d3def27cfd7ddcd391cb4140a77a2f (diff) | |
download | pkgsrc-1f3004789b8ebae920ce18d8e800dfd1797caef8.tar.gz |
pkgtools/pkglint: update to 21.2.2
Changes since 21.2.1:
Check the variable names of OPSYS-specific variables for typos, such as
'Dragonfly' with a lowercase 'f'. Suggested by David A. Holland in
PR pkg/56352.
Warn if variables like CFLAGS are assigned using the operator '='.
Suggested by David A. Holland in PR pkg/56352.
Diffstat (limited to 'pkgtools')
-rw-r--r-- | pkgtools/pkglint/Makefile | 5 | ||||
-rw-r--r-- | pkgtools/pkglint/files/makepat/pat.go | 195 | ||||
-rw-r--r-- | pkgtools/pkglint/files/makepat/pat_test.go | 53 | ||||
-rw-r--r-- | pkgtools/pkglint/files/mkassignchecker.go | 77 | ||||
-rw-r--r-- | pkgtools/pkglint/files/mkassignchecker_test.go | 47 | ||||
-rw-r--r-- | pkgtools/pkglint/files/pkgsrc.go | 13 | ||||
-rw-r--r-- | pkgtools/pkglint/files/textproc/lexer.go | 6 | ||||
-rw-r--r-- | pkgtools/pkglint/files/vartype.go | 32 |
8 files changed, 336 insertions, 92 deletions
diff --git a/pkgtools/pkglint/Makefile b/pkgtools/pkglint/Makefile index 366af698eed..7b68321dfa5 100644 --- a/pkgtools/pkglint/Makefile +++ b/pkgtools/pkglint/Makefile @@ -1,7 +1,6 @@ -# $NetBSD: Makefile,v 1.691 2021/07/13 11:36:37 bsiegert Exp $ +# $NetBSD: Makefile,v 1.692 2021/08/08 22:04:15 rillig Exp $ -PKGNAME= pkglint-21.2.1 -PKGREVISION= 1 +PKGNAME= pkglint-21.2.2 CATEGORIES= pkgtools DISTNAME= tools MASTER_SITES= ${MASTER_SITE_GITHUB:=golang/} diff --git a/pkgtools/pkglint/files/makepat/pat.go b/pkgtools/pkglint/files/makepat/pat.go index 848c97e6b23..63181d15622 100644 --- a/pkgtools/pkglint/files/makepat/pat.go +++ b/pkgtools/pkglint/files/makepat/pat.go @@ -32,20 +32,16 @@ func Compile(pattern string) (*Pattern, error) { lex := textproc.NewLexer(pattern) for !lex.EOF() { + ch := lex.NextByte() - if lex.SkipByte('*') { + switch ch { + case '*': p.AddTransition(s, 0, 255, s) - continue - } - - if lex.SkipByte('?') { + case '?': next := p.AddState(false) p.AddTransition(s, 0, 255, next) s = next - continue - } - - if lex.SkipByte('\\') { + case '\\': if lex.EOF() { return nil, errors.New("unfinished escape sequence") } @@ -53,23 +49,17 @@ func Compile(pattern string) (*Pattern, error) { next := p.AddState(false) p.AddTransition(s, ch, ch, next) s = next - continue - } - - ch := lex.NextByte() - if ch != '[' { + case '[': + next, err := compileCharClass(&p, lex, ch, s) + if err != nil { + return nil, err + } + s = next + default: next := p.AddState(false) p.AddTransition(s, ch, ch, next) s = next - continue - } - - next, err := compileCharClass(&p, lex, ch, s) - if err != nil { - return nil, err } - - s = next } p.states[s].end = true @@ -144,13 +134,17 @@ func (p *Pattern) AddTransition(from StateID, min, max byte, to StateID) { // Match tests whether a pattern matches the given string. func (p *Pattern) Match(s string) bool { + if len(p.states) == 0 { + return false + } + curr := make([]bool, len(p.states)) next := make([]bool, len(p.states)) curr[0] = true for _, ch := range []byte(s) { ok := false - for i, _ := range next { + for i := range next { next[i] = false } @@ -183,21 +177,33 @@ func (p *Pattern) Match(s string) bool { // match at the same time. func Intersect(p1, p2 *Pattern) *Pattern { var res Pattern - for i1 := 0; i1 < len(p1.states); i1++ { - for i2 := 0; i2 < len(p2.states); i2++ { - res.AddState(p1.states[i1].end && p2.states[i2].end) + + newState := make(map[[2]StateID]StateID) + + // stateFor returns the state ID in the intersection, + // creating it if necessary. + stateFor := func(s1, s2 StateID) StateID { + key := [2]StateID{s1, s2} + ns, ok := newState[key] + if !ok { + ns = res.AddState(p1.states[s1].end && p2.states[s2].end) + newState[key] = ns } + return ns } - for i1 := 0; i1 < len(p1.states); i1++ { - for i2 := 0; i2 < len(p2.states); i2++ { - for _, t1 := range p1.states[i1].transitions { - for _, t2 := range p2.states[i2].transitions { + // Each pattern needs a start node. + stateFor(0, 0) + + for i1, s1 := range p1.states { + for i2, s2 := range p2.states { + for _, t1 := range s1.transitions { + for _, t2 := range s2.transitions { min := bmax(t1.min, t2.min) max := bmin(t1.max, t2.max) if min <= max { - from := StateID(i1*len(p2.states) + i2) - to := t1.to*StateID(len(p2.states)) + t2.to + from := stateFor(StateID(i1), StateID(i2)) + to := stateFor(t1.to, t2.to) res.AddTransition(from, min, max, to) } } @@ -209,33 +215,101 @@ func Intersect(p1, p2 *Pattern) *Pattern { } func (p *Pattern) optimized() *Pattern { - var opt Pattern + reachable := p.reachable() + relevant := p.relevant(reachable) + return p.compressed(relevant) +} - var todo []StateID - hasNewID := make([]bool, len(p.states)) - newIDs := make([]StateID, len(p.states)) +// reachable returns all states that are reachable from the start state. +// In optimized patterns, each state is reachable. +func (p *Pattern) reachable() []bool { + reachable := make([]bool, len(p.states)) + + progress := make([]int, len(p.states)) // 0 = unseen, 1 = to do, 2 = done + + progress[0] = 1 + + for { + changed := false + for i, pr := range progress { + if pr == 1 { + reachable[i] = true + progress[i] = 2 + changed = true + for _, tr := range p.states[i].transitions { + if progress[tr.to] == 0 { + progress[tr.to] = 1 + } + } + } + } + + if !changed { + break + } + } - todo = append(todo, 0) - newIDs[0] = opt.AddState(p.states[0].end) - hasNewID[0] = true + return reachable +} - for len(todo) > 0 { - oldStateID := todo[len(todo)-1] - todo = todo[:len(todo)-1] +// relevant returns all states from which and end state is reachable. +// In optimized patterns, each state is relevant. +func (p *Pattern) relevant(reachable []bool) []bool { + relevant := make([]bool, len(p.states)) - oldState := p.states[oldStateID] + progress := make([]int, len(p.states)) // 0 = unseen, 1 = to do, 2 = done - for _, t := range oldState.transitions { - if !hasNewID[t.to] { - hasNewID[t.to] = true - newIDs[t.to] = opt.AddState(p.states[t.to].end) - todo = append(todo, t.to) + for i, state := range p.states { + if state.end && reachable[i] { + progress[i] = 1 + } + } + + for { + changed := false + for to, pr := range progress { + if pr != 1 { + continue } - opt.AddTransition(newIDs[oldStateID], t.min, t.max, newIDs[t.to]) + progress[to] = 2 + relevant[to] = true + changed = true + for from, st := range p.states { + for _, tr := range st.transitions { + if tr.to == StateID(to) && reachable[from] && + progress[from] == 0 { + progress[from] = 1 + } + } + } + } + + if !changed { + break } } - // TODO: remove transitions that point to a dead end + return relevant +} + +// compressed creates a pattern that contains only the relevant states. +func (p *Pattern) compressed(relevant []bool) *Pattern { + var opt Pattern + + newIDs := make([]StateID, len(p.states)) + for i, r := range relevant { + if r { + newIDs[i] = opt.AddState(p.states[i].end) + } + } + + for from, s := range p.states { + for _, t := range s.transitions { + if relevant[from] && relevant[t.to] { + opt.AddTransition(newIDs[from], t.min, t.max, newIDs[t.to]) + } + } + } return &opt } @@ -246,25 +320,12 @@ func (p *Pattern) optimized() *Pattern { // [^] // Intersect("*.c", "*.h") func (p *Pattern) CanMatch() bool { - reachable := make([]bool, len(p.states)) - reachable[0] = true - -again: - changed := false - for i, s := range p.states { - if reachable[i] { - for _, t := range s.transitions { - if !reachable[t.to] { - reachable[t.to] = true - changed = true - } - } - } - } - if changed { - goto again + if len(p.states) == 0 { + return false } + reachable := p.reachable() + for i, s := range p.states { if reachable[i] && s.end { return true diff --git a/pkgtools/pkglint/files/makepat/pat_test.go b/pkgtools/pkglint/files/makepat/pat_test.go index 3b9ec9ae56b..d9110633df3 100644 --- a/pkgtools/pkglint/files/makepat/pat_test.go +++ b/pkgtools/pkglint/files/makepat/pat_test.go @@ -44,10 +44,11 @@ func Test_compileCharClass(t *testing.T) { p, err := Compile(tt.pattern) if err != nil { t.Fail() - } - got := p.Match(tt.str) - if got != tt.want { - t.Errorf("got %v, want %v", got, tt.want) + } else { + got := p.Match(tt.str) + if got != tt.want { + t.Errorf("got %v, want %v", got, tt.want) + } } }) } @@ -176,6 +177,8 @@ func Test_Intersect(t *testing.T) { {"N-9.99.*", "N-[1-9].*", "", false, true}, {"N-9.99.*", "N-[1-9][0-9].*", "", false, false}, {"*.c", "*.h", "", false, false}, + {"a*", "*b", "ab", true, true}, + {"a*bc", "ab*c", "abc", true, true}, } for _, tt := range tests { t.Run(tt.str, func(t *testing.T) { @@ -220,6 +223,48 @@ func Test_Pattern_optimized(t *testing.T) { } } +func Test_Pattern_reachable(t *testing.T) { + p, err := Compile("N-*") + if err != nil { + t.Fatal(err) + } + + reachable := p.reachable() + + if !reflect.DeepEqual(reachable, []bool{true, true, true}) { + t.Errorf("%#v", reachable) + } +} + +func Test_Pattern_relevant(t *testing.T) { + p, err := Compile("N-*") + if err != nil { + t.Fatal(err) + } + + reachable := p.reachable() + relevant := p.relevant(reachable) + + if !reflect.DeepEqual(relevant, []bool{true, true, true}) { + t.Errorf("%#v", relevant) + } +} + +func Test_Pattern_compressed(t *testing.T) { + p, err := Compile("N-*") + if err != nil { + t.Fatal(err) + } + + reachable := p.reachable() + relevant := p.relevant(reachable) + compressed := p.compressed(relevant) + + if !reflect.DeepEqual(compressed, p) { + t.Errorf("%#v", compressed) + } +} + func Test_Pattern_CanMatch(t *testing.T) { tests := []struct { p1 string diff --git a/pkgtools/pkglint/files/mkassignchecker.go b/pkgtools/pkglint/files/mkassignchecker.go index e2b074ce5f7..a059e78ef42 100644 --- a/pkgtools/pkglint/files/mkassignchecker.go +++ b/pkgtools/pkglint/files/mkassignchecker.go @@ -1,6 +1,7 @@ package pkglint import ( + "netbsd.org/pkglint/textproc" "strconv" "strings" ) @@ -29,6 +30,7 @@ func (ck *MkAssignChecker) checkLeft() { } ck.checkLeftNotUsed() + ck.checkLeftOpsys() ck.checkLeftDeprecated() ck.checkLeftBsdPrefs() if !ck.checkLeftUserSettable() { @@ -38,7 +40,7 @@ func (ck *MkAssignChecker) checkLeft() { ck.checkLeftRationale() NewMkLineChecker(ck.MkLines, ck.MkLine).checkTextVarUse( - ck.MkLine.Varname(), + varname, NewVartype(BtVariableName, NoVartypeOptions, NewACLEntry("*", aclpAll)), VucLoadTime) } @@ -91,6 +93,33 @@ func (ck *MkAssignChecker) checkLeftNotUsed() { "see mk/subst.mk for an example of such a documentation comment.") } +// checkLeftOpsys checks whether the variable name is one of the OPSYS +// variables, which get merged with their corresponding VAR.${OPSYS} in +// bsd.pkg.mk. +func (ck *MkAssignChecker) checkLeftOpsys() { + varname := ck.MkLine.Varname() + varbase := varnameBase(varname) + if !G.Pkgsrc.IsOpsysVar(varbase) { + return + } + + varparam := varnameParam(varname) + if varparam == "" || varparam == "*" || + textproc.Lower.Contains(varparam[0]) { + return + } + + platforms := G.Pkgsrc.VariableType(ck.MkLines, "OPSYS").basicType + if platforms.HasEnum(varparam) { + return + } + + ck.MkLine.Warnf( + "Since %s is an OPSYS variable, "+ + "its parameter %q should be one of %s.", + varbase, varparam, platforms.AllowedEnums()) +} + func (ck *MkAssignChecker) checkLeftDeprecated() { varname := ck.MkLine.Varname() if fix := G.Pkgsrc.Deprecated[varname]; fix != "" { @@ -344,6 +373,7 @@ func (ck *MkAssignChecker) checkLeftRationale() { func (ck *MkAssignChecker) checkOp() { ck.checkOpShell() + ck.checkOpAppendOnly() } func (ck *MkAssignChecker) checkOpShell() { @@ -393,6 +423,51 @@ func (ck *MkAssignChecker) checkOpShell() { "or .for directive.") } +// https://gnats.netbsd.org/56352 +func (ck *MkAssignChecker) checkOpAppendOnly() { + + if ck.MkLine.Op() != opAssign { + return + } + + varname := ck.MkLine.Varname() + varbase := varnameBase(varname) + + // See pkgtools/bootstrap-mk-files/files/sys.mk + switch varbase { + case + "CFLAGS", // C + "OBJCFLAGS", // Objective C + "FFLAGS", // Fortran + "RFLAGS", // Ratfor + "LFLAGS", // Lex + "LDFLAGS", // Linker + "LINTFLAGS", // Lint + "PFLAGS", // Pascal + "YFLAGS", // Yacc + "LDADD": // Just for symmetry + break + default: + return + } + + // At this point, it does not matter whether bsd.prefs.mk has been + // included or not since the above variables get their default values + // in sys.mk already, which is loaded even before the very first line + // of the package Makefile. + + // The parameterized OPSYS variants do not get any default values before + // the package Makefile is included. Therefore, as long as bsd.prefs.mk + // has not been included, the operator '=' can still be used. Testing for + // bsd.prefs.mk is only half the story, any other accidental overwrites + // are caught by RedundantScope. + if varbase != varname && !ck.MkLines.Tools.SeenPrefs { + return + } + + ck.MkLine.Warnf("Assignments to %q should use \"+=\", not \"=\".", varname) +} + // checkLeft checks everything to the right of the assignment operator. func (ck *MkAssignChecker) checkRight() { mkline := ck.MkLine diff --git a/pkgtools/pkglint/files/mkassignchecker_test.go b/pkgtools/pkglint/files/mkassignchecker_test.go index 3bf22bb2902..9a8a9278eec 100644 --- a/pkgtools/pkglint/files/mkassignchecker_test.go +++ b/pkgtools/pkglint/files/mkassignchecker_test.go @@ -200,6 +200,36 @@ func (s *Suite) Test_MkAssignChecker_checkLeftNotUsed__infra(c *check.C) { "WARN: ~/category/package/Makefile:22: UNDOCUMENTED is used but not defined.") } +// https://gnats.netbsd.org/56352 +func (s *Suite) Test_MkAssignChecker_checkLeftOpsys(c *check.C) { + t := s.Init(c) + + t.SetUpVartypes() + + mklines := t.NewMkLines("filename.mk", + MkCvsID, + "", + "CPPFLAGS.mumble+=\t-DMACRO", + "CPPFLAGS.Linux+=\t-DMACRO", + "CFLAGS.NebTSD+=\t\t-Wall", + "CFLAGS.NetBSD+=\t\t-Wall", + "CXXFLAGS.DragonFly=\t-Wall", + "CXXFLAGS.DragonFlyBSD=\t-Wall", + "LDFLAGS.SunOS+=\t\t-lX11 -lm", + "LDFLAGS.SunOS+=\t\t-lX11 -lm", + "LDFLAGS.*+=\t\t-lfallback") + + mklines.Check() + + t.CheckOutputLines( + "WARN: filename.mk:5: Since CFLAGS is an OPSYS variable, "+ + "its parameter \"NebTSD\" should be one of "+ + "Cygwin DragonFly FreeBSD Linux NetBSD SunOS.", + "WARN: filename.mk:8: Since CXXFLAGS is an OPSYS variable, "+ + "its parameter \"DragonFlyBSD\" should be one of "+ + "Cygwin DragonFly FreeBSD Linux NetBSD SunOS.") +} + func (s *Suite) Test_MkAssignChecker_checkLeftDeprecated(c *check.C) { t := s.Init(c) @@ -725,6 +755,23 @@ func (s *Suite) Test_MkAssignChecker_checkOpShell(c *check.C) { "WARN: ~/category/package/standalone.mk:15: Please use \"${ECHO}\" instead of \"echo\".") } +func (s *Suite) Test_MkAssignChecker_checkOpAppendOnly(c *check.C) { + t := s.Init(c) + + t.SetUpVartypes() + mklines := t.NewMkLines("filename.mk", + MkCvsID, + "", + "CFLAGS=\t\t-O2", + "CFLAGS.SunOS=\t-O0") + + mklines.Check() + + t.CheckOutputLines( + "WARN: filename.mk:3: " + + "Assignments to \"CFLAGS\" should use \"+=\", not \"=\".") +} + func (s *Suite) Test_MkAssignChecker_checkRight(c *check.C) { t := s.Init(c) diff --git a/pkgtools/pkglint/files/pkgsrc.go b/pkgtools/pkglint/files/pkgsrc.go index e131318fc23..959be58542f 100644 --- a/pkgtools/pkglint/files/pkgsrc.go +++ b/pkgtools/pkglint/files/pkgsrc.go @@ -1169,6 +1169,19 @@ func (src *Pkgsrc) IsBuildDef(varname string) bool { return src.buildDefs[varname] } +func (src *Pkgsrc) IsOpsysVar(varbase string) bool { + // See mk/bsd.pkg.mk, "OPSYSVARS". + switch varbase { + case "CFLAGS", "CXXFLAGS", "CPPFLAGS", "LDFLAGS", "LIBS", + "CMAKE_ARGS", "CONFIGURE_ARGS", "CONFIGURE_ENV", + "BUILDLINK_TRANSFORM", "SUBST_CLASSES", + "BUILD_TARGET", "MAKE_ENV", "MAKE_FLAGS", "USE_TOOLS": + return true + } + // TODO: Packages can add their own variables to OPSYSVARS as well. + return false +} + // ReadDir lists the files and subdirectories from the given directory // (relative to the pkgsrc root). // diff --git a/pkgtools/pkglint/files/textproc/lexer.go b/pkgtools/pkglint/files/textproc/lexer.go index 982e21f50da..2ecddb9d990 100644 --- a/pkgtools/pkglint/files/textproc/lexer.go +++ b/pkgtools/pkglint/files/textproc/lexer.go @@ -76,7 +76,7 @@ func (l *Lexer) NextString(prefix string) string { return "" } -// SkipText skips over the given string, if the remaining string starts +// SkipString skips over the given string, if the remaining string starts // with it. It returns whether it actually skipped. func (l *Lexer) SkipString(prefix string) bool { skipped := strings.HasPrefix(l.rest, prefix) @@ -199,8 +199,8 @@ func (l *Lexer) SkipRegexp(re *regexp.Regexp) bool { } // NextRegexp tests whether the remaining string matches the given regular -// expression, and in that case, skips over it and returns the matched substrings, -// as in regexp.FindStringSubmatch. +// expression, and in that case, chops it off and returns the matched +// substrings, as in regexp.FindStringSubmatch. // If the regular expression does not match, returns nil. func (l *Lexer) NextRegexp(re *regexp.Regexp) []string { if !strings.HasPrefix(re.String(), "^") { diff --git a/pkgtools/pkglint/files/vartype.go b/pkgtools/pkglint/files/vartype.go index dfe4aa3d109..7056634761a 100644 --- a/pkgtools/pkglint/files/vartype.go +++ b/pkgtools/pkglint/files/vartype.go @@ -29,22 +29,26 @@ const ( // and as lists of arbitrary things. List vartypeOptions = 1 << iota - // The variable is not defined by the pkgsrc infrastructure. + // Guessed means that the variable is not defined by the pkgsrc + // infrastructure. // It follows the common naming convention, therefore its type can be guessed. // Sometimes, with files and paths, this leads to wrong decisions. Guessed - // The variable can, or in some cases must, be defined by the package. + // PackageSettable means that the variable can, or in some cases must, + // be defined by the package. // For several of these variables, the pkgsrc infrastructure provides // a reasonable default value, either in bsd.prefs.mk or in bsd.pkg.mk. PackageSettable - // The variable can be defined by the pkgsrc user in mk.conf. + // UserSettable means that the variable can be defined by the pkgsrc + // user in mk.conf. // Its value is available at load time after bsd.prefs.mk has been included. UserSettable - // This variable is provided by either the pkgsrc infrastructure in - // mk/*, or by <sys.mk>, which is included at the very beginning. + // SystemProvided means that this variable is provided by either the + // pkgsrc infrastructure in mk/*, or by <sys.mk>, which is included + // at the very beginning. // // TODO: Clearly distinguish between: // * sys.mk @@ -56,14 +60,14 @@ const ( // expressive enough. This is related to the scope and lifetime of // variables and should be modelled separately. // - // See DefinedInSysMk. + // See DefinedIfInScope. SystemProvided - // This variable may be provided in the command line by the pkgsrc - // user when building a package. + // CommandLineProvided means that this variable may be provided in the + // command line by the pkgsrc user when building a package. // - // Since the values of these variables are not written down in any - // file, they must not influence the generated binary packages. + // Since the values of these variables are not recorded in any file, + // they must not influence the generated binary packages. // // See UserSettable. CommandLineProvided @@ -72,8 +76,8 @@ const ( // describing why they are set. Typical examples are NOT_FOR_* variables. NeedsRationale - // When something is appended to this variable, each additional - // value should be on a line of its own. + // OnePerLine means that when something is appended to this variable, + // each additional value should be on a line of its own. OnePerLine // AlwaysInScope is true when the variable is always available. @@ -134,8 +138,8 @@ const ( // X11_TYPE (user-settable) NonemptyIfDefined - // Unique is true if it doesn't make sense to append the same - // value more than once to the variable. + // Unique marks variables where it doesn't make sense to append the same + // value more than once. // // A typical example is CATEGORIES. Unique |