From 66bc08fd1ac240ee792307c3520d8fd21dc5030f Mon Sep 17 00:00:00 2001 From: rillig Date: Wed, 21 Aug 2019 16:45:16 +0000 Subject: pkgtools/pkglint: update to 5.7.21 Changes since 5.7.20: * PKG_OPTIONS that are handled using patterns are correctly identified. * Simple R packages should follow the canonical variable order. * Fixed some edge cases for aligning variable assignments. * Improved detection of allowed values for USE_LANGUAGES. --- pkgtools/pkglint/Makefile | 4 +- pkgtools/pkglint/files/check_test.go | 12 ++ pkgtools/pkglint/files/mkline_test.go | 7 +- pkgtools/pkglint/files/mklinechecker.go | 11 +- pkgtools/pkglint/files/mklinechecker_test.go | 5 +- pkgtools/pkglint/files/mkshparser.go | 6 +- pkgtools/pkglint/files/mkshparser_test.go | 39 +++- pkgtools/pkglint/files/mktypes.go | 8 +- pkgtools/pkglint/files/options.go | 30 +++- pkgtools/pkglint/files/options_test.go | 63 +++++++ pkgtools/pkglint/files/package.go | 2 + pkgtools/pkglint/files/package_test.go | 17 +- pkgtools/pkglint/files/varalignblock.go | 33 ++-- pkgtools/pkglint/files/varalignblock_test.go | 254 ++++++++++++++++++++++++++- pkgtools/pkglint/files/vardefs.go | 29 ++- pkgtools/pkglint/files/vardefs_test.go | 38 +++- pkgtools/pkglint/files/vartype.go | 4 + pkgtools/pkglint/files/vartypecheck.go | 67 +++++-- pkgtools/pkglint/files/vartypecheck_test.go | 63 ++++++- 19 files changed, 604 insertions(+), 88 deletions(-) (limited to 'pkgtools') diff --git a/pkgtools/pkglint/Makefile b/pkgtools/pkglint/Makefile index 369202bc188..35dc6458272 100644 --- a/pkgtools/pkglint/Makefile +++ b/pkgtools/pkglint/Makefile @@ -1,6 +1,6 @@ -# $NetBSD: Makefile,v 1.593 2019/08/16 21:00:17 rillig Exp $ +# $NetBSD: Makefile,v 1.594 2019/08/21 16:45:16 rillig Exp $ -PKGNAME= pkglint-5.7.20 +PKGNAME= pkglint-5.7.21 CATEGORIES= pkgtools DISTNAME= tools MASTER_SITES= ${MASTER_SITE_GITHUB:=golang/} diff --git a/pkgtools/pkglint/files/check_test.go b/pkgtools/pkglint/files/check_test.go index aa07665e0db..29f32c6485a 100644 --- a/pkgtools/pkglint/files/check_test.go +++ b/pkgtools/pkglint/files/check_test.go @@ -309,6 +309,18 @@ func (t *Tester) SetUpPkgsrc() { t.CreateFileLines("mk/bsd.fast.prefs.mk", MkCvsID) + // This file is used for initializing the allowed values for + // USE_LANGUAGES; see VarTypeRegistry.compilerLanguages. + t.CreateFileLines("mk/compiler.mk", + "_CXX_STD_VERSIONS=\tc++ c++14", + ".if ${USE_LANGUAGES:Mada} || \\", + " ${USE_LANGUAGES:Mc} || \\", + " ${USE_LANGUAGES:Mc99} || \\", + " ${USE_LANGUAGES:Mobjc} || \\", + " ${USE_LANGUAGES:Mfortran} || \\", + " ${USE_LANGUAGES:Mfortran77}", + ".endif") + // Category Makefiles require this file for the common definitions. t.CreateFileLines("mk/misc/category.mk") diff --git a/pkgtools/pkglint/files/mkline_test.go b/pkgtools/pkglint/files/mkline_test.go index 7b774e8d16c..82fb1a97a02 100644 --- a/pkgtools/pkglint/files/mkline_test.go +++ b/pkgtools/pkglint/files/mkline_test.go @@ -457,9 +457,10 @@ func (s *Suite) Test_MkLine__aligned(c *check.C) { "\tvalue", true) - // In commented multilines, the continuation lines may or may not start - // with a comment character. Bmake doesn't care, but for human readers - // it is confusing to omit the leading comment character. + // In commented multilines, bmake doesn't care whether the + // continuation lines does or doesn't start with a comment character. + // For human readers though, it is confusing to omit the leading + // comment character. // // For determining whether a multiline is aligned, the initial comment // character is ignored. diff --git a/pkgtools/pkglint/files/mklinechecker.go b/pkgtools/pkglint/files/mklinechecker.go index 237210ad6fb..9dfa903f033 100644 --- a/pkgtools/pkglint/files/mklinechecker.go +++ b/pkgtools/pkglint/files/mklinechecker.go @@ -599,7 +599,7 @@ func (ck MkLineChecker) checkVaruseModifiersRange(varuse *MkVarUse) { if len(mods) == 3 { if m, _, from, to, options := mods[0].MatchSubst(); m && from == "^" && matches(to, `^\w+$`) && options == "1" { magic := to - if m, positive, pattern := mods[1].MatchMatch(); m && positive && pattern == magic+"*" { + if m, positive, pattern, _ := mods[1].MatchMatch(); m && positive && pattern == magic+"*" { if m, _, from, to, options = mods[2].MatchSubst(); m && from == "^"+magic && to == "" && options == "" { fix := ck.MkLine.Autofix() fix.Notef("The modifier %q can be written as %q.", varuse.Mod(), ":[1]") @@ -1583,7 +1583,8 @@ func (ck MkLineChecker) simplifyCondition(varuse *MkVarUse, fromEmpty bool, notE pattern + condStr(fromEmpty, ")", "}") - to := "${" + varname + "} " + op + " " + pattern + quote := condStr(matches(pattern, `[^\-/0-9@A-Za-z]`), "\"", "") + to := "${" + varname + "} " + op + " " + quote + pattern + quote // TODO: Check in more cases whether the parentheses are really necessary. // In a !!${VAR} expression, parentheses are necessary. @@ -1599,11 +1600,11 @@ func (ck MkLineChecker) simplifyCondition(varuse *MkVarUse, fromEmpty bool, notE modifiers := varuse.modifiers for _, modifier := range modifiers { - if m, positive, pattern := modifier.MatchMatch(); m && (positive || len(modifiers) == 1) { + if m, positive, pattern, exact := modifier.MatchMatch(); m && (positive || len(modifiers) == 1) { ck.checkVartype(varname, opUseMatch, pattern, "") vartype := G.Pkgsrc.VariableType(ck.MkLines, varname) - if matches(pattern, `^[\w-/]+$`) && vartype != nil && !vartype.List() { + if exact && matches(pattern, `^[\w-/]+$`) && vartype != nil && !vartype.List() { fix := ck.MkLine.Autofix() fix.Notef("%s should be compared using %s instead of matching against %q.", @@ -1649,7 +1650,7 @@ func (ck MkLineChecker) checkDirectiveCondCompareVarStr(varuse *MkVarUse, op str ck.checkCompareVarStr(varname, op, str) case 1: - if m, _, pattern := varmods[0].MatchMatch(); m { + if m, _, pattern, _ := varmods[0].MatchMatch(); m { ck.checkVartype(varname, opUseMatch, pattern, "") // After applying the :M or :N modifier, every expression may end up empty, diff --git a/pkgtools/pkglint/files/mklinechecker_test.go b/pkgtools/pkglint/files/mklinechecker_test.go index 319a3b8f93d..e2f258ffd06 100644 --- a/pkgtools/pkglint/files/mklinechecker_test.go +++ b/pkgtools/pkglint/files/mklinechecker_test.go @@ -2184,6 +2184,9 @@ func (s *Suite) Test_MkLineChecker_checkVarUseQuoting(c *check.C) { mklines := t.SetUpFileMkLines("options.mk", MkCvsID, "GOPATH=\t${WRKDIR}", + "", + "CONFIGURE_ENV+=\tNAME=${R_PKGNAME} VER=${R_PKGVER}", + "", "do-build:", "\tcd ${WRKSRC} && GOPATH=${GOPATH} PATH=${PATH} :") @@ -2197,7 +2200,7 @@ func (s *Suite) Test_MkLineChecker_checkVarUseQuoting(c *check.C) { // of pkgsrc, and these may contain special characters. t.CheckOutputLines( - "WARN: ~/options.mk:4: The variable PATH should be quoted as part of a shell word.") + "WARN: ~/options.mk:7: The variable PATH should be quoted as part of a shell word.") } func (s *Suite) Test_MkLineChecker_checkVarUseQuoting__mstar(c *check.C) { diff --git a/pkgtools/pkglint/files/mkshparser.go b/pkgtools/pkglint/files/mkshparser.go index b248c6b9dc9..301154e64c2 100644 --- a/pkgtools/pkglint/files/mkshparser.go +++ b/pkgtools/pkglint/files/mkshparser.go @@ -11,12 +11,12 @@ func parseShellProgram(line *Line, program string) (*MkShList, error) { lexer := NewShellLexer(tokens, rest) parser := shyyParserImpl{} - succeeded := parser.Parse(lexer) + zeroMeansSuccess := parser.Parse(lexer) switch { - case succeeded == 0 && lexer.error == "": + case zeroMeansSuccess == 0 && lexer.error == "": return lexer.result, nil - case succeeded == 0: + case zeroMeansSuccess == 0: return nil, fmt.Errorf("splitIntoShellTokens couldn't parse %q", rest) default: return nil, &ParseError{append([]string{lexer.current}, lexer.remaining...)} diff --git a/pkgtools/pkglint/files/mkshparser_test.go b/pkgtools/pkglint/files/mkshparser_test.go index 31dc889e350..be937ab42b7 100644 --- a/pkgtools/pkglint/files/mkshparser_test.go +++ b/pkgtools/pkglint/files/mkshparser_test.go @@ -19,6 +19,7 @@ func (s *Suite) Test_parseShellProgram__parse_error_for_dollar(c *check.C) { t.CheckEquals(err, expError) } else { t.CheckDeepEquals(err, expError) + t.CheckDeepEquals(err.Error(), expError.Error()) // Just for code coverage t.CheckDeepEquals(program, expProgram) } @@ -48,6 +49,12 @@ func (s *Suite) Test_parseShellProgram__parse_error_for_dollar(c *check.C) { nil, nil, nil...) + + test( + "case ;;", + nil, + &ParseError{[]string{";;"}}, + nil...) } type ShSuite struct { @@ -396,11 +403,17 @@ func (s *ShSuite) Test_ShellParser__case_clause(c *check.C) { b.Words("*"), b.List(), sepNone)))) - // The default case may be omitted if PATTERNS can never be empty. + // The default case may even be omitted. s.test("case $$expr in ${PATTERNS:@p@ (${p}) action ;; @} esac", b.List().AddCommand(b.Case( b.Token("$$expr"), b.CaseItemVar("${PATTERNS:@p@ (${p}) action ;; @}")))) + + // Only variables that end with a :@ modifier may be used in this + // construct. All others are tokenized as normal words and lead + // to a syntax error in the shell parser. + s.testFail("case $$expr in ${PATTERNS} esac", + []string{}...) } func (s *ShSuite) Test_ShellParser__if_clause(c *check.C) { @@ -589,6 +602,21 @@ func (s *ShSuite) test(program string, expected *MkShList) { } } +func (s *ShSuite) testFail(program string, expectedRemaining ...string) { + t := s.t + + tokens, rest := splitIntoShellTokens(dummyLine, program) + t.CheckEquals(rest, "") + lexer := ShellLexer{remaining: tokens, atCommandStart: true} + parser := shyyParserImpl{} + + zeroMeansSuccess := parser.Parse(&lexer) + + if t.CheckEquals(zeroMeansSuccess, 1) && t.Check(lexer.error, check.Not(check.Equals), "") { + t.CheckDeepEquals(lexer.remaining, expectedRemaining) + } +} + func (s *ShSuite) Test_ShellLexer_Lex__redirects(c *check.C) { t := s.t @@ -708,6 +736,15 @@ func (s *Suite) Test_ShellLexer_Lex__case_patterns(c *check.C) { tkIN, tkWORD, tkESAC) + + test( + "case $$expr in ${PATTERNS:Mpattern} esac", + + tkCASE, + tkWORD, + tkIN, + tkWORD, + tkWORD) // No tkESAC since there is no :@ modifier. } type MkShBuilder struct { diff --git a/pkgtools/pkglint/files/mktypes.go b/pkgtools/pkglint/files/mktypes.go index 96348ba4259..77bd1c4b527 100644 --- a/pkgtools/pkglint/files/mktypes.go +++ b/pkgtools/pkglint/files/mktypes.go @@ -92,11 +92,13 @@ func (m MkVarUseModifier) Subst(str string) (string, bool) { // :Mpattern => true, true, "pattern" // :Npattern => true, false, "pattern" // :X => false -func (m MkVarUseModifier) MatchMatch() (ok bool, positive bool, pattern string) { +func (m MkVarUseModifier) MatchMatch() (ok bool, positive bool, pattern string, exact bool) { if hasPrefix(m.Text, "M") || hasPrefix(m.Text, "N") { - return true, m.Text[0] == 'M', m.Text[1:] + // See devel/bmake/files/str.c:^Str_Match + exact := !strings.ContainsAny(m.Text[1:], "*?[\\") + return true, m.Text[0] == 'M', m.Text[1:], exact } - return false, false, "" + return false, false, "", false } func (m MkVarUseModifier) IsToLower() bool { return m.Text == "tl" } diff --git a/pkgtools/pkglint/files/options.go b/pkgtools/pkglint/files/options.go index 12e68e90218..ea238b348f7 100755 --- a/pkgtools/pkglint/files/options.go +++ b/pkgtools/pkglint/files/options.go @@ -1,5 +1,7 @@ package pkglint +import "path" + func CheckLinesOptionsMk(mklines *MkLines) { ck := OptionsLinesChecker{ mklines, @@ -125,13 +127,27 @@ func (ck *OptionsLinesChecker) handleLowerLine(mkline *MkLine) { func (ck *OptionsLinesChecker) handleLowerCondition(mkline *MkLine, cond *MkCond) { recordUsedOption := func(varuse *MkVarUse) { - if varuse.varname == "PKG_OPTIONS" && len(varuse.modifiers) == 1 { - if m, positive, pattern := varuse.modifiers[0].MatchMatch(); m && positive { - option := pattern - if !containsVarRef(option) { - ck.handledOptions[option] = mkline - ck.optionsInDeclarationOrder = append(ck.optionsInDeclarationOrder, option) - } + if varuse.varname != "PKG_OPTIONS" || len(varuse.modifiers) != 1 { + return + } + + m, positive, pattern, exact := varuse.modifiers[0].MatchMatch() + if !m || !positive || containsVarRef(pattern) { + return + } + + if exact { + option := pattern + ck.handledOptions[option] = mkline + ck.optionsInDeclarationOrder = append(ck.optionsInDeclarationOrder, option) + return + } + + for declaredOption := range ck.declaredOptions { + matched, err := path.Match(pattern, declaredOption) + if err == nil && matched { + ck.handledOptions[declaredOption] = mkline + ck.optionsInDeclarationOrder = append(ck.optionsInDeclarationOrder, declaredOption) } } } diff --git a/pkgtools/pkglint/files/options_test.go b/pkgtools/pkglint/files/options_test.go index d57114b0559..77dba48ce9d 100755 --- a/pkgtools/pkglint/files/options_test.go +++ b/pkgtools/pkglint/files/options_test.go @@ -324,3 +324,66 @@ func (s *Suite) Test_CheckLinesOptionsMk__autofix(c *check.C) { ". endif", ".endif") } + +// A few packages (such as www/w3m) define several options that are +// handled by a single .if block in the lower part. +func (s *Suite) Test_CheckLinesOptionsMk__combined_option_handling(c *check.C) { + t := s.Init(c) + + t.SetUpOption("opt-variant1", "") + t.SetUpOption("opt-variant2", "") + t.SetUpOption("other", "") + t.CreateFileLines("mk/bsd.options.mk") + t.SetUpPackage("category/package", + ".include \"options.mk\"") + t.CreateFileLines("category/package/options.mk", + MkCvsID, + "", + "PKG_OPTIONS_VAR=\tPKG_OPTIONS.package", + "PKG_SUPPORTED_OPTIONS=\topt-variant1 opt-variant2", + "", + ".include \"../../mk/bsd.options.mk\"", + "", + ".if ${PKG_OPTIONS:Mopt-variant*}", + ".endif") + t.FinishSetUp() + t.Chdir("category/package") + + G.Check(".") + + // Before 5.7.21 on 2019-08-17, pkglint issued an error about the + // "invalid option name opt-variant*" and warnings about the + // unhandled options "opt-variant1" and "opt-variant2". + t.CheckOutputEmpty() +} + +func (s *Suite) Test_CheckLinesOptionsMk__combined_option_handling_coverage(c *check.C) { + t := s.Init(c) + + t.SetUpOption("opt-variant", "") + t.CreateFileLines("mk/bsd.options.mk") + t.SetUpPackage("category/package", + ".include \"options.mk\"") + t.CreateFileLines("category/package/options.mk", + MkCvsID, + "", + "PKG_OPTIONS_VAR=\tPKG_OPTIONS.package", + "PKG_SUPPORTED_OPTIONS=\topt-variant", + "", + ".include \"../../mk/bsd.options.mk\"", + "", + ".if ${PKG_OPTIONS:Mopt-[}", // intentional syntax error + ".endif", + "", + ".if ${PKG_OPTIONS:Mother-*}", + ".endif") + t.FinishSetUp() + t.Chdir("category/package") + + G.Check(".") + + // The warning appears because the pattern "opt-[" is malformed + // and therefore doesn't match the option. + t.CheckOutputLines( + "WARN: options.mk:4: Option \"opt-variant\" should be handled below in an .if block.") +} diff --git a/pkgtools/pkglint/files/package.go b/pkgtools/pkglint/files/package.go index 7c8e28d4a33..7f2449fb327 100644 --- a/pkgtools/pkglint/files/package.go +++ b/pkgtools/pkglint/files/package.go @@ -950,6 +950,8 @@ func (pkg *Package) CheckVarorder(mklines *MkLines) { {"GITHUB_TAG", optional}, {"DISTNAME", optional}, {"PKGNAME", optional}, + {"R_PKGNAME", optional}, + {"R_PKGVER", optional}, {"PKGREVISION", optional}, {"CATEGORIES", once}, {"MASTER_SITES", many}, diff --git a/pkgtools/pkglint/files/package_test.go b/pkgtools/pkglint/files/package_test.go index 00a78817276..d3b0a308e1e 100644 --- a/pkgtools/pkglint/files/package_test.go +++ b/pkgtools/pkglint/files/package_test.go @@ -1639,12 +1639,15 @@ func (s *Suite) Test_Package_checkGnuConfigureUseLanguages__realistic_compiler_m "", ".include \"../../mk/compiler.mk\"") t.CreateFileLines("mk/compiler.mk", - MkCvsID, - ".include \"bsd.prefs.mk\"", + "_CXX_STD_VERSIONS=\tc++ c++14", + ".if ${USE_LANGUAGES:Mada} \\", + " || ${USE_LANGUAGES:Mc} \\", + " || ${USE_LANGUAGES:Mfortran77}", + ".endif", "", - "USE_LANGUAGES?=\tc", - "USE_LANGUAGES+=\tc", - "USE_LANGUAGES+=\tc++") + // This line is ignored since it comes from the pkgsrc infrastructure. + "USE_LANGUAGES?=\t\tc") + t.FinishSetUp() G.Check(t.File("category/package")) @@ -1776,8 +1779,6 @@ func (s *Suite) Test_Package_checkUseLanguagesCompilerMk__too_late(c *check.C) { t.SetUpPackage("category/package", ".include \"../../mk/compiler.mk\"", "USE_LANGUAGES=\tc c99 fortran ada c++14") - t.CreateFileLines("mk/compiler.mk", - MkCvsID) t.FinishSetUp() G.Check(t.File("category/package")) @@ -1798,8 +1799,6 @@ func (s *Suite) Test_Package_checkUseLanguagesCompilerMk__compiler_mk(c *check.C t.CreateFileLines("category/package/compiler.mk", MkCvsID, "USE_LANGUAGES=\tc++") - t.CreateFileLines("mk/compiler.mk", - MkCvsID) t.FinishSetUp() G.Check(t.File("category/package")) diff --git a/pkgtools/pkglint/files/varalignblock.go b/pkgtools/pkglint/files/varalignblock.go index 11dac56d5cc..d07f021b44d 100644 --- a/pkgtools/pkglint/files/varalignblock.go +++ b/pkgtools/pkglint/files/varalignblock.go @@ -173,6 +173,10 @@ func (*VaralignBlock) split(textnl string, initial bool) varalignSplitResult { lexer := p.lexer parseLeadingComment := func() string { + if hasPrefix(lexer.Rest(), "# ") { + return "" + } + mark := lexer.Mark() if !lexer.SkipByte('#') && initial && lexer.SkipByte(' ') { @@ -379,7 +383,7 @@ func (va *VaralignBlock) realign(info *varalignLine, newWidth int) { } } -func (va *VaralignBlock) realignMultiEmptyInitial(info *varalignLine, newWidth int) { +func (*VaralignBlock) realignMultiEmptyInitial(info *varalignLine, newWidth int) { leadingComment := info.parts.leadingComment varnameOp := info.parts.varnameOp oldSpace := info.parts.spaceBeforeValue @@ -400,15 +404,15 @@ func (va *VaralignBlock) realignMultiEmptyInitial(info *varalignLine, newWidth i } hasSpace := strings.IndexByte(oldSpace, ' ') != -1 - oldColumn := tabWidth(leadingComment + varnameOp + oldSpace) + oldColumn := info.varnameOpSpaceWidth() column := tabWidth(leadingComment + varnameOp + newSpace) - assert(column >= oldColumn || column > info.varnameOpWidth()) - // TODO: explicitly mention "single space", "tabs to the newWidth", "tabs to column 72" fix := info.mkline.Autofix() - if hasSpace && column != oldColumn { + if newSpace == " " { + fix.Notef("This outlier variable should be aligned with a single space.") + } else if hasSpace && column != oldColumn { fix.Notef("This variable value should be aligned with tabs, not spaces, to column %d.", column+1) } else if column != oldColumn { fix.Notef("This variable value should be aligned to column %d.", column+1) @@ -431,15 +435,7 @@ func (va *VaralignBlock) realignMultiEmptyFollow(info *varalignLine, newWidth in } } - newWidth = oldWidth + va.indentDiff - if newWidth < 8 { - newWidth = oldWidth & -8 - if newWidth < 8 { - newWidth = 8 - } - } - - newSpace := indent(newWidth) + newSpace := indent(imax(oldWidth+va.indentDiff, 8)) if newSpace == oldSpace { return } @@ -531,12 +527,11 @@ func (va *VaralignBlock) realignSingle(info *varalignLine, newWidth int) { oldColumn := tabWidth(leadingComment + varnameOp + oldSpace) column := tabWidth(leadingComment + varnameOp + newSpace) - if info.parts.value == "" && info.parts.trailingComment == "" && !info.continuation() { - return - } - fix := info.mkline.Autofix() - if hasSpace && column != oldColumn { + if newSpace == " " { + fix.Notef("This outlier variable value should be aligned with a single space.") + va.explainWrongColumn(fix) + } else if hasSpace && column != oldColumn { fix.Notef("This variable value should be aligned with tabs, not spaces, to column %d.", column+1) va.explainWrongColumn(fix) } else if column != oldColumn { diff --git a/pkgtools/pkglint/files/varalignblock_test.go b/pkgtools/pkglint/files/varalignblock_test.go index d74d30ad2b7..3b4e31a69c0 100644 --- a/pkgtools/pkglint/files/varalignblock_test.go +++ b/pkgtools/pkglint/files/varalignblock_test.go @@ -29,6 +29,8 @@ func (vt *VaralignTester) Input(lines ...string) { vt.input = lines } // Internals remembers the expected internal state of the varalignBlockInfos, // to better trace down at which points the decisions are made. +// +// Each line has the format " ". func (vt *VaralignTester) Internals(lines ...string) { vt.internals = lines } // Diagnostics remembers the expected diagnostics. @@ -1053,7 +1055,7 @@ func (s *Suite) Test_VaralignBlock__outlier_in_follow_continuation(c *check.C) { "38 38", " 24") vt.Diagnostics( - "NOTE: ~/Makefile:2: This variable value should be aligned to column 40.") + "NOTE: ~/Makefile:2: This outlier variable should be aligned with a single space.") vt.Autofixes( "AUTOFIX: ~/Makefile:2: Replacing \"\" with \" \".") vt.Fixed( @@ -1800,6 +1802,60 @@ func (s *Suite) Test_VaralignBlock__outlier_14(c *check.C) { vt.Run() } +func (s *Suite) Test_VaralignBlock__outlier_with_several_spaces(c *check.C) { + vt := NewVaralignTester(s, c) + vt.Input( + "SHORT=\tvalue", + "VERY_VERY_LONG_VARIABLE_NAME= value") + vt.Internals( + "06 08", + "29 32") + vt.Diagnostics( + "NOTE: ~/Makefile:2: This outlier variable value should be aligned with a single space.") + vt.Autofixes( + "AUTOFIX: ~/Makefile:2: Replacing \" \" with \" \".") + vt.Fixed( + "SHORT= value", + "VERY_VERY_LONG_VARIABLE_NAME= value") + vt.Run() +} + +func (s *Suite) Test_VaralignBlock__single_space_in_short_line(c *check.C) { + vt := NewVaralignTester(s, c) + vt.Input( + "SHORT= value", + "LONG_NAME=\tvalue") + vt.Internals( + "06 07", + "10 16") + vt.Diagnostics( + "NOTE: ~/Makefile:1: This variable value should be aligned with tabs, not spaces, to column 17.") + vt.Autofixes( + "AUTOFIX: ~/Makefile:1: Replacing \" \" with \"\\t\\t\".") + vt.Fixed( + "SHORT= value", + "LONG_NAME= value") + vt.Run() +} + +func (s *Suite) Test_VaralignBlock__outlier_without_space(c *check.C) { + vt := NewVaralignTester(s, c) + vt.Input( + "SHORT=\tvalue", + "LONG.678901234567890=value") + vt.Internals( + "06 08", + "21 21") + vt.Diagnostics( + "NOTE: ~/Makefile:2: This outlier variable value should be aligned with a single space.") + vt.Autofixes( + "AUTOFIX: ~/Makefile:2: Replacing \"\" with \" \".") + vt.Fixed( + "SHORT= value", + "LONG.678901234567890= value") + vt.Run() +} + // The INSTALLATION_DIRS line is so long that it is considered an outlier, // since compared to the DIST line, it is at least two tabs away. // Pkglint before 2018-01-26 suggested that it "should be aligned to column 9", @@ -2224,6 +2280,25 @@ func (s *Suite) Test_VaralignBlock__command_with_arguments(c *check.C) { vt.Run() } +// Variables with empty values and no comments are completely ignored, +// since they have nothing to be aligned with the other lines. +func (s *Suite) Test_VaralignBlock__empty_value(c *check.C) { + vt := NewVaralignTester(s, c) + vt.Input( + "EMPTY_VALUE=", + "VAR=\t\tvalue") + vt.Internals( + "04 16") + vt.Diagnostics( + nil...) + vt.Autofixes( + nil...) + vt.Fixed( + "EMPTY_VALUE=", + "VAR= value") + vt.Run() +} + func (s *Suite) Test_VaralignBlock_Process__autofix(c *check.C) { t := s.Init(c) @@ -2345,14 +2420,105 @@ func (s *Suite) Test_VaralignBlock_realignMultiEmptyInitial(c *check.C) { mklines := t.NewMkLines("filename.mk", MkCvsID, "VAR=\t${VAR}", - // FIXME: It's not possible to align with tabs to column 21. "LONG_VARIABLE_NAME= \t \\", "\t${LONG_VARIABLE_NAME}") mklines.Check() t.CheckOutputLines( - "NOTE: filename.mk:3: This variable value should be aligned with tabs, not spaces, to column 21.") + "NOTE: filename.mk:3: This outlier variable should be aligned with a single space.") +} + +func (s *Suite) Test_VaralignBlock_realignMultiEmptyInitial__spaces(c *check.C) { + vt := NewVaralignTester(s, c) + vt.Input( + "VAR= \\", + "\tvalue", + // This line is necessary to trigger the realignment; see VaralignBlock.Finish. + "VAR= value") + vt.Internals( + "04 08", + " 08", + "04 05") + vt.Diagnostics( + "NOTE: ~/Makefile:1: Variable values should be aligned with tabs, not spaces.", + "NOTE: ~/Makefile:3: This variable value should be aligned with tabs, not spaces, to column 9.") + vt.Autofixes( + "AUTOFIX: ~/Makefile:1: Replacing \" \" with \"\\t\".", + "AUTOFIX: ~/Makefile:3: Replacing \" \" with \"\\t\".") + vt.Fixed( + "VAR= \\", + " value", + "VAR= value") + vt.Run() +} + +func (s *Suite) Test_VaralignBlock_realignMultiInitial__spaces(c *check.C) { + vt := NewVaralignTester(s, c) + vt.Input( + "VAR= value1 \\", + " value2") + vt.Internals( + "04 08", + " 08") + vt.Diagnostics( + "NOTE: ~/Makefile:1: Variable values should be aligned with tabs, not spaces.", + "NOTE: ~/Makefile:2: This continuation line should be indented with \"\\t\".") + vt.Autofixes( + "AUTOFIX: ~/Makefile:1: Replacing \" \" with \"\\t\".", + "AUTOFIX: ~/Makefile:2: Replacing \" \" with \"\\t\".") + vt.Fixed( + "VAR= value1 \\", + " value2") + vt.Run() +} + +// This example is quite unrealistic since typically the first line is +// the least indented. +// +// All follow-up lines are indented with at least one tab, to make clear +// they are continuation lines. +func (s *Suite) Test_VaralignBlock_realignMultiEmptyFollow(c *check.C) { + vt := NewVaralignTester(s, c) + vt.Input( + "VAR= \\", + " value1 \\", + " value2 \\", + " value3 \\", + "value4 \\", + "\\", + "# comment") + vt.Internals( + "04 05", + " 08", + " 10", + " 06", + " 00", + " 00", + " 00") + vt.Diagnostics( + "NOTE: ~/Makefile:2: This continuation line should be indented with \"\\t\".", + "NOTE: ~/Makefile:3: This continuation line should be indented with \"\\t \".", + "NOTE: ~/Makefile:4: This continuation line should be indented with \"\\t\".", + "NOTE: ~/Makefile:5: This continuation line should be indented with \"\\t\".", + "NOTE: ~/Makefile:6: This continuation line should be indented with \"\\t\".", + "NOTE: ~/Makefile:7: This continuation line should be indented with \"\\t\".") + vt.Autofixes( + "AUTOFIX: ~/Makefile:2: Replacing \" \" with \"\\t\".", + "AUTOFIX: ~/Makefile:3: Replacing \" \" with \"\\t \".", + "AUTOFIX: ~/Makefile:4: Replacing \" \" with \"\\t\".", + "AUTOFIX: ~/Makefile:5: Replacing \"\" with \"\\t\".", + "AUTOFIX: ~/Makefile:6: Replacing \"\" with \"\\t\".", + "AUTOFIX: ~/Makefile:7: Replacing \"\" with \"\\t\".") + vt.Fixed( + "VAR= \\", + " value1 \\", + " value2 \\", + " value3 \\", + " value4 \\", + " \\", + " # comment") + vt.Run() } func (s *Suite) Test_VaralignBlock_split(c *check.C) { @@ -2546,6 +2712,68 @@ func (s *Suite) Test_VaralignBlock_split(c *check.C) { continuation: "\\", }) + // A follow-up line may start with a comment character. There are + // two possible interpretations: + // + // 1. It is a leading comment, and the rest of the line is parsed + // as usual. + // + // 2. It is a continuation of the value, and therefore the value ends + // here; everything after this line is part of the trailing comment. + // + // The character that follows the comment character decides which + // interpretation is used. A space makes the comment a trailing + // comment since that's the way these trailing comments typically look. + // Any other character makes it a leading comment. + + test("#\tcomment", false, + varalignSplitResult{ + leadingComment: "#", + varnameOp: "", + spaceBeforeValue: "\t", + value: "comment", + spaceAfterValue: "", + trailingComment: "", + spaceAfterComment: "", + continuation: "", + }) + + test("#\tcomment \\", false, + varalignSplitResult{ + leadingComment: "#", + varnameOp: "", + spaceBeforeValue: "\t", + value: "comment", + spaceAfterValue: " ", + trailingComment: "", + spaceAfterComment: "", + continuation: "\\", + }) + + test("# comment", false, + varalignSplitResult{ + leadingComment: "", + varnameOp: "", + spaceBeforeValue: "", + value: "", + spaceAfterValue: "", + trailingComment: "# comment", + spaceAfterComment: "", + continuation: "", + }) + + test("# comment \\", false, + varalignSplitResult{ + leadingComment: "", + varnameOp: "", + spaceBeforeValue: "", + value: "", + spaceAfterValue: "", + trailingComment: "# comment", + spaceAfterComment: " ", + continuation: "\\", + }) + // Commented variable assignments are only valid if they // directly follow the comment sign. // @@ -2555,6 +2783,26 @@ func (s *Suite) Test_VaralignBlock_split(c *check.C) { func() { test("# VAR= value", true, varalignSplitResult{}) }) } +// This test runs canonicalInitial directly since as of August 2019 +// that function is only used in a single place, and from this place +// varnameOpSpaceWidth is always bigger than width. +func (s *Suite) Test_varalignLine_canonicalInitial(c *check.C) { + t := s.Init(c) + + var v varalignLine + v.parts.varnameOp = "LONG.123456789=" + v.parts.spaceBeforeValue = " " + t.CheckEquals(v.canonicalInitial(16), false) + + v.parts.varnameOp = "LONG.1234567890=" + + t.CheckEquals(v.canonicalInitial(16), true) + + v.parts.spaceBeforeValue = "" + + t.CheckEquals(v.canonicalInitial(16), false) +} + func (s *Suite) Test_varalignLine_canonicalFollow(c *check.C) { t := s.Init(c) diff --git a/pkgtools/pkglint/files/vardefs.go b/pkgtools/pkglint/files/vardefs.go index c03c9b546f6..0f04f34758e 100644 --- a/pkgtools/pkglint/files/vardefs.go +++ b/pkgtools/pkglint/files/vardefs.go @@ -311,7 +311,12 @@ func (reg *VarTypeRegistry) infralist(varname string, basicType *BasicType) { // compilerLanguages reads the available languages that are typically // bundled in a single compiler framework, such as GCC or Clang. func (reg *VarTypeRegistry) compilerLanguages(src *Pkgsrc) *BasicType { - mklines := LoadMk(src.File("mk/compiler.mk"), NotEmpty) + options := NotEmpty + if !G.Testing { + options = NotEmpty | MustSucceed + } + mklines := LoadMk(src.File("mk/compiler.mk"), options) + languages := make(map[string]bool) if mklines != nil { for _, mkline := range mklines.mklines { @@ -321,15 +326,19 @@ func (reg *VarTypeRegistry) compilerLanguages(src *Pkgsrc) *BasicType { languages[intern(word)] = true } } - } - } - alwaysAvailable := [...]string{ - "ada", "c", "c99", "c++", "c++11", "c++14", - "fortran", "fortran77", "java", "objc", "obj-c++"} - - for _, language := range alwaysAvailable { - languages[language] = true + if mkline.IsDirective() && mkline.Cond() != nil { + mkline.Cond().Walk(&MkCondCallback{ + VarUse: func(varuse *MkVarUse) { + if varuse.varname == "USE_LANGUAGES" && len(varuse.modifiers) == 1 { + ok, _, pattern, exact := varuse.modifiers[0].MatchMatch() + if ok && exact && !containsVarRef(pattern) { + languages[intern(pattern)] = true + } + } + }}) + } + } } joined := keysJoined(languages) @@ -1529,6 +1538,8 @@ func (reg *VarTypeRegistry) Init(src *Pkgsrc) { "*: use, use-loadtime") reg.sys("RUN", BtShellCommand) reg.sys("RUN_LDCONFIG", BtYesNo) + reg.pkg("R_PKGNAME", BtRPkgName) + reg.pkg("R_PKGVER", BtRPkgVer) reg.pkglist("SCRIPTS_ENV", BtShellWord) reg.usrlist("SETGID_GAMES_PERMS", BtPerms) reg.usrlist("SETUID_ROOT_PERMS", BtPerms) diff --git a/pkgtools/pkglint/files/vardefs_test.go b/pkgtools/pkglint/files/vardefs_test.go index 8663e141734..eaf2a31b073 100644 --- a/pkgtools/pkglint/files/vardefs_test.go +++ b/pkgtools/pkglint/files/vardefs_test.go @@ -12,6 +12,35 @@ func (s *Suite) Test_VarTypeRegistry_Init(c *check.C) { t.CheckEquals(src.vartypes.Canon("USE_BUILTIN.*").basicType.name, "YesNoIndirectly") } +func (s *Suite) Test_VarTypeRegistry_compilerLanguages(c *check.C) { + t := s.Init(c) + + G.Testing = false // Just for code coverage + t.CreateFileLines("mk/compiler.mk", + MkCvsID, + "", + "_CXX_STD_VERSIONS= gnu++14", + ".for _version_ in ${_CXX_STD_VERSIONS}", + ". if !empty(USE_LANGUAGES:M${_version_})", + "USE_LANGUAGES+= c++", + ". endif", + ".endfor", + "", + ".if ${USE_LANGUAGES:Mexpr-lang} || !empty(USE_LANGUAGES:Mempty-lang)", + ".endif", + "", + // Just for code coverage + ".if ${OTHER} || ${USE_LANGUAGES} \\", + " || ${USE_LANGUAGES:O} || ${USE_LANGUAGES:Mc++-*}", + ".endif") + reg := NewVarTypeRegistry() + + compilerLanguages := reg.compilerLanguages(&G.Pkgsrc) + + enumValues := compilerLanguages.AllowedEnums() + t.CheckEquals(enumValues, "empty-lang expr-lang gnu++14") +} + func (s *Suite) Test_VarTypeRegistry_enumFrom(c *check.C) { t := s.Init(c) @@ -38,7 +67,10 @@ func (s *Suite) Test_VarTypeRegistry_enumFrom(c *check.C) { ". if !empty(USE_LANGUAGES:M${_version_})", "USE_LANGUAGES+= c++", ". endif", - ".endfor") + ".endfor", + "", + ".if ${USE_LANGUAGES:Mexpr-lang} || !empty(USE_LANGUAGES:Mempty-lang)", + ".endif") t.SetUpVartypes() @@ -49,8 +81,8 @@ func (s *Suite) Test_VarTypeRegistry_enumFrom(c *check.C) { test("EMACS_VERSIONS_ACCEPTED", "enum: emacs29 emacs31 (list, package-settable)") test("PKG_JVM", "enum: jdk16 openjdk7 openjdk8 oracle-jdk8 sun-jdk7 (system-provided)") - test("USE_LANGUAGES", "enum: ada c c++ c++03 c++0x c++11 c++14 c99 "+ - "fortran fortran77 gnu++03 gnu++0x gnu++11 gnu++14 java obj-c++ objc (list, package-settable)") + test("USE_LANGUAGES", "enum: c++03 c++0x c++11 c++14 empty-lang expr-lang "+ + "gnu++03 gnu++0x gnu++11 gnu++14 (list, package-settable)") test("PKGSRC_COMPILER", "enum: ccache distcc f2c g95 gcc ido mipspro-ucode sunpro (list, user-settable)") } diff --git a/pkgtools/pkglint/files/vartype.go b/pkgtools/pkglint/files/vartype.go index d930dbf8895..a781c7b944c 100644 --- a/pkgtools/pkglint/files/vartype.go +++ b/pkgtools/pkglint/files/vartype.go @@ -264,6 +264,8 @@ func (bt *BasicType) NeedsQ() bool { BtPkgRevision, BtPrefixPathname, BtPythonDependency, + BtRPkgName, + BtRPkgVer, BtRelativePkgDir, BtRelativePkgPath, BtStage, @@ -338,6 +340,8 @@ var ( BtPkgRevision = &BasicType{"PkgRevision", (*VartypeCheck).PkgRevision} BtPrefixPathname = &BasicType{"PrefixPathname", (*VartypeCheck).PrefixPathname} BtPythonDependency = &BasicType{"PythonDependency", (*VartypeCheck).PythonDependency} + BtRPkgName = &BasicType{"RPkgName", (*VartypeCheck).RPkgName} + BtRPkgVer = &BasicType{"RPkgVer", (*VartypeCheck).RPkgVer} BtRelativePkgDir = &BasicType{"RelativePkgDir", (*VartypeCheck).RelativePkgDir} BtRelativePkgPath = &BasicType{"RelativePkgPath", (*VartypeCheck).RelativePkgPath} BtRestricted = &BasicType{"Restricted", (*VartypeCheck).Restricted} diff --git a/pkgtools/pkglint/files/vartypecheck.go b/pkgtools/pkglint/files/vartypecheck.go index ae3e35e332b..8603c18a9ef 100644 --- a/pkgtools/pkglint/files/vartypecheck.go +++ b/pkgtools/pkglint/files/vartypecheck.go @@ -793,12 +793,30 @@ func (cv *VartypeCheck) Option() { return } - if m, optname := match1(value, `^-?([a-z][-0-9a-z+]*)$`); m { - if !cv.MkLines.once.FirstTimeSlice("option:", optname) { - return - } + validName := regex.Pattern(`^-?([a-z][-0-9a-z_+]*)$`) + if cv.Op == opUseMatch { + validName = `^-?([a-z][*+\-0-9?\[\]_a-z]*)$` + } + + // TODO: Distinguish between: + // - a bare option name (must start with a letter), + // - an option selection (may have a leading hyphen). + m, optname := match1(value, validName) + if !m { + cv.Errorf("Invalid option name %q. Option names must start with a lowercase letter and be all-lowercase.", value) + return + } + + if !cv.MkLines.once.FirstTimeSlice("option:", optname) { + return + } + + if contains(optname, "_") { + cv.Warnf("Use of the underscore character in option names is deprecated.") + return + } - // There's a difference between empty and absent here. + if !strings.ContainsAny(optname, "*?[") { if _, found := G.Pkgsrc.PkgOptions[optname]; !found { cv.Warnf("Unknown option %q.", optname) cv.Explain( @@ -807,15 +825,7 @@ func (cv *VartypeCheck) Option() { "update that file yourself or suggest a description for this option", "on the tech-pkg@NetBSD.org mailing list.") } - return - } - - if matches(value, `^-?([a-z][-0-9a-z_\+]*)$`) { - cv.Warnf("Use of the underscore character in option names is deprecated.") - return } - - cv.Errorf("Invalid option name %q. Option names must start with a lowercase letter and be all-lowercase.", value) } // Pathlist checks variables like the PATH environment variable. @@ -911,6 +921,10 @@ func (cv *VartypeCheck) Pkgname() { "A valid package name has the form packagename-version, where version", "consists only of digits, letters and dots.") } + + if matches(value, `nb\d+$`) { + cv.Errorf("The \"nb\" part of the version number belongs in PKGREVISION.") + } } func (cv *VartypeCheck) PkgOptionsVar() { @@ -1019,6 +1033,33 @@ func (cv *VartypeCheck) PythonDependency() { } } +func (cv *VartypeCheck) RPkgName() { + if cv.Op == opUseMatch { + return + } + + if cv.Value != cv.ValueNoVar { + cv.Warnf("The R package name should not contain variables.") + return + } + + if hasPrefix(cv.Value, "R-") { + cv.Warnf("The %s does not need the %q prefix.", cv.Varname, "R-") + } + + invalid := replaceAll(cv.Value, `[\w\-.+]`, "") + if invalid != "" { + cv.Warnf("The R package name contains the invalid characters %q.", invalid) + } +} + +func (cv *VartypeCheck) RPkgVer() { + value := cv.Value + if cv.Op != opUseMatch && value == cv.ValueNoVar && !matches(value, `^\d[\w-.]*$`) { + cv.Warnf("Invalid R version number %q.", value) + } +} + // RelativePkgDir refers to a package directory, e.g. ../../category/pkgbase. func (cv *VartypeCheck) RelativePkgDir() { MkLineChecker{cv.MkLines, cv.MkLine}.CheckRelativePkgdir(cv.Value) diff --git a/pkgtools/pkglint/files/vartypecheck_test.go b/pkgtools/pkglint/files/vartypecheck_test.go index c99387f1ad3..51161040aeb 100644 --- a/pkgtools/pkglint/files/vartypecheck_test.go +++ b/pkgtools/pkglint/files/vartypecheck_test.go @@ -1063,6 +1063,12 @@ func (s *Suite) Test_VartypeCheck_Pkgname(c *check.C) { vt.Output( "WARN: filename.mk:8: \"pkgbase-z1\" is not a valid package name.") + vt.Values( + "pkgbase-1.0nb17") + + vt.Output( + "ERROR: filename.mk:11: The \"nb\" part of the version number belongs in PKGREVISION.") + vt.Op(opUseMatch) vt.Values( "pkgbase-[0-9]*") @@ -1124,6 +1130,18 @@ func (s *Suite) Test_VartypeCheck_PkgRevision(c *check.C) { vt.OutputEmpty() } +func (s *Suite) Test_VartypeCheck_PrefixPathname(c *check.C) { + vt := NewVartypeCheckTester(s.Init(c), (*VartypeCheck).PrefixPathname) + + vt.Varname("PKGMANDIR") + vt.Values( + "man/man1", + "share/locale") + + vt.Output( + "WARN: filename.mk:1: Please use \"${PKGMANDIR}/man1\" instead of \"man/man1\".") +} + func (s *Suite) Test_VartypeCheck_PythonDependency(c *check.C) { vt := NewVartypeCheckTester(s.Init(c), (*VartypeCheck).PythonDependency) @@ -1138,16 +1156,47 @@ func (s *Suite) Test_VartypeCheck_PythonDependency(c *check.C) { "WARN: filename.mk:3: Invalid Python dependency \"cairo,X\".") } -func (s *Suite) Test_VartypeCheck_PrefixPathname(c *check.C) { - vt := NewVartypeCheckTester(s.Init(c), (*VartypeCheck).PrefixPathname) +func (s *Suite) Test_VartypeCheck_RPkgName(c *check.C) { + vt := NewVartypeCheckTester(s.Init(c), (*VartypeCheck).RPkgName) - vt.Varname("PKGMANDIR") + vt.Varname("R_PKGNAME") vt.Values( - "man/man1", - "share/locale") + "package", + "${VAR}", + "a,b,c", + "under_score", + "R-package") vt.Output( - "WARN: filename.mk:1: Please use \"${PKGMANDIR}/man1\" instead of \"man/man1\".") + "WARN: filename.mk:2: The R package name should not contain variables.", + "WARN: filename.mk:3: The R package name contains the invalid characters \",,\".", + "WARN: filename.mk:5: The R_PKGNAME does not need the \"R-\" prefix.") + + vt.Op(opUseMatch) + vt.Values( + "R-package") + + vt.OutputEmpty() +} + +func (s *Suite) Test_VartypeCheck_RPkgVer(c *check.C) { + vt := NewVartypeCheckTester(s.Init(c), (*VartypeCheck).RPkgVer) + + vt.Varname("R_PKGVER") + vt.Values( + "1.0", + "1-2-3", + "${VERSION}", + "1-:") + + vt.Output( + "WARN: filename.mk:4: Invalid R version number \"1-:\".") + + vt.Op(opUseMatch) + vt.Values( + "1-:") + + vt.OutputEmpty() } func (s *Suite) Test_VartypeCheck_RelativePkgPath(c *check.C) { @@ -1688,6 +1737,6 @@ func (vt *VartypeCheckTester) OutputEmpty() { func (vt *VartypeCheckTester) nextSection() { if vt.lineno%10 != 1 { - vt.lineno = vt.lineno - vt.lineno%10 + 11 + vt.lineno += 9 - (vt.lineno+8)%10 } } -- cgit v1.2.3