diff options
author | rillig <rillig@pkgsrc.org> | 2019-01-13 19:55:52 +0000 |
---|---|---|
committer | rillig <rillig@pkgsrc.org> | 2019-01-13 19:55:52 +0000 |
commit | 6ff3280b7b7dc7d0451f54fa081477885b3468fa (patch) | |
tree | 14ffcecb732e69ca7447d66c11f98ae0d58b4002 /pkgtools/pkglint/files | |
parent | 74c1dce01aea03b6d5d449e15507220926a2d6a5 (diff) | |
download | pkgsrc-6ff3280b7b7dc7d0451f54fa081477885b3468fa.tar.gz |
pkgtools/pkglint: update to 5.6.11
Changes since 5.6.10:
* Improved the wording of several warnings
* Fixed parsing of complicated dependency patterns such as
{ssh{,6}-[0-9]*,openssh-[0-9]*}. Pkglint still doesn't understand
them but at least it doesn't mark them as "unknown" anymore.
* Lots of refactoring, as usual. This is the last part of the big
refactoring, therefore future changes to pkglint are expected to be
smaller than in the previous 3 months.
Diffstat (limited to 'pkgtools/pkglint/files')
60 files changed, 2863 insertions, 1893 deletions
diff --git a/pkgtools/pkglint/files/alternatives_test.go b/pkgtools/pkglint/files/alternatives_test.go index 760dc2f9071..95a51135c5f 100644 --- a/pkgtools/pkglint/files/alternatives_test.go +++ b/pkgtools/pkglint/files/alternatives_test.go @@ -5,7 +5,7 @@ import "gopkg.in/check.v1" func (s *Suite) Test_CheckFileAlternatives__PLIST(c *check.C) { t := s.Init(c) - t.SetupPackage("category/package") + t.SetUpPackage("category/package") t.Chdir("category/package") t.CreateFileLines("ALTERNATIVES", "sbin/sendmail @PREFIX@/sbin/sendmail.postfix@POSTFIXVER@", @@ -37,7 +37,7 @@ func (s *Suite) Test_CheckFileAlternatives__PLIST(c *check.C) { "ERROR: ALTERNATIVES:7: Alternative implementation \"@VARBASE@/game/scores\" "+ "must appear in the PLIST as \"${VARBASE}/game/scores\".") - t.SetupCommandLine("--autofix") + t.SetUpCommandLine("--autofix") G.Check(".") diff --git a/pkgtools/pkglint/files/autofix_test.go b/pkgtools/pkglint/files/autofix_test.go index 8c947119423..f7d0db78920 100644 --- a/pkgtools/pkglint/files/autofix_test.go +++ b/pkgtools/pkglint/files/autofix_test.go @@ -22,8 +22,8 @@ func (s *Suite) Test_Autofix_Warnf__duplicate(c *check.C) { func (s *Suite) Test_Autofix__default_leaves_line_unchanged(c *check.C) { t := s.Init(c) - t.SetupCommandLine("--source") - mklines := t.SetupFileMkLines("Makefile", + t.SetUpCommandLine("--source") + mklines := t.SetUpFileMkLines("Makefile", "# row 1 \\", "continuation of row 1") line := mklines.lines.Lines[0] @@ -50,8 +50,8 @@ func (s *Suite) Test_Autofix__default_leaves_line_unchanged(c *check.C) { func (s *Suite) Test_Autofix__show_autofix_modifies_line(c *check.C) { t := s.Init(c) - t.SetupCommandLine("--source", "--show-autofix") - mklines := t.SetupFileMkLines("Makefile", + t.SetUpCommandLine("--source", "--show-autofix") + mklines := t.SetUpFileMkLines("Makefile", "# row 1 \\", "continuation of row 1") line := mklines.lines.Lines[0] @@ -86,8 +86,8 @@ func (s *Suite) Test_Autofix__show_autofix_modifies_line(c *check.C) { func (s *Suite) Test_Autofix_ReplaceAfter__autofix(c *check.C) { t := s.Init(c) - t.SetupCommandLine("--autofix", "--source") - mklines := t.SetupFileMkLines("Makefile", + t.SetUpCommandLine("--autofix", "--source") + mklines := t.SetUpFileMkLines("Makefile", "# line 1 \\", "continuation 1 \\", "continuation 2") @@ -108,8 +108,8 @@ func (s *Suite) Test_Autofix_ReplaceAfter__autofix(c *check.C) { func (s *Suite) Test_Autofix_ReplaceRegex__show_autofix(c *check.C) { t := s.Init(c) - t.SetupCommandLine("--show-autofix") - lines := t.SetupFileLines("Makefile", + t.SetUpCommandLine("--show-autofix") + lines := t.SetUpFileLines("Makefile", "line1", "line2", "line3") @@ -137,8 +137,8 @@ func (s *Suite) Test_Autofix_ReplaceRegex__show_autofix(c *check.C) { func (s *Suite) Test_Autofix_ReplaceRegex__autofix(c *check.C) { t := s.Init(c) - t.SetupCommandLine("--autofix", "--source") - lines := t.SetupFileLines("Makefile", + t.SetUpCommandLine("--autofix", "--source") + lines := t.SetUpFileLines("Makefile", "line1", "line2", "line3") @@ -177,8 +177,8 @@ func (s *Suite) Test_Autofix_ReplaceRegex__autofix(c *check.C) { func (s *Suite) Test_Autofix_ReplaceRegex__show_autofix_and_source(c *check.C) { t := s.Init(c) - t.SetupCommandLine("--show-autofix", "--source") - lines := t.SetupFileLines("Makefile", + t.SetUpCommandLine("--show-autofix", "--source") + lines := t.SetUpFileLines("Makefile", "line1", "line2", "line3") @@ -213,8 +213,8 @@ func (s *Suite) Test_Autofix_ReplaceRegex__show_autofix_and_source(c *check.C) { func (s *Suite) Test_SaveAutofixChanges(c *check.C) { t := s.Init(c) - t.SetupCommandLine("--autofix") - lines := t.SetupFileLines("example.txt", + t.SetUpCommandLine("--autofix") + lines := t.SetUpFileLines("example.txt", "line1 := value1", "line2 := value2", "line3 := value3") @@ -238,8 +238,8 @@ func (s *Suite) Test_SaveAutofixChanges(c *check.C) { func (s *Suite) Test_SaveAutofixChanges__no_changes_necessary(c *check.C) { t := s.Init(c) - t.SetupCommandLine("--autofix") - lines := t.SetupFileLines("DESCR", + t.SetUpCommandLine("--autofix") + lines := t.SetUpFileLines("DESCR", "Line 1", "Line 2") @@ -259,7 +259,7 @@ func (s *Suite) Test_SaveAutofixChanges__no_changes_necessary(c *check.C) { func (s *Suite) Test_Autofix__multiple_fixes(c *check.C) { t := s.Init(c) - t.SetupCommandLine("--show-autofix", "--explain") + t.SetUpCommandLine("--show-autofix", "--explain") line := t.NewLine("filename", 1, "original") @@ -338,7 +338,7 @@ func (s *Suite) Test_Autofix_Explain__without_explain_option(c *check.C) { func (s *Suite) Test_Autofix_Explain__default(c *check.C) { t := s.Init(c) - t.SetupCommandLine("--explain") + t.SetUpCommandLine("--explain") line := t.NewLine("Makefile", 74, "line1") fix := line.Autofix() @@ -358,7 +358,7 @@ func (s *Suite) Test_Autofix_Explain__default(c *check.C) { func (s *Suite) Test_Autofix_Explain__show_autofix(c *check.C) { t := s.Init(c) - t.SetupCommandLine("--show-autofix", "--explain") + t.SetUpCommandLine("--show-autofix", "--explain") line := t.NewLine("Makefile", 74, "line1") fix := line.Autofix() @@ -379,7 +379,7 @@ func (s *Suite) Test_Autofix_Explain__show_autofix(c *check.C) { func (s *Suite) Test_Autofix_Explain__autofix(c *check.C) { t := s.Init(c) - t.SetupCommandLine("--autofix", "--explain") + t.SetUpCommandLine("--autofix", "--explain") line := t.NewLine("Makefile", 74, "line1") fix := line.Autofix() @@ -396,7 +396,7 @@ func (s *Suite) Test_Autofix_Explain__autofix(c *check.C) { func (s *Suite) Test_Autofix_Explain__SilentAutofixFormat(c *check.C) { t := s.Init(c) - t.SetupCommandLine("--explain") + t.SetUpCommandLine("--explain") line := t.NewLine("example.txt", 1, "Text") fix := line.Autofix() @@ -411,7 +411,7 @@ func (s *Suite) Test_Autofix_Explain__SilentAutofixFormat(c *check.C) { func (s *Suite) Test_Autofix_Explain__silent_with_diagnostic(c *check.C) { t := s.Init(c) - t.SetupCommandLine("--explain") + t.SetUpCommandLine("--explain") line := t.NewLine("example.txt", 1, "Text") fix := line.Autofix() @@ -438,8 +438,8 @@ func (s *Suite) Test_Autofix_Explain__silent_with_diagnostic(c *check.C) { func (s *Suite) Test_Autofix__show_autofix_and_source_continuation_line(c *check.C) { t := s.Init(c) - t.SetupCommandLine("--show-autofix", "--source") - mklines := t.SetupFileMkLines("Makefile", + t.SetUpCommandLine("--show-autofix", "--source") + mklines := t.SetUpFileMkLines("Makefile", MkRcsID, "# before \\", "The old song \\", @@ -468,7 +468,7 @@ func (s *Suite) Test_Autofix__show_autofix_and_source_continuation_line(c *check func (s *Suite) Test_Autofix_InsertBefore(c *check.C) { t := s.Init(c) - t.SetupCommandLine("--show-autofix", "--source") + t.SetUpCommandLine("--show-autofix", "--source") line := t.NewLine("Makefile", 30, "original") fix := line.Autofix() @@ -486,7 +486,7 @@ func (s *Suite) Test_Autofix_InsertBefore(c *check.C) { func (s *Suite) Test_Autofix_Delete(c *check.C) { t := s.Init(c) - t.SetupCommandLine("--show-autofix", "--source") + t.SetUpCommandLine("--show-autofix", "--source") line := t.NewLine("Makefile", 30, "to be deleted") fix := line.Autofix() @@ -504,8 +504,8 @@ func (s *Suite) Test_Autofix_Delete(c *check.C) { func (s *Suite) Test_Autofix_Delete__continuation_line(c *check.C) { t := s.Init(c) - t.SetupCommandLine("--show-autofix", "--source") - mklines := t.SetupFileMkLines("Makefile", + t.SetUpCommandLine("--show-autofix", "--source") + mklines := t.SetUpFileMkLines("Makefile", MkRcsID, "# line 1 \\", "continued") @@ -527,7 +527,7 @@ func (s *Suite) Test_Autofix_Delete__continuation_line(c *check.C) { func (s *Suite) Test_Autofix_Delete__combined_with_insert(c *check.C) { t := s.Init(c) - t.SetupCommandLine("--show-autofix", "--source") + t.SetUpCommandLine("--show-autofix", "--source") line := t.NewLine("Makefile", 30, "to be deleted") fix := line.Autofix() @@ -552,7 +552,7 @@ func (s *Suite) Test_Autofix_Delete__combined_with_insert(c *check.C) { func (s *Suite) Test_Autofix__suppress_unfixable_warnings(c *check.C) { t := s.Init(c) - t.SetupCommandLine("--show-autofix", "--source") + t.SetUpCommandLine("--show-autofix", "--source") lines := t.NewLines("Makefile", "line1", "line2", @@ -629,7 +629,7 @@ func (s *Suite) Test_Autofix_Custom__in_memory(c *check.C) { t.CheckOutputLines( "WARN: Makefile:1: Please write in ALL-UPPERCASE.") - t.SetupCommandLine("--show-autofix") + t.SetUpCommandLine("--show-autofix") doFix(lines.Lines[1]) @@ -638,7 +638,7 @@ func (s *Suite) Test_Autofix_Custom__in_memory(c *check.C) { "AUTOFIX: Makefile:2: Converting to uppercase") c.Check(lines.Lines[1].Text, equals, "LINE2") - t.SetupCommandLine("--autofix") + t.SetUpCommandLine("--autofix") doFix(lines.Lines[2]) @@ -651,9 +651,9 @@ func (s *Suite) Test_Autofix_Custom__in_memory(c *check.C) { func (s *Suite) Test_Autofix_skip(c *check.C) { t := s.Init(c) - t.SetupCommandLine("--only", "few", "--autofix") + t.SetUpCommandLine("--only", "few", "--autofix") - mklines := t.SetupFileMkLines("filename", + mklines := t.SetUpFileMkLines("filename", "VAR=\t111 222 333 444 555 \\", "666") lines := mklines.lines @@ -691,7 +691,7 @@ func (s *Suite) Test_Autofix_skip(c *check.C) { func (s *Suite) Test_Autofix_Apply__only(c *check.C) { t := s.Init(c) - t.SetupCommandLine("--autofix", "--source", "--only", "interesting") + t.SetUpCommandLine("--autofix", "--source", "--only", "interesting") line := t.NewLine("Makefile", 27, "The old song") // Is completely ignored, including any autofixes. @@ -746,7 +746,7 @@ func (s *Suite) Test_Autofix_Apply__panic(c *check.C) { func (s *Suite) Test_Autofix_Apply__explanation_followed_by_note(c *check.C) { t := s.Init(c) - t.SetupCommandLine("--source") + t.SetUpCommandLine("--source") line := t.NewLine("README.txt", 123, "text") fix := line.Autofix() @@ -768,8 +768,8 @@ func (s *Suite) Test_Autofix_Apply__explanation_followed_by_note(c *check.C) { func (s *Suite) Test_SaveAutofixChanges__file_removed(c *check.C) { t := s.Init(c) - t.SetupCommandLine("--autofix") - lines := t.SetupFileLines("subdir/file.txt", + t.SetUpCommandLine("--autofix") + lines := t.SetUpFileLines("subdir/file.txt", "line 1") _ = os.RemoveAll(t.File("subdir")) @@ -792,8 +792,8 @@ func (s *Suite) Test_SaveAutofixChanges__file_busy_Windows(c *check.C) { return } - t.SetupCommandLine("--autofix") - lines := t.SetupFileLines("subdir/file.txt", + t.SetUpCommandLine("--autofix") + lines := t.SetUpFileLines("subdir/file.txt", "line 1") // As long as the file is kept open, it cannot be overwritten or deleted. @@ -821,8 +821,8 @@ func (s *Suite) Test_SaveAutofixChanges__file_busy_Windows(c *check.C) { func (s *Suite) Test_SaveAutofixChanges__cannot_overwrite(c *check.C) { t := s.Init(c) - t.SetupCommandLine("--autofix") - lines := t.SetupFileLines("file.txt", + t.SetUpCommandLine("--autofix") + lines := t.SetUpFileLines("file.txt", "line 1") c.Check(os.RemoveAll(t.File("file.txt")), check.IsNil) @@ -845,12 +845,12 @@ func (s *Suite) Test_SaveAutofixChanges__cannot_overwrite(c *check.C) { func (s *Suite) Test_Autofix__lonely_source(c *check.C) { t := s.Init(c) - t.SetupCommandLine("-Wall", "--source") + t.SetUpCommandLine("-Wall", "--source") G.Logger.Opts.LogVerbose = false // For realistic conditions; otherwise all diagnostics are logged. - t.SetupPackage("x11/xorg-cf-files", + t.SetUpPackage("x11/xorg-cf-files", ".include \"../../x11/xorgproto/buildlink3.mk\"") - t.SetupPackage("x11/xorgproto", + t.SetUpPackage("x11/xorgproto", "DISTNAME=\txorgproto-1.0") t.CreateFileDummyBuildlink3("x11/xorgproto/buildlink3.mk") t.CreateFileLines("x11/xorgproto/builtin.mk", @@ -878,10 +878,10 @@ func (s *Suite) Test_Autofix__lonely_source(c *check.C) { func (s *Suite) Test_Autofix__lonely_source_2(c *check.C) { t := s.Init(c) - t.SetupCommandLine("-Wall", "--source", "--explain") + t.SetUpCommandLine("-Wall", "--source", "--explain") G.Logger.Opts.LogVerbose = false // For realistic conditions; otherwise all diagnostics are logged. - t.SetupPackage("print/tex-bibtex8", + t.SetUpPackage("print/tex-bibtex8", "MAKE_FLAGS+=\tCFLAGS=${CFLAGS.${PKGSRC_COMPILER}}") G.Pkgsrc.LoadInfrastructure() t.Chdir(".") diff --git a/pkgtools/pkglint/files/buildlink3.go b/pkgtools/pkglint/files/buildlink3.go index 5994af2c85f..24b2a02769c 100644 --- a/pkgtools/pkglint/files/buildlink3.go +++ b/pkgtools/pkglint/files/buildlink3.go @@ -162,7 +162,7 @@ func (ck *Buildlink3Checker) checkVarassign(exp *MkExpecter, mkline MkLine, pkgb if varname == "BUILDLINK_ABI_DEPENDS."+pkgbase { ck.abiLine = mkline - parser := NewParser(mkline.Line, value, false) + parser := NewMkParser(nil, value, false) if dp := parser.Dependency(); dp != nil && parser.EOF() { ck.abi = dp } @@ -171,7 +171,7 @@ func (ck *Buildlink3Checker) checkVarassign(exp *MkExpecter, mkline MkLine, pkgb if varname == "BUILDLINK_API_DEPENDS."+pkgbase { ck.apiLine = mkline - parser := NewParser(mkline.Line, value, false) + parser := NewMkParser(nil, value, false) if dp := parser.Dependency(); dp != nil && parser.EOF() { ck.api = dp } @@ -218,6 +218,7 @@ func (ck *Buildlink3Checker) checkVaruseInPkgbase(pkgbase string, pkgbaseLine Mk checkSpecificVar("${PHP_PKG_PREFIX}", "php") if !warned { + // TODO: Replace regex with proper VarUse. if m, varuse := match1(pkgbase, `(\$\{\w+\})`); m { pkgbaseLine.Warnf("Please replace %q with a simple string (also in other variables in this file).", varuse) warned = true diff --git a/pkgtools/pkglint/files/buildlink3_test.go b/pkgtools/pkglint/files/buildlink3_test.go index fdaa98fac51..e21e31bbe5a 100644 --- a/pkgtools/pkglint/files/buildlink3_test.go +++ b/pkgtools/pkglint/files/buildlink3_test.go @@ -5,10 +5,10 @@ import "gopkg.in/check.v1" func (s *Suite) Test_CheckLinesBuildlink3Mk__unfinished_url2pkg(c *check.C) { t := s.Init(c) - t.SetupVartypes() + t.SetUpVartypes() t.CreateFileLines("x11/Xbae/Makefile") t.CreateFileLines("mk/motif.buildlink3.mk") - mklines := t.SetupFileMkLines("category/package/buildlink3.mk", + mklines := t.SetUpFileMkLines("category/package/buildlink3.mk", MkRcsID, "# XXX This file was created automatically using createbuildlink-@PKGVERSION@", "", @@ -42,7 +42,7 @@ func (s *Suite) Test_CheckLinesBuildlink3Mk__unfinished_url2pkg(c *check.C) { func (s *Suite) Test_CheckLinesBuildlink3Mk__name_mismatch_Haskell_incomplete(c *check.C) { t := s.Init(c) - t.SetupPackage("x11/hs-X11", + t.SetUpPackage("x11/hs-X11", "DISTNAME=\tX11-1.0") t.Chdir("x11/hs-X11") t.CreateFileLines("buildlink3.mk", @@ -79,7 +79,7 @@ func (s *Suite) Test_CheckLinesBuildlink3Mk__name_mismatch_Haskell_complete(c *c t.CreateFileLines("mk/haskell.mk", MkRcsID, "PKGNAME?=\ths-${DISTNAME}") - t.SetupPackage("x11/hs-X11", + t.SetUpPackage("x11/hs-X11", "DISTNAME=\tX11-1.0", "", ".include \"../../mk/haskell.mk\"") @@ -107,7 +107,7 @@ func (s *Suite) Test_CheckLinesBuildlink3Mk__name_mismatch_Haskell_complete(c *c func (s *Suite) Test_CheckLinesBuildlink3Mk__name_mismatch_multiple_inclusion(c *check.C) { t := s.Init(c) - t.SetupVartypes() + t.SetUpVartypes() mklines := t.NewMkLines("buildlink3.mk", MkRcsID, "", @@ -131,7 +131,7 @@ func (s *Suite) Test_CheckLinesBuildlink3Mk__name_mismatch_multiple_inclusion(c func (s *Suite) Test_CheckLinesBuildlink3Mk__name_mismatch_abi_api(c *check.C) { t := s.Init(c) - t.SetupVartypes() + t.SetUpVartypes() mklines := t.NewMkLines("buildlink3.mk", MkRcsID, "", @@ -158,7 +158,7 @@ func (s *Suite) Test_CheckLinesBuildlink3Mk__name_mismatch_abi_api(c *check.C) { func (s *Suite) Test_CheckLinesBuildlink3Mk__abi_api_versions(c *check.C) { t := s.Init(c) - t.SetupVartypes() + t.SetUpVartypes() mklines := t.NewMkLines("buildlink3.mk", MkRcsID, "", @@ -187,9 +187,9 @@ func (s *Suite) Test_CheckLinesBuildlink3Mk__abi_api_versions(c *check.C) { func (s *Suite) Test_Buildlink3Checker_checkVarassign__abi_api_versions_brace(c *check.C) { t := s.Init(c) - t.SetupVartypes() + t.SetUpVartypes() t.CreateFileLines("multimedia/totem/Makefile") - mklines := t.SetupFileMkLines("multimedia/totem/buildlink3.mk", + mklines := t.SetUpFileMkLines("multimedia/totem/buildlink3.mk", MkRcsID, "", "BUILDLINK_TREE+=\ttotem", @@ -215,7 +215,7 @@ func (s *Suite) Test_Buildlink3Checker_checkVarassign__abi_api_versions_brace(c func (s *Suite) Test_CheckLinesBuildlink3Mk__missing_BUILDLINK_TREE_at_beginning(c *check.C) { t := s.Init(c) - t.SetupVartypes() + t.SetUpVartypes() mklines := t.NewMkLines("buildlink3.mk", MkRcsID, "", @@ -234,7 +234,7 @@ func (s *Suite) Test_CheckLinesBuildlink3Mk__missing_BUILDLINK_TREE_at_beginning func (s *Suite) Test_CheckLinesBuildlink3Mk__missing_BUILDLINK_TREE_at_end(c *check.C) { t := s.Init(c) - t.SetupVartypes() + t.SetUpVartypes() mklines := t.NewMkLines("buildlink3.mk", MkRcsID, "", @@ -260,7 +260,7 @@ func (s *Suite) Test_CheckLinesBuildlink3Mk__missing_BUILDLINK_TREE_at_end(c *ch func (s *Suite) Test_CheckLinesBuildlink3Mk__DEPMETHOD_placement(c *check.C) { t := s.Init(c) - t.SetupVartypes() + t.SetUpVartypes() mklines := t.NewMkLines("buildlink3.mk", MkRcsID, "", @@ -287,7 +287,7 @@ func (s *Suite) Test_CheckLinesBuildlink3Mk__DEPMETHOD_placement(c *check.C) { func (s *Suite) Test_CheckLinesBuildlink3Mk__multiple_inclusion_wrong(c *check.C) { t := s.Init(c) - t.SetupVartypes() + t.SetUpVartypes() mklines := t.NewMkLines("buildlink3.mk", MkRcsID, "", @@ -308,7 +308,7 @@ func (s *Suite) Test_CheckLinesBuildlink3Mk__multiple_inclusion_wrong(c *check.C func (s *Suite) Test_CheckLinesBuildlink3Mk__missing_endif(c *check.C) { t := s.Init(c) - t.SetupVartypes() + t.SetUpVartypes() mklines := t.NewMkLines("buildlink3.mk", MkRcsID, "", @@ -327,7 +327,7 @@ func (s *Suite) Test_CheckLinesBuildlink3Mk__missing_endif(c *check.C) { func (s *Suite) Test_CheckLinesBuildlink3Mk__invalid_dependency_patterns(c *check.C) { t := s.Init(c) - t.SetupVartypes() + t.SetUpVartypes() mklines := t.NewMkLines("buildlink3.mk", MkRcsID, "", @@ -354,7 +354,7 @@ func (s *Suite) Test_CheckLinesBuildlink3Mk__invalid_dependency_patterns(c *chec func (s *Suite) Test_CheckLinesBuildlink3Mk__PKGBASE_with_variable(c *check.C) { t := s.Init(c) - t.SetupVartypes() + t.SetUpVartypes() mklines := t.NewMkLines("buildlink3.mk", MkRcsID, "", @@ -379,7 +379,7 @@ func (s *Suite) Test_CheckLinesBuildlink3Mk__PKGBASE_with_variable(c *check.C) { func (s *Suite) Test_CheckLinesBuildlink3Mk__PKGBASE_with_unknown_variable(c *check.C) { t := s.Init(c) - t.SetupVartypes() + t.SetUpVartypes() mklines := t.NewMkLines("buildlink3.mk", MkRcsID, "", @@ -432,8 +432,8 @@ func (s *Suite) Test_CheckLinesBuildlink3Mk__PKGBASE_with_unknown_variable(c *ch func (s *Suite) Test_Buildlink3Checker_checkMainPart__nested_if(c *check.C) { t := s.Init(c) - t.SetupVartypes() - mklines := t.SetupFileMkLines("category/package/buildlink3.mk", + t.SetUpVartypes() + mklines := t.SetUpFileMkLines("category/package/buildlink3.mk", MkRcsID, "", "BUILDLINK_TREE+=\ths-X11", @@ -459,8 +459,8 @@ func (s *Suite) Test_Buildlink3Checker_checkMainPart__nested_if(c *check.C) { func (s *Suite) Test_Buildlink3Checker_checkMainPart__comment_at_end_of_file(c *check.C) { t := s.Init(c) - t.SetupVartypes() - mklines := t.SetupFileMkLines("category/package/buildlink3.mk", + t.SetUpVartypes() + mklines := t.SetUpFileMkLines("category/package/buildlink3.mk", MkRcsID, "", "BUILDLINK_TREE+=\ths-X11", diff --git a/pkgtools/pkglint/files/category_test.go b/pkgtools/pkglint/files/category_test.go index c1b017a355a..834a2b34ff4 100644 --- a/pkgtools/pkglint/files/category_test.go +++ b/pkgtools/pkglint/files/category_test.go @@ -5,7 +5,7 @@ import "gopkg.in/check.v1" func (s *Suite) Test_CheckdirCategory__totally_broken(c *check.C) { t := s.Init(c) - t.SetupVartypes() + t.SetUpVartypes() t.CreateFileLines("archivers/Makefile", "# $", "SUBDIR+=pkg1", @@ -39,7 +39,7 @@ func (s *Suite) Test_CheckdirCategory__totally_broken(c *check.C) { func (s *Suite) Test_CheckdirCategory__invalid_comment(c *check.C) { t := s.Init(c) - t.SetupVartypes() + t.SetUpVartypes() t.CreateFileLines("archivers/Makefile", MkRcsID, "", @@ -68,8 +68,8 @@ func (s *Suite) Test_CheckdirCategory__invalid_comment(c *check.C) { func (s *Suite) Test_CheckdirCategory__wip(c *check.C) { t := s.Init(c) - t.SetupPkgsrc() - t.SetupVartypes() + t.SetUpPkgsrc() + t.SetUpVartypes() t.CreateFileLines("mk/misc/category.mk") t.CreateFileLines("wip/package/Makefile") t.CreateFileLines("wip/Makefile", @@ -93,8 +93,8 @@ func (s *Suite) Test_CheckdirCategory__wip(c *check.C) { func (s *Suite) Test_CheckdirCategory__subdirs(c *check.C) { t := s.Init(c) - t.SetupPkgsrc() - t.SetupVartypes() + t.SetUpPkgsrc() + t.SetUpVartypes() t.CreateFileLines("mk/misc/category.mk") t.CreateFileLines("category/in-wrong-order/Makefile") t.CreateFileLines("category/duplicate/Makefile") @@ -141,9 +141,9 @@ func (s *Suite) Test_CheckdirCategory__subdirs(c *check.C) { func (s *Suite) Test_CheckdirCategory__subdirs_file_system_at_the_bottom(c *check.C) { t := s.Init(c) - t.SetupCommandLine("-Wall", "--show-autofix") - t.SetupPkgsrc() - t.SetupVartypes() + t.SetUpCommandLine("-Wall", "--show-autofix") + t.SetUpPkgsrc() + t.SetUpVartypes() t.CreateFileLines("mk/misc/category.mk") t.CreateFileLines("category/mk-and-fs/Makefile") t.CreateFileLines("category/zzz-fs-only/Makefile") @@ -166,8 +166,8 @@ func (s *Suite) Test_CheckdirCategory__subdirs_file_system_at_the_bottom(c *chec func (s *Suite) Test_CheckdirCategory__indentation(c *check.C) { t := s.Init(c) - t.SetupPkgsrc() - t.SetupVartypes() + t.SetUpPkgsrc() + t.SetUpVartypes() t.CreateFileLines("mk/misc/category.mk") t.CreateFileLines("category/package1/Makefile") t.CreateFileLines("category/package2/Makefile") @@ -186,3 +186,15 @@ func (s *Suite) Test_CheckdirCategory__indentation(c *check.C) { t.CheckOutputLines( "NOTE: ~/category/Makefile:5: This variable value should be aligned with tabs, not spaces, to column 17.") } + +func (s *Suite) Test_CheckdirCategory__no_Makefile(c *check.C) { + t := s.Init(c) + + t.SetUpPkgsrc() + t.CreateFileLines("category/other-file") + + G.Check(t.File("category")) + + t.CheckOutputLines( + "ERROR: ~/category/Makefile: Cannot be read.") +} diff --git a/pkgtools/pkglint/files/check_test.go b/pkgtools/pkglint/files/check_test.go index 4b1d3dac8bb..3fa59b7aef1 100644 --- a/pkgtools/pkglint/files/check_test.go +++ b/pkgtools/pkglint/files/check_test.go @@ -29,10 +29,10 @@ type Suite struct { // Init creates and returns a test helper that allows to: // // * create files for the test: -// CreateFileLines, SetupPkgsrc, SetupPackage +// CreateFileLines, SetUpPkgsrc, SetUpPackage // // * load these files into Line and MkLine objects (for tests spanning multiple files): -// SetupFileLines, SetupFileMkLines +// SetUpFileLines, SetUpFileMkLines // // * create new in-memory Line and MkLine objects (for simple tests): // NewLine, NewLines, NewMkLine, NewMkLines @@ -40,7 +40,7 @@ type Suite struct { // * check the files that have been changed by the --autofix feature: // CheckFileLines // -// * check the pkglint diagnostics: CheckLinesEmpty, CheckLinesOutput +// * check the pkglint diagnostics: CheckOutputEmpty, CheckOutputLines func (s *Suite) Init(c *check.C) *Tester { // Note: the check.C object from SetUpTest cannot be used here, @@ -70,7 +70,7 @@ func (s *Suite) SetUpTest(c *check.C) { G.Pkgsrc = NewPkgsrc(t.File(".")) t.c = c - t.SetupCommandLine("-Wall") // To catch duplicate warnings + t.SetUpCommandLine("-Wall") // To catch duplicate warnings t.c = nil // To improve code coverage and ensure that trace.Result works @@ -127,12 +127,12 @@ type Tester struct { relCwd string // See Tester.Chdir } -// SetupCommandLine simulates a command line for the remainder of the test. +// SetUpCommandLine simulates a command line for the remainder of the test. // See Pkglint.ParseCommandLine. // -// If SetupCommandLine is not called explicitly in a test, the command line +// If SetUpCommandLine is not called explicitly in a test, the command line // "-Wall" is used, to provide a high code coverage in the tests. -func (t *Tester) SetupCommandLine(args ...string) { +func (t *Tester) SetUpCommandLine(args ...string) { // Prevent tracing from being disabled; see EnableSilentTracing. prevTracing := trace.Tracing @@ -152,59 +152,59 @@ func (t *Tester) SetupCommandLine(args ...string) { G.Logger.Opts.LogVerbose = true } -// SetupVartypes registers a few hundred variables like MASTER_SITES, +// SetUpVartypes registers a few hundred variables like MASTER_SITES, // WRKSRC, SUBST_SED.*, so that their data types are known to pkglint. // // Without calling this, there will be many warnings about undefined // or unused variables, or unknown shell commands. // -// See SetupTool for registering tools like echo, awk, perl. -func (t *Tester) SetupVartypes() { +// See SetUpTool for registering tools like echo, awk, perl. +func (t *Tester) SetUpVartypes() { G.Pkgsrc.InitVartypes() } -func (t *Tester) SetupMasterSite(varname string, urls ...string) { +func (t *Tester) SetUpMasterSite(varname string, urls ...string) { for _, url := range urls { G.Pkgsrc.registerMasterSite(varname, url) } } -// SetupOption pretends that the package option is defined in mk/defaults/options.description. -func (t *Tester) SetupOption(name, description string) { +// SetUpOption pretends that the package option is defined in mk/defaults/options.description. +func (t *Tester) SetUpOption(name, description string) { G.Pkgsrc.PkgOptions[name] = description } -func (t *Tester) SetupTool(name, varname string, validity Validity) *Tool { +func (t *Tester) SetUpTool(name, varname string, validity Validity) *Tool { return G.Pkgsrc.Tools.def(name, varname, false, validity) } -// SetupFileLines creates a temporary file and writes the given lines to it. +// SetUpFileLines creates a temporary file and writes the given lines to it. // The file is then read in, without interpreting line continuations. // -// See SetupFileMkLines for loading a Makefile fragment. -func (t *Tester) SetupFileLines(relativeFileName string, lines ...string) Lines { +// See SetUpFileMkLines for loading a Makefile fragment. +func (t *Tester) SetUpFileLines(relativeFileName string, lines ...string) Lines { filename := t.CreateFileLines(relativeFileName, lines...) return Load(filename, MustSucceed) } -// SetupFileLines creates a temporary file and writes the given lines to it. +// SetUpFileLines creates a temporary file and writes the given lines to it. // The file is then read in, handling line continuations for Makefiles. // -// See SetupFileLines for loading an ordinary file. -func (t *Tester) SetupFileMkLines(relativeFileName string, lines ...string) MkLines { +// See SetUpFileLines for loading an ordinary file. +func (t *Tester) SetUpFileMkLines(relativeFileName string, lines ...string) MkLines { filename := t.CreateFileLines(relativeFileName, lines...) return LoadMk(filename, MustSucceed) } -// SetupPkgsrc sets up a minimal but complete pkgsrc installation in the +// SetUpPkgsrc sets up a minimal but complete pkgsrc installation in the // temporary folder, so that pkglint runs without any errors. -// Individual files may be overwritten by calling other Setup* methods. +// Individual files may be overwritten by calling other SetUp* methods. // // This setup is especially interesting for testing Pkglint.Main. // // If the test works on a lower level than Pkglint.Main, // LoadInfrastructure must be called to actually load the infrastructure files. -func (t *Tester) SetupPkgsrc() { +func (t *Tester) SetUpPkgsrc() { // This file is needed to locate the pkgsrc root directory. // See findPkgsrcTopdir. @@ -262,9 +262,9 @@ func (t *Tester) SetupPkgsrc() { t.CreateFileLines("mk/misc/category.mk") } -// SetupCategory makes the given category valid by creating a dummy Makefile. +// SetUpCategory makes the given category valid by creating a dummy Makefile. // After that, it can be mentioned in the CATEGORIES variable of a package. -func (t *Tester) SetupCategory(name string) { +func (t *Tester) SetUpCategory(name string) { G.Assertf(!contains(name, "/"), "Category must not contain a slash.") if _, err := os.Stat(t.File(name + "/Makefile")); os.IsNotExist(err) { @@ -273,7 +273,7 @@ func (t *Tester) SetupCategory(name string) { } } -// SetupPackage sets up all files for a package (including the pkgsrc +// SetUpPackage sets up all files for a package (including the pkgsrc // infrastructure) so that it does not produce any warnings. // // The given makefileLines start in line 20. Except if they are variable @@ -283,12 +283,12 @@ func (t *Tester) SetupCategory(name string) { // // After calling this method, individual files can be overwritten as necessary. // Then, G.Pkgsrc.LoadInfrastructure should be called to load all the files. -func (t *Tester) SetupPackage(pkgpath string, makefileLines ...string) string { +func (t *Tester) SetUpPackage(pkgpath string, makefileLines ...string) string { category := path.Dir(pkgpath) - t.SetupPkgsrc() - t.SetupVartypes() - t.SetupCategory(category) + t.SetUpPkgsrc() + t.SetUpVartypes() + t.SetUpCategory(category) t.CreateFileLines(pkgpath+"/DESCR", "Package description") @@ -428,7 +428,7 @@ func (t *Tester) File(relativeFileName string) string { // of the temporary directory, creating it if necessary. // // After this call, all files loaded from the temporary directory via -// SetupFileLines or CreateFileLines or similar methods will use path names +// SetUpFileLines or CreateFileLines or similar methods will use path names // relative to this directory. // // After the test, the previous working directory is restored, so that @@ -567,14 +567,14 @@ func (t *Tester) NewShellLine(filename string, lineno int, text string) *ShellLi // NewLines returns a list of simple lines that belong together. // -// To work with line continuations like in Makefiles, use SetupFileMkLines. +// To work with line continuations like in Makefiles, use SetUpFileMkLines. func (t *Tester) NewLines(filename string, lines ...string) Lines { return t.NewLinesAt(filename, 1, lines...) } // NewLinesAt returns a list of simple lines that belong together. // -// To work with line continuations like in Makefiles, use SetupFileMkLines. +// To work with line continuations like in Makefiles, use SetUpFileMkLines. func (t *Tester) NewLinesAt(filename string, firstLine int, texts ...string) Lines { lines := make([]Line, len(texts)) for i, text := range texts { @@ -588,7 +588,7 @@ func (t *Tester) NewLinesAt(filename string, firstLine int, texts ...string) Lin // taking continuation lines into account. // // No actual file is created for the lines; -// see SetupFileMkLines for loading Makefile fragments with line continuations. +// see SetUpFileMkLines for loading Makefile fragments with line continuations. func (t *Tester) NewMkLines(filename string, lines ...string) MkLines { var rawText strings.Builder for _, line := range lines { @@ -648,7 +648,7 @@ func (t *Tester) CheckOutputLines(expectedLines ...string) { // where they are shown together with the trace log. // // This is useful when stepping through the code, especially -// in combination with SetupCommandLine("--debug"). +// in combination with SetUpCommandLine("--debug"). // // In JetBrains GoLand, the tracing output is suppressed after the first // failed check, see https://youtrack.jetbrains.com/issue/GO-6154. diff --git a/pkgtools/pkglint/files/cmd/pkglint/pkglint.go b/pkgtools/pkglint/files/cmd/pkglint/pkglint.go index 4f355a0c0ee..8b4abedc693 100644 --- a/pkgtools/pkglint/files/cmd/pkglint/pkglint.go +++ b/pkgtools/pkglint/files/cmd/pkglint/pkglint.go @@ -5,6 +5,8 @@ import ( "os" ) +var exit = os.Exit + func main() { - os.Exit(pkglint.Main()) + exit(pkglint.Main()) } diff --git a/pkgtools/pkglint/files/cmd/pkglint/pkglint_test.go b/pkgtools/pkglint/files/cmd/pkglint/pkglint_test.go new file mode 100644 index 00000000000..f0da546376a --- /dev/null +++ b/pkgtools/pkglint/files/cmd/pkglint/pkglint_test.go @@ -0,0 +1,48 @@ +package main + +import ( + "io/ioutil" + "os" + "testing" + + "gopkg.in/check.v1" +) + +type Suite struct{} + +func Test(t *testing.T) { + check.Suite(new(Suite)) + check.TestingT(t) +} + +// This test goes into great lengths to bring the code coverage to 100%. +// Without this test, the main function would be a trivial one-liner. +// To make that one-liner testable, the call to os.Exit must be mockable. +func (s *Suite) Test_main(c *check.C) { + tmpdir := c.MkDir() + out, err := os.Create(tmpdir + "/out") + c.Assert(err, check.IsNil) + + prevStdout := os.Stdout + prevArgs := os.Args + prevExit := exit + defer func() { + os.Stdout = prevStdout + os.Args = prevArgs + exit = prevExit + }() + + os.Stdout = out + os.Args = []string{"pkglint", "--version"} + exit = func(code int) { c.Check(code, check.Equals, 0) } + + main() + + err = out.Close() + c.Assert(err, check.IsNil) + + output, err := ioutil.ReadFile(out.Name()) + c.Assert(err, check.IsNil) + + c.Check(string(output), check.Matches, `^(@VERSION@|\d+(\.\d+)+)\n$`) +} diff --git a/pkgtools/pkglint/files/distinfo_test.go b/pkgtools/pkglint/files/distinfo_test.go index 2cd7b361902..364eebcb72e 100644 --- a/pkgtools/pkglint/files/distinfo_test.go +++ b/pkgtools/pkglint/files/distinfo_test.go @@ -11,7 +11,7 @@ func (s *Suite) Test_CheckLinesDistinfo(c *check.C) { "patch contents") t.CreateFileLines("patches/patch-ab", "patch contents") - lines := t.SetupFileLines("distinfo", + lines := t.SetUpFileLines("distinfo", "should be the RCS ID", "should be empty", "MD5 (distfile.tar.gz) = 12345678901234567890123456789012", @@ -43,9 +43,9 @@ func (s *Suite) Test_CheckLinesDistinfo(c *check.C) { func (s *Suite) Test_distinfoLinesChecker_checkGlobalDistfileMismatch(c *check.C) { t := s.Init(c) - t.SetupPkgsrc() - t.SetupPackage("category/package1") - t.SetupPackage("category/package2") + t.SetUpPkgsrc() + t.SetUpPackage("category/package1") + t.SetUpPackage("category/package2") t.CreateFileLines("category/package1/distinfo", RcsID, "", @@ -88,12 +88,12 @@ func (s *Suite) Test_distinfoLinesChecker_checkGlobalDistfileMismatch(c *check.C func (s *Suite) Test_CheckLinesDistinfo__uncommitted_patch(c *check.C) { t := s.Init(c) - t.SetupPackage("category/package") + t.SetUpPackage("category/package") t.Chdir("category/package") t.CreateFileDummyPatch("patches/patch-aa") t.CreateFileLines("CVS/Entries", "/distinfo/...") - t.SetupFileLines("distinfo", + t.SetUpFileLines("distinfo", RcsID, "", "SHA1 (patch-aa) = ebbf34b0641bcb508f17d5a27f2bf2a536d810ac") @@ -107,12 +107,12 @@ func (s *Suite) Test_CheckLinesDistinfo__uncommitted_patch(c *check.C) { func (s *Suite) Test_CheckLinesDistinfo__unrecorded_patches(c *check.C) { t := s.Init(c) - t.SetupPackage("category/package") + t.SetUpPackage("category/package") t.Chdir("category/package") t.CreateFileLines("patches/CVS/Entries") t.CreateFileDummyPatch("patches/patch-aa") t.CreateFileDummyPatch("patches/patch-src-Makefile") - t.SetupFileLines("distinfo", + t.SetUpFileLines("distinfo", RcsID, "", "SHA1 (distfile.tar.gz) = ...", @@ -133,14 +133,14 @@ func (s *Suite) Test_CheckLinesDistinfo__unrecorded_patches(c *check.C) { func (s *Suite) Test_CheckLinesDistinfo__relative_path_in_distinfo(c *check.C) { t := s.Init(c) - t.SetupPackage("category/package", + t.SetUpPackage("category/package", "DISTINFO_FILE=\t../../other/common/distinfo", "PATCHDIR=\t../../devel/patches/patches") t.Remove("category/package/distinfo") t.CreateFileLines("devel/patches/patches/CVS/Entries") t.CreateFileDummyPatch("devel/patches/patches/patch-aa") t.CreateFileDummyPatch("devel/patches/patches/patch-only-in-patches") - t.SetupFileLines("other/common/distinfo", + t.SetUpFileLines("other/common/distinfo", RcsID, "", "SHA1 (patch-aa) = ...", @@ -163,14 +163,14 @@ func (s *Suite) Test_CheckLinesDistinfo__relative_path_in_distinfo(c *check.C) { func (s *Suite) Test_CheckLinesDistinfo__distinfo_and_patches_in_separate_directory(c *check.C) { t := s.Init(c) - t.SetupPackage("category/package", + t.SetUpPackage("category/package", "DISTINFO_FILE=\t../../other/common/distinfo", "PATCHDIR=\t../../other/common/patches") t.Remove("category/package/distinfo") t.CreateFileLines("other/common/patches/CVS/Entries") t.CreateFileDummyPatch("other/common/patches/patch-aa") t.CreateFileDummyPatch("other/common/patches/patch-only-in-patches") - t.SetupFileLines("other/common/distinfo", + t.SetUpFileLines("other/common/distinfo", RcsID, "", "SHA1 (patch-aa) = ...", @@ -193,7 +193,7 @@ func (s *Suite) Test_CheckLinesDistinfo__manual_patches(c *check.C) { t.Chdir("category/package") t.CreateFileLines("patches/manual-libtool.m4") - lines := t.SetupFileLines("distinfo", + lines := t.SetUpFileLines("distinfo", RcsID, "", "SHA1 (patch-aa) = ...") @@ -223,8 +223,8 @@ func (s *Suite) Test_CheckLinesDistinfo__manual_patches(c *check.C) { func (s *Suite) Test_CheckLinesDistinfo__missing_php_patches(c *check.C) { t := s.Init(c) - t.SetupPkgsrc() - t.SetupCommandLine("-Wall,no-space") + t.SetUpPkgsrc() + t.SetUpCommandLine("-Wall,no-space") t.CreateFileLines("licenses/unknown-license") t.CreateFileLines("lang/php/ext.mk", MkRcsID, diff --git a/pkgtools/pkglint/files/files_test.go b/pkgtools/pkglint/files/files_test.go index 6f09e214947..c6581e44b46 100644 --- a/pkgtools/pkglint/files/files_test.go +++ b/pkgtools/pkglint/files/files_test.go @@ -47,7 +47,7 @@ func (s *Suite) Test_convertToLogicalLines__continuation_in_last_line(c *check.C func (s *Suite) Test_convertToLogicalLines__comments(c *check.C) { t := s.Init(c) - mklines := t.SetupFileMkLines("comment.mk", + mklines := t.SetUpFileMkLines("comment.mk", "# This is a comment", "", "#\\", @@ -130,7 +130,7 @@ func (s *Suite) Test_convertToLogicalLines__missing_newline_at_eof_in_continuati func (s *Suite) Test_convertToLogicalLines__missing_newline_at_eof_with_source(c *check.C) { t := s.Init(c) - t.SetupCommandLine("-Wall", "--source") + t.SetUpCommandLine("-Wall", "--source") rawText := "" + "last line\\" diff --git a/pkgtools/pkglint/files/licenses/licenses_test.go b/pkgtools/pkglint/files/licenses/licenses_test.go index c6fc1b2a940..168363c000b 100644 --- a/pkgtools/pkglint/files/licenses/licenses_test.go +++ b/pkgtools/pkglint/files/licenses/licenses_test.go @@ -11,42 +11,43 @@ import ( type Suite struct{} func (s *Suite) Test_Parse(c *check.C) { - checkParse := func(cond string, expected string) { + test := func(cond string, expected string) { c.Check(toJSON(Parse(cond)), check.Equals, expected) } - checkParseDeep := func(cond string, expected *Condition) { + + testDeep := func(cond string, expected *Condition) { c.Check(Parse(cond), check.DeepEquals, expected) } - checkParseDeep("gnu-gpl-v2", NewName("gnu-gpl-v2")) + testDeep("gnu-gpl-v2", NewName("gnu-gpl-v2")) - checkParse("gnu-gpl-v2", "{Name:gnu-gpl-v2}") + test("gnu-gpl-v2", "{Name:gnu-gpl-v2}") - checkParse("a AND b", "{And:true,Children:[{Name:a},{Name:b}]}") - checkParse("a OR b", "{Or:true,Children:[{Name:a},{Name:b}]}") + test("a AND b", "{And:true,Children:[{Name:a},{Name:b}]}") + test("a OR b", "{Or:true,Children:[{Name:a},{Name:b}]}") - checkParse("a OR (b AND c)", "{Or:true,Children:[{Name:a},{Paren:{And:true,Children:[{Name:b},{Name:c}]}}]}") - checkParse("(a OR b) AND c", "{And:true,Children:[{Paren:{Or:true,Children:[{Name:a},{Name:b}]}},{Name:c}]}") + test("a OR (b AND c)", "{Or:true,Children:[{Name:a},{Paren:{And:true,Children:[{Name:b},{Name:c}]}}]}") + test("(a OR b) AND c", "{And:true,Children:[{Paren:{Or:true,Children:[{Name:a},{Name:b}]}},{Name:c}]}") - checkParse("a AND b AND c AND d", "{And:true,Children:[{Name:a},{Name:b},{Name:c},{Name:d}]}") - checkParseDeep( + test("a AND b AND c AND d", "{And:true,Children:[{Name:a},{Name:b},{Name:c},{Name:d}]}") + testDeep( "a AND b AND c AND d", NewAnd(NewName("a"), NewName("b"), NewName("c"), NewName("d"))) - checkParse("a OR b OR c OR d", "{Or:true,Children:[{Name:a},{Name:b},{Name:c},{Name:d}]}") - checkParseDeep( + test("a OR b OR c OR d", "{Or:true,Children:[{Name:a},{Name:b},{Name:c},{Name:d}]}") + testDeep( "a OR b OR c OR d", NewOr(NewName("a"), NewName("b"), NewName("c"), NewName("d"))) - checkParse("(a OR b) AND (c AND d)", "{And:true,Children:[{Paren:{Or:true,Children:[{Name:a},{Name:b}]}},{Paren:{And:true,Children:[{Name:c},{Name:d}]}}]}") - checkParseDeep( + test("(a OR b) AND (c AND d)", "{And:true,Children:[{Paren:{Or:true,Children:[{Name:a},{Name:b}]}},{Paren:{And:true,Children:[{Name:c},{Name:d}]}}]}") + testDeep( "(a OR b) AND (c AND d)", NewAnd( NewParen(NewOr(NewName("a"), NewName("b"))), NewParen(NewAnd(NewName("c"), NewName("d"))))) - checkParse("a AND b OR c AND d", "{And:true,Or:true,Children:[{Name:a},{Name:b},{Name:c},{Name:d}]}") - checkParse("((a AND (b AND c)))", "{Paren:{Children:[{Paren:{And:true,Children:[{Name:a},{Paren:{And:true,Children:[{Name:b},{Name:c}]}}]}}]}}") + test("a AND b OR c AND d", "{And:true,Or:true,Children:[{Name:a},{Name:b},{Name:c},{Name:d}]}") + test("((a AND (b AND c)))", "{Paren:{Children:[{Paren:{And:true,Children:[{Name:a},{Paren:{And:true,Children:[{Name:b},{Name:c}]}}]}}]}}") c.Check(Parse("a AND b OR c AND d").String(), check.Equals, "a MIXED b MIXED c MIXED d") diff --git a/pkgtools/pkglint/files/licenses_test.go b/pkgtools/pkglint/files/licenses_test.go index c6c4c4da33c..f97d7aef2b9 100644 --- a/pkgtools/pkglint/files/licenses_test.go +++ b/pkgtools/pkglint/files/licenses_test.go @@ -47,8 +47,8 @@ func (s *Suite) Test_LicenseChecker_Check(c *check.C) { func (s *Suite) Test_LicenseChecker_checkName__LICENSE_FILE(c *check.C) { t := s.Init(c) - t.SetupPkgsrc() - t.SetupPackage("category/package", + t.SetUpPkgsrc() + t.SetUpPackage("category/package", "LICENSE=\tmy-license", "", "LICENSE_FILE=\tmy-license") diff --git a/pkgtools/pkglint/files/linechecker_test.go b/pkgtools/pkglint/files/linechecker_test.go index 0c07e5159cb..c74ef97a7d7 100644 --- a/pkgtools/pkglint/files/linechecker_test.go +++ b/pkgtools/pkglint/files/linechecker_test.go @@ -7,7 +7,7 @@ import ( func (s *Suite) Test_LineChecker_CheckAbsolutePathname(c *check.C) { t := s.Init(c) - t.SetupCommandLine("-Wabsname", "--explain") + t.SetUpCommandLine("-Wabsname", "--explain") mklines := t.NewMkLines("Makefile", MkRcsID, "\tbindir=/bin", @@ -72,7 +72,7 @@ func (s *Suite) Test_LineChecker_CheckAbsolutePathname(c *check.C) { func (s *Suite) Test_LineChecker_CheckAbsolutePathname__disabled_by_default(c *check.C) { t := s.Init(c) - t.SetupCommandLine( /* none, which means -Wall is suppressed */ ) + t.SetUpCommandLine( /* none, which means -Wall is suppressed */ ) line := t.NewLine("Makefile", 1, "# dummy") LineChecker{line}.CheckAbsolutePathname("bindir=/bin") diff --git a/pkgtools/pkglint/files/lines_test.go b/pkgtools/pkglint/files/lines_test.go index 55dce78dd41..393efda7617 100644 --- a/pkgtools/pkglint/files/lines_test.go +++ b/pkgtools/pkglint/files/lines_test.go @@ -30,8 +30,8 @@ func (s *Suite) Test_Lines_CheckRcsID(c *check.C) { func (s *Suite) Test_Lines_CheckRcsID__wip(c *check.C) { t := s.Init(c) - t.SetupPkgsrc() - t.SetupPackage("wip/package", + t.SetUpPkgsrc() + t.SetUpPackage("wip/package", "CATEGORIES=\tchinese") t.CreateFileLines("wip/package/file1.mk", "# $"+"NetBSD: dummy $") diff --git a/pkgtools/pkglint/files/logging_test.go b/pkgtools/pkglint/files/logging_test.go index 8c0a6ffce99..e4be6eb7041 100644 --- a/pkgtools/pkglint/files/logging_test.go +++ b/pkgtools/pkglint/files/logging_test.go @@ -114,8 +114,8 @@ func (s *Suite) Test_Logger_Diag__explanation(c *check.C) { func (s *Suite) Test_Logger__show_source_separator(c *check.C) { t := s.Init(c) - t.SetupCommandLine("--source") - lines := t.SetupFileLines("DESCR", + t.SetUpCommandLine("--source") + lines := t.SetUpFileLines("DESCR", "The first line", "The second line", "The third line") @@ -152,8 +152,8 @@ func (s *Suite) Test_Logger__show_source_separator(c *check.C) { func (s *Suite) Test__show_source_separator_show_autofix(c *check.C) { t := s.Init(c) - t.SetupCommandLine("--source", "--show-autofix") - lines := t.SetupFileLines("DESCR", + t.SetUpCommandLine("--source", "--show-autofix") + lines := t.SetUpFileLines("DESCR", "The first line", "The second line", "The third line") @@ -190,8 +190,8 @@ func (s *Suite) Test__show_source_separator_show_autofix(c *check.C) { func (s *Suite) Test__show_source_separator_autofix(c *check.C) { t := s.Init(c) - t.SetupCommandLine("--source", "--autofix") - lines := t.SetupFileLines("DESCR", + t.SetUpCommandLine("--source", "--autofix") + lines := t.SetUpFileLines("DESCR", "The first line", "The second line", "The third line") @@ -221,7 +221,7 @@ func (s *Suite) Test__show_source_separator_autofix(c *check.C) { func (s *Suite) Test_Logger_Explain__only(c *check.C) { t := s.Init(c) - t.SetupCommandLine("--only", "interesting", "--explain") + t.SetUpCommandLine("--only", "interesting", "--explain") line := t.NewLine("Makefile", 27, "The old song") // Neither the warning nor the corresponding explanation are logged. @@ -241,7 +241,7 @@ func (s *Suite) Test_Logger_Explain__only(c *check.C) { func (s *Suite) Test_Logger_Explain__show_autofix(c *check.C) { t := s.Init(c) - t.SetupCommandLine("--explain", "--show-autofix") + t.SetUpCommandLine("--explain", "--show-autofix") line := t.NewLine("Makefile", 27, "The old song") line.Warnf("Warning without fix.") @@ -268,7 +268,7 @@ func (s *Suite) Test_Logger_Explain__show_autofix(c *check.C) { func (s *Suite) Test_Logger_Explain__show_autofix_and_source(c *check.C) { t := s.Init(c) - t.SetupCommandLine("--explain", "--show-autofix", "--source") + t.SetUpCommandLine("--explain", "--show-autofix", "--source") line := t.NewLine("Makefile", 27, "The old song") line.Warnf("Warning without fix.") @@ -299,7 +299,7 @@ func (s *Suite) Test_Logger_Explain__show_autofix_and_source(c *check.C) { func (s *Suite) Test_Logger_Explain__autofix_and_source(c *check.C) { t := s.Init(c) - t.SetupCommandLine("--explain", "--autofix", "--source") + t.SetUpCommandLine("--explain", "--autofix", "--source") line := t.NewLine("Makefile", 27, "The old song") line.Warnf("Warning without fix.") @@ -329,7 +329,7 @@ func (s *Suite) Test_Logger_Explain__autofix_and_source(c *check.C) { func (s *Suite) Test_Logger_Explain__empty_lines(c *check.C) { t := s.Init(c) - t.SetupCommandLine("--explain") + t.SetUpCommandLine("--explain") line := t.NewLine("Makefile", 27, "The old song") line.Warnf("A normal warning.") @@ -350,7 +350,7 @@ func (s *Suite) Test_Logger_Explain__empty_lines(c *check.C) { func (s *Suite) Test_Logger_ShowSummary__explanations_with_only(c *check.C) { t := s.Init(c) - t.SetupCommandLine("--only", "interesting") + t.SetUpCommandLine("--only", "interesting") line := t.NewLine("Makefile", 27, "The old song") // Neither the warning nor the corresponding explanation are logged. @@ -540,7 +540,7 @@ func (s *Suite) Test_Logger_ShowSummary__autofix_available_with_autofix_option(c func (s *Suite) Test_Logger_Logf__duplicate_messages(c *check.C) { t := s.Init(c) - t.SetupCommandLine("--explain") + t.SetUpCommandLine("--explain") G.Logger.Opts.LogVerbose = false line := t.NewLine("README.txt", 123, "text") @@ -563,7 +563,7 @@ func (s *Suite) Test_Logger_Logf__duplicate_messages(c *check.C) { func (s *Suite) Test_Logger_Logf__duplicate_explanations(c *check.C) { t := s.Init(c) - t.SetupCommandLine("--explain") + t.SetUpCommandLine("--explain") line := t.NewLine("README.txt", 123, "text") // In rare cases, different diagnostics may have the same explanation. @@ -583,7 +583,7 @@ func (s *Suite) Test_Logger_Logf__duplicate_explanations(c *check.C) { func (s *Suite) Test_Logger_Logf__gcc_format(c *check.C) { t := s.Init(c) - t.SetupCommandLine("--gcc-output-format") + t.SetUpCommandLine("--gcc-output-format") G.Logf(Note, "filename", "123", "Both filename and line number.", "Both filename and line number.") G.Logf(Note, "", "123", "No filename, only line number.", "No filename, only line number.") @@ -600,7 +600,7 @@ func (s *Suite) Test_Logger_Logf__gcc_format(c *check.C) { func (s *Suite) Test_Logger_Logf__traditional_format(c *check.C) { t := s.Init(c) - t.SetupCommandLine("--gcc-output-format=no") + t.SetUpCommandLine("--gcc-output-format=no") G.Logf(Note, "filename", "123", "Both filename and line number.", "Both filename and line number.") G.Logf(Note, "", "123", "No filename, only line number.", "No filename, only line number.") @@ -618,7 +618,7 @@ func (s *Suite) Test_Logger_Logf__traditional_format(c *check.C) { func (s *Suite) Test_Logger_Logf__strange_characters(c *check.C) { t := s.Init(c) - t.SetupCommandLine("--gcc-output-format", "--source", "--explain") + t.SetUpCommandLine("--gcc-output-format", "--source", "--explain") G.Logf(Note, "filename", "123", "Format.", "Unicode \U0001F645 and ANSI \x1B are never logged.") G.Explain( @@ -634,7 +634,7 @@ func (s *Suite) Test_Logger_Logf__strange_characters(c *check.C) { func (s *Suite) Test_Logger_Diag__show_source(c *check.C) { t := s.Init(c) - t.SetupCommandLine("--show-autofix", "--source") + t.SetUpCommandLine("--show-autofix", "--source") line := t.NewLine("filename", 123, "text") fix := line.Autofix() @@ -655,7 +655,7 @@ func (s *Suite) Test_Logger_Diag__show_source(c *check.C) { func (s *Suite) Test_Logger_Diag__show_source_with_whole_file(c *check.C) { t := s.Init(c) - t.SetupCommandLine("--source") + t.SetUpCommandLine("--source") line := NewLineWhole("filename") line.Warnf("This line does not have any RawLine attached.") @@ -669,7 +669,7 @@ func (s *Suite) Test_Logger_Diag__show_source_with_whole_file(c *check.C) { func (s *Suite) Test_Logger_Diag__source_duplicates(c *check.C) { t := s.Init(c) - t.SetupPkgsrc() + t.SetUpPkgsrc() t.CreateFileLines("category/dependency/patches/patch-aa", RcsID, "", @@ -678,9 +678,9 @@ func (s *Suite) Test_Logger_Diag__source_duplicates(c *check.C) { "@@ -1,1 +1,1 @@", "-old line", "+new line") - t.SetupPackage("category/package1", + t.SetUpPackage("category/package1", "PATCHDIR=\t../../category/dependency/patches") - t.SetupPackage("category/package2", + t.SetUpPackage("category/package2", "PATCHDIR=\t../../category/dependency/patches") G.Main("pkglint", "--source", "-Wall", t.File("category/package1"), t.File("category/package2")) @@ -702,16 +702,16 @@ func (s *Suite) Test_Logger_Diag__source_duplicates(c *check.C) { func (s *Suite) Test_Logger_shallBeLogged(c *check.C) { t := s.Init(c) - t.SetupCommandLine( /* none */ ) + t.SetUpCommandLine( /* none */ ) c.Check(G.shallBeLogged("Options should not contain whitespace."), equals, true) - t.SetupCommandLine("--only", "whitespace") + t.SetUpCommandLine("--only", "whitespace") c.Check(G.shallBeLogged("Options should not contain whitespace."), equals, true) c.Check(G.shallBeLogged("Options should not contain space."), equals, false) - t.SetupCommandLine( /* none again */ ) + t.SetUpCommandLine( /* none again */ ) c.Check(G.shallBeLogged("Options should not contain whitespace."), equals, true) c.Check(G.shallBeLogged("Options should not contain space."), equals, true) @@ -722,7 +722,7 @@ func (s *Suite) Test_Logger_shallBeLogged(c *check.C) { func (s *Suite) Test_Logger_Logf__duplicate_autofix(c *check.C) { t := s.Init(c) - t.SetupCommandLine("--explain", "--autofix") + t.SetUpCommandLine("--explain", "--autofix") G.Logger.Opts.LogVerbose = false // See SetUpTest line := t.NewLine("README.txt", 123, "text") diff --git a/pkgtools/pkglint/files/mkline.go b/pkgtools/pkglint/files/mkline.go index 2ea46f0281a..674dffed3b2 100644 --- a/pkgtools/pkglint/files/mkline.go +++ b/pkgtools/pkglint/files/mkline.go @@ -78,7 +78,7 @@ func NewMkLine(line Line) *MkLineImpl { } if m, commented, varname, spaceAfterVarname, op, valueAlign, value, spaceAfterValue, comment := MatchVarassign(text); m { - if G.Opts.WarnSpace && spaceAfterVarname != "" { + if spaceAfterVarname != "" { switch { case hasSuffix(varname, "+") && op == "=": break @@ -95,7 +95,7 @@ func NewMkLine(line Line) *MkLineImpl { // XXX: This check should be moved somewhere else. NewMkLine should only be concerned with parsing. if comment != "" && value != "" && spaceAfterValue == "" { - line.Warnf("The # character starts a comment.") + line.Warnf("The # character starts a Makefile comment.") G.Explain( "In a variable assignment, an unescaped # starts a comment that", "continues until the end of the line.", @@ -143,7 +143,7 @@ func NewMkLine(line Line) *MkLineImpl { } // XXX: Replace this regular expression with proper parsing. - // There might be a ${VAR:M*.c} in these variables, which currently confuses the "parser". + // There might be a ${VAR:M*.c} in these variables, which the below regular expression cannot handle. if m, targets, whitespace, sources := match3(text, `^([^\t :]+(?:[\t ]*[^\t :]+)*)([\t ]*):[\t ]*([^#]*?)(?:[\t ]*#.*)?$`); m { // XXX: This check should be moved somewhere else. NewMkLine should only be concerned with parsing. if whitespace != "" { @@ -534,7 +534,7 @@ func (mkline *MkLineImpl) ResolveVarsInRelativePath(relativePath string) string // Relative pkgsrc paths usually only contain two or three levels. // A possible reason for reaching this assertion is: // Tests that access the file system must create their lines - // using t.SetupFileMkLines, not using t.NewMkLines. + // using t.SetUpFileMkLines, not using t.NewMkLines. G.Assertf(!contains(pkgsrcdir, "../../../../.."), "Relative path %q for %q is too deep below the pkgsrc root %q.", pkgsrcdir, basedir, G.Pkgsrc.File(".")) diff --git a/pkgtools/pkglint/files/mkline_test.go b/pkgtools/pkglint/files/mkline_test.go index cad1b44a2c7..1d958d54964 100644 --- a/pkgtools/pkglint/files/mkline_test.go +++ b/pkgtools/pkglint/files/mkline_test.go @@ -140,7 +140,7 @@ func (s *Suite) Test_NewMkLine__merge_conflict(c *check.C) { func (s *Suite) Test_NewMkLine__autofix_space_after_varname(c *check.C) { t := s.Init(c) - t.SetupCommandLine("-Wspace") + t.SetUpCommandLine("-Wspace") filename := t.CreateFileLines("Makefile", MkRcsID, "VARNAME +=\t${VARNAME}", @@ -155,7 +155,7 @@ func (s *Suite) Test_NewMkLine__autofix_space_after_varname(c *check.C) { // FIXME: Don't say anything here because the spaced form is clearer that the compressed form. "NOTE: ~/Makefile:4: Unnecessary space after variable name \"VARNAME+\".") - t.SetupCommandLine("-Wspace", "--autofix") + t.SetUpCommandLine("-Wspace", "--autofix") CheckFileMk(filename) @@ -227,7 +227,7 @@ func (s *Suite) Test_MkLine_Cond(c *check.C) { func (s *Suite) Test_VarUseContext_String(c *check.C) { t := s.Init(c) - t.SetupVartypes() + t.SetUpVartypes() vartype := G.Pkgsrc.VariableType("PKGNAME") vuc := VarUseContext{vartype, vucTimeUnknown, vucQuotBackt, false} @@ -259,7 +259,7 @@ func (s *Suite) Test_NewMkLine__number_sign(c *check.C) { c.Check(mklineVarassignUnescaped.Value(), equals, "'s,") t.CheckOutputLines( - "WARN: filename:1: The # character starts a comment.") + "WARN: filename:1: The # character starts a Makefile comment.") } func (s *Suite) Test_NewMkLine__varassign_leading_space(c *check.C) { @@ -318,7 +318,7 @@ func (s *Suite) Test_MkLine_VariableNeedsQuoting__unknown_rhs(c *check.C) { t := s.Init(c) mkline := t.NewMkLine("filename", 1, "PKGNAME:= ${UNKNOWN}") - t.SetupVartypes() + t.SetUpVartypes() vuc := VarUseContext{G.Pkgsrc.VariableType("PKGNAME"), vucTimeParse, vucQuotUnknown, false} nq := mkline.VariableNeedsQuoting("UNKNOWN", nil, &vuc) @@ -329,8 +329,8 @@ func (s *Suite) Test_MkLine_VariableNeedsQuoting__unknown_rhs(c *check.C) { func (s *Suite) Test_MkLine_VariableNeedsQuoting__append_URL_to_list_of_URLs(c *check.C) { t := s.Init(c) - t.SetupVartypes() - t.SetupMasterSite("MASTER_SITE_SOURCEFORGE", "http://downloads.sourceforge.net/sourceforge/") + t.SetUpVartypes() + t.SetUpMasterSite("MASTER_SITE_SOURCEFORGE", "http://downloads.sourceforge.net/sourceforge/") mkline := t.NewMkLine("Makefile", 95, "MASTER_SITES=\t${HOMEPAGE}") vuc := VarUseContext{G.Pkgsrc.vartypes["MASTER_SITES"], vucTimeRun, vucQuotPlain, false} @@ -346,8 +346,8 @@ func (s *Suite) Test_MkLine_VariableNeedsQuoting__append_URL_to_list_of_URLs(c * func (s *Suite) Test_MkLine_VariableNeedsQuoting__append_list_to_list(c *check.C) { t := s.Init(c) - t.SetupVartypes() - t.SetupMasterSite("MASTER_SITE_SOURCEFORGE", "http://downloads.sourceforge.net/sourceforge/") + t.SetUpVartypes() + t.SetUpMasterSite("MASTER_SITE_SOURCEFORGE", "http://downloads.sourceforge.net/sourceforge/") mkline := t.NewMkLine("Makefile", 96, "MASTER_SITES=\t${MASTER_SITE_SOURCEFORGE:=squirrel-sql/}") MkLineChecker{mkline}.checkVarassign() @@ -359,7 +359,7 @@ func (s *Suite) Test_MkLine_VariableNeedsQuoting__append_list_to_list(c *check.C func (s *Suite) Test_MkLine_VariableNeedsQuoting__eval_shell(c *check.C) { t := s.Init(c) - t.SetupVartypes() + t.SetUpVartypes() mkline := t.NewMkLine("builtin.mk", 3, "USE_BUILTIN.Xfixes!=\t${PKG_ADMIN} pmatch 'pkg-[0-9]*' ${BUILTIN_PKG.Xfixes:Q}") @@ -373,7 +373,7 @@ func (s *Suite) Test_MkLine_VariableNeedsQuoting__eval_shell(c *check.C) { func (s *Suite) Test_MkLine_VariableNeedsQuoting__command_in_single_quotes(c *check.C) { t := s.Init(c) - t.SetupVartypes() + t.SetUpVartypes() mkline := t.NewMkLine("Makefile", 3, "SUBST_SED.hpath=\t-e 's|^\\(INSTALL[\t:]*=\\).*|\\1${INSTALL}|'") @@ -387,9 +387,9 @@ func (s *Suite) Test_MkLine_VariableNeedsQuoting__command_in_single_quotes(c *ch func (s *Suite) Test_MkLine_VariableNeedsQuoting__command_in_command(c *check.C) { t := s.Init(c) - t.SetupVartypes() - t.SetupTool("find", "FIND", AtRunTime) - t.SetupTool("sort", "SORT", AtRunTime) + t.SetUpVartypes() + t.SetUpTool("find", "FIND", AtRunTime) + t.SetUpTool("sort", "SORT", AtRunTime) G.Pkg = NewPackage(t.File("category/pkgbase")) G.Mk = t.NewMkLines("Makefile", MkRcsID, @@ -405,7 +405,7 @@ func (s *Suite) Test_MkLine_VariableNeedsQuoting__command_in_command(c *check.C) func (s *Suite) Test_MkLine_VariableNeedsQuoting__word_as_part_of_word(c *check.C) { t := s.Init(c) - t.SetupVartypes() + t.SetUpVartypes() G.Mk = t.NewMkLines("Makefile", MkRcsID, "EGDIR=\t${EGDIR}/${MACHINE_GNU_PLATFORM}") @@ -424,9 +424,9 @@ func (s *Suite) Test_MkLine_VariableNeedsQuoting__word_as_part_of_word(c *check. func (s *Suite) Test_MkLine_VariableNeedsQuoting__command_as_command_argument(c *check.C) { t := s.Init(c) - t.SetupTool("perl", "PERL5", AtRunTime) - t.SetupTool("bash", "BASH", AtRunTime) - t.SetupVartypes() + t.SetUpTool("perl", "PERL5", AtRunTime) + t.SetUpTool("bash", "BASH", AtRunTime) + t.SetUpVartypes() mklines := t.NewMkLines("Makefile", MkRcsID, "\t${RUN} cd ${WRKSRC} && ( ${ECHO} ${PERL5:Q} ; ${ECHO} ) | ${BASH} ./install", @@ -443,7 +443,7 @@ func (s *Suite) Test_MkLine_VariableNeedsQuoting__command_as_command_argument(c func (s *Suite) Test_MkLine_VariableNeedsQuoting__URL_as_part_of_word_in_list(c *check.C) { t := s.Init(c) - t.SetupVartypes() + t.SetUpVartypes() G.Mk = t.NewMkLines("Makefile", MkRcsID, "MASTER_SITES=${HOMEPAGE}archive/") @@ -460,9 +460,9 @@ func (s *Suite) Test_MkLine_VariableNeedsQuoting__URL_as_part_of_word_in_list(c func (s *Suite) Test_MkLine_VariableNeedsQuoting__command_in_subshell(c *check.C) { t := s.Init(c) - t.SetupVartypes() - t.SetupTool("awk", "AWK", AtRunTime) - t.SetupTool("echo", "ECHO", AtRunTime) + t.SetUpVartypes() + t.SetUpTool("awk", "AWK", AtRunTime) + t.SetUpTool("echo", "ECHO", AtRunTime) G.Mk = t.NewMkLines("xpi.mk", MkRcsID, "\t id=$$(${AWK} '{print}' < ${WRKSRC}/idfile) && echo \"$$id\"", @@ -482,7 +482,7 @@ func (s *Suite) Test_MkLine_VariableNeedsQuoting__command_in_subshell(c *check.C func (s *Suite) Test_MkLine_VariableNeedsQuoting__LDFLAGS_in_single_quotes(c *check.C) { t := s.Init(c) - t.SetupVartypes() + t.SetUpVartypes() G.Mk = t.NewMkLines("x11/mlterm/Makefile", MkRcsID, "SUBST_SED.link=-e 's|(LIBTOOL_LINK).*(LIBS)|& ${LDFLAGS:M*:Q}|g'", @@ -504,7 +504,7 @@ func (s *Suite) Test_MkLine_VariableNeedsQuoting__LDFLAGS_in_single_quotes(c *ch func (s *Suite) Test_MkLine_VariableNeedsQuoting__package_options(c *check.C) { t := s.Init(c) - t.SetupVartypes() + t.SetUpVartypes() G.Mk = t.NewMkLines("Makefile", MkRcsID, "PKG_SUGGESTED_OPTIONS+=\t${PKG_DEFAULT_OPTIONS:Mcdecimal} ${PKG_OPTIONS.py-trytond:Mcdecimal}") @@ -518,9 +518,9 @@ func (s *Suite) Test_MkLine_VariableNeedsQuoting__package_options(c *check.C) { func (s *Suite) Test_MkLine_VariableNeedsQuoting__tool_in_quotes_in_subshell_in_shellwords(c *check.C) { t := s.Init(c) - t.SetupTool("echo", "ECHO", AtRunTime) - t.SetupTool("sh", "SH", AtRunTime) - t.SetupVartypes() + t.SetUpTool("echo", "ECHO", AtRunTime) + t.SetUpTool("sh", "SH", AtRunTime) + t.SetUpVartypes() G.Mk = t.NewMkLines("x11/labltk/Makefile", MkRcsID, "CONFIGURE_ARGS+=\t-tklibs \"`${SH} -c '${ECHO} $$TK_LD_FLAGS'`\"") @@ -534,7 +534,7 @@ func (s *Suite) Test_MkLine_VariableNeedsQuoting__tool_in_quotes_in_subshell_in_ func (s *Suite) Test_MkLine_VariableNeedsQuoting__LDADD_in_BUILDLINK_TRANSFORM(c *check.C) { t := s.Init(c) - t.SetupVartypes() + t.SetUpVartypes() G.Mk = t.NewMkLines("x11/qt5-qtbase/Makefile.common", "BUILDLINK_TRANSFORM+=opt:-ldl:${BUILDLINK_LDADD.dl:M*}") @@ -548,7 +548,7 @@ func (s *Suite) Test_MkLine_VariableNeedsQuoting__LDADD_in_BUILDLINK_TRANSFORM(c func (s *Suite) Test_MkLine_VariableNeedsQuoting__command_in_message(c *check.C) { t := s.Init(c) - t.SetupVartypes() + t.SetUpVartypes() G.Mk = t.NewMkLines("benchmarks/iozone/Makefile", "SUBST_MESSAGE.crlf=\tStripping EOL CR in ${REPLACE_PERL}") @@ -561,7 +561,7 @@ func (s *Suite) Test_MkLine_VariableNeedsQuoting__command_in_message(c *check.C) func (s *Suite) Test_MkLine_VariableNeedsQuoting__guessed_list_variable_in_quotes(c *check.C) { t := s.Init(c) - t.SetupVartypes() + t.SetUpVartypes() G.Mk = t.NewMkLines("audio/jack-rack/Makefile", MkRcsID, "LADSPA_PLUGIN_PATH=\t${PREFIX}/lib/ladspa", @@ -576,7 +576,7 @@ func (s *Suite) Test_MkLine_VariableNeedsQuoting__guessed_list_variable_in_quote func (s *Suite) Test_MkLine_VariableNeedsQuoting__list_in_list(c *check.C) { t := s.Init(c) - t.SetupVartypes() + t.SetUpVartypes() G.Mk = t.NewMkLines("x11/eterm/Makefile", MkRcsID, "DISTFILES=\t${DEFAULT_DISTFILES} ${PIXMAP_FILES}") @@ -591,8 +591,8 @@ func (s *Suite) Test_MkLine_VariableNeedsQuoting__list_in_list(c *check.C) { func (s *Suite) Test_MkLine_VariableNeedsQuoting__PKGNAME_and_URL_list_in_URL_list(c *check.C) { t := s.Init(c) - t.SetupMasterSite("MASTER_SITE_GNOME", "http://ftp.gnome.org/") - t.SetupVartypes() + t.SetUpMasterSite("MASTER_SITE_GNOME", "http://ftp.gnome.org/") + t.SetUpVartypes() G.Mk = t.NewMkLines("x11/gtk3/Makefile", MkRcsID, "MASTER_SITES=\tftp://ftp.gtk.org/${PKGNAME}/ ${MASTER_SITE_GNOME:=subdir/}") @@ -605,8 +605,8 @@ func (s *Suite) Test_MkLine_VariableNeedsQuoting__PKGNAME_and_URL_list_in_URL_li func (s *Suite) Test_MkLine_VariableNeedsQuoting__tool_in_CONFIGURE_ENV(c *check.C) { t := s.Init(c) - t.SetupVartypes() - t.SetupTool("tar", "TAR", AtRunTime) + t.SetUpVartypes() + t.SetUpTool("tar", "TAR", AtRunTime) mklines := t.NewMkLines("Makefile", MkRcsID, "", @@ -625,8 +625,8 @@ func (s *Suite) Test_MkLine_VariableNeedsQuoting__tool_in_CONFIGURE_ENV(c *check func (s *Suite) Test_MkLine_VariableNeedsQuoting__backticks(c *check.C) { t := s.Init(c) - t.SetupVartypes() - t.SetupTool("cat", "CAT", AtRunTime) + t.SetUpVartypes() + t.SetUpTool("cat", "CAT", AtRunTime) mklines := t.NewMkLines("Makefile", MkRcsID, "", @@ -653,10 +653,10 @@ func (s *Suite) Test_MkLine_VariableNeedsQuoting__backticks(c *check.C) { func (s *Suite) Test_MkLine_VariableNeedsQuoting__only_remove_known(c *check.C) { t := s.Init(c) - t.SetupCommandLine("-Wall", "--autofix") - t.SetupVartypes() + t.SetUpCommandLine("-Wall", "--autofix") + t.SetUpVartypes() - mklines := t.SetupFileMkLines("Makefile", + mklines := t.SetUpFileMkLines("Makefile", MkRcsID, "", "demo: .PHONY", @@ -680,10 +680,10 @@ func (s *Suite) Test_MkLine_VariableNeedsQuoting__only_remove_known(c *check.C) func (s *Suite) Test_MkLine_VariableNeedsQuoting__shellword_part(c *check.C) { t := s.Init(c) - t.SetupCommandLine("-Wall,no-space") - t.SetupVartypes() + t.SetUpCommandLine("-Wall,no-space") + t.SetUpVartypes() - mklines := t.SetupFileMkLines("Makefile", + mklines := t.SetUpFileMkLines("Makefile", MkRcsID, "", "SUBST_CLASSES+= class", @@ -702,11 +702,11 @@ func (s *Suite) Test_MkLine_VariableNeedsQuoting__shellword_part(c *check.C) { func (s *Suite) Test_MkLine_VariableNeedsQuoting__tool_in_shell_command(c *check.C) { t := s.Init(c) - t.SetupCommandLine("-Wall,no-space") - t.SetupVartypes() - t.SetupTool("bash", "BASH", AtRunTime) + t.SetUpCommandLine("-Wall,no-space") + t.SetUpVartypes() + t.SetUpTool("bash", "BASH", AtRunTime) - mklines := t.SetupFileMkLines("Makefile", + mklines := t.SetUpFileMkLines("Makefile", MkRcsID, "", "CONFIG_SHELL= ${BASH}") @@ -721,10 +721,10 @@ func (s *Suite) Test_MkLine_VariableNeedsQuoting__tool_in_shell_command(c *check func (s *Suite) Test_MkLine_VariableNeedsQuoting__uncovered_cases(c *check.C) { t := s.Init(c) - t.SetupCommandLine("-Wall,no-space") - t.SetupVartypes() + t.SetUpCommandLine("-Wall,no-space") + t.SetUpVartypes() - mklines := t.SetupFileMkLines("Makefile", + mklines := t.SetUpFileMkLines("Makefile", MkRcsID, "", "GO_SRCPATH= ${HOMEPAGE:S,https://,,}", @@ -748,8 +748,8 @@ func (s *Suite) Test_MkLine_VariableNeedsQuoting__uncovered_cases(c *check.C) { func (s *Suite) Test_MkLine__shell_varuse_in_backt_dquot(c *check.C) { t := s.Init(c) - t.SetupVartypes() - t.SetupTool("grep", "GREP", AtRunTime) + t.SetUpVartypes() + t.SetUpTool("grep", "GREP", AtRunTime) mklines := t.NewMkLines("x11/motif/Makefile", MkRcsID, "post-patch:", @@ -765,7 +765,7 @@ func (s *Suite) Test_MkLine__shell_varuse_in_backt_dquot(c *check.C) { func (s *Suite) Test_MkLine__comment_in_comment(c *check.C) { t := s.Init(c) - t.SetupVartypes() + t.SetUpVartypes() mklines := t.NewMkLines("Makefile", MkRcsID, "COMMENT=\tPKCS#5 v2.0 PBKDF2 Module") @@ -773,7 +773,7 @@ func (s *Suite) Test_MkLine__comment_in_comment(c *check.C) { mklines.Check() t.CheckOutputLines( - "WARN: Makefile:2: The # character starts a comment.") + "WARN: Makefile:2: The # character starts a Makefile comment.") } // Ensures that the conditional variables of a line can be set even @@ -830,6 +830,65 @@ func (s *Suite) Test_MkLine_ValueSplit(c *check.C) { "words") } +func (s *Suite) Test_MkLine_Fields__varassign(c *check.C) { + t := s.Init(c) + + test := func(value string, expected ...string) { + mkline := t.NewMkLine("Makefile", 1, "PATH=\t"+value) + fields := mkline.Fields() + c.Check(fields, deepEquals, expected) + + // Repeated calls get the cached value. + if len(fields) > 0 { + cached := mkline.Fields() + c.Check(&cached[0], equals, &fields[0]) + } + } + + test("# empty", + nil...) + + test("word", + "word") + + test("word '${VAR}single ${VAR}' \"\t\"", + "word", + "'${VAR}single", "${VAR}'", // FIXME: should be a single word. + "\"", "\"") // FIXME: should be a single word. +} + +func (s *Suite) Test_MkLine_Fields__for(c *check.C) { + t := s.Init(c) + + test := func(value string, expected ...string) { + mkline := t.NewMkLine("Makefile", 1, ".for "+value) + fields := mkline.Fields() + c.Check(fields, deepEquals, expected) + + // Repeated calls get the cached value. + if len(fields) > 0 { + cached := mkline.Fields() + c.Check(&cached[0], equals, &fields[0]) + } + } + + // Unrealistic, but needed for full code coverage. + test("# empty", + nil...) + + // Still unrealistic. + test("i in # empty", + "i", + "in") + + test("i in word '${VAR}single ${VAR}' \"\t\"", + "i", + "in", + "word", + "'${VAR}single", "${VAR}'", // FIXME: should be a single word. + "\"", "\"") // FIXME: should be a single word. +} + func (s *Suite) Test_MkLine_ValueFields(c *check.C) { t := s.Init(c) @@ -911,34 +970,34 @@ func (s *Suite) Test_MkLine_ResolveVarsInRelativePath(c *check.C) { t.CreateFileLines("lang/php72/Makefile") t.CreateFileLines("emulators/suse100_base/Makefile") t.CreateFileLines("lang/python36/Makefile") - mklines := t.SetupFileMkLines("Makefile", + mklines := t.SetUpFileMkLines("Makefile", MkRcsID) mkline := mklines.mklines[0] - checkResolve := func(before string, after string) { + test := func(before string, after string) { c.Check(mkline.ResolveVarsInRelativePath(before), equals, after) } - checkResolve("", ".") - checkResolve("${LUA_PKGSRCDIR}", "../../lang/lua53") - checkResolve("${PHPPKGSRCDIR}", "../../lang/php72") - checkResolve("${SUSE_DIR_PREFIX}", "suse100") - checkResolve("${PYPKGSRCDIR}", "../../lang/python36") - checkResolve("${PYPACKAGE}", "python36") - checkResolve("${FILESDIR}", "${FILESDIR}") - checkResolve("${PKGDIR}", "${PKGDIR}") + test("", ".") + test("${LUA_PKGSRCDIR}", "../../lang/lua53") + test("${PHPPKGSRCDIR}", "../../lang/php72") + test("${SUSE_DIR_PREFIX}", "suse100") + test("${PYPKGSRCDIR}", "../../lang/python36") + test("${PYPACKAGE}", "python36") + test("${FILESDIR}", "${FILESDIR}") + test("${PKGDIR}", "${PKGDIR}") G.Pkg = NewPackage(t.File("category/package")) - checkResolve("${FILESDIR}", "files") - checkResolve("${PKGDIR}", ".") + test("${FILESDIR}", "files") + test("${PKGDIR}", ".") } func (s *Suite) Test_MkLine_ResolveVarsInRelativePath__directory_depth(c *check.C) { t := s.Init(c) - t.SetupVartypes() - mklines := t.SetupFileMkLines("multimedia/totem/bla.mk", + t.SetUpVartypes() + mklines := t.SetUpFileMkLines("multimedia/totem/bla.mk", MkRcsID, "BUILDLINK_PKGSRCDIR.totem?=\t../../multimedia/totem") @@ -951,7 +1010,7 @@ func (s *Suite) Test_MkLine_ResolveVarsInRelativePath__directory_depth(c *check. func (s *Suite) Test_MatchVarassign(c *check.C) { s.Init(c) - checkVarassign := func(text string, commented bool, varname, spaceAfterVarname, op, align, value, spaceAfterValue, comment string) { + test := func(text string, commented bool, varname, spaceAfterVarname, op, align, value, spaceAfterValue, comment string) { type VarAssign struct { commented bool varname, spaceAfterVarname string @@ -968,36 +1027,38 @@ func (s *Suite) Test_MatchVarassign(c *check.C) { actual := VarAssign{acommented, avarname, aspaceAfterVarname, aop, aalign, avalue, aspaceAfterValue, acomment} c.Check(actual, equals, expected) } - checkNotVarassign := func(text string) { + + testInvalid := func(text string) { m, _, _, _, _, _, _, _, _ := MatchVarassign(text) if m { c.Errorf("Text %q matches variable assignment but shouldn't.", text) } } - checkVarassign("C++=c11", false, "C+", "", "+=", "C++=", "c11", "", "") - checkVarassign("V=v", false, "V", "", "=", "V=", "v", "", "") - checkVarassign("VAR=#comment", false, "VAR", "", "=", "VAR=", "", "", "#comment") - checkVarassign("VAR=\\#comment", false, "VAR", "", "=", "VAR=", "#comment", "", "") - checkVarassign("VAR=\\\\\\##comment", false, "VAR", "", "=", "VAR=", "\\\\#", "", "#comment") - checkVarassign("VAR=\\", false, "VAR", "", "=", "VAR=", "\\", "", "") - checkVarassign("VAR += value", false, "VAR", " ", "+=", "VAR += ", "value", "", "") - checkVarassign(" VAR=value", false, "VAR", "", "=", " VAR=", "value", "", "") - checkVarassign("VAR=value #comment", false, "VAR", "", "=", "VAR=", "value", " ", "#comment") - - checkNotVarassign("\tVAR=value") - checkNotVarassign("?=value") - checkNotVarassign("<=value") - checkNotVarassign("#") - checkNotVarassign("VAR.$$=value") + test("C++=c11", false, "C+", "", "+=", "C++=", "c11", "", "") + test("V=v", false, "V", "", "=", "V=", "v", "", "") + test("VAR=#comment", false, "VAR", "", "=", "VAR=", "", "", "#comment") + test("VAR=\\#comment", false, "VAR", "", "=", "VAR=", "#comment", "", "") + test("VAR=\\\\\\##comment", false, "VAR", "", "=", "VAR=", "\\\\#", "", "#comment") + test("VAR=\\", false, "VAR", "", "=", "VAR=", "\\", "", "") + test("VAR += value", false, "VAR", " ", "+=", "VAR += ", "value", "", "") + test(" VAR=value", false, "VAR", "", "=", " VAR=", "value", "", "") + test("VAR=value #comment", false, "VAR", "", "=", "VAR=", "value", " ", "#comment") + test("NFILES=${FILES:[#]}", false, "NFILES", "", "=", "NFILES=", "${FILES:[#]}", "", "") + + testInvalid("\tVAR=value") + testInvalid("?=value") + testInvalid("<=value") + testInvalid("#") + testInvalid("VAR.$$=value") // A commented variable assignment must start immediately after the comment character. // There must be no additional whitespace before the variable name. - checkVarassign("#VAR=value", true, "VAR", "", "=", "#VAR=", "value", "", "") + test("#VAR=value", true, "VAR", "", "=", "#VAR=", "value", "", "") // A single space is typically used for writing documentation, not for commenting out code. // Therefore this line doesn't count as commented variable assignment. - checkNotVarassign("# VAR=value") + testInvalid("# VAR=value") } func (s *Suite) Test_NewMkOperator(c *check.C) { diff --git a/pkgtools/pkglint/files/mklinechecker.go b/pkgtools/pkglint/files/mklinechecker.go index 2f0e99dcdc6..ed4aa7358c5 100644 --- a/pkgtools/pkglint/files/mklinechecker.go +++ b/pkgtools/pkglint/files/mklinechecker.go @@ -218,8 +218,7 @@ func (ck MkLineChecker) checkDirectiveFor(forVars map[string]bool, indentation * // // The guessed flag could also be determined more correctly. As of November 2018, // running pkglint over the whole pkgsrc tree did not produce any different result - // whether guessed was true or false, so currently it is not worth investing - // any work. + // whether guessed was true or false. forLoopType := Vartype{lkShell, BtUnknown, []ACLEntry{{"*", aclpAllRead}}, false} forLoopContext := VarUseContext{&forLoopType, vucTimeParse, vucQuotFor, false} for _, itemsVar := range mkline.DetermineUsedVariables() { @@ -291,7 +290,7 @@ func (ck MkLineChecker) checkVarassignLeftPermissions() { return } if trace.Tracing { - defer trace.Call()() + defer trace.Call0()() } mkline := ck.MkLine @@ -871,7 +870,7 @@ func (ck MkLineChecker) checkVarassignLeftNotUsed() { // has the correct data type and quoting. func (ck MkLineChecker) checkVarassignRightVaruse() { if trace.Tracing { - defer trace.Call()() + defer trace.Call0()() } mkline := ck.MkLine @@ -884,7 +883,7 @@ func (ck MkLineChecker) checkVarassignRightVaruse() { vartype := G.Pkgsrc.VariableType(mkline.Varname()) if op == opAssignShell { - vartype = shellcommandsContextType + vartype = shellCommandsType } if vartype != nil && vartype.IsShell() { @@ -969,6 +968,7 @@ func (ck MkLineChecker) checkVarassignMisc() { } if varname == "DIST_SUBDIR" || varname == "WRKSRC" { + // TODO: Replace regex with proper VarUse. if m, revVarname := match1(value, `\$\{(PKGNAME|PKGVERSION)[:\}]`); m { mkline.Warnf("%s should not be used in %s as it includes the PKGREVISION. "+ "Please use %[1]s_NOREV instead.", revVarname, varname) @@ -1077,7 +1077,7 @@ func (ck MkLineChecker) CheckVartypeBasic(varname string, checker *BasicType, op mkline := ck.MkLine valueNoVar := mkline.WithoutMakeVariables(value) - ctx := VartypeCheck{mkline, mkline.Line, varname, op, value, valueNoVar, comment, guessed} + ctx := VartypeCheck{mkline, varname, op, value, valueNoVar, comment, guessed} checker.checker(&ctx) } diff --git a/pkgtools/pkglint/files/mklinechecker_test.go b/pkgtools/pkglint/files/mklinechecker_test.go index 7ce4f7350aa..6278ddc8ceb 100644 --- a/pkgtools/pkglint/files/mklinechecker_test.go +++ b/pkgtools/pkglint/files/mklinechecker_test.go @@ -18,7 +18,7 @@ func (s *Suite) Test_MkLineChecker_checkVarassignLeft(c *check.C) { func (s *Suite) Test_MkLineChecker_Check__url2pkg(c *check.C) { t := s.Init(c) - t.SetupVartypes() + t.SetUpVartypes() mkline := t.NewMkLine("filename.mk", 1, "# url2pkg-marker") @@ -31,10 +31,10 @@ func (s *Suite) Test_MkLineChecker_Check__url2pkg(c *check.C) { func (s *Suite) Test_MkLineChecker_Check__buildlink3_include_prefs(c *check.C) { t := s.Init(c) - t.SetupVartypes() + t.SetUpVartypes() t.CreateFileLines("mk/bsd.prefs.mk") - mklines := t.SetupFileMkLines("category/package/buildlink3.mk", + mklines := t.SetUpFileMkLines("category/package/buildlink3.mk", ".include \"../../mk/bsd.prefs.mk\"") // If the buildlink3.mk file doesn't actually exist, resolving the // relative path fails since that depends on the actual file system, @@ -52,13 +52,13 @@ func (s *Suite) Test_MkLineChecker_Check__buildlink3_include_prefs(c *check.C) { func (s *Suite) Test_MkLineChecker_checkInclude(c *check.C) { t := s.Init(c) - t.SetupVartypes() + t.SetUpVartypes() t.CreateFileLines("pkgtools/x11-links/buildlink3.mk") t.CreateFileLines("graphics/jpeg/buildlink3.mk") t.CreateFileLines("devel/intltool/buildlink3.mk") t.CreateFileLines("devel/intltool/builtin.mk") - mklines := t.SetupFileMkLines("category/package/filename.mk", + mklines := t.SetUpFileMkLines("category/package/filename.mk", MkRcsID, "", ".include \"../../pkgtools/x11-links/buildlink3.mk\"", @@ -98,7 +98,7 @@ func (s *Suite) Test_MkLineChecker_checkInclude__Makefile_exists(c *check.C) { t := s.Init(c) t.CreateFileLines("other/existing/Makefile") - t.SetupPackage("category/package", + t.SetUpPackage("category/package", ".include \"../../other/existing/Makefile\"", ".include \"../../other/not-found/Makefile\"") @@ -111,7 +111,7 @@ func (s *Suite) Test_MkLineChecker_checkInclude__Makefile_exists(c *check.C) { func (s *Suite) Test_MkLineChecker_checkDirective(c *check.C) { t := s.Init(c) - t.SetupVartypes() + t.SetUpVartypes() mklines := t.NewMkLines("category/package/filename.mk", MkRcsID, @@ -150,7 +150,7 @@ func (s *Suite) Test_MkLineChecker_checkDirective(c *check.C) { func (s *Suite) Test_MkLineChecker_checkDirective__for_loop_varname(c *check.C) { t := s.Init(c) - t.SetupVartypes() + t.SetUpVartypes() mklines := t.NewMkLines("filename.mk", MkRcsID, @@ -179,7 +179,7 @@ func (s *Suite) Test_MkLineChecker_checkDirective__for_loop_varname(c *check.C) func (s *Suite) Test_MkLineChecker_checkDependencyRule(c *check.C) { t := s.Init(c) - t.SetupVartypes() + t.SetUpVartypes() mklines := t.NewMkLines("category/package/filename.mk", MkRcsID, @@ -200,8 +200,8 @@ func (s *Suite) Test_MkLineChecker_checkDependencyRule(c *check.C) { func (s *Suite) Test_MkLineChecker_checkVartype__simple_type(c *check.C) { t := s.Init(c) - t.SetupCommandLine("-Wtypes") - t.SetupVartypes() + t.SetUpCommandLine("-Wtypes") + t.SetUpVartypes() // Since COMMENT is defined in vardefs.go its type is certain instead of guessed. vartype := G.Pkgsrc.VariableType("COMMENT") @@ -221,7 +221,7 @@ func (s *Suite) Test_MkLineChecker_checkVartype__simple_type(c *check.C) { func (s *Suite) Test_MkLineChecker_checkVartype(c *check.C) { t := s.Init(c) - t.SetupVartypes() + t.SetUpVartypes() mkline := t.NewMkLine("filename", 1, "DISTNAME=gcc-${GCC_VERSION}") MkLineChecker{mkline}.checkVartype("DISTNAME", opAssign, "gcc-${GCC_VERSION}", "") @@ -235,8 +235,8 @@ func (s *Suite) Test_MkLineChecker_checkVartype(c *check.C) { func (s *Suite) Test_MkLineChecker_checkVartype__skip(c *check.C) { t := s.Init(c) - t.SetupCommandLine("-Wno-types") - t.SetupVartypes() + t.SetUpCommandLine("-Wno-types") + t.SetUpVartypes() mkline := t.NewMkLine("filename", 1, "DISTNAME=invalid:::distname") MkLineChecker{mkline}.Check() @@ -247,7 +247,7 @@ func (s *Suite) Test_MkLineChecker_checkVartype__skip(c *check.C) { func (s *Suite) Test_MkLineChecker_checkVartype__append_to_non_list(c *check.C) { t := s.Init(c) - t.SetupVartypes() + t.SetUpVartypes() mklines := t.NewMkLines("filename.mk", MkRcsID, "DISTNAME+=\tsuffix", @@ -267,7 +267,7 @@ func (s *Suite) Test_MkLineChecker_checkVarassign__URL_with_shell_special_charac t := s.Init(c) G.Pkg = NewPackage(t.File("graphics/gimp-fix-ca")) - t.SetupVartypes() + t.SetUpVartypes() mkline := t.NewMkLine("filename", 10, "MASTER_SITES=http://registry.gimp.org/file/fix-ca.c?action=download&id=9884&file=") MkLineChecker{mkline}.checkVarassign() @@ -278,10 +278,10 @@ func (s *Suite) Test_MkLineChecker_checkVarassign__URL_with_shell_special_charac func (s *Suite) Test_MkLineChecker_checkDirectiveCond(c *check.C) { t := s.Init(c) - t.SetupCommandLine("-Wtypes") - t.SetupVartypes() + t.SetUpCommandLine("-Wtypes") + t.SetUpVartypes() - testCond := func(cond string, output ...string) { + test := func(cond string, output ...string) { MkLineChecker{t.NewMkLine("filename", 1, cond)}.checkDirectiveCond() if len(output) > 0 { t.CheckOutputLines(output...) @@ -290,35 +290,35 @@ func (s *Suite) Test_MkLineChecker_checkDirectiveCond(c *check.C) { } } - testCond(".if !empty(PKGSRC_COMPILER:Mmycc)", + test(".if !empty(PKGSRC_COMPILER:Mmycc)", "WARN: filename:1: The pattern \"mycc\" cannot match any of "+ "{ ccache ccc clang distcc f2c gcc hp icc ido "+ "mipspro mipspro-ucode pcc sunpro xlc } for PKGSRC_COMPILER.") - testCond(".elif ${A} != ${B}") + test(".elif ${A} != ${B}") - testCond(".if ${HOMEPAGE} == \"mailto:someone@example.org\"", + test(".if ${HOMEPAGE} == \"mailto:someone@example.org\"", "WARN: filename:1: \"mailto:someone@example.org\" is not a valid URL.") - testCond(".if !empty(PKGSRC_RUN_TEST:M[Y][eE][sS])", + test(".if !empty(PKGSRC_RUN_TEST:M[Y][eE][sS])", "WARN: filename:1: PKGSRC_RUN_TEST should be matched "+ "against \"[yY][eE][sS]\" or \"[nN][oO]\", not \"[Y][eE][sS]\".") - testCond(".if !empty(IS_BUILTIN.Xfixes:M[yY][eE][sS])") + test(".if !empty(IS_BUILTIN.Xfixes:M[yY][eE][sS])") - testCond(".if !empty(${IS_BUILTIN.Xfixes:M[yY][eE][sS]})", + test(".if !empty(${IS_BUILTIN.Xfixes:M[yY][eE][sS]})", "WARN: filename:1: The empty() function takes a variable name as parameter, "+ "not a variable expression.") - testCond(".if ${PKGSRC_COMPILER} == \"msvc\"", + test(".if ${PKGSRC_COMPILER} == \"msvc\"", "WARN: filename:1: \"msvc\" is not valid for PKGSRC_COMPILER. "+ "Use one of { ccache ccc clang distcc f2c gcc hp icc ido mipspro mipspro-ucode pcc sunpro xlc } instead.", "WARN: filename:1: Use ${PKGSRC_COMPILER:Mmsvc} instead of the == operator.") - testCond(".if ${PKG_LIBTOOL:Mlibtool}", + test(".if ${PKG_LIBTOOL:Mlibtool}", "NOTE: filename:1: PKG_LIBTOOL should be compared using == instead of matching against \":Mlibtool\".") - testCond(".if ${MACHINE_PLATFORM:MUnknownOS-*-*} || ${MACHINE_ARCH:Mx86}", + test(".if ${MACHINE_PLATFORM:MUnknownOS-*-*} || ${MACHINE_ARCH:Mx86}", "WARN: filename:1: "+ "The pattern \"UnknownOS\" cannot match any of "+ "{ AIX BSDOS Bitrig Cygwin Darwin DragonFly FreeBSD FreeMiNT GNUkFreeBSD HPUX Haiku "+ @@ -334,13 +334,32 @@ func (s *Suite) Test_MkLineChecker_checkDirectiveCond(c *check.C) { "} for MACHINE_ARCH.", "NOTE: filename:1: MACHINE_ARCH should be compared using == instead of matching against \":Mx86\".") - testCond(".if ${MASTER_SITES:Mftp://*} == \"ftp://netbsd.org/\"") + test(".if ${MASTER_SITES:Mftp://*} == \"ftp://netbsd.org/\"", + nil...) + + // The only interesting line from the below tracing output is the one + // containing "checkCompareVarStr". + t.EnableTracingToLog() + test(".if ${VAR:Mpattern1:Mpattern2} == comparison", + "TRACE: + MkLineChecker.checkDirectiveCond(\"${VAR:Mpattern1:Mpattern2} == comparison\")", + "TRACE: 1 + (*MkParser).mkCondAtom(\"${VAR:Mpattern1:Mpattern2} == comparison\")", + "TRACE: 1 - (*MkParser).mkCondAtom(\"${VAR:Mpattern1:Mpattern2} == comparison\")", + "TRACE: 1 checkCompareVarStr ${VAR:Mpattern1:Mpattern2} == comparison", + "TRACE: 1 + MkLineChecker.CheckVaruse(filename:1, ${VAR:Mpattern1:Mpattern2}, (no-type time:parse quoting:plain wordpart:false))", + "TRACE: 1 2 + (*Pkgsrc).VariableType(\"VAR\")", + "TRACE: 1 2 3 No type definition found for \"VAR\".", + "TRACE: 1 2 - (*Pkgsrc).VariableType(\"VAR\", \"=>\", (*pkglint.Vartype)(nil))", + "TRACE: 1 2 + (*MkLineImpl).VariableNeedsQuoting(\"VAR\", (*pkglint.Vartype)(nil), (no-type time:parse quoting:plain wordpart:false))", + "TRACE: 1 2 - (*MkLineImpl).VariableNeedsQuoting(\"VAR\", (*pkglint.Vartype)(nil), (no-type time:parse quoting:plain wordpart:false), \"=>\", unknown)", + "TRACE: 1 - MkLineChecker.CheckVaruse(filename:1, ${VAR:Mpattern1:Mpattern2}, (no-type time:parse quoting:plain wordpart:false))", + "TRACE: - MkLineChecker.checkDirectiveCond(\"${VAR:Mpattern1:Mpattern2} == comparison\")") + t.EnableSilentTracing() } func (s *Suite) Test_MkLineChecker_checkVarassign(c *check.C) { t := s.Init(c) - t.SetupVartypes() + t.SetUpVartypes() G.Mk = t.NewMkLines("Makefile", MkRcsID, @@ -355,8 +374,8 @@ func (s *Suite) Test_MkLineChecker_checkVarassign(c *check.C) { func (s *Suite) Test_MkLineChecker_checkVarassignLeftPermissions(c *check.C) { t := s.Init(c) - t.SetupCommandLine("-Wall,no-space") - t.SetupVartypes() + t.SetUpCommandLine("-Wall,no-space") + t.SetUpVartypes() mklines := t.NewMkLines("options.mk", MkRcsID, "PKG_DEVELOPER?= yes", @@ -374,7 +393,7 @@ func (s *Suite) Test_MkLineChecker_checkVarassignLeftPermissions(c *check.C) { func (s *Suite) Test_MkLineChecker_checkVarassignLeftPermissions__infrastructure(c *check.C) { t := s.Init(c) - t.SetupVartypes() + t.SetUpVartypes() t.CreateFileLines("mk/infra.mk", MkRcsID, "", @@ -389,7 +408,7 @@ func (s *Suite) Test_MkLineChecker_checkVarassignLeftPermissions__infrastructure func (s *Suite) Test_MkLineChecker_checkVarassignRightVaruse(c *check.C) { t := s.Init(c) - t.SetupVartypes() + t.SetUpVartypes() mkline := t.NewMkLine("module.mk", 123, "PLIST_SUBST+=\tLOCALBASE=${LOCALBASE:Q}") @@ -403,12 +422,13 @@ func (s *Suite) Test_MkLineChecker_checkVarassignRightVaruse(c *check.C) { func (s *Suite) Test_MkLineChecker_checkVarusePermissions(c *check.C) { t := s.Init(c) - t.SetupVartypes() + t.SetUpVartypes() mklines := t.NewMkLines("options.mk", MkRcsID, "COMMENT=\t${GAMES_USER}", "COMMENT:=\t${PKGBASE}", "PYPKGPREFIX=${PKGBASE}") + G.Pkgsrc.loadDefaultBuildDefs() G.Pkgsrc.UserDefinedVars.Define("GAMES_USER", mklines.mklines[0]) mklines.Check() @@ -423,7 +443,7 @@ func (s *Suite) Test_MkLineChecker_checkVarusePermissions(c *check.C) { func (s *Suite) Test_MkLineChecker_checkVarusePermissions__load_time(c *check.C) { t := s.Init(c) - t.SetupVartypes() + t.SetUpVartypes() mklines := t.NewMkLines("options.mk", MkRcsID, "WRKSRC:=${.CURDIR}", @@ -443,8 +463,8 @@ func (s *Suite) Test_MkLineChecker_checkVarusePermissions__load_time(c *check.C) func (s *Suite) Test_MkLineChecker_checkVarusePermissions__load_time_guessed(c *check.C) { t := s.Init(c) - t.SetupVartypes() - t.SetupTool("install", "", AtRunTime) + t.SetUpVartypes() + t.SetUpTool("install", "", AtRunTime) mklines := t.NewMkLines("install-docfiles.mk", MkRcsID, "DOCFILES=\ta b c", @@ -469,7 +489,7 @@ func (s *Suite) Test_MkLineChecker_checkVarusePermissions__load_time_guessed(c * func (s *Suite) Test_MkLineChecker_checkVarusePermissions__PKGREVISION(c *check.C) { t := s.Init(c) - t.SetupVartypes() + t.SetUpVartypes() mklines := t.NewMkLines("any.mk", MkRcsID, // PKGREVISION may only be set in Makefile, not used at load time; see vardefs.go. @@ -486,7 +506,7 @@ func (s *Suite) Test_MkLineChecker_checkVarusePermissions__PKGREVISION(c *check. func (s *Suite) Test_MkLineChecker_Check__warn_varuse_LOCALBASE(c *check.C) { t := s.Init(c) - t.SetupVartypes() + t.SetUpVartypes() mkline := t.NewMkLine("options.mk", 56, "PKGNAME=${LOCALBASE}") MkLineChecker{mkline}.Check() @@ -500,7 +520,7 @@ func (s *Suite) Test_MkLineChecker_CheckRelativePkgdir(c *check.C) { t.CreateFileLines("other/package/Makefile") // Must be in the filesystem because of directory references. - mklines := t.SetupFileMkLines("category/package/Makefile", + mklines := t.SetUpFileMkLines("category/package/Makefile", "# dummy") ck := MkLineChecker{mklines.mklines[0]} @@ -538,7 +558,7 @@ func (s *Suite) Test_MkLineChecker__unclosed_varuse(c *check.C) { func (s *Suite) Test_MkLineChecker_Check__varuse_modifier_L(c *check.C) { t := s.Init(c) - t.SetupVartypes() + t.SetUpVartypes() mklines := t.NewMkLines("x11/xkeyboard-config/Makefile", "FILES_SUBST+=XKBCOMP_SYMLINK=${${XKBBASE}/xkbcomp:L:Q}", "FILES_SUBST+=XKBCOMP_SYMLINK=${${XKBBASE}/xkbcomp:Q}") @@ -565,7 +585,7 @@ func (s *Suite) Test_MkLineChecker_Check__varuse_modifier_L(c *check.C) { func (s *Suite) Test_MkLineChecker_checkDirectiveCond__comparison_with_shell_command(c *check.C) { t := s.Init(c) - t.SetupVartypes() + t.SetUpVartypes() mklines := t.NewMkLines("security/openssl/Makefile", MkRcsID, ".if ${PKGSRC_COMPILER} == \"gcc\" && ${CC} == \"cc\"", @@ -581,7 +601,7 @@ func (s *Suite) Test_MkLineChecker_checkDirectiveCond__comparison_with_shell_com func (s *Suite) Test_MkLineChecker_checkDirectiveCondEmpty(c *check.C) { t := s.Init(c) - t.SetupVartypes() + t.SetUpVartypes() mkline := t.NewMkLine("module.mk", 123, ".if ${PKGPATH} == \"category/package\"") ck := MkLineChecker{mkline} @@ -615,7 +635,7 @@ func (s *Suite) Test_MkLineChecker_checkDirectiveCondEmpty(c *check.C) { func (s *Suite) Test_MkLineChecker_checkDirectiveCond__comparing_PKGSRC_COMPILER_with_eqeq(c *check.C) { t := s.Init(c) - t.SetupVartypes() + t.SetUpVartypes() G.Mk = t.NewMkLines("audio/pulseaudio/Makefile", MkRcsID, ".if ${OPSYS} == \"Darwin\" && ${PKGSRC_COMPILER} == \"clang\"", @@ -630,7 +650,7 @@ func (s *Suite) Test_MkLineChecker_checkDirectiveCond__comparing_PKGSRC_COMPILER func (s *Suite) Test_MkLineChecker_checkVartype__CFLAGS_with_backticks(c *check.C) { t := s.Init(c) - t.SetupVartypes() + t.SetUpVartypes() G.Mk = t.NewMkLines("chat/pidgin-icb/Makefile", MkRcsID, "CFLAGS+=\t`pkg-config pidgin --cflags`") @@ -654,7 +674,7 @@ func (s *Suite) Test_MkLineChecker_checkVartype__CFLAGS_with_backticks(c *check. func (s *Suite) Test_MkLineChecker_checkVartype__CFLAGS(c *check.C) { t := s.Init(c) - t.SetupVartypes() + t.SetUpVartypes() mklines := t.NewMkLines("Makefile", MkRcsID, "CPPFLAGS.SunOS+=\t-DPIPECOMMAND=\\\"/usr/sbin/sendmail -bs %s\\\"") @@ -669,8 +689,8 @@ func (s *Suite) Test_MkLineChecker_checkVartype__CFLAGS(c *check.C) { func (s *Suite) Test_MkLineChecker_checkDirectiveIndentation__autofix(c *check.C) { t := s.Init(c) - t.SetupCommandLine("--autofix", "-Wspace") - lines := t.SetupFileLines("filename.mk", + t.SetUpCommandLine("--autofix", "-Wspace") + lines := t.SetUpFileLines("filename.mk", MkRcsID, ".if defined(A)", ".for a in ${A}", @@ -702,9 +722,9 @@ func (s *Suite) Test_MkLineChecker_checkDirectiveIndentation__autofix(c *check.C func (s *Suite) Test_MkLineChecker_checkDirectiveIndentation__autofix_multiline(c *check.C) { t := s.Init(c) - t.SetupCommandLine("-Wall", "--autofix") - t.SetupVartypes() - mklines := t.SetupFileMkLines("options.mk", + t.SetUpCommandLine("-Wall", "--autofix") + t.SetUpVartypes() + mklines := t.SetUpFileMkLines("options.mk", MkRcsID, ".if ${PKGNAME} == pkgname", ".if \\", @@ -730,8 +750,8 @@ func (s *Suite) Test_MkLineChecker_checkDirectiveIndentation__autofix_multiline( func (s *Suite) Test_MkLineChecker_CheckVaruseShellword(c *check.C) { t := s.Init(c) - t.SetupVartypes() - mklines := t.SetupFileMkLines("options.mk", + t.SetUpVartypes() + mklines := t.SetUpFileMkLines("options.mk", MkRcsID, "GOPATH=\t${WRKDIR}", "do-build:", @@ -753,9 +773,9 @@ func (s *Suite) Test_MkLineChecker_CheckVaruseShellword(c *check.C) { func (s *Suite) Test_MkLineChecker_CheckVaruseShellword__mstar(c *check.C) { t := s.Init(c) - t.SetupCommandLine("-Wall,no-space") - t.SetupVartypes() - mklines := t.SetupFileMkLines("options.mk", + t.SetUpCommandLine("-Wall,no-space") + t.SetUpVartypes() + mklines := t.SetUpFileMkLines("options.mk", MkRcsID, "CONFIGURE_ARGS+= ${CFLAGS:Q}", "CONFIGURE_ARGS+= ${CFLAGS:M*:Q}", @@ -769,7 +789,7 @@ func (s *Suite) Test_MkLineChecker_CheckVaruseShellword__mstar(c *check.C) { mklines.Check() // FIXME: There should be some notes and warnings about missing :M*; - // these are currently prevented by the PERL5 case in VariableNeedsQuoting. + // these are prevented by the PERL5 case in VariableNeedsQuoting. t.CheckOutputLines( "WARN: ~/options.mk:4: ADA_FLAGS is used but not defined.") } @@ -777,8 +797,8 @@ func (s *Suite) Test_MkLineChecker_CheckVaruseShellword__mstar(c *check.C) { func (s *Suite) Test_MkLineChecker_CheckVaruseShellword__mstar_not_needed(c *check.C) { t := s.Init(c) - t.SetupCommandLine("-Wall,no-space") - pkg := t.SetupPackage("category/package", + t.SetUpCommandLine("-Wall,no-space") + pkg := t.SetUpPackage("category/package", "MAKE_FLAGS+=\tCFLAGS=${CFLAGS:M*:Q}", "MAKE_FLAGS+=\tLFLAGS=${LDFLAGS:M*:Q}") G.Pkgsrc.LoadInfrastructure() @@ -799,7 +819,7 @@ func (s *Suite) Test_MkLineChecker_CheckVaruseShellword__mstar_not_needed(c *che func (s *Suite) Test_MkLineChecker_CheckVaruseShellword__q_not_needed(c *check.C) { t := s.Init(c) - pkg := t.SetupPackage("category/package", + pkg := t.SetUpPackage("category/package", "MASTER_SITES=\t${HOMEPAGE:Q}") G.Pkgsrc.LoadInfrastructure() @@ -814,9 +834,9 @@ func (s *Suite) Test_MkLineChecker_CheckVaruseShellword__q_not_needed(c *check.C func (s *Suite) Test_MkLineChecker_CheckVaruse__eq_nonlist(c *check.C) { t := s.Init(c) - t.SetupVartypes() - t.SetupMasterSite("MASTER_SITE_GITHUB", "https://github.com/") - mklines := t.SetupFileMkLines("options.mk", + t.SetUpVartypes() + t.SetUpMasterSite("MASTER_SITE_GITHUB", "https://github.com/") + mklines := t.SetUpFileMkLines("options.mk", MkRcsID, "WRKSRC=\t\t${WRKDIR:=/subdir}", "MASTER_SITES=\t${MASTER_SITE_GITHUB:=organization/}") @@ -830,9 +850,9 @@ func (s *Suite) Test_MkLineChecker_CheckVaruse__eq_nonlist(c *check.C) { func (s *Suite) Test_MkLineChecker_CheckVaruse__for(c *check.C) { t := s.Init(c) - t.SetupVartypes() - t.SetupMasterSite("MASTER_SITE_GITHUB", "https://github.com/") - mklines := t.SetupFileMkLines("options.mk", + t.SetUpVartypes() + t.SetUpMasterSite("MASTER_SITE_GITHUB", "https://github.com/") + mklines := t.SetUpFileMkLines("options.mk", MkRcsID, ".for var in a b c", "\t: ${var}", @@ -851,8 +871,8 @@ func (s *Suite) Test_MkLineChecker_CheckVaruse__for(c *check.C) { func (s *Suite) Test_MkLineChecker_CheckVaruse__varcanon(c *check.C) { t := s.Init(c) - t.SetupVartypes() - t.SetupPkgsrc() + t.SetUpVartypes() + t.SetUpPkgsrc() t.CreateFileLines("mk/sys-vars.mk", MkRcsID, "CPPPATH.Linux=\t/usr/bin/cpp") @@ -882,13 +902,13 @@ func (s *Suite) Test_MkLineChecker_CheckVaruse__varcanon(c *check.C) { func (s *Suite) Test_MkLineChecker_CheckVaruse__defined_in_infrastructure(c *check.C) { t := s.Init(c) - t.SetupPkgsrc() - t.SetupVartypes() + t.SetUpPkgsrc() + t.SetUpVartypes() t.CreateFileLines("mk/deeply/nested/infra.mk", MkRcsID, "INFRA_VAR?=\tvalue") G.Pkgsrc.LoadInfrastructure() - mklines := t.SetupFileMkLines("category/package/module.mk", + mklines := t.SetUpFileMkLines("category/package/module.mk", MkRcsID, "do-fetch:", "\t: ${INFRA_VAR} ${UNDEFINED}") @@ -904,14 +924,14 @@ func (s *Suite) Test_MkLineChecker_CheckVaruse__build_defs(c *check.C) { // XXX: This paragraph should not be necessary since VARBASE and X11_TYPE // are also defined in vardefs.go. - t.SetupPkgsrc() + t.SetUpPkgsrc() t.CreateFileLines("mk/defaults/mk.conf", "VARBASE?= /usr/pkg/var") G.Pkgsrc.LoadInfrastructure() - t.SetupCommandLine("-Wall,no-space") - t.SetupVartypes() - mklines := t.SetupFileMkLines("options.mk", + t.SetUpCommandLine("-Wall,no-space") + t.SetUpVartypes() + mklines := t.SetUpFileMkLines("options.mk", MkRcsID, "COMMENT= ${VARBASE} ${X11_TYPE}", "PKG_FAIL_REASON+= ${VARBASE} ${X11_TYPE}", @@ -926,8 +946,8 @@ func (s *Suite) Test_MkLineChecker_CheckVaruse__build_defs(c *check.C) { func (s *Suite) Test_MkLineChecker_CheckVaruse__complicated_range(c *check.C) { t := s.Init(c) - t.SetupCommandLine("--show-autofix", "--source") - t.SetupVartypes() + t.SetUpCommandLine("--show-autofix", "--source") + t.SetUpVartypes() mkline := t.NewMkLine("mk/compiler/gcc.mk", 150, "CC:=\t${CC:C/^/_asdf_/1:M_asdf_*:S/^_asdf_//}") @@ -946,7 +966,7 @@ func (s *Suite) Test_MkLineChecker_CheckVaruse__complicated_range(c *check.C) { func (s *Suite) Test_MkLineChecker_CheckVaruse__deprecated_PKG_DEBUG(c *check.C) { t := s.Init(c) - t.SetupVartypes() + t.SetUpVartypes() G.Pkgsrc.initDeprecatedVars() mkline := t.NewMkLine("module.mk", 123, @@ -962,7 +982,7 @@ func (s *Suite) Test_MkLineChecker_CheckVaruse__deprecated_PKG_DEBUG(c *check.C) func (s *Suite) Test_MkLineChecker_checkVaruseUndefined__indirect_variables(c *check.C) { t := s.Init(c) - t.SetupTool("echo", "ECHO", AfterPrefsMk) + t.SetUpTool("echo", "ECHO", AfterPrefsMk) mkline := t.NewMkLine("net/uucp/Makefile", 123, "\techo ${UUCP_${var}}") MkLineChecker{mkline}.Check() @@ -981,11 +1001,11 @@ func (s *Suite) Test_MkLineChecker_checkVaruseUndefined__indirect_variables(c *c func (s *Suite) Test_MkLineChecker_checkVarassignMisc(c *check.C) { t := s.Init(c) - t.SetupPkgsrc() + t.SetUpPkgsrc() G.Pkgsrc.LoadInfrastructure() - t.SetupCommandLine("-Wall,no-space") - t.SetupVartypes() - mklines := t.SetupFileMkLines("module.mk", + t.SetUpCommandLine("-Wall,no-space") + t.SetUpVartypes() + mklines := t.SetUpFileMkLines("module.mk", MkRcsID, "EGDIR= ${PREFIX}/etc/rc.d", "_TOOLS_VARNAME.sed= SED", @@ -993,7 +1013,7 @@ func (s *Suite) Test_MkLineChecker_checkVarassignMisc(c *check.C) { "WRKSRC= ${PKGNAME}", "SITES_distfile.tar.gz= ${MASTER_SITE_GITHUB:=user/}", // TODO: The first of the below assignments should be flagged as redundant by RedundantScope; - // that check is currently only implemented for package Makefiles, not for other files. + // as of January 2019, that check is only implemented for package Makefiles, not for other files. "PYTHON_VERSIONS_ACCEPTED= -13", "PYTHON_VERSIONS_ACCEPTED= 27 36") @@ -1020,11 +1040,11 @@ func (s *Suite) Test_MkLineChecker_checkVarassignMisc(c *check.C) { func (s *Suite) Test_MkLineChecker_checkText(c *check.C) { t := s.Init(c) - t.SetupPkgsrc() + t.SetUpPkgsrc() G.Pkgsrc.LoadInfrastructure() - t.SetupCommandLine("-Wall,no-space") - mklines := t.SetupFileMkLines("module.mk", + t.SetUpCommandLine("-Wall,no-space") + mklines := t.SetUpFileMkLines("module.mk", MkRcsID, "CFLAGS+= -Wl,--rpath,${PREFIX}/lib", "PKG_FAIL_REASON+= \"Group ${GAMEGRP} doesn't exist.\"") @@ -1041,8 +1061,8 @@ func (s *Suite) Test_MkLineChecker_checkText(c *check.C) { func (s *Suite) Test_MkLineChecker_checkText__WRKSRC(c *check.C) { t := s.Init(c) - t.SetupCommandLine("-Wall", "--explain") - mklines := t.SetupFileMkLines("module.mk", + t.SetUpCommandLine("-Wall", "--explain") + mklines := t.SetUpFileMkLines("module.mk", MkRcsID, "pre-configure:", "\tcd ${WRKSRC}/..") @@ -1071,11 +1091,11 @@ func (s *Suite) Test_MkLineChecker_checkText__WRKSRC(c *check.C) { func (s *Suite) Test_MkLineChecker_CheckRelativePath(c *check.C) { t := s.Init(c) - t.SetupPkgsrc() + t.SetUpPkgsrc() G.Pkgsrc.LoadInfrastructure() t.CreateFileLines("wip/package/Makefile") t.CreateFileLines("wip/package/module.mk") - mklines := t.SetupFileMkLines("category/package/module.mk", + mklines := t.SetUpFileMkLines("category/package/module.mk", MkRcsID, "DEPENDS+= wip-package-[0-9]*:../../wip/package", ".include \"../../wip/package/module.mk\"", diff --git a/pkgtools/pkglint/files/mklines.go b/pkgtools/pkglint/files/mklines.go index f01305e060d..1ae0b141e00 100644 --- a/pkgtools/pkglint/files/mklines.go +++ b/pkgtools/pkglint/files/mklines.go @@ -31,7 +31,7 @@ func NewMkLines(lines Lines) MkLines { mklines[i] = NewMkLine(line) } - tools := NewTools(lines.FileName) + tools := NewTools() tools.Fallback(G.Pkgsrc.Tools) return &MkLinesImpl{ diff --git a/pkgtools/pkglint/files/mklines_test.go b/pkgtools/pkglint/files/mklines_test.go index d9f329ef01b..57f1c8c1b11 100644 --- a/pkgtools/pkglint/files/mklines_test.go +++ b/pkgtools/pkglint/files/mklines_test.go @@ -8,8 +8,8 @@ import ( func (s *Suite) Test_MkLines_Check__unusual_target(c *check.C) { t := s.Init(c) - t.SetupVartypes() - t.SetupTool("cc", "CC", AtRunTime) + t.SetUpVartypes() + t.SetUpTool("cc", "CC", AtRunTime) mklines := t.NewMkLines("Makefile", MkRcsID, "", @@ -25,7 +25,7 @@ func (s *Suite) Test_MkLines_Check__unusual_target(c *check.C) { func (s *Suite) Test_MkLines__quoting_LDFLAGS_for_GNU_configure(c *check.C) { t := s.Init(c) - t.SetupVartypes() + t.SetUpVartypes() G.Pkg = NewPackage(t.File("category/pkgbase")) mklines := t.NewMkLines("Makefile", MkRcsID, @@ -42,10 +42,10 @@ func (s *Suite) Test_MkLines__quoting_LDFLAGS_for_GNU_configure(c *check.C) { func (s *Suite) Test_MkLines__for_loop_multiple_variables(c *check.C) { t := s.Init(c) - t.SetupVartypes() - t.SetupTool("echo", "ECHO", AtRunTime) - t.SetupTool("find", "FIND", AtRunTime) - t.SetupTool("pax", "PAX", AtRunTime) + t.SetUpVartypes() + t.SetUpTool("echo", "ECHO", AtRunTime) + t.SetUpTool("find", "FIND", AtRunTime) + t.SetUpTool("pax", "PAX", AtRunTime) mklines := t.NewMkLines("Makefile", // From audio/squeezeboxserver MkRcsID, "", @@ -69,7 +69,7 @@ func (s *Suite) Test_MkLines__for_loop_multiple_variables(c *check.C) { func (s *Suite) Test_MkLines__comparing_YesNo_variable_to_string(c *check.C) { t := s.Init(c) - t.SetupVartypes() + t.SetUpVartypes() mklines := t.NewMkLines("databases/gdbm_compat/builtin.mk", MkRcsID, ".if ${USE_BUILTIN.gdbm} == \"no\"", @@ -88,8 +88,8 @@ func (s *Suite) Test_MkLines__comparing_YesNo_variable_to_string(c *check.C) { func (s *Suite) Test_MkLines__varuse_sh_modifier(c *check.C) { t := s.Init(c) - t.SetupVartypes() - t.SetupTool("sed", "SED", AfterPrefsMk) + t.SetUpVartypes() + t.SetUpTool("sed", "SED", AfterPrefsMk) mklines := t.NewMkLines("lang/qore/module.mk", MkRcsID, "qore-version=\tqore --short-version | ${SED} -e s/-.*//", @@ -118,7 +118,7 @@ func (s *Suite) Test_MkLines__varuse_sh_modifier(c *check.C) { func (s *Suite) Test_MkLines__varuse_parameterized(c *check.C) { t := s.Init(c) - t.SetupVartypes() + t.SetUpVartypes() mklines := t.NewMkLines("converters/wv2/Makefile", MkRcsID, "CONFIGURE_ARGS+=\t\t${CONFIGURE_ARGS.${ICONV_TYPE}-iconv}", @@ -155,7 +155,7 @@ func (s *Suite) Test_MkLines__varuse_parameterized(c *check.C) { func (s *Suite) Test_MkLines__loop_modifier(c *check.C) { t := s.Init(c) - t.SetupVartypes() + t.SetUpVartypes() mklines := t.NewMkLines("chat/xchat/Makefile", MkRcsID, "GCONF_SCHEMAS=\tapps_xchat_url_handler.schemas", @@ -172,7 +172,7 @@ func (s *Suite) Test_MkLines__loop_modifier(c *check.C) { func (s *Suite) Test_MkLines__PKG_SKIP_REASON_depending_on_OPSYS(c *check.C) { t := s.Init(c) - t.SetupVartypes() + t.SetUpVartypes() mklines := t.NewMkLines("Makefile", MkRcsID, "PKG_SKIP_REASON+=\t\"Fails everywhere\"", @@ -189,8 +189,8 @@ func (s *Suite) Test_MkLines__PKG_SKIP_REASON_depending_on_OPSYS(c *check.C) { func (s *Suite) Test_MkLines_Check__use_list_variable_as_part_of_word(c *check.C) { t := s.Init(c) - t.SetupVartypes() - t.SetupTool("tr", "", AtRunTime) + t.SetUpVartypes() + t.SetUpTool("tr", "", AtRunTime) mklines := t.NewMkLines("converters/chef/Makefile", MkRcsID, "\tcd ${WRKSRC} && tr '\\r' '\\n' < ${DISTDIR}/${DIST_SUBDIR}/${DISTFILES} > chef.l") @@ -204,7 +204,7 @@ func (s *Suite) Test_MkLines_Check__use_list_variable_as_part_of_word(c *check.C func (s *Suite) Test_MkLines_Check__absolute_pathname_depending_on_OPSYS(c *check.C) { t := s.Init(c) - t.SetupVartypes() + t.SetUpVartypes() mklines := t.NewMkLines("games/heretic2-demo/Makefile", MkRcsID, ".if ${OPSYS} == \"DragonFly\"", @@ -227,7 +227,7 @@ func (s *Suite) Test_MkLines_Check__absolute_pathname_depending_on_OPSYS(c *chec func (s *Suite) Test_MkLines_CheckForUsedComment(c *check.C) { t := s.Init(c) - t.SetupCommandLine("--show-autofix") + t.SetUpCommandLine("--show-autofix") test := func(pkgpath string, lines []string, diagnostics []string) { mklines := t.NewMkLines("Makefile.common", lines...) @@ -306,8 +306,8 @@ func (s *Suite) Test_MkLines_CheckForUsedComment(c *check.C) { func (s *Suite) Test_MkLines_collectDefinedVariables(c *check.C) { t := s.Init(c) - t.SetupCommandLine("-Wall,no-space") - t.SetupPkgsrc() + t.SetUpCommandLine("-Wall,no-space") + t.SetUpPkgsrc() t.CreateFileLines("mk/tools/defaults.mk", "USE_TOOLS+= autoconf autoconf213") G.Pkgsrc.LoadInfrastructure() @@ -346,11 +346,11 @@ func (s *Suite) Test_MkLines_collectDefinedVariables(c *check.C) { func (s *Suite) Test_MkLines_collectDefinedVariables__BUILTIN_FIND_FILES_VAR(c *check.C) { t := s.Init(c) - t.SetupCommandLine("-Wall,no-space") - t.SetupPackage("category/package") + t.SetUpCommandLine("-Wall,no-space") + t.SetUpPackage("category/package") t.CreateFileLines("mk/buildlink3/bsd.builtin.mk", MkRcsID) - mklines := t.SetupFileMkLines("category/package/builtin.mk", + mklines := t.SetUpFileMkLines("category/package/builtin.mk", MkRcsID, "", "BUILTIN_FIND_FILES_VAR:= H_XFT2", @@ -409,7 +409,7 @@ func (s *Suite) Test_MkLines_collectUsedVariables__nested(c *check.C) { func (s *Suite) Test_MkLines__private_tool_undefined(c *check.C) { t := s.Init(c) - t.SetupVartypes() + t.SetUpVartypes() mklines := t.NewMkLines("filename", MkRcsID, "", @@ -424,7 +424,7 @@ func (s *Suite) Test_MkLines__private_tool_undefined(c *check.C) { func (s *Suite) Test_MkLines__private_tool_defined(c *check.C) { t := s.Init(c) - t.SetupVartypes() + t.SetUpVartypes() mklines := t.NewMkLines("filename", MkRcsID, "TOOLS_CREATE+=\tmd5sum", @@ -441,7 +441,7 @@ func (s *Suite) Test_MkLines__private_tool_defined(c *check.C) { func (s *Suite) Test_MkLines_Check__indentation(c *check.C) { t := s.Init(c) - t.SetupVartypes() + t.SetUpVartypes() mklines := t.NewMkLines("options.mk", MkRcsID, ". if !defined(GUARD_MK)", @@ -492,9 +492,9 @@ func (s *Suite) Test_MkLines_Check__indentation(c *check.C) { func (s *Suite) Test_MkLines_Check__indentation_include(c *check.C) { t := s.Init(c) - t.SetupVartypes() + t.SetUpVartypes() t.CreateFileLines("included.mk") - mklines := t.SetupFileMkLines("module.mk", + mklines := t.SetUpFileMkLines("module.mk", MkRcsID, "", ".if ${PKGPATH} == \"category/package\"", @@ -514,7 +514,7 @@ func (s *Suite) Test_MkLines_Check__indentation_include(c *check.C) { func (s *Suite) Test_MkLines_Check__endif_comment(c *check.C) { t := s.Init(c) - t.SetupVartypes() + t.SetUpVartypes() mklines := t.NewMkLines("opsys.mk", MkRcsID, "", @@ -551,7 +551,7 @@ func (s *Suite) Test_MkLines_Check__endif_comment(c *check.C) { func (s *Suite) Test_MkLines_Check__unfinished_directives(c *check.C) { t := s.Init(c) - t.SetupVartypes() + t.SetUpVartypes() mklines := t.NewMkLines("opsys.mk", MkRcsID, "", @@ -572,7 +572,7 @@ func (s *Suite) Test_MkLines_Check__unfinished_directives(c *check.C) { func (s *Suite) Test_MkLines_Check__unbalanced_directives(c *check.C) { t := s.Init(c) - t.SetupVartypes() + t.SetUpVartypes() mklines := t.NewMkLines("opsys.mk", MkRcsID, "", @@ -594,7 +594,7 @@ func (s *Suite) Test_MkLines_Check__unbalanced_directives(c *check.C) { func (s *Suite) Test_MkLines_Check__incomplete_subst_at_end(c *check.C) { t := s.Init(c) - t.SetupVartypes() + t.SetUpVartypes() mklines := t.NewMkLines("subst.mk", MkRcsID, "", @@ -614,11 +614,11 @@ func (s *Suite) Test_MkLines_Check__incomplete_subst_at_end(c *check.C) { func (s *Suite) Test_MkLines__wip_category_Makefile(c *check.C) { t := s.Init(c) - t.SetupCommandLine("-Wall", "--explain") - t.SetupVartypes() - t.SetupTool("rm", "RM", AtRunTime) + t.SetUpCommandLine("-Wall", "--explain") + t.SetUpVartypes() + t.SetUpTool("rm", "RM", AtRunTime) t.CreateFileLines("mk/misc/category.mk") - mklines := t.SetupFileMkLines("wip/Makefile", + mklines := t.SetUpFileMkLines("wip/Makefile", MkRcsID, "", "COMMENT=\tWIP pkgsrc packages", @@ -656,8 +656,8 @@ func (s *Suite) Test_MkLines__wip_category_Makefile(c *check.C) { func (s *Suite) Test_MkLines_collectDocumentedVariables(c *check.C) { t := s.Init(c) - t.SetupVartypes() - t.SetupTool("rm", "RM", AtRunTime) + t.SetUpVartypes() + t.SetUpTool("rm", "RM", AtRunTime) mklines := t.NewMkLines("Makefile", MkRcsID, "#", @@ -707,7 +707,7 @@ func (s *Suite) Test_MkLines_collectDocumentedVariables(c *check.C) { func (s *Suite) Test_MkLines__shell_command_indentation(c *check.C) { t := s.Init(c) - t.SetupVartypes() + t.SetUpVartypes() mklines := t.NewMkLines("Makefile", MkRcsID, "#", @@ -726,8 +726,8 @@ func (s *Suite) Test_MkLines__shell_command_indentation(c *check.C) { func (s *Suite) Test_MkLines__unknown_options(c *check.C) { t := s.Init(c) - t.SetupVartypes() - t.SetupOption("known", "") + t.SetUpVartypes() + t.SetUpOption("known", "") mklines := t.NewMkLines("options.mk", MkRcsID, "#", @@ -957,14 +957,14 @@ func (s *Suite) Test_MkLines_CheckRedundantAssignments__shell_and_eval_literal(c func (s *Suite) Test_MkLines_Check__PLIST_VARS(c *check.C) { t := s.Init(c) - t.SetupCommandLine("-Wno-space") - t.SetupVartypes() - t.SetupOption("both", "") - t.SetupOption("only-added", "") - t.SetupOption("only-defined", "") + t.SetUpCommandLine("-Wno-space") + t.SetUpVartypes() + t.SetUpOption("both", "") + t.SetUpOption("only-added", "") + t.SetUpOption("only-defined", "") t.CreateFileLines("mk/bsd.options.mk") - mklines := t.SetupFileMkLines("category/package/options.mk", + mklines := t.SetUpFileMkLines("category/package/options.mk", MkRcsID, "", "PKG_OPTIONS_VAR= PKG_OPTIONS.pkg", @@ -993,12 +993,12 @@ func (s *Suite) Test_MkLines_Check__PLIST_VARS(c *check.C) { func (s *Suite) Test_MkLines_Check__PLIST_VARS_indirect(c *check.C) { t := s.Init(c) - t.SetupCommandLine("-Wno-space") - t.SetupVartypes() - t.SetupOption("option1", "") - t.SetupOption("option2", "") + t.SetUpCommandLine("-Wno-space") + t.SetUpVartypes() + t.SetUpOption("option1", "") + t.SetUpOption("option2", "") - mklines := t.SetupFileMkLines("module.mk", + mklines := t.SetUpFileMkLines("module.mk", MkRcsID, "", "MY_PLIST_VARS= option1 option2", @@ -1029,11 +1029,11 @@ func (s *Suite) Test_MkLines_Check__PLIST_VARS_indirect(c *check.C) { func (s *Suite) Test_MkLines_Check__PLIST_VARS_indirect_2(c *check.C) { t := s.Init(c) - t.SetupCommandLine("-Wno-space") - t.SetupVartypes() - t.SetupOption("a", "") - t.SetupOption("b", "") - t.SetupOption("c", "") + t.SetUpCommandLine("-Wno-space") + t.SetUpVartypes() + t.SetUpOption("a", "") + t.SetUpOption("b", "") + t.SetUpOption("c", "") mklines := t.NewMkLines("module.mk", MkRcsID, @@ -1056,8 +1056,8 @@ func (s *Suite) Test_MkLines_Check__PLIST_VARS_indirect_2(c *check.C) { func (s *Suite) Test_MkLines_collectElse(c *check.C) { t := s.Init(c) - t.SetupCommandLine("-Wno-space") - t.SetupVartypes() + t.SetUpCommandLine("-Wno-space") + t.SetUpVartypes() mklines := t.NewMkLines("module.mk", MkRcsID, @@ -1083,8 +1083,8 @@ func (s *Suite) Test_MkLines_collectElse(c *check.C) { func (s *Suite) Test_MkLines_Check__defined_and_used_variables(c *check.C) { t := s.Init(c) - t.SetupCommandLine("-Wno-space") - t.SetupVartypes() + t.SetUpCommandLine("-Wno-space") + t.SetUpVartypes() mklines := t.NewMkLines("module.mk", MkRcsID, @@ -1110,8 +1110,8 @@ func (s *Suite) Test_MkLines_Check__defined_and_used_variables(c *check.C) { func (s *Suite) Test_MkLines_Check__hacks_mk(c *check.C) { t := s.Init(c) - t.SetupCommandLine("-Wall,no-space") - t.SetupVartypes() + t.SetUpCommandLine("-Wall,no-space") + t.SetUpVartypes() mklines := t.NewMkLines("hacks.mk", MkRcsID, "", @@ -1127,8 +1127,8 @@ func (s *Suite) Test_MkLines_Check__hacks_mk(c *check.C) { func (s *Suite) Test_MkLines_Check__MASTER_SITE_in_HOMEPAGE(c *check.C) { t := s.Init(c) - t.SetupMasterSite("MASTER_SITE_GITHUB", "https://github.com/") - t.SetupVartypes() + t.SetUpMasterSite("MASTER_SITE_GITHUB", "https://github.com/") + t.SetUpVartypes() G.Mk = t.NewMkLines("devel/catch/Makefile", MkRcsID, "HOMEPAGE=\t${MASTER_SITE_GITHUB:=philsquared/Catch/}", @@ -1150,7 +1150,7 @@ func (s *Suite) Test_MkLines_Check__MASTER_SITE_in_HOMEPAGE(c *check.C) { func (s *Suite) Test_MkLines_Check__VERSION_as_word_part_in_MASTER_SITES(c *check.C) { t := s.Init(c) - t.SetupVartypes() + t.SetUpVartypes() mklines := t.NewMkLines("geography/viking/Makefile", MkRcsID, "MASTER_SITES=\t${MASTER_SITE_SOURCEFORGE:=viking/}${VERSION}/") @@ -1166,7 +1166,7 @@ func (s *Suite) Test_MkLines_Check__VERSION_as_word_part_in_MASTER_SITES(c *chec func (s *Suite) Test_MkLines_Check__shell_command_as_word_part_in_ENV_list(c *check.C) { t := s.Init(c) - t.SetupVartypes() + t.SetUpVartypes() mklines := t.NewMkLines("x11/lablgtk1/Makefile", MkRcsID, "CONFIGURE_ENV+=\tCC=${CC}") @@ -1181,8 +1181,8 @@ func (s *Suite) Test_MkLines_Check__shell_command_as_word_part_in_ENV_list(c *ch func (s *Suite) Test_MkLines_Check__extra_warnings(c *check.C) { t := s.Init(c) - t.SetupCommandLine("-Wextra") - t.SetupVartypes() + t.SetUpCommandLine("-Wextra") + t.SetUpVartypes() G.Pkg = NewPackage(t.File("category/pkgbase")) G.Mk = t.NewMkLines("options.mk", MkRcsID, @@ -1212,8 +1212,8 @@ func (s *Suite) Test_MkLines_Check__extra_warnings(c *check.C) { func (s *Suite) Test_MkLines_ForEach__conditional_variables(c *check.C) { t := s.Init(c) - t.SetupCommandLine("-Wall,no-space") - t.SetupVartypes() + t.SetUpCommandLine("-Wall,no-space") + t.SetUpVartypes() mklines := t.NewMkLines("conditional.mk", MkRcsID, "", @@ -1251,8 +1251,8 @@ func (s *Suite) Test_MkLines_ForEach__conditional_variables(c *check.C) { func (s *Suite) Test_MkLines_checkVarassignPlist__indirect(c *check.C) { t := s.Init(c) - t.SetupVartypes() - mklines := t.SetupFileMkLines("plist.mk", + t.SetUpVartypes() + mklines := t.SetUpFileMkLines("plist.mk", MkRcsID, "", "MY_PLIST_VARS=\tvar1 var2", @@ -1269,7 +1269,7 @@ func (s *Suite) Test_MkLines_checkVarassignPlist__indirect(c *check.C) { func (s *Suite) Test_VaralignBlock_Process__autofix(c *check.C) { t := s.Init(c) - t.SetupCommandLine("-Wspace", "--show-autofix") + t.SetUpCommandLine("-Wspace", "--show-autofix") mklines := t.NewMkLines("file.mk", "VAR= value", // Indentation 7, fixed to 8. @@ -1339,7 +1339,7 @@ func (s *Suite) Test_VaralignBlock_Process__reduce_indentation(c *check.C) { func (s *Suite) Test_VaralignBlock_Process__longest_line_no_space(c *check.C) { t := s.Init(c) - t.SetupCommandLine("-Wspace") + t.SetUpCommandLine("-Wspace") mklines := t.NewMkLines("file.mk", "SUBST_CLASSES+= aaaaaaaa", "SUBST_STAGE.aaaaaaaa= pre-configure", @@ -1362,7 +1362,7 @@ func (s *Suite) Test_VaralignBlock_Process__longest_line_no_space(c *check.C) { func (s *Suite) Test_VaralignBlock_Process__only_spaces(c *check.C) { t := s.Init(c) - t.SetupCommandLine("-Wspace") + t.SetUpCommandLine("-Wspace") mklines := t.NewMkLines("file.mk", "SUBST_CLASSES+= aaaaaaaa", "SUBST_STAGE.aaaaaaaa= pre-configure", diff --git a/pkgtools/pkglint/files/mklines_varalign_test.go b/pkgtools/pkglint/files/mklines_varalign_test.go index 3e144278f09..422c4909eb0 100755 --- a/pkgtools/pkglint/files/mklines_varalign_test.go +++ b/pkgtools/pkglint/files/mklines_varalign_test.go @@ -55,9 +55,9 @@ func (vt *VaralignTester) run(autofix bool) { if vt.ShowSource { cmdline = append(cmdline, "--source") } - t.SetupCommandLine(cmdline...) + t.SetUpCommandLine(cmdline...) - mklines := t.SetupFileMkLines("Makefile", vt.input...) + mklines := t.SetUpFileMkLines("Makefile", vt.input...) var varalign VaralignBlock for _, mkline := range mklines.mklines { diff --git a/pkgtools/pkglint/files/mkparser.go b/pkgtools/pkglint/files/mkparser.go index 2f07d4b8cb3..0262025f633 100644 --- a/pkgtools/pkglint/files/mkparser.go +++ b/pkgtools/pkglint/files/mkparser.go @@ -9,7 +9,17 @@ import ( // MkParser wraps a Parser and provides methods for parsing // things related to Makefiles. type MkParser struct { - *Parser + Line Line + lexer *textproc.Lexer + EmitWarnings bool +} + +func (p *MkParser) EOF() bool { + return p.lexer.EOF() +} + +func (p *MkParser) Rest() string { + return p.lexer.Rest() } // NewMkParser creates a new parser for the given text. @@ -20,7 +30,7 @@ type MkParser struct { // TODO: Remove the emitWarnings argument in order to separate parsing from checking. func NewMkParser(line Line, text string, emitWarnings bool) *MkParser { G.Assertf((line != nil) == emitWarnings, "line must be given iff emitWarnings is set") - return &MkParser{NewParser(line, text, emitWarnings)} + return &MkParser{line, textproc.NewLexer(text), emitWarnings} } // MkTokens splits a text like in the following example: @@ -253,10 +263,12 @@ loop: } lexer.Reset(modifierMark) + // FIXME: Why skip over unknown modifiers here? This accepts :S,a,b,c,d,e,f but shouldn't. - re := G.res.Compile(regex.Pattern(`^([^:$` + string(closing) + `]|\$\$)+`)) + re := G.res.Compile(regex.Pattern(ifelseStr(closing == '}', `^([^:$}]|\$\$)+`, `^([^:$)]|\$\$)+`))) for p.VarUse() != nil || lexer.SkipRegexp(re) { } + if suffixSubst := lexer.Since(modifierMark); contains(suffixSubst, "=") { appendModifier(suffixSubst) continue @@ -265,10 +277,12 @@ loop: return modifiers } +// varUseModifierSubst parses a :S,from,to, or a :C,from,to, modifier. func (p *MkParser) varUseModifierSubst(lexer *textproc.Lexer, closing byte) bool { - lexer.Skip(1) + lexer.Skip(1 /* the initial S or C */) + sep := lexer.PeekByte() // bmake allows _any_ separator, even letters. - if sep == -1 { + if sep == -1 || byte(sep) == closing { return false } @@ -306,8 +320,11 @@ func (p *MkParser) varUseModifierSubst(lexer *textproc.Lexer, closing byte) bool return true } +// varUseModifierAt parses a variable modifier like ":@v@echo ${v};@", +// which expands the variable value in a loop. func (p *MkParser) varUseModifierAt(lexer *textproc.Lexer, closing byte, varname string) bool { - lexer.Skip(1) + lexer.Skip(1 /* the initial @ */) + loopVar := lexer.NextBytesSet(AlnumDot) if loopVar == "" || !lexer.SkipByte('@') { return false @@ -325,6 +342,7 @@ func (p *MkParser) varUseModifierAt(lexer *textproc.Lexer, closing byte, varname } // MkCond parses a condition like ${OPSYS} == "NetBSD". +// // See devel/bmake/files/cond.c. func (p *MkParser) MkCond() MkCond { and := p.mkCondAnd() @@ -515,6 +533,145 @@ func (p *MkParser) Varname() string { return lexer.Since(mark) } +func (p *MkParser) PkgbasePattern() string { + lexer := p.lexer + start := lexer.Mark() + + for { + if p.VarUse() != nil || + lexer.SkipRegexp(G.res.Compile(`^[\w.*+,{}]+`)) || + lexer.SkipRegexp(G.res.Compile(`^\[[\d-]+\]`)) { + continue + } + + lookahead := lexer.Copy() + if !lookahead.SkipByte('-') { + break + } + + if lookahead.SkipRegexp(G.res.Compile(`^\d`)) || + // TODO: Replace regex with proper VarUse. + lookahead.SkipRegexp(G.res.Compile(`^\$\{\w*VER\w*\}`)) || + lookahead.SkipByte('[') { + + // The parser is looking at a hyphen followed by a version number. + // This means the pkgbase stops before the hyphen. + break + } + + lexer.Skip(1 /* the hyphen */) + } + + pkgbase := lexer.Since(start) + if strings.Count(pkgbase, "{") == strings.Count(pkgbase, "}") { + return pkgbase + } + + // Unbalanced braces, as in "{ssh{,6}-[0-9]". + lexer.Reset(start) + return "" +} + +type DependencyPattern struct { + Pkgbase string // "freeciv-client", "{gcc48,gcc48-libs}", "${EMACS_REQD}" + LowerOp string // ">=", ">" + Lower string // "2.5.0", "${PYVER}" + UpperOp string // "<", "<=" + Upper string // "3.0", "${PYVER}" + Wildcard string // "[0-9]*", "1.5.*", "${PYVER}" +} + +func (p *MkParser) Dependency() *DependencyPattern { + lexer := p.lexer + + parseVersion := func() string { + mark := lexer.Mark() + + for p.VarUse() != nil { + } + if lexer.Since(mark) != "" { + return lexer.Since(mark) + } + + m := lexer.NextRegexp(G.res.Compile(`^\d[\w.]*`)) + if m != nil { + return m[0] + } + + return "" + } + + var dp DependencyPattern + mark := lexer.Mark() + dp.Pkgbase = p.PkgbasePattern() + if dp.Pkgbase == "" { + return nil + } + + mark2 := lexer.Mark() + op := lexer.NextString(">=") + if op == "" { + op = lexer.NextString(">") + } + + if op != "" { + version := parseVersion() + if version != "" { + dp.LowerOp = op + dp.Lower = version + } else { + lexer.Reset(mark2) + } + } + + op = lexer.NextString("<=") + if op == "" { + op = lexer.NextString("<") + } + + if op != "" { + version := parseVersion() + if version != "" { + dp.UpperOp = op + dp.Upper = version + } else { + lexer.Reset(mark2) + } + } + + if dp.LowerOp != "" || dp.UpperOp != "" { + return &dp + } + + if lexer.SkipByte('-') && lexer.Rest() != "" { + versionMark := lexer.Mark() + + // FIXME: Use VarUse. + for lexer.SkipRegexp(G.res.Compile(`^(\$\{\w+\}|[\w\[\]*_.\-])`)) { + } + + if !lexer.SkipString("{,nb*}") { + lexer.SkipString("{,nb[0-9]*}") + } + + dp.Wildcard = lexer.Since(versionMark) + return &dp + } + + if pkgbaseParser := NewMkParser(nil, dp.Pkgbase, false); pkgbaseParser.VarUse() != nil && pkgbaseParser.EOF() { + return &dp + } + + if hasSuffix(dp.Pkgbase, "-*") { + dp.Pkgbase = strings.TrimSuffix(dp.Pkgbase, "-*") + dp.Wildcard = "*" + return &dp + } + + lexer.Reset(mark) + return nil +} + // MkCond is a condition in a Makefile, such as ${OPSYS} == NetBSD. // // The representation is somewhere between syntactic and semantic. diff --git a/pkgtools/pkglint/files/mkparser_test.go b/pkgtools/pkglint/files/mkparser_test.go index 439d5072bea..5b4acb6c302 100644 --- a/pkgtools/pkglint/files/mkparser_test.go +++ b/pkgtools/pkglint/files/mkparser_test.go @@ -325,7 +325,7 @@ func (s *Suite) Test_MkParser_VarUse(c *check.C) { func (s *Suite) Test_MkParser_VarUse__ambiguous(c *check.C) { t := s.Init(c) - t.SetupCommandLine("--explain") + t.SetUpCommandLine("--explain") mkline := t.NewMkLine("module.mk", 123, "\t$Varname $X") p := NewMkParser(mkline.Line, mkline.ShellCommand(), true) @@ -503,9 +503,9 @@ func (s *Suite) Test_MkParser_MkCond(c *check.C) { func (s *Suite) Test_MkParser_VarUse__parentheses_autofix(c *check.C) { t := s.Init(c) - t.SetupCommandLine("--autofix") - t.SetupVartypes() - lines := t.SetupFileLines("Makefile", + t.SetUpCommandLine("--autofix") + t.SetUpVartypes() + lines := t.SetUpFileLines("Makefile", MkRcsID, "COMMENT=$(P1) $(P2)) $(P3:Q) ${BRACES} $(A.$(B.$(C)))") mklines := NewMkLines(lines) @@ -522,6 +522,163 @@ func (s *Suite) Test_MkParser_VarUse__parentheses_autofix(c *check.C) { "COMMENT=${P1} ${P2}) ${P3:Q} ${BRACES} $(A.$(B.${C}))") } +func (s *Suite) Test_MkParser_varUseModifierSubst(c *check.C) { + t := s.Init(c) + + varUse := NewMkVarUse + test := func(text string, varUse *MkVarUse, rest string) { + mkline := t.NewMkLine("Makefile", 20, "\t"+text) + p := NewMkParser(mkline.Line, mkline.ShellCommand(), true) + + actual := p.VarUse() + + t.Check(actual, deepEquals, varUse) + t.Check(p.Rest(), equals, rest) + t.CheckOutputEmpty() + } + + test("${VAR:S", nil, "${VAR:S") // Just for code coverage. + test("${VAR:S}", varUse("VAR"), "") // FIXME: should not consume anything. + test("${VAR:S,}", varUse("VAR"), "") // FIXME: should not consume anything. + test("${VAR:S,from,to}", varUse("VAR"), "") // FIXME: should not consume anything. + test("${VAR:S,from,to,}", varUse("VAR", "S,from,to,"), "") + test("${VAR:S,^from$,to,}", varUse("VAR", "S,^from$,to,"), "") + test("${VAR:S,@F@,${F},}", varUse("VAR", "S,@F@,${F},"), "") +} + +func (s *Suite) Test_MkParser_varUseModifierAt(c *check.C) { + t := s.Init(c) + + varUse := NewMkVarUse + test := func(text string, varUse *MkVarUse, rest string, diagnostics ...string) { + mkline := t.NewMkLine("Makefile", 20, "\t"+text) + p := NewMkParser(mkline.Line, mkline.ShellCommand(), true) + + actual := p.VarUse() + + t.Check(actual, deepEquals, varUse) + t.Check(p.Rest(), equals, rest) + if len(diagnostics) > 0 { + t.CheckOutputLines(diagnostics...) + } else { + t.CheckOutputEmpty() + } + } + + test("${VAR:@", nil, "${VAR:@") // Just for code coverage. + test("${VAR:@i@${i}}", varUse("VAR", "@i@${i}"), "", + "WARN: Makefile:20: Modifier ${VAR:@i@...@} is missing the final \"@\".") + test("${VAR:@i@${i}@}", varUse("VAR", "@i@${i}@"), "") +} + +func (s *Suite) Test_MkParser_PkgbasePattern(c *check.C) { + + testRest := func(pattern, expected, rest string) { + parser := NewMkParser(nil, pattern, false) + actual := parser.PkgbasePattern() + c.Check(actual, equals, expected) + c.Check(parser.Rest(), equals, rest) + } + + testRest("fltk", "fltk", "") + testRest("fltk|", "fltk", "|") + testRest("boost-build-1.59.*", "boost-build", "-1.59.*") + testRest("${PHP_PKG_PREFIX}-pdo-5.*", "${PHP_PKG_PREFIX}-pdo", "-5.*") + testRest("${PYPKGPREFIX}-metakit-[0-9]*", "${PYPKGPREFIX}-metakit", "-[0-9]*") + + // This is a valid dependency pattern, but it's more complicated + // than the patterns pkglint can handle as of January 2019. + // + // This pattern doesn't have a single package base, which means it cannot be parsed at all. + testRest("{ssh{,6}-[0-9]*,openssh-[0-9]*}", "", "{ssh{,6}-[0-9]*,openssh-[0-9]*}") +} + +func (s *Suite) Test_MkParser_Dependency(c *check.C) { + + testRest := func(pattern string, expected DependencyPattern, rest string) { + parser := NewMkParser(nil, pattern, false) + dp := parser.Dependency() + if c.Check(dp, check.NotNil) { + c.Check(*dp, equals, expected) + c.Check(parser.Rest(), equals, rest) + } + } + + testNil := func(pattern string) { + parser := NewMkParser(nil, pattern, false) + dp := parser.Dependency() + if c.Check(dp, check.IsNil) { + c.Check(parser.Rest(), equals, pattern) + } + } + + test := func(pattern string, expected DependencyPattern) { + testRest(pattern, expected, "") + } + + test("fltk>=1.1.5rc1<1.3", + DependencyPattern{"fltk", ">=", "1.1.5rc1", "<", "1.3", ""}) + + test("libwcalc-1.0*", + DependencyPattern{"libwcalc", "", "", "", "", "1.0*"}) + + test("${PHP_PKG_PREFIX}-pdo-5.*", + DependencyPattern{"${PHP_PKG_PREFIX}-pdo", "", "", "", "", "5.*"}) + + test("${PYPKGPREFIX}-metakit-[0-9]*", + DependencyPattern{"${PYPKGPREFIX}-metakit", "", "", "", "", "[0-9]*"}) + + test("boost-build-1.59.*", + DependencyPattern{"boost-build", "", "", "", "", "1.59.*"}) + + test("${_EMACS_REQD}", + DependencyPattern{"${_EMACS_REQD}", "", "", "", "", ""}) + + test("{gcc46,gcc46-libs}>=4.6.0", + DependencyPattern{"{gcc46,gcc46-libs}", ">=", "4.6.0", "", "", ""}) + + test("perl5-*", + DependencyPattern{"perl5", "", "", "", "", "*"}) + + test("verilog{,-current}-[0-9]*", + DependencyPattern{"verilog{,-current}", "", "", "", "", "[0-9]*"}) + + test("mpg123{,-esound,-nas}>=0.59.18", + DependencyPattern{"mpg123{,-esound,-nas}", ">=", "0.59.18", "", "", ""}) + + test("mysql*-{client,server}-[0-9]*", + DependencyPattern{"mysql*-{client,server}", "", "", "", "", "[0-9]*"}) + + test("postgresql8[0-35-9]-${module}-[0-9]*", + DependencyPattern{"postgresql8[0-35-9]-${module}", "", "", "", "", "[0-9]*"}) + + test("ncurses-${NC_VERS}{,nb*}", + DependencyPattern{"ncurses", "", "", "", "", "${NC_VERS}{,nb*}"}) + + test("xulrunner10>=${MOZ_BRANCH}${MOZ_BRANCH_MINOR}", + DependencyPattern{"xulrunner10", ">=", "${MOZ_BRANCH}${MOZ_BRANCH_MINOR}", "", "", ""}) + + testRest("gnome-control-center>=2.20.1{,nb*}", + DependencyPattern{"gnome-control-center", ">=", "2.20.1", "", "", ""}, "{,nb*}") + + testNil(">=2.20.1{,nb*}") + + testNil("pkgbase<=") + + // TODO: support this edge case someday. + // "{ssh{,6}-[0-9]*,openssh-[0-9]*}" is not representable using the current data structure + + // TODO: More test cases from current pkgsrc: + // R-jsonlite>=0.9.6* + // + // {ezmlm>=0.53,ezmlm-idx>=0.40} + // {samba>=2.0,ja-samba>=2.0} + // {mecab-ipadic>=2.7.0,mecab-jumandic>=5.1} + // + // ${_EMACS_CONFLICTS.${_EMACS_FLAVOR}} + // ${DISTNAME:S/gnome-vfs/gnome-vfs2-${GNOME_VFS_NAME}/} +} + func (s *Suite) Test_MkCondWalker_Walk(c *check.C) { t := s.Init(c) diff --git a/pkgtools/pkglint/files/mkshparser.go b/pkgtools/pkglint/files/mkshparser.go index f6e04017675..1b48e656eac 100644 --- a/pkgtools/pkglint/files/mkshparser.go +++ b/pkgtools/pkglint/files/mkshparser.go @@ -1,6 +1,6 @@ package pkglint -import "strconv" +import "fmt" func parseShellProgram(line Line, program string) (*MkShList, error) { if trace.Tracing { @@ -13,10 +13,14 @@ func parseShellProgram(line Line, program string) (*MkShList, error) { succeeded := parser.Parse(lexer) - if succeeded == 0 && lexer.error == "" { + switch { + case succeeded == 0 && lexer.error == "": return lexer.result, nil + case succeeded == 0 && rest != "": + return nil, fmt.Errorf("splitIntoShellTokens couldn't parse %q", rest) + default: + return nil, &ParseError{append([]string{lexer.current}, lexer.remaining...)} } - return nil, &ParseError{append([]string{lexer.current}, lexer.remaining...)} } type ParseError struct { @@ -157,7 +161,7 @@ func (lex *ShellLexer) Lex(lval *shyySymType) (ttype int) { } if m, fdstr, op := match2(token, `^(\d+)(<<-|<<|<>|<&|>>|>&|>\||<|>)$`); m { - fd, _ := strconv.Atoi(fdstr) + fd := toInt(fdstr, -1) lval.IONum = fd lex.ioRedirect = op return tkIO_NUMBER diff --git a/pkgtools/pkglint/files/mkshparser_test.go b/pkgtools/pkglint/files/mkshparser_test.go index 22ef5064f30..21a7a2629fb 100644 --- a/pkgtools/pkglint/files/mkshparser_test.go +++ b/pkgtools/pkglint/files/mkshparser_test.go @@ -2,23 +2,54 @@ package pkglint import ( "encoding/json" + "fmt" "gopkg.in/check.v1" - "strconv" ) -func (s *Suite) Test_parseShellProgram__parse_error_for_unfinished_shell_variable(c *check.C) { +func (s *Suite) Test_parseShellProgram__parse_error_for_dollar(c *check.C) { t := s.Init(c) - mkline := t.NewMkLine("module.mk", 1, "\t$${") + test := func(text string, expProgram *MkShList, expError error, expDiagnostics ...string) { + shline := t.NewShellLine("module.mk", 123, "\t"+text) - list, err := parseShellProgram(mkline.Line, mkline.ShellCommand()) + if len(expDiagnostics) > 0 { + defer t.CheckOutputLines(expDiagnostics...) + } else { + defer t.CheckOutputEmpty() + } + + program, err := parseShellProgram(shline.mkline.Line, text) + + if err == nil { + c.Check(err, equals, expError) + } else { + c.Check(err, deepEquals, expError) + c.Check(program, deepEquals, expProgram) + } + } + + test("$$", + nil, + nil, + nil...) - c.Check(list, check.IsNil) - // XXX: []string{"$${"} would be an even better error message - c.Check(err.Error(), equals, "parse error at []string{\"\"}") + test( + "$${", + nil, + fmt.Errorf("splitIntoShellTokens couldn't parse \"$${\""), + "WARN: module.mk:123: Unclosed shell variable starting at \"$${\".") - t.CheckOutputLines( - "WARN: module.mk:1: Internal pkglint error in ShTokenizer.ShAtom at \"$${\" (quoting=plain).") + test( + "$$;", + nil, + nil, + nil...) + + test( + "shell$$;", + nil, + nil, + nil...) } type ShSuite struct { @@ -602,10 +633,7 @@ func (b *MkShBuilder) SimpleCommand(words ...string) *MkShCommand { if assignments && matches(word, `^[A-Za-z_]\w*=`) { cmd.Assignments = append(cmd.Assignments, b.Token(word)) } else if m, fdstr, op, rest := match3(word, `^(\d*)(<<-|<<|<&|>>|>&|>\||<|>)(.*)$`); m { - fd, err := strconv.Atoi(fdstr) - if err != nil { - fd = -1 - } + fd := toInt(fdstr, -1) cmd.Redirections = append(cmd.Redirections, b.Redirection(fd, op, rest)) } else { assignments = false diff --git a/pkgtools/pkglint/files/mktypes.go b/pkgtools/pkglint/files/mktypes.go index 20ddec04447..1ded441ae53 100644 --- a/pkgtools/pkglint/files/mktypes.go +++ b/pkgtools/pkglint/files/mktypes.go @@ -110,6 +110,11 @@ func (m MkVarUseModifier) Subst(str string) string { return result } +// MatchMatch tries to match the modifier to a :M or a :N pattern matching. +// Examples: +// :Mpattern => true, true, "pattern" +// :Npattern => true, false, "pattern" +// :X => false func (m MkVarUseModifier) MatchMatch() (ok bool, positive bool, pattern string) { if hasPrefix(m.Text, "M") || hasPrefix(m.Text, "N") { return true, m.Text[0] == 'M', m.Text[1:] diff --git a/pkgtools/pkglint/files/options_test.go b/pkgtools/pkglint/files/options_test.go index 25ad28beb7c..e5300b25a31 100755 --- a/pkgtools/pkglint/files/options_test.go +++ b/pkgtools/pkglint/files/options_test.go @@ -5,20 +5,20 @@ import "gopkg.in/check.v1" func (s *Suite) Test_CheckLinesOptionsMk(c *check.C) { t := s.Init(c) - t.SetupCommandLine("-Wall,no-space") - t.SetupVartypes() - t.SetupOption("mc-charset", "") - t.SetupOption("mysql", "") - t.SetupOption("ncurses", "") - t.SetupOption("negative", "Demonstrates negated .if/.else") - t.SetupOption("slang", "") - t.SetupOption("sqlite", "") - t.SetupOption("x11", "") + t.SetUpCommandLine("-Wall,no-space") + t.SetUpVartypes() + t.SetUpOption("mc-charset", "") + t.SetUpOption("mysql", "") + t.SetUpOption("ncurses", "") + t.SetUpOption("negative", "Demonstrates negated .if/.else") + t.SetUpOption("slang", "") + t.SetUpOption("sqlite", "") + t.SetUpOption("x11", "") t.CreateFileLines("mk/bsd.options.mk", MkRcsID) - mklines := t.SetupFileMkLines("category/package/options.mk", + mklines := t.SetUpFileMkLines("category/package/options.mk", MkRcsID, "", "PKG_OPTIONS_VAR= PKG_OPTIONS.mc", @@ -75,13 +75,13 @@ func (s *Suite) Test_CheckLinesOptionsMk(c *check.C) { func (s *Suite) Test_CheckLinesOptionsMk__unexpected_line(c *check.C) { t := s.Init(c) - t.SetupCommandLine("-Wno-space") - t.SetupVartypes() + t.SetUpCommandLine("-Wno-space") + t.SetUpVartypes() t.CreateFileLines("mk/bsd.options.mk", MkRcsID) - mklines := t.SetupFileMkLines("category/package/options.mk", + mklines := t.SetUpFileMkLines("category/package/options.mk", MkRcsID, "", "PKG_OPTIONS_VAR= PKG_OPTIONS.mc", @@ -98,17 +98,17 @@ func (s *Suite) Test_CheckLinesOptionsMk__unexpected_line(c *check.C) { func (s *Suite) Test_CheckLinesOptionsMk__malformed_condition(c *check.C) { t := s.Init(c) - t.SetupCommandLine("-Wno-space") - t.SetupVartypes() - t.SetupOption("mc-charset", "") - t.SetupOption("ncurses", "") - t.SetupOption("slang", "") - t.SetupOption("x11", "") + t.SetUpCommandLine("-Wno-space") + t.SetUpVartypes() + t.SetUpOption("mc-charset", "") + t.SetUpOption("ncurses", "") + t.SetUpOption("slang", "") + t.SetUpOption("x11", "") t.CreateFileLines("mk/bsd.options.mk", MkRcsID) - mklines := t.SetupFileMkLines("category/package/options.mk", + mklines := t.SetUpFileMkLines("category/package/options.mk", MkRcsID, "", "PKG_OPTIONS_VAR= PKG_OPTIONS.mc", diff --git a/pkgtools/pkglint/files/package.go b/pkgtools/pkglint/files/package.go index a3a5453efe5..032436104cb 100644 --- a/pkgtools/pkglint/files/package.go +++ b/pkgtools/pkglint/files/package.go @@ -317,7 +317,9 @@ func (pkg *Package) readMakefile(filename string, mainLines MkLines, allLines Mk // // Disabled for now since it increases the running time by about 20% // and produces many new warnings, which must be evaluated first. - if false && path.Base(filename) == "buildlink3.mk" { + // + // TODO: Remove the G.Testing below. + if G.Testing && path.Base(filename) == "buildlink3.mk" { builtin := path.Join(path.Dir(filename), "builtin.mk") if fileExists(builtin) { pkg.readMakefile(builtin, mainLines, allLines, "") @@ -553,7 +555,7 @@ func (pkg *Package) checkUpdate() { return } - for _, sugg := range G.Pkgsrc.GetSuggestedPackageUpdates() { + for _, sugg := range G.Pkgsrc.SuggestedUpdates() { if pkg.EffectivePkgbase != sugg.Pkgname { continue } diff --git a/pkgtools/pkglint/files/package_test.go b/pkgtools/pkglint/files/package_test.go index 3f8f43366eb..32eb53e4194 100644 --- a/pkgtools/pkglint/files/package_test.go +++ b/pkgtools/pkglint/files/package_test.go @@ -71,7 +71,7 @@ func (s *Suite) Test_Package_pkgnameFromDistname(c *check.C) { func (s *Suite) Test_Package_CheckVarorder(c *check.C) { t := s.Init(c) - t.SetupCommandLine("-Worder") + t.SetUpCommandLine("-Worder") pkg := NewPackage(t.File("x11/9term")) pkg.CheckVarorder(t.NewMkLines("Makefile", @@ -105,7 +105,7 @@ func (s *Suite) Test_Package_CheckVarorder(c *check.C) { func (s *Suite) Test_Package_CheckVarorder__comments_do_not_crash(c *check.C) { t := s.Init(c) - t.SetupCommandLine("-Worder") + t.SetUpCommandLine("-Worder") pkg := NewPackage(t.File("x11/9term")) pkg.CheckVarorder(t.NewMkLines("Makefile", @@ -128,7 +128,7 @@ func (s *Suite) Test_Package_CheckVarorder__comments_do_not_crash(c *check.C) { func (s *Suite) Test_Package_CheckVarorder__comments_are_ignored(c *check.C) { t := s.Init(c) - t.SetupCommandLine("-Worder") + t.SetUpCommandLine("-Worder") pkg := NewPackage(t.File("x11/9term")) @@ -149,7 +149,7 @@ func (s *Suite) Test_Package_CheckVarorder__comments_are_ignored(c *check.C) { func (s *Suite) Test_Package_CheckVarorder__skip_if_there_are_directives(c *check.C) { t := s.Init(c) - t.SetupCommandLine("-Worder") + t.SetUpCommandLine("-Worder") pkg := NewPackage(t.File("category/package")) @@ -174,7 +174,7 @@ func (s *Suite) Test_Package_CheckVarorder__skip_if_there_are_directives(c *chec func (s *Suite) Test_Package_CheckVarorder__GITHUB_PROJECT_at_the_top(c *check.C) { t := s.Init(c) - t.SetupCommandLine("-Worder") + t.SetUpCommandLine("-Worder") pkg := NewPackage(t.File("x11/9term")) pkg.CheckVarorder(t.NewMkLines("Makefile", @@ -195,7 +195,7 @@ func (s *Suite) Test_Package_CheckVarorder__GITHUB_PROJECT_at_the_top(c *check.C func (s *Suite) Test_Package_CheckVarorder__GITHUB_PROJECT_at_the_bottom(c *check.C) { t := s.Init(c) - t.SetupCommandLine("-Worder") + t.SetUpCommandLine("-Worder") pkg := NewPackage(t.File("x11/9term")) pkg.CheckVarorder(t.NewMkLines("Makefile", @@ -216,7 +216,7 @@ func (s *Suite) Test_Package_CheckVarorder__GITHUB_PROJECT_at_the_bottom(c *chec func (s *Suite) Test_Package_CheckVarorder__license(c *check.C) { t := s.Init(c) - t.SetupCommandLine("-Worder") + t.SetUpCommandLine("-Worder") t.CreateFileLines("mk/bsd.pkg.mk", "# dummy") t.CreateFileLines("x11/Makefile", MkRcsID) @@ -232,7 +232,7 @@ func (s *Suite) Test_Package_CheckVarorder__license(c *check.C) { "", ".include \"../../mk/bsd.pkg.mk\"") - t.SetupVartypes() + t.SetUpVartypes() G.Check(t.File("x11/9term")) @@ -245,7 +245,7 @@ func (s *Suite) Test_Package_CheckVarorder__license(c *check.C) { func (s *Suite) Test_Package_CheckVarorder__MASTER_SITES(c *check.C) { t := s.Init(c) - t.SetupCommandLine("-Worder") + t.SetUpCommandLine("-Worder") pkg := NewPackage(t.File("category/package")) pkg.CheckVarorder(t.NewMkLines("Makefile", @@ -266,8 +266,8 @@ func (s *Suite) Test_Package_CheckVarorder__MASTER_SITES(c *check.C) { func (s *Suite) Test_Package_CheckVarorder__diagnostics(c *check.C) { t := s.Init(c) - t.SetupCommandLine("-Worder") - t.SetupVartypes() + t.SetUpCommandLine("-Worder") + t.SetUpVartypes() pkg := NewPackage(t.File("category/package")) pkg.CheckVarorder(t.NewMkLines("Makefile", @@ -353,8 +353,8 @@ func (s *Suite) Test_Package_determineEffectivePkgVars__precedence(c *check.C) { func (s *Suite) Test_Package_determineEffectivePkgVars__same(c *check.C) { t := s.Init(c) - t.SetupCommandLine("-Wall,no-order") - pkg := t.SetupPackage("category/package", + t.SetUpCommandLine("-Wall,no-order") + pkg := t.SetUpPackage("category/package", "DISTNAME=\tdistname-1.0", "PKGNAME=\tdistname-1.0") @@ -368,8 +368,8 @@ func (s *Suite) Test_Package_determineEffectivePkgVars__same(c *check.C) { func (s *Suite) Test_Package_determineEffectivePkgVars__invalid_DISTNAME(c *check.C) { t := s.Init(c) - t.SetupCommandLine("-Wall,no-order") - pkg := t.SetupPackage("category/package", + t.SetUpCommandLine("-Wall,no-order") + pkg := t.SetUpPackage("category/package", "DISTNAME=\tpkgname-version") G.Check(pkg) @@ -406,9 +406,9 @@ func (s *Suite) Test_Package_checkPossibleDowngrade(c *check.C) { func (s *Suite) Test_Package_loadPackageMakefile__dump(c *check.C) { t := s.Init(c) - t.SetupCommandLine("--dumpmakefile") - t.SetupVartypes() - t.SetupPkgsrc() + t.SetUpCommandLine("--dumpmakefile") + t.SetUpVartypes() + t.SetUpPkgsrc() t.CreateFileLines("category/Makefile") t.CreateFileLines("category/package/PLIST", PlistRcsID, @@ -445,8 +445,8 @@ func (s *Suite) Test_Package_loadPackageMakefile__dump(c *check.C) { func (s *Suite) Test_Package__varuse_at_load_time(c *check.C) { t := s.Init(c) - t.SetupPkgsrc() - t.SetupTool("printf", "", AtRunTime) + t.SetUpPkgsrc() + t.SetUpTool("printf", "", AtRunTime) t.CreateFileLines("licenses/2-clause-bsd", "# dummy") t.CreateFileLines("misc/Makefile") @@ -510,7 +510,7 @@ func (s *Suite) Test_Package__varuse_at_load_time(c *check.C) { "", ".include \"../../mk/bsd.pkg.mk\"") - t.SetupCommandLine("-q", "-Wall,no-space") + t.SetUpCommandLine("-q", "-Wall,no-space") G.Pkgsrc.LoadInfrastructure() G.Check(t.File("category/pkgbase")) @@ -564,7 +564,7 @@ func (s *Suite) Test_Package_loadPackageMakefile__PECL_VERSION(c *check.C) { ".if defined(USE_PHP_EXT_PATCHES)", "PATCHDIR= ${.CURDIR}/${PHPPKGSRCDIR}/patches", ".endif") - pkg := t.SetupPackage("category/package", + pkg := t.SetUpPackage("category/package", "PECL_VERSION=\t1.1.2", ".include \"../../lang/php/ext.mk\"") @@ -574,9 +574,9 @@ func (s *Suite) Test_Package_loadPackageMakefile__PECL_VERSION(c *check.C) { func (s *Suite) Test_Package_checkIncludeConditionally__conditional_and_unconditional_include(c *check.C) { t := s.Init(c) - t.SetupVartypes() - t.SetupOption("zlib", "") - t.SetupPackage("category/package", + t.SetUpVartypes() + t.SetUpOption("zlib", "") + t.SetUpPackage("category/package", ".include \"../../devel/zlib/buildlink3.mk\"", ".if ${OPSYS} == \"Linux\"", ".include \"../../sysutils/coreutils/buildlink3.mk\"", @@ -609,8 +609,8 @@ func (s *Suite) Test_Package_checkIncludeConditionally__conditional_and_uncondit func (s *Suite) Test_Package__include_without_exists(c *check.C) { t := s.Init(c) - t.SetupVartypes() - t.SetupPackage("category/package", + t.SetUpVartypes() + t.SetUpPackage("category/package", ".include \"options.mk\"") G.checkdirPackage(t.File("category/package")) @@ -623,8 +623,8 @@ func (s *Suite) Test_Package__include_without_exists(c *check.C) { func (s *Suite) Test_Package__include_after_exists(c *check.C) { t := s.Init(c) - t.SetupVartypes() - t.SetupPackage("category/package", + t.SetUpVartypes() + t.SetUpPackage("category/package", ".if exists(options.mk)", ". include \"options.mk\"", ".endif") @@ -640,8 +640,8 @@ func (s *Suite) Test_Package__include_after_exists(c *check.C) { func (s *Suite) Test_Package_readMakefile__include_other_after_exists(c *check.C) { t := s.Init(c) - t.SetupVartypes() - t.SetupPackage("category/package", + t.SetUpVartypes() + t.SetUpPackage("category/package", ".if exists(options.mk)", ". include \"another.mk\"", ".endif") @@ -656,8 +656,8 @@ func (s *Suite) Test_Package_readMakefile__include_other_after_exists(c *check.C func (s *Suite) Test_Package__redundant_master_sites(c *check.C) { t := s.Init(c) - t.SetupVartypes() - t.SetupMasterSite("MASTER_SITE_R_CRAN", "http://cran.r-project.org/src/") + t.SetUpVartypes() + t.SetUpMasterSite("MASTER_SITE_R_CRAN", "http://cran.r-project.org/src/") t.CreateFileLines("mk/bsd.pkg.mk") t.CreateFileLines("licenses/gnu-gpl-v2", "The licenses for most software are designed to take away ...") @@ -691,11 +691,11 @@ func (s *Suite) Test_Package__redundant_master_sites(c *check.C) { func (s *Suite) Test_Package_checkUpdate(c *check.C) { t := s.Init(c) - t.SetupPackage("category/pkg1", + t.SetUpPackage("category/pkg1", "PKGNAME= package1-1.0") - t.SetupPackage("category/pkg2", + t.SetUpPackage("category/pkg2", "PKGNAME= package2-1.0") - t.SetupPackage("category/pkg3", + t.SetUpPackage("category/pkg3", "PKGNAME= package3-5.0") t.CreateFileLines("doc/TODO", "Suggested package updates", @@ -724,7 +724,7 @@ func (s *Suite) Test_Package_checkUpdate(c *check.C) { func (s *Suite) Test_NewPackage(c *check.C) { t := s.Init(c) - t.SetupPkgsrc() + t.SetUpPkgsrc() t.CreateFileLines("category/Makefile", MkRcsID) @@ -742,8 +742,8 @@ func (s *Suite) Test_NewPackage(c *check.C) { func (s *Suite) Test__distinfo_from_other_package(c *check.C) { t := s.Init(c) - t.SetupCommandLine("-Wall,no-space") - t.SetupPkgsrc() + t.SetUpCommandLine("-Wall,no-space") + t.SetUpPkgsrc() t.Chdir(".") t.CreateFileLines("x11/gst-x11/Makefile", MkRcsID, @@ -772,8 +772,8 @@ func (s *Suite) Test__distinfo_from_other_package(c *check.C) { func (s *Suite) Test_Package_checkfilePackageMakefile__GNU_CONFIGURE(c *check.C) { t := s.Init(c) - t.SetupCommandLine("-Wall,no-space") - pkg := t.SetupPackage("category/package", + t.SetUpCommandLine("-Wall,no-space") + pkg := t.SetUpPackage("category/package", "GNU_CONFIGURE=\tyes", "USE_LANGUAGES=\t#") @@ -788,8 +788,8 @@ func (s *Suite) Test_Package_checkfilePackageMakefile__GNU_CONFIGURE(c *check.C) func (s *Suite) Test_Package_checkfilePackageMakefile__GNU_CONFIGURE_ok(c *check.C) { t := s.Init(c) - t.SetupCommandLine("-Wall,no-space") - pkg := t.SetupPackage("category/package", + t.SetUpCommandLine("-Wall,no-space") + pkg := t.SetUpPackage("category/package", "GNU_CONFIGURE=\tyes", "USE_LANGUAGES=\t# none, really") @@ -801,8 +801,8 @@ func (s *Suite) Test_Package_checkfilePackageMakefile__GNU_CONFIGURE_ok(c *check func (s *Suite) Test_Package_checkfilePackageMakefile__REPLACE_PERL(c *check.C) { t := s.Init(c) - t.SetupCommandLine("-Wall,no-space") - pkg := t.SetupPackage("category/package", + t.SetUpCommandLine("-Wall,no-space") + pkg := t.SetUpPackage("category/package", "REPLACE_PERL=\t*.pl", "NO_CONFIGURE=\tyes") @@ -815,7 +815,7 @@ func (s *Suite) Test_Package_checkfilePackageMakefile__REPLACE_PERL(c *check.C) func (s *Suite) Test_Package_checkfilePackageMakefile__META_PACKAGE_with_distinfo(c *check.C) { t := s.Init(c) - pkg := t.SetupPackage("category/package", + pkg := t.SetUpPackage("category/package", "META_PACKAGE=\tyes") G.Check(pkg) @@ -828,7 +828,7 @@ func (s *Suite) Test_Package_checkfilePackageMakefile__META_PACKAGE_with_distinf func (s *Suite) Test_Package_checkfilePackageMakefile__USE_IMAKE_and_USE_X11(c *check.C) { t := s.Init(c) - pkg := t.SetupPackage("category/package", + pkg := t.SetUpPackage("category/package", "USE_X11=\tyes", "USE_IMAKE=\tyes") @@ -841,8 +841,8 @@ func (s *Suite) Test_Package_checkfilePackageMakefile__USE_IMAKE_and_USE_X11(c * func (s *Suite) Test_Package_readMakefile__skipping(c *check.C) { t := s.Init(c) - t.SetupCommandLine("-Wall,no-space") - pkg := t.SetupPackage("category/package", + t.SetUpCommandLine("-Wall,no-space") + pkg := t.SetUpPackage("category/package", ".include \"${MYSQL_PKGSRCDIR:S/-client$/-server/}/buildlink3.mk\"") t.EnableTracingToLog() @@ -872,7 +872,7 @@ func (s *Suite) Test_Package_readMakefile__skipping(c *check.C) { func (s *Suite) Test_Package_readMakefile__not_found(c *check.C) { t := s.Init(c) - pkg := t.SetupPackage("category/package", + pkg := t.SetUpPackage("category/package", ".include \"../../devel/zlib/buildlink3.mk\"") t.CreateFileLines("devel/zlib/buildlink3.mk", ".include \"../../enoent/enoent/buildlink3.mk\"") @@ -888,7 +888,7 @@ func (s *Suite) Test_Package_readMakefile__relative(c *check.C) { t.CreateFileLines("category/package/extra.mk", MkRcsID) - pkg := t.SetupPackage("category/package", + pkg := t.SetUpPackage("category/package", ".include \"../package/extra.mk\"") G.Check(pkg) @@ -904,15 +904,15 @@ func (s *Suite) Test_Package_readMakefile__relative(c *check.C) { func (s *Suite) Test_Package_checkLocallyModified(c *check.C) { t := s.Init(c) - // no-order since SetupPackage doesn't place OWNER correctly. - t.SetupCommandLine("-Wall,no-order") + // no-order since SetUpPackage doesn't place OWNER correctly. + t.SetUpCommandLine("-Wall,no-order") G.Username = "example-user" t.CreateFileLines("category/package/CVS/Entries", "/Makefile//modified//") // In packages without specific MAINTAINER, everyone may commit changes. - pkg := t.SetupPackage("category/package", + pkg := t.SetUpPackage("category/package", "MAINTAINER=\tpkgsrc-users@NetBSD.org") G.Check(pkg) @@ -921,7 +921,7 @@ func (s *Suite) Test_Package_checkLocallyModified(c *check.C) { // A package with a MAINTAINER may be edited with care. - t.SetupPackage("category/package", + t.SetUpPackage("category/package", "MAINTAINER=\tmaintainer@example.org") G.Check(pkg) @@ -932,7 +932,7 @@ func (s *Suite) Test_Package_checkLocallyModified(c *check.C) { // A package with an OWNER may NOT be edited by others. - pkg = t.SetupPackage("category/package", + pkg = t.SetUpPackage("category/package", "#MAINTAINER=\t# undefined", "OWNER=\towner@example.org") @@ -944,7 +944,7 @@ func (s *Suite) Test_Package_checkLocallyModified(c *check.C) { // In a package with both OWNER and MAINTAINER, OWNER wins. - pkg = t.SetupPackage("category/package", + pkg = t.SetUpPackage("category/package", "MAINTAINER=\tmaintainer@example.org", "OWNER=\towner@example.org") diff --git a/pkgtools/pkglint/files/parser.go b/pkgtools/pkglint/files/parser.go deleted file mode 100644 index ab9e6f9ea65..00000000000 --- a/pkgtools/pkglint/files/parser.go +++ /dev/null @@ -1,123 +0,0 @@ -package pkglint - -import ( - "netbsd.org/pkglint/textproc" - "strings" -) - -type Parser struct { - Line Line - lexer *textproc.Lexer - EmitWarnings bool -} - -func NewParser(line Line, s string, emitWarnings bool) *Parser { - return &Parser{line, textproc.NewLexer(s), emitWarnings} -} - -func (p *Parser) EOF() bool { - return p.lexer.EOF() -} - -func (p *Parser) Rest() string { - return p.lexer.Rest() -} - -func (p *Parser) PkgbasePattern() (pkgbase string) { - lexer := p.lexer - - for { - mark := lexer.Mark() - - if lexer.SkipRegexp(G.res.Compile(`^\$\{\w+\}`)) || - lexer.SkipRegexp(G.res.Compile(`^[\w.*+,{}]+`)) || - lexer.SkipRegexp(G.res.Compile(`^\[[\d-]+\]`)) { - pkgbase += lexer.Since(mark) - continue - } - - if lexer.SkipByte('-') { - if lexer.SkipRegexp(G.res.Compile(`^\d`)) || - lexer.SkipRegexp(G.res.Compile(`^\$\{\w*VER\w*\}`)) || - lexer.SkipByte('[') { - lexer.Reset(mark) - return - } - pkgbase += "-" - continue - } - - lexer.Reset(mark) - return - } -} - -type DependencyPattern struct { - Pkgbase string // "freeciv-client", "{gcc48,gcc48-libs}", "${EMACS_REQD}" - LowerOp string // ">=", ">" - Lower string // "2.5.0", "${PYVER}" - UpperOp string // "<", "<=" - Upper string // "3.0", "${PYVER}" - Wildcard string // "[0-9]*", "1.5.*", "${PYVER}" -} - -func (p *Parser) Dependency() *DependencyPattern { - lexer := p.lexer - - var dp DependencyPattern - mark := lexer.Mark() - dp.Pkgbase = p.PkgbasePattern() - if dp.Pkgbase == "" { - return nil - } - - mark2 := lexer.Mark() - op := lexer.NextString(">=") - if op == "" { - op = lexer.NextString(">") - } - if op != "" { - if m := lexer.NextRegexp(G.res.Compile(`^(?:(?:\$\{\w+\})+|\d[\w.]*)`)); m != nil { - dp.LowerOp = op - dp.Lower = m[0] - } else { - lexer.Reset(mark2) - } - } - - op = lexer.NextString("<=") - if op == "" { - op = lexer.NextString("<") - } - if op != "" { - if m := lexer.NextRegexp(G.res.Compile(`^(?:(?:\$\{\w+\})+|\d[\w.]*)`)); m != nil { - dp.UpperOp = op - dp.Upper = m[0] - } else { - lexer.Reset(mark2) - } - } - - if dp.LowerOp != "" || dp.UpperOp != "" { - return &dp - } - - if lexer.SkipByte('-') && lexer.Rest() != "" { - dp.Wildcard = lexer.Rest() - lexer.Skip(len(lexer.Rest())) - return &dp - } - - if hasPrefix(dp.Pkgbase, "${") && hasSuffix(dp.Pkgbase, "}") { - return &dp - } - - if hasSuffix(dp.Pkgbase, "-*") { - dp.Pkgbase = strings.TrimSuffix(dp.Pkgbase, "-*") - dp.Wildcard = "*" - return &dp - } - - lexer.Reset(mark) - return nil -} diff --git a/pkgtools/pkglint/files/parser_test.go b/pkgtools/pkglint/files/parser_test.go deleted file mode 100644 index e5754a9bde9..00000000000 --- a/pkgtools/pkglint/files/parser_test.go +++ /dev/null @@ -1,97 +0,0 @@ -package pkglint - -import ( - "gopkg.in/check.v1" -) - -func (s *Suite) Test_Parser_PkgbasePattern(c *check.C) { - - testRest := func(pattern, expected, rest string) { - parser := NewParser(dummyLine, pattern, false) - actual := parser.PkgbasePattern() - c.Check(actual, equals, expected) - c.Check(parser.Rest(), equals, rest) - } - - testRest("fltk", "fltk", "") - testRest("fltk|", "fltk", "|") - testRest("boost-build-1.59.*", "boost-build", "-1.59.*") - testRest("${PHP_PKG_PREFIX}-pdo-5.*", "${PHP_PKG_PREFIX}-pdo", "-5.*") - testRest("${PYPKGPREFIX}-metakit-[0-9]*", "${PYPKGPREFIX}-metakit", "-[0-9]*") -} - -func (s *Suite) Test_Parser_Dependency(c *check.C) { - - testRest := func(pattern string, expected DependencyPattern, rest string) { - parser := NewParser(dummyLine, pattern, false) - dp := parser.Dependency() - if c.Check(dp, check.NotNil) { - c.Check(*dp, equals, expected) - c.Check(parser.Rest(), equals, rest) - } - } - - testNil := func(pattern string) { - parser := NewParser(dummyLine, pattern, false) - dp := parser.Dependency() - if c.Check(dp, check.IsNil) { - c.Check(parser.Rest(), equals, pattern) - } - } - - test := func(pattern string, expected DependencyPattern) { - testRest(pattern, expected, "") - } - - test("fltk>=1.1.5rc1<1.3", - DependencyPattern{"fltk", ">=", "1.1.5rc1", "<", "1.3", ""}) - - test("libwcalc-1.0*", - DependencyPattern{"libwcalc", "", "", "", "", "1.0*"}) - - test("${PHP_PKG_PREFIX}-pdo-5.*", - DependencyPattern{"${PHP_PKG_PREFIX}-pdo", "", "", "", "", "5.*"}) - - test("${PYPKGPREFIX}-metakit-[0-9]*", - DependencyPattern{"${PYPKGPREFIX}-metakit", "", "", "", "", "[0-9]*"}) - - test("boost-build-1.59.*", - DependencyPattern{"boost-build", "", "", "", "", "1.59.*"}) - - test("${_EMACS_REQD}", - DependencyPattern{"${_EMACS_REQD}", "", "", "", "", ""}) - - test("{gcc46,gcc46-libs}>=4.6.0", - DependencyPattern{"{gcc46,gcc46-libs}", ">=", "4.6.0", "", "", ""}) - - test("perl5-*", - DependencyPattern{"perl5", "", "", "", "", "*"}) - - test("verilog{,-current}-[0-9]*", - DependencyPattern{"verilog{,-current}", "", "", "", "", "[0-9]*"}) - - test("mpg123{,-esound,-nas}>=0.59.18", - DependencyPattern{"mpg123{,-esound,-nas}", ">=", "0.59.18", "", "", ""}) - - test("mysql*-{client,server}-[0-9]*", - DependencyPattern{"mysql*-{client,server}", "", "", "", "", "[0-9]*"}) - - test("postgresql8[0-35-9]-${module}-[0-9]*", - DependencyPattern{"postgresql8[0-35-9]-${module}", "", "", "", "", "[0-9]*"}) - - test("ncurses-${NC_VERS}{,nb*}", - DependencyPattern{"ncurses", "", "", "", "", "${NC_VERS}{,nb*}"}) - - test("xulrunner10>=${MOZ_BRANCH}${MOZ_BRANCH_MINOR}", - DependencyPattern{"xulrunner10", ">=", "${MOZ_BRANCH}${MOZ_BRANCH_MINOR}", "", "", ""}) - - testRest("gnome-control-center>=2.20.1{,nb*}", - DependencyPattern{"gnome-control-center", ">=", "2.20.1", "", "", ""}, "{,nb*}") - - testNil(">=2.20.1{,nb*}") - - testNil("pkgbase<=") - - // TODO: support this edge case someday. - // "{ssh{,6}-[0-9]*,openssh-[0-9]*}" is not representable using the current data structure -} diff --git a/pkgtools/pkglint/files/patches_test.go b/pkgtools/pkglint/files/patches_test.go index cfc5db52045..a7d66d7e6fe 100644 --- a/pkgtools/pkglint/files/patches_test.go +++ b/pkgtools/pkglint/files/patches_test.go @@ -34,7 +34,7 @@ func (s *Suite) Test_CheckLinesPatch__without_empty_line__autofix(c *check.C) { t := s.Init(c) t.Chdir("category/package") - patchLines := t.SetupFileLines("patch-WithoutEmptyLines", + patchLines := t.SetUpFileLines("patch-WithoutEmptyLines", RcsID, "Text", "--- file.orig", @@ -50,7 +50,7 @@ func (s *Suite) Test_CheckLinesPatch__without_empty_line__autofix(c *check.C) { // The hash is taken from a breakpoint at the beginning of AutofixDistinfo, oldSha1 "SHA1 (some patch) = 49abd735b7e706ea9ed6671628bb54e91f7f5ffb") - t.SetupCommandLine("-Wall", "--autofix") + t.SetUpCommandLine("-Wall", "--autofix") G.Pkg = NewPackage(".") CheckLinesPatch(patchLines) @@ -82,7 +82,7 @@ func (s *Suite) Test_CheckLinesPatch__without_empty_line__autofix(c *check.C) { func (s *Suite) Test_CheckLinesPatch__no_comment_and_no_empty_lines(c *check.C) { t := s.Init(c) - patchLines := t.SetupFileLines("patch-WithoutEmptyLines", + patchLines := t.SetUpFileLines("patch-WithoutEmptyLines", RcsID, "--- file.orig", "+++ file", @@ -357,7 +357,7 @@ func (s *Suite) Test_CheckLinesPatch__only_context_header_but_no_content(c *chec func (s *Suite) Test_CheckLinesPatch__Makefile_with_absolute_pathnames(c *check.C) { t := s.Init(c) - t.SetupCommandLine("-Wabsname", "-Wno-extra") + t.SetUpCommandLine("-Wabsname", "-Wno-extra") lines := t.NewLines("patch-unified", RcsID, "", @@ -491,7 +491,7 @@ func (s *Suite) Test_CheckLinesPatch__context_lines_with_tab_instead_of_space(c func (s *Suite) Test_CheckLinesPatch__autofix_empty_patch(c *check.C) { t := s.Init(c) - t.SetupCommandLine("-Wall", "--autofix") + t.SetUpCommandLine("-Wall", "--autofix") lines := t.NewLines("patch-aa", RcsID) @@ -505,7 +505,7 @@ func (s *Suite) Test_CheckLinesPatch__autofix_empty_patch(c *check.C) { func (s *Suite) Test_CheckLinesPatch__autofix_long_empty_patch(c *check.C) { t := s.Init(c) - t.SetupCommandLine("-Wall", "--autofix") + t.SetUpCommandLine("-Wall", "--autofix") lines := t.NewLines("patch-aa", RcsID, "") @@ -518,8 +518,8 @@ func (s *Suite) Test_CheckLinesPatch__autofix_long_empty_patch(c *check.C) { func (s *Suite) Test_CheckLinesPatch__crlf_autofix(c *check.C) { t := s.Init(c) - t.SetupCommandLine("-Wall", "--autofix") - lines := t.SetupFileLines("patch-aa", + t.SetUpCommandLine("-Wall", "--autofix") + lines := t.SetUpFileLines("patch-aa", RcsID, "", "Documentation", @@ -542,7 +542,7 @@ func (s *Suite) Test_CheckLinesPatch__crlf_autofix(c *check.C) { func (s *Suite) Test_CheckLinesPatch__autogenerated(c *check.C) { t := s.Init(c) - lines := t.SetupFileLines("patch-aa", + lines := t.SetUpFileLines("patch-aa", RcsID, "", "Documentation", @@ -562,7 +562,7 @@ func (s *Suite) Test_CheckLinesPatch__autogenerated(c *check.C) { func (s *Suite) Test_CheckLinesPatch__empty_context_lines_in_hunk(c *check.C) { t := s.Init(c) - lines := t.SetupFileLines("patch-aa", + lines := t.SetUpFileLines("patch-aa", RcsID, "", "Documentation", @@ -587,7 +587,7 @@ func (s *Suite) Test_CheckLinesPatch__empty_context_lines_in_hunk(c *check.C) { func (s *Suite) Test_CheckLinesPatch__invalid_line_in_hunk(c *check.C) { t := s.Init(c) - lines := t.SetupFileLines("patch-aa", + lines := t.SetUpFileLines("patch-aa", RcsID, "", "Documentation", @@ -609,7 +609,7 @@ func (s *Suite) Test_CheckLinesPatch__invalid_line_in_hunk(c *check.C) { func (s *Suite) Test_PatchChecker_checklineAdded__shell(c *check.C) { t := s.Init(c) - lines := t.SetupFileLines("patch-aa", + lines := t.SetUpFileLines("patch-aa", RcsID, "", "Documentation", @@ -628,7 +628,7 @@ func (s *Suite) Test_PatchChecker_checklineAdded__shell(c *check.C) { func (s *Suite) Test_PatchChecker_checklineAdded__text(c *check.C) { t := s.Init(c) - lines := t.SetupFileLines("patch-aa", + lines := t.SetUpFileLines("patch-aa", RcsID, "", "Documentation", @@ -647,7 +647,7 @@ func (s *Suite) Test_PatchChecker_checklineAdded__text(c *check.C) { func (s *Suite) Test_PatchChecker_checklineAdded__unknown(c *check.C) { t := s.Init(c) - lines := t.SetupFileLines("patch-aa", + lines := t.SetUpFileLines("patch-aa", RcsID, "", "Documentation", @@ -666,7 +666,7 @@ func (s *Suite) Test_PatchChecker_checklineAdded__unknown(c *check.C) { func (s *Suite) Test_PatchChecker_checktextRcsid(c *check.C) { t := s.Init(c) - lines := t.SetupFileLines("patch-aa", + lines := t.SetUpFileLines("patch-aa", RcsID, "", "Documentation", diff --git a/pkgtools/pkglint/files/pkglint.go b/pkgtools/pkglint/files/pkglint.go index c648c45d896..ac0f9287d85 100644 --- a/pkgtools/pkglint/files/pkglint.go +++ b/pkgtools/pkglint/files/pkglint.go @@ -30,8 +30,8 @@ type Pkglint struct { Infrastructure bool // Is the currently checked file from the pkgsrc infrastructure? Testing bool // Is pkglint in self-testing mode (only during development)? Username string // For checking against OWNER and MAINTAINER - CvsEntriesDir string // Cached to avoid I/O - CvsEntriesLines Lines + cvsEntriesDir string // Cached to avoid I/O + cvsEntriesLines Lines Logger @@ -154,7 +154,7 @@ func (pkglint *Pkglint) Main(argv ...string) (exitCode int) { defer f.Close() err = pprof.StartCPUProfile(f) - G.Assertf(err == nil, "Cannot start profiling: %s", err) + G.AssertNil(err, "Cannot start profiling") defer pprof.StopCPUProfile() pkglint.res.Profiling() @@ -309,7 +309,7 @@ func (pkglint *Pkglint) Check(dirent string) { isReg := st.Mode().IsRegular() dir := dirent - if isReg { + if !isDir { dir = path.Dir(dirent) } @@ -324,17 +324,17 @@ func (pkglint *Pkglint) Check(dirent string) { return } - switch { - case isDir && isEmptyDir(dirent): - return - - case isReg: + if isReg { depth := strings.Count(pkgsrcRel, "/") pkglint.checkExecutable(dirent, st.Mode()) pkglint.checkReg(dirent, basename, depth) return } + if isDir && isEmptyDir(dirent) { + return + } + switch pkgsrcdir { case "../..": pkglint.checkdirPackage(dir) @@ -441,21 +441,26 @@ func (pkglint *Pkglint) checkdirPackage(dir string) { continue } - if path.Base(filename) == "Makefile" { - if st, err := os.Lstat(filename); err == nil { - pkglint.checkExecutable(filename, st.Mode()) - } + st, err := os.Lstat(filename) + switch { + case err != nil: + // For missing custom distinfo file, an error message is already generated + // for the line where DISTINFO_FILE is defined. + // + // For all other cases it is next to impossible to reach this branch + // since all those files come from calls to dirglob. + break + + case path.Base(filename) == "Makefile": + pkglint.checkExecutable(filename, st.Mode()) if pkglint.Opts.CheckMakefile { pkg.checkfilePackageMakefile(filename, mklines) } - } else { - st, err := os.Lstat(filename) - if err != nil { - NewLineWhole(filename).Errorf("Cannot determine file type: %s", err) - } else { - pkglint.checkDirent(filename, st.Mode()) - } + + default: + pkglint.checkDirent(filename, st.Mode()) } + if contains(filename, "/patches/patch-") { havePatches = true } else if hasSuffix(filename, "/distinfo") { @@ -483,10 +488,20 @@ func (pkglint *Pkglint) Assertf(cond bool, format string, args ...interface{}) { } } -// Returns the pkgsrc top-level directory, relative to the given file or directory. -func findPkgsrcTopdir(filename string) string { +// AssertNil ensures that the given error is nil. +// +// Other than Assertf, this method does not require any comparison operator in the calling code. +// This makes it possible to get 100% branch coverage for cases that "really can never fail". +func (pkglint *Pkglint) AssertNil(err error, format string, args ...interface{}) { + if err != nil { + panic("Pkglint internal error: " + sprintf(format, args...) + ": " + err.Error()) + } +} + +// Returns the pkgsrc top-level directory, relative to the given directory. +func findPkgsrcTopdir(dirname string) string { for _, dir := range [...]string{".", "..", "../..", "../../.."} { - if fileExists(filename + "/" + dir + "/mk/bsd.pkg.mk") { + if fileExists(dirname + "/" + dir + "/mk/bsd.pkg.mk") { return dir } } @@ -644,6 +659,7 @@ func (pkglint *Pkglint) checkReg(filename, basename string, depth int) { if depth == 2 && !pkglint.Wip { if contains(basename, "README") || contains(basename, "TODO") { NewLineWhole(filename).Errorf("Packages in main pkgsrc must not have a %s file.", basename) + // TODO: Add a convincing explanation. return } } @@ -706,7 +722,7 @@ func (pkglint *Pkglint) checkReg(filename, basename string, depth int) { } } - case matches(basename, `^patch-[-A-Za-z0-9_.~+]*[A-Za-z0-9_]$`): + case matches(basename, `^patch-[-\w.~+]*\w$`): if pkglint.Opts.CheckPatches { if lines := Load(filename, NotEmpty|LogErrors); lines != nil { CheckLinesPatch(lines) @@ -721,7 +737,9 @@ func (pkglint *Pkglint) checkReg(filename, basename string, depth int) { case matches(filename, `(?:^|/)patches/[^/]*$`): NewLineWhole(filename).Warnf("Patch files should be named \"patch-\", followed by letters, '-', '_', '.', and digits only.") - case matches(basename, `^(?:.*\.mk|Makefile.*)$`) && !matches(filename, `files/`) && !matches(filename, `patches/`): + case (hasPrefix(basename, "Makefile") || hasSuffix(basename, ".mk")) && + !contains(filename, "files/") && + !contains(filename, "patches/"): if pkglint.Opts.CheckMk { CheckFileMk(filename) } @@ -754,38 +772,36 @@ func (pkglint *Pkglint) checkReg(filename, basename string, depth int) { } func (pkglint *Pkglint) checkExecutable(filename string, mode os.FileMode) { - switch { - case !mode.IsRegular(): - // Directories and other entries may be executable. - - case mode.Perm()&0111 == 0: - // Good. + if mode.Perm()&0111 == 0 { + // Not executable at all. + return + } - case isCommitted(filename): + if isCommitted(filename) { // Too late to be fixed by the package developer, since // CVS remembers the executable bit in the repo file. // At this point, it can only be reset by the CVS admins. + return + } - default: - line := NewLineWhole(filename) - fix := line.Autofix() - fix.Warnf("Should not be executable.") - fix.Explain( - "No package file should ever be executable.", - "Even the INSTALL and DEINSTALL scripts are usually not usable", - "in the form they have in the package,", - "as the pathnames get adjusted during installation.", - "So there is no need to have any file executable.") - fix.Custom(func(showAutofix, autofix bool) { - fix.Describef(0, "Clearing executable bits") - if autofix { - if err := os.Chmod(filename, mode&^0111); err != nil { - line.Errorf("Cannot clear executable bits: %s", err) - } + line := NewLineWhole(filename) + fix := line.Autofix() + fix.Warnf("Should not be executable.") + fix.Explain( + "No package file should ever be executable.", + "Even the INSTALL and DEINSTALL scripts are usually not usable", + "in the form they have in the package,", + "as the pathnames get adjusted during installation.", + "So there is no need to have any file executable.") + fix.Custom(func(showAutofix, autofix bool) { + fix.Describef(0, "Clearing executable bits") + if autofix { + if err := os.Chmod(filename, mode&^0111); err != nil { + line.Errorf("Cannot clear executable bits: %s", err) } - }) - fix.Apply() - } + } + }) + fix.Apply() } func CheckLinesTrailingEmptyLines(lines Lines) { @@ -804,9 +820,10 @@ func CheckLinesTrailingEmptyLines(lines Lines) { // Tool returns the tool definition from the closest scope (file, global), or nil. // The command can be "sed" or "gsed" or "${SED}". // If a tool is returned, usable tells whether that tool has been added -// to USE_TOOLS in the current scope. +// to USE_TOOLS in the current scope (file or package). func (pkglint *Pkglint) Tool(command string, time ToolTime) (tool *Tool, usable bool) { varname := "" + // TODO: Replace regex with proper VarUse. if m, toolVarname := match1(command, `^\$\{(\w+)\}$`); m { varname = toolVarname } @@ -833,10 +850,10 @@ func (pkglint *Pkglint) Tool(command string, time ToolTime) (tool *Tool, usable // ToolByVarname looks up the tool by its variable name, e.g. "SED". // -// The returned tool may come either from the current Makefile or the -// current package. It is not guaranteed to be usable, only defined; -// that must be checked by the calling code, see Tool.UsableAtLoadTime and -// Tool.UsableAtRunTime. +// The returned tool may come either from the current file or the current package. +// It is not guaranteed to be usable (added to USE_TOOLS), only defined; +// that must be checked by the calling code, +// see Tool.UsableAtLoadTime and Tool.UsableAtRunTime. func (pkglint *Pkglint) ToolByVarname(varname string) *Tool { return pkglint.tools().ByVarname(varname) } @@ -848,3 +865,19 @@ func (pkglint *Pkglint) tools() *Tools { return pkglint.Pkgsrc.Tools } } + +func (pkglint *Pkglint) loadCvsEntries(filename string) Lines { + dir := path.Dir(filename) + if dir == pkglint.cvsEntriesDir { + return pkglint.cvsEntriesLines + } + + lines := Load(dir+"/CVS/Entries", 0) + if lines == nil { + return nil + } + + pkglint.cvsEntriesDir = dir + pkglint.cvsEntriesLines = lines + return lines +} diff --git a/pkgtools/pkglint/files/pkglint_test.go b/pkgtools/pkglint/files/pkglint_test.go index 54a1d34ec04..8c50c9eff83 100644 --- a/pkgtools/pkglint/files/pkglint_test.go +++ b/pkgtools/pkglint/files/pkglint_test.go @@ -4,60 +4,18 @@ import ( "gopkg.in/check.v1" "io/ioutil" "os" + "path" + "path/filepath" "strings" ) func (s *Suite) Test_Pkglint_Main__help(c *check.C) { t := s.Init(c) - exitcode := G.Main("pkglint", "-h") + exitCode := G.Main("pkglint", "-h") - c.Check(exitcode, equals, 0) - c.Check(t.Output(), check.Matches, `^\Qusage: pkglint [options] dir...\E\n(?s).+`) -} - -func (s *Suite) Test_Pkglint_Main__version(c *check.C) { - t := s.Init(c) - - exitcode := G.Main("pkglint", "--version") - - c.Check(exitcode, equals, 0) - t.CheckOutputLines( - confVersion) -} - -func (s *Suite) Test_Pkglint_Main__no_args(c *check.C) { - t := s.Init(c) - - exitcode := G.Main("pkglint") - - c.Check(exitcode, equals, 1) - t.CheckOutputLines( - "FATAL: \".\" must be inside a pkgsrc tree.") -} - -func (s *Suite) Test_Pkglint_Main__only(c *check.C) { - t := s.Init(c) - - exitcode := G.ParseCommandLine([]string{"pkglint", "-Wall", "-o", ":Q", "--version"}) - - if exitcode != -1 { - c.Check(exitcode, equals, 0) - } - c.Check(G.Opts.LogOnly, deepEquals, []string{":Q"}) - t.CheckOutputLines( - confVersion) -} - -func (s *Suite) Test_Pkglint_Main__unknown_option(c *check.C) { - t := s.Init(c) - - exitcode := G.Main("pkglint", "--unknown-option") - - c.Check(exitcode, equals, 1) + c.Check(exitCode, equals, 0) t.CheckOutputLines( - "pkglint: unknown option: --unknown-option", - "", "usage: pkglint [options] dir...", "", " -C, --check=check,... enable or disable specific checks", @@ -113,10 +71,59 @@ func (s *Suite) Test_Pkglint_Main__unknown_option(c *check.C) { " (Prefix a flag with \"no-\" to disable it.)") } +func (s *Suite) Test_Pkglint_Main__version(c *check.C) { + t := s.Init(c) + + exitcode := G.Main("pkglint", "--version") + + c.Check(exitcode, equals, 0) + t.CheckOutputLines( + confVersion) +} + +func (s *Suite) Test_Pkglint_Main__no_args(c *check.C) { + t := s.Init(c) + + exitcode := G.Main("pkglint") + + // The "." from the error message is the implicit argument added in Pkglint.Main. + c.Check(exitcode, equals, 1) + t.CheckOutputLines( + "FATAL: \".\" must be inside a pkgsrc tree.") +} + +func (s *Suite) Test_Pkglint_Main__only(c *check.C) { + t := s.Init(c) + + exitcode := G.ParseCommandLine([]string{"pkglint", "-Wall", "--only", ":Q", "--version"}) + + if exitcode != -1 { + c.Check(exitcode, equals, 0) + } + c.Check(G.Opts.LogOnly, deepEquals, []string{":Q"}) + t.CheckOutputLines( + confVersion) +} + +func (s *Suite) Test_Pkglint_Main__unknown_option(c *check.C) { + t := s.Init(c) + + exitcode := G.Main("pkglint", "--unknown-option") + + c.Check(exitcode, equals, 1) + c.Check(t.Output(), check.Matches, + `\Qpkglint: unknown option: --unknown-option\E\n`+ + `\Q\E\n`+ + `\Qusage: pkglint [options] dir...\E\n`+ + `(?s).+`) + // See Test_Pkglint_Main__help for the complete output. +} + +// This test covers the code path for unexpected panics. func (s *Suite) Test_Pkglint_Main__panic(c *check.C) { t := s.Init(c) - pkg := t.SetupPackage("category/package") + pkg := t.SetUpPackage("category/package") G.out = nil // Force an error that cannot happen in practice. @@ -127,15 +134,18 @@ func (s *Suite) Test_Pkglint_Main__panic(c *check.C) { // Demonstrates which infrastructure files are necessary to actually run // pkglint in a realistic scenario. -// For most tests, this setup is too much work, therefore they -// initialize only those parts of the infrastructure they really -// need. // // Especially covers Pkglint.ShowSummary and Pkglint.checkReg. func (s *Suite) Test_Pkglint_Main__complete_package(c *check.C) { t := s.Init(c) - t.SetupPkgsrc() + // Since the general infrastructure setup is useful for several tests, + // it is available as a separate method. + // + // In this test, several of the infrastructure files are later + // overwritten with more realistic and interesting content. + // This is typical of the pkglint tests. + t.SetUpPkgsrc() // FIXME: pkglint should warn that the latest version in this file // (1.10) doesn't match the current version in the package (1.11). @@ -154,10 +164,6 @@ func (s *Suite) Test_Pkglint_Main__complete_package(c *check.C) { "", "\to checkperms-1.13 [supports more file formats]") - // The LICENSE in the package Makefile is searched here. - t.CreateFileLines("licenses/bsd-2", - "# dummy") - // The MASTER_SITES in the package Makefile are searched here. // See Pkgsrc.loadMasterSites. t.CreateFileLines("mk/fetch/sites.mk", @@ -165,12 +171,20 @@ func (s *Suite) Test_Pkglint_Main__complete_package(c *check.C) { "", "MASTER_SITE_GITHUB+=\thttps://github.com/") - // The existence of this file makes the category "sysutils" valid. + // After setting up the pkgsrc infrastructure, the files for + // a complete pkgsrc package are created individually. + // + // In this test each file is created manually for demonstration purposes. + // Other tests typically call t.SetUpPackage, which does most of the work + // shown here while allowing to adjust the package Makefile a little bit. + + // The existence of this file makes the category "sysutils" valid, + // so that it can be used in CATEGORIES in the package Makefile. // The category "tools" on the other hand is not valid. t.CreateFileLines("sysutils/Makefile", MkRcsID) - // The package Makefile is quite simple, containing just the + // The package Makefile in this test is quite simple, containing just the // standard variable definitions. The data for checking the variable // values is partly defined in the pkgsrc infrastructure files // (as defined in the previous lines), and partly in the pkglint @@ -185,7 +199,7 @@ func (s *Suite) Test_Pkglint_Main__complete_package(c *check.C) { "MAINTAINER=\tpkgsrc-users@NetBSD.org", "HOMEPAGE=\thttps://github.com/rillig/checkperms/", "COMMENT=\tCheck file permissions", - "LICENSE=\tbsd-2", + "LICENSE=\t2-clause-bsd", "", ".include \"../../mk/bsd.pkg.mk\"") @@ -226,15 +240,17 @@ func (s *Suite) Test_Pkglint_Main__complete_package(c *check.C) { "", "SHA1 (checkperms-1.12.tar.gz) = 34c084b4d06bcd7a8bba922ff57677e651eeced5", "RMD160 (checkperms-1.12.tar.gz) = cd95029aa930b6201e9580b3ab7e36dd30b8f925", - "SHA512 (checkperms-1.12.tar.gz) = 43e37b5963c63fdf716acdb470928d7e21a7bdfd"+ - "dd6c85cf626a11acc7f45fa52a53d4bcd83d543150328fe8cec5587987d2d9a7c5f0aaeb02ac1127ab41f8ae", + "SHA512 (checkperms-1.12.tar.gz) = "+ + "43e37b5963c63fdf716acdb470928d7e21a7bdfddd6c85cf626a11acc7f45fa5"+ + "2a53d4bcd83d543150328fe8cec5587987d2d9a7c5f0aaeb02ac1127ab41f8ae", "Size (checkperms-1.12.tar.gz) = 6621 bytes", "SHA1 (patch-checkperms.c) = asdfasdf") // Invalid SHA-1 checksum G.Main("pkglint", "-Wall", "-Call", t.File("sysutils/checkperms")) t.CheckOutputLines( - "WARN: ~/sysutils/checkperms/Makefile:3: This package should be updated to 1.13 ([supports more file formats]).", + "WARN: ~/sysutils/checkperms/Makefile:3: "+ + "This package should be updated to 1.13 ([supports more file formats]).", "ERROR: ~/sysutils/checkperms/Makefile:4: Invalid category \"tools\".", "ERROR: ~/sysutils/checkperms/README: Packages in main pkgsrc must not have a README file.", "ERROR: ~/sysutils/checkperms/TODO: Packages in main pkgsrc must not have a TODO file.", @@ -257,18 +273,19 @@ func (s *Suite) Test_Pkglint_Main__complete_package(c *check.C) { // // go tool cover -html=pkglint.cov -o coverage.html // -// To measure the branch coverage of the pkglint tests: +// To measure the branch coverage of pkglint checking a complete pkgsrc installation, +// install https://github.com/rillig/gobco and adjust the following code: // // env \ // PKGLINT_TESTDIR=C:/Users/rillig/git/pkgsrc \ // PKGLINT_TESTCMDLINE="-r -Wall -Call -e" \ -// gobco -vet=off -test.covermode=count \ +// gobco -test.covermode=count \ // -test.coverprofile=pkglint-pkgsrc.pprof \ -// -timeout=3600s -check.f '^' \ +// -timeout=3600s -check.f '^Test_Pkglint__realistic' \ // > pkglint-pkgsrc.out // // See https://github.com/rillig/gobco for the tool to measure the branch coverage. -func (s *Suite) Test_Pkglint__coverage(c *check.C) { +func (s *Suite) Test_Pkglint__realistic(c *check.C) { if cwd := os.Getenv("PKGLINT_TESTDIR"); cwd != "" { err := os.Chdir(cwd) @@ -291,6 +308,13 @@ func (s *Suite) Test_Pkglint_Check__outside(c *check.C) { G.Check(t.File(".")) + // In a realistic scenario, pkglint will only reach this point + // when the first command line argument is valid but a following + // argument is outside the pkgsrc tree. + // + // If the first argument is already outside of any pkgsrc tree, + // pkglint will exit with a fatal error message since it doesn't + // know where to load the infrastructure files from. t.CheckOutputLines( "ERROR: ~: Cannot determine the pkgsrc root directory for \"~\".") } @@ -298,7 +322,7 @@ func (s *Suite) Test_Pkglint_Check__outside(c *check.C) { func (s *Suite) Test_Pkglint_Check__empty_directory(c *check.C) { t := s.Init(c) - t.SetupPkgsrc() + t.SetUpPkgsrc() t.CreateFileLines("category/package/CVS/Entries") G.Check(t.File("category/package")) @@ -310,7 +334,7 @@ func (s *Suite) Test_Pkglint_Check__empty_directory(c *check.C) { func (s *Suite) Test_Pkglint_Check__files_directory(c *check.C) { t := s.Init(c) - t.SetupPkgsrc() + t.SetUpPkgsrc() t.CreateFileLines("category/package/files/README.md") G.Check(t.File("category/package/files")) @@ -320,40 +344,80 @@ func (s *Suite) Test_Pkglint_Check__files_directory(c *check.C) { "ERROR: ~/category/package/files: Cannot check directories outside a pkgsrc tree.") } +func (s *Suite) Test_Pkglint_Check__patches_directory(c *check.C) { + t := s.Init(c) + + t.SetUpPkgsrc() + t.CreateFileDummyPatch("category/package/patches/patch-README.md") + + G.Check(t.File("category/package/patches")) + + // This diagnostic is not really correct, but it's an edge case anyway. + t.CheckOutputLines( + "ERROR: ~/category/package/patches: Cannot check directories outside a pkgsrc tree.") +} + +// See devel/libtool for an example package that uses manual patches. func (s *Suite) Test_Pkglint_Check__manual_patch(c *check.C) { t := s.Init(c) - t.SetupPkgsrc() + t.SetUpPackage("category/package") + t.CreateFileLines("category/package/patches/unknown-file") t.CreateFileLines("category/package/patches/manual-configure") - t.CreateFileLines("category/package/Makefile", - MkRcsID) G.Check(t.File("category/package")) + // Pkglint doesn't inspect the manual patch files, it also doesn't mark them as unknown files. + t.CheckOutputLines( + "WARN: ~/category/package/patches/unknown-file: Patch files should be named \"patch-\", " + + "followed by letters, '-', '_', '.', and digits only.") +} + +func (s *Suite) Test_Pkglint_Check__doc_TODO(c *check.C) { + t := s.Init(c) + + t.SetUpPkgsrc() + + G.Check(G.Pkgsrc.File("doc/TODO")) + + // The file doc/TODO cannot be checked explicitly and individually. + // It is loaded as part of the pkgsrc infrastructure and is thus + // checked implicitly whenever a package or an individual file is checked. t.CheckOutputLines( - "WARN: ~/category/package/Makefile: Neither PLIST nor PLIST.common exist, and PLIST_SRC is unset.", - "WARN: ~/category/package/distinfo: File not found. Please run \""+confMake+" makesum\" or define NO_CHECKSUM=yes in the package Makefile.", - "ERROR: ~/category/package/Makefile: Each package must define its LICENSE.", - "WARN: ~/category/package/Makefile: Each package should define a COMMENT.") + "WARN: ~/doc/TODO: Unexpected file found.") } +// This test covers the different code paths for deciding whether a directory +// should be checked as the top-level, a category or a package. func (s *Suite) Test_Pkglint_Check(c *check.C) { t := s.Init(c) + t.SetUpVartypes() + t.CreateFileLines("mk/misc/category.mk") t.CreateFileLines("mk/bsd.pkg.mk") t.CreateFileLines("category/package/Makefile") - t.CreateFileLines("category/Makefile") - t.CreateFileLines("Makefile") + t.CreateFileLines("category/Makefile", + MkRcsID, + "", + "COMMENT=\tCategory\u0007", + "", + "SUBDIR+=\tpackage", + "", + ".include \"../mk/misc/category.mk\"") + t.CreateFileLines("Makefile", + MkRcsID, + "COMMENT=\tToplevel\u0005") G.Check(t.File(".")) t.CheckOutputLines( - "ERROR: ~/Makefile: Must not be empty.") + "WARN: ~/Makefile:2: Line contains invalid characters (U+0005).") G.Check(t.File("category")) t.CheckOutputLines( - "ERROR: ~/category/Makefile: Must not be empty.") + "WARN: ~/category/Makefile:3: Line contains invalid characters (U+0007).", + "WARN: ~/category/Makefile:3: COMMENT contains invalid characters (U+0007).") G.Check(t.File("category/package")) @@ -366,6 +430,8 @@ func (s *Suite) Test_Pkglint_Check(c *check.C) { "ERROR: ~/category/package/nonexistent: No such file or directory.") } +// Pkglint must never be trapped in an endless loop, even when +// resolving the value of a variable that refers back to itself. func (s *Suite) Test_resolveVariableRefs__circular_reference(c *check.C) { t := s.Init(c) @@ -373,6 +439,8 @@ func (s *Suite) Test_resolveVariableRefs__circular_reference(c *check.C) { G.Pkg = NewPackage(t.File("category/pkgbase")) G.Pkg.vars.Define("GCC_VERSION", mkline) + // TODO: It may be better to define MkLines.Resolve and Package.Resolve, + // to clearly state the scope of the involved variables. resolved := resolveVariableRefs("gcc-${GCC_VERSION}") c.Check(resolved, equals, "gcc-${GCC_VERSION}") @@ -389,6 +457,9 @@ func (s *Suite) Test_resolveVariableRefs__multilevel(c *check.C) { defineVar(mkline2, "SECOND") defineVar(mkline3, "THIRD") + // TODO: Add a similar test in which some of the variables are defined + // conditionally or with differing values, just to see what pkglint does + // in such a case. resolved := resolveVariableRefs("you ${FIRST}") c.Check(resolved, equals, "you got it") @@ -412,7 +483,7 @@ func (s *Suite) Test_resolveVariableRefs__special_chars(c *check.C) { func (s *Suite) Test_CheckLinesDescr(c *check.C) { t := s.Init(c) - t.SetupVartypes() + t.SetUpVartypes() lines := t.NewLines("DESCR", "word "+strings.Repeat("X", 80), strings.Repeat("X", 90), // No warning since there are no spaces. @@ -469,8 +540,8 @@ func (s *Suite) Test_CheckLinesMessage__malformed(c *check.C) { func (s *Suite) Test_CheckLinesMessage__autofix(c *check.C) { t := s.Init(c) - t.SetupCommandLine("-Wall", "--autofix") - lines := t.SetupFileLines("MESSAGE", + t.SetUpCommandLine("-Wall", "--autofix") + lines := t.SetUpFileLines("MESSAGE", "1", "2", "3", @@ -480,13 +551,11 @@ func (s *Suite) Test_CheckLinesMessage__autofix(c *check.C) { CheckLinesMessage(lines) t.CheckOutputLines( - "AUTOFIX: ~/MESSAGE:1: Inserting a line "+ - "\"===========================================================================\" "+ - "before this line.", + "AUTOFIX: ~/MESSAGE:1: Inserting a line \"=============================="+ + "=============================================\" before this line.", "AUTOFIX: ~/MESSAGE:1: Inserting a line \"$"+"NetBSD$\" before this line.", - "AUTOFIX: ~/MESSAGE:5: Inserting a line "+ - "\"===========================================================================\" "+ - "after this line.") + "AUTOFIX: ~/MESSAGE:5: Inserting a line \"=============================="+ + "=============================================\" after this line.") t.CheckFileLines("MESSAGE", "===========================================================================", RcsID, @@ -503,8 +572,8 @@ func (s *Suite) Test_CheckLinesMessage__autofix(c *check.C) { func (s *Suite) Test_Pkglint_checkReg__alternatives(c *check.C) { t := s.Init(c) - t.SetupPkgsrc() - lines := t.SetupFileLines("category/package/ALTERNATIVES", + t.SetUpPkgsrc() + lines := t.SetUpFileLines("category/package/ALTERNATIVES", "bin/tar bin/gnu-tar") G.Main("pkglint", lines.FileName) @@ -518,12 +587,13 @@ func (s *Suite) Test_Pkglint_checkReg__alternatives(c *check.C) { func (s *Suite) Test_Pkglint__profiling(c *check.C) { t := s.Init(c) - t.SetupPkgsrc() + t.SetUpPkgsrc() t.Chdir(".") G.Main("pkglint", "--profiling") // Pkglint always writes the profiling data into the current directory. + // TODO: Make the location of the profiling log a mandatory parameter. c.Check(fileExists("pkglint.pprof"), equals, true) err := os.Remove("pkglint.pprof") @@ -539,37 +609,23 @@ func (s *Suite) Test_Pkglint__profiling(c *check.C) { func (s *Suite) Test_Pkglint__profiling_error(c *check.C) { t := s.Init(c) - t.SetupPkgsrc() + t.SetUpPkgsrc() t.Chdir(".") t.CreateFileLines("pkglint.pprof/file") exitcode := G.Main("pkglint", "--profiling") c.Check(exitcode, equals, 1) - c.Check(t.Output(), check.Matches, `^FATAL: Cannot create profiling file: open pkglint\.pprof: .*\n$`) + c.Check(t.Output(), check.Matches, + `^FATAL: Cannot create profiling file: open pkglint\.pprof: .*\n$`) } func (s *Suite) Test_Pkglint_checkReg__in_current_working_directory(c *check.C) { t := s.Init(c) - t.SetupPkgsrc() - t.SetupVartypes() - t.CreateFileLines("licenses/mit") + t.SetUpPackage("category/package") t.Chdir("category/package") t.CreateFileLines("log") - t.CreateFileLines("Makefile", - MkRcsID, - "", - "NO_CHECKSUM= yes", - "COMMENT= Useful utilities", - "LICENSE= mit", - "", - ".include \"../../mk/bsd.pkg.mk\"") - t.CreateFileLines("PLIST", - PlistRcsID, - "bin/program") - t.CreateFileLines("DESCR", - "Package description") G.Main("pkglint") @@ -601,21 +657,22 @@ func (s *Suite) Test_Pkglint_Tool__prefer_mk_over_pkgsrc(c *check.C) { func (s *Suite) Test_Pkglint_Tool__lookup_by_name_fallback(c *check.C) { t := s.Init(c) - mkline := t.NewMkLine("dummy.mk", 123, "DUMMY=\tvalue") G.Mk = t.NewMkLines("Makefile", MkRcsID) - global := G.Pkgsrc.Tools.Define("tool", "TOOL", mkline) - - global.Validity = Nowhere + t.SetUpTool("tool", "", Nowhere) loadTimeTool, loadTimeUsable := G.Tool("tool", LoadTime) runTimeTool, runTimeUsable := G.Tool("tool", RunTime) - c.Check(*loadTimeTool, equals, *global) + // The tool is returned even though it may not be used at the moment. + // The calling code must explicitly check for usability. + + c.Check(loadTimeTool.String(), equals, "tool:::Nowhere") c.Check(loadTimeUsable, equals, false) - c.Check(*runTimeTool, equals, *global) + c.Check(runTimeTool.String(), equals, "tool:::Nowhere") c.Check(runTimeUsable, equals, false) } +// TODO: Document the purpose of this test. func (s *Suite) Test_Pkglint_Tool__lookup_by_varname(c *check.C) { t := s.Init(c) @@ -636,6 +693,7 @@ func (s *Suite) Test_Pkglint_Tool__lookup_by_varname(c *check.C) { c.Check(runTimeUsable, equals, true) } +// TODO: Document the purpose of this test. func (s *Suite) Test_Pkglint_Tool__lookup_by_varname_fallback(c *check.C) { t := s.Init(c) @@ -651,6 +709,7 @@ func (s *Suite) Test_Pkglint_Tool__lookup_by_varname_fallback(c *check.C) { c.Check(runTimeUsable, equals, false) } +// TODO: Document the purpose of this test. func (s *Suite) Test_Pkglint_Tool__lookup_by_varname_fallback_runtime(c *check.C) { t := s.Init(c) @@ -689,11 +748,11 @@ func (s *Suite) Test_Pkglint_ToolByVarname(c *check.C) { c.Check(G.ToolByVarname("TOOL").String(), equals, "tool:TOOL::AtRunTime") } -func (s *Suite) Test_CheckFileOther(c *check.C) { +func (s *Suite) Test_Pkglint_checkReg__other(c *check.C) { t := s.Init(c) - t.SetupCommandLine("-Call", "-Wall,no-space") - pkg := t.SetupPackage("category/package") + t.SetUpCommandLine("-Call", "-Wall,no-space") + pkg := t.SetUpPackage("category/package") t.CreateFileLines("category/package/INSTALL", "#! /bin/sh") t.CreateFileLines("category/package/DEINSTALL", @@ -707,8 +766,8 @@ func (s *Suite) Test_CheckFileOther(c *check.C) { func (s *Suite) Test_Pkglint_Check__invalid_files_before_import(c *check.C) { t := s.Init(c) - t.SetupCommandLine("-Call", "-Wall,no-space", "--import") - pkg := t.SetupPackage("category/package") + t.SetUpCommandLine("-Call", "-Wall,no-space", "--import") + pkg := t.SetUpPackage("category/package") t.CreateFileLines("category/package/work/log") t.CreateFileLines("category/package/Makefile~") t.CreateFileLines("category/package/Makefile.orig") @@ -726,8 +785,8 @@ func (s *Suite) Test_Pkglint_Check__invalid_files_before_import(c *check.C) { func (s *Suite) Test_Pkglint_checkDirent__errors(c *check.C) { t := s.Init(c) - t.SetupCommandLine("-Call", "-Wall,no-space") - t.SetupPkgsrc() + t.SetUpCommandLine("-Call", "-Wall,no-space") + t.SetUpPkgsrc() t.CreateFileLines("category/package/files/subdir/file") t.CreateFileLines("category/package/files/subdir/subsub/file") G.Pkgsrc.LoadInfrastructure() @@ -745,8 +804,8 @@ func (s *Suite) Test_Pkglint_checkDirent__errors(c *check.C) { func (s *Suite) Test_Pkglint_checkDirent__file_selection(c *check.C) { t := s.Init(c) - t.SetupCommandLine("-Call", "-Wall,no-space") - t.SetupPkgsrc() + t.SetUpCommandLine("-Call", "-Wall,no-space") + t.SetUpPkgsrc() t.CreateFileLines("doc/CHANGES-2018", RcsID) t.CreateFileLines("category/package/buildlink3.mk", @@ -772,16 +831,7 @@ func (s *Suite) Test_Pkglint_checkReg__readme_and_todo(c *check.C) { t.CreateFileLines("category/package/files/README", "Extra file that is installed later.") - t.CreateFileLines("category/package/patches/patch-README", - RcsID, - "", - "Documentation", - "", - "--- old", - "+++ new", - "@@ -1,1 +1,1 @@", - "-old", - "+new") + t.CreateFileDummyPatch("category/package/patches/patch-README") t.CreateFileLines("category/package/Makefile", MkRcsID, "CATEGORIES=category", @@ -798,18 +848,26 @@ func (s *Suite) Test_Pkglint_checkReg__readme_and_todo(c *check.C) { t.CreateFileLines("category/package/distinfo", RcsID, "", - "SHA1 (patch-README) = b9101ebf0bca8ce243ed6433b65555fa6a5ecd52") + "SHA1 (patch-README) = ebbf34b0641bcb508f17d5a27f2bf2a536d810ac") // Copy category/package/** to wip/package. - for _, basename := range []string{"files/README", "patches/patch-README", "Makefile", "PLIST", "README", "TODO", "distinfo"} { - src := "category/package/" + basename - dst := "wip/package/" + basename - text, err := ioutil.ReadFile(t.File(src)) - c.Check(err, check.IsNil) - t.CreateFileLines(dst, strings.TrimSuffix(string(text), "\n")) - } + err := filepath.Walk( + t.File("category/package"), + func(pathname string, info os.FileInfo, err error) error { + if info.Mode().IsRegular() { + src := filepath.ToSlash(pathname) + dst := strings.Replace(src, "category/package", "wip/package", 1) + text, e := ioutil.ReadFile(src) + c.Check(e, check.IsNil) + _ = os.MkdirAll(path.Dir(dst), 0700) + e = ioutil.WriteFile(dst, []byte(text), 0600) + c.Check(e, check.IsNil) + } + return err + }) + c.Check(err, check.IsNil) - t.SetupPkgsrc() + t.SetUpPkgsrc() G.Pkgsrc.LoadInfrastructure() t.Chdir(".") @@ -870,6 +928,8 @@ func (s *Suite) Test_Pkglint_checkReg__spec(c *check.C) { "WARN: ~/category/package/spec: Only packages in regress/ may have spec files.") } +// Since all required information is passed to G.checkDirent via parameters, +// this test produces the expected results even though none of these files actually exists. func (s *Suite) Test_Pkglint_checkDirent__skipped(c *check.C) { t := s.Init(c) @@ -907,8 +967,8 @@ func (s *Suite) Test_Pkglint_checkdirPackage(c *check.C) { func (s *Suite) Test_Pkglint_checkdirPackage__PKGDIR(c *check.C) { t := s.Init(c) - t.SetupVartypes() - t.SetupPkgsrc() + t.SetUpVartypes() + t.SetUpPkgsrc() t.CreateFileLines("category/Makefile") t.CreateFileLines("other/package/Makefile", MkRcsID) @@ -942,7 +1002,7 @@ func (s *Suite) Test_Pkglint_checkdirPackage__PKGDIR(c *check.C) { func (s *Suite) Test_Pkglint_checkdirPackage__patch_without_distinfo(c *check.C) { t := s.Init(c) - pkg := t.SetupPackage("category/package") + pkg := t.SetUpPackage("category/package") t.CreateFileDummyPatch("category/package/patches/patch-aa") t.Remove("category/package/distinfo") @@ -965,7 +1025,7 @@ func (s *Suite) Test_Pkglint_checkdirPackage__meta_package_without_license(c *ch MkRcsID, "", "META_PACKAGE=\tyes") - t.SetupVartypes() + t.SetUpVartypes() G.checkdirPackage(".") @@ -978,7 +1038,7 @@ func (s *Suite) Test_Pkglint_checkdirPackage__meta_package_without_license(c *ch func (s *Suite) Test_Pkglint_checkdirPackage__filename_with_variable(c *check.C) { t := s.Init(c) - pkg := t.SetupPackage("category/package", + pkg := t.SetUpPackage("category/package", ".include \"../../mk/bsd.prefs.mk\"", "", "RUBY_VERSIONS_ACCEPTED=\t22 23 24 25", // As of 2018. @@ -989,7 +1049,7 @@ func (s *Suite) Test_Pkglint_checkdirPackage__filename_with_variable(c *check.C) "RUBY_PKGDIR=\t../../lang/ruby-${RUBY_VER}-base", "DISTINFO_FILE=\t${RUBY_PKGDIR}/distinfo") - // Pkglint cannot currently resolve the location of DISTINFO_FILE completely + // As of January 2019, pkglint cannot resolve the location of DISTINFO_FILE completely // because the variable \"rv\" comes from a .for loop. // // TODO: iterate over variables in simple .for loops like the above. @@ -1002,8 +1062,8 @@ func (s *Suite) Test_Pkglint_checkdirPackage__filename_with_variable(c *check.C) func (s *Suite) Test_Pkglint_checkdirPackage__ALTERNATIVES(c *check.C) { t := s.Init(c) - t.SetupCommandLine("-Wall,no-space") - pkg := t.SetupPackage("category/package") + t.SetUpCommandLine("-Wall,no-space") + pkg := t.SetUpPackage("category/package") t.CreateFileLines("category/package/ALTERNATIVES", "bin/wrapper bin/wrapper-impl") @@ -1016,6 +1076,22 @@ func (s *Suite) Test_Pkglint_checkdirPackage__ALTERNATIVES(c *check.C) { "Alternative implementation \"bin/wrapper-impl\" must be an absolute path.") } +func (s *Suite) Test_Pkglint_checkdirPackage__nonexistent_DISTINFO_FILE(c *check.C) { + t := s.Init(c) + + t.SetUpPackage("category/package", + "DISTINFO_FILE=\tnonexistent") + G.Pkgsrc.LoadInfrastructure() + + G.Check(t.File("category/package")) + + t.CheckOutputLines( + "WARN: ~/category/package/nonexistent: File not found. "+ + "Please run \""+bmake("makesum")+"\" "+ + "or define NO_CHECKSUM=yes in the package Makefile.", + "ERROR: ~/category/package/Makefile:20: Relative path \"nonexistent\" does not exist.") +} + func (s *Suite) Test_CheckFileMk__enoent(c *check.C) { t := s.Init(c) @@ -1035,7 +1111,7 @@ func (s *Suite) Test_Pkglint_checkExecutable(c *check.C) { t.CheckOutputLines( "WARN: ~/file.mk: Should not be executable.") - t.SetupCommandLine("--autofix") + t.SetUpCommandLine("--autofix") G.checkExecutable(filename, 0555) @@ -1064,7 +1140,7 @@ func (s *Suite) Test_Main(c *check.C) { outProfiling, err := os.Create(t.CreateFileLines("out.profiling")) c.Check(err, check.IsNil) - t.SetupPackage("category/package") + t.SetUpPackage("category/package") t.Chdir("category/package") runMain := func(out *os.File, commandLine ...string) { diff --git a/pkgtools/pkglint/files/pkgsrc.go b/pkgtools/pkglint/files/pkgsrc.go index fba2faba51f..4ad19d482a6 100644 --- a/pkgtools/pkglint/files/pkgsrc.go +++ b/pkgtools/pkglint/files/pkgsrc.go @@ -7,7 +7,6 @@ import ( "os" "path/filepath" "sort" - "strconv" "strings" ) @@ -49,7 +48,7 @@ func NewPkgsrc(dir string) *Pkgsrc { src := Pkgsrc{ dir, make(map[string]bool), - NewTools("Pkgsrc"), + NewTools(), make(map[string]string), make(map[string]string), make(map[string]string), @@ -63,17 +62,15 @@ func NewPkgsrc(dir string) *Pkgsrc { nil, // Only initialized when pkglint is run for a whole pkgsrc installation nil} - addDefaultBuildDefs(&src) - return &src } -func addDefaultBuildDefs(src *Pkgsrc) { +func (src *Pkgsrc) loadDefaultBuildDefs() { // Some user-defined variables do not influence the binary // package at all and therefore do not have to be added to // BUILD_DEFS; therefore they are marked as "already added". - src.AddBuildDefs( + src.addBuildDefs( "DISTDIR", "FETCH_CMD", "FETCH_OUTPUT_ARGS", @@ -82,7 +79,7 @@ func addDefaultBuildDefs(src *Pkgsrc) { // The following variables are used so often that not every // package should need to add it to BUILD_DEFS manually. - src.AddBuildDefs( + src.addBuildDefs( "PKGSRC_COMPILER", "PKGSRC_USE_SSP", "UNPRIVILEGED", @@ -90,14 +87,15 @@ func addDefaultBuildDefs(src *Pkgsrc) { // The following variables are so obscure that they are // probably not used in practice. - src.AddBuildDefs( + src.addBuildDefs( "MANINSTALL") // The following variables are added to _BUILD_DEFS by the pkgsrc // infrastructure and thus don't need to be added by the package again. // To regenerate the below list: // grep -hr '^_BUILD_DEFS+=' mk/ | tr ' \t' '\n\n' | sed -e 's,.*=,,' -e '/^_/d' -e '/^$/d' -e 's,.*,"&"\,,' | sort -u - src.AddBuildDefs( + // TODO: Run the equivalent of the above command at startup. + src.addBuildDefs( "ABI", "BUILTIN_PKGS", "CFLAGS", @@ -156,6 +154,7 @@ func (src *Pkgsrc) LoadInfrastructure() { src.loadTools() src.initDeprecatedVars() src.loadUntypedVars() + src.loadDefaultBuildDefs() } // Latest returns the latest package matching the given pattern. @@ -189,6 +188,7 @@ func (src *Pkgsrc) ListVersions(category string, re regex.Pattern, repl string, "Regular expression %q must be anchored at both ends.", re) } + // TODO: Maybe convert cache key to a struct, to save allocations. cacheKey := category + "/" + string(re) + " => " + repl if latest, found := src.listVersions[cacheKey]; found { return latest @@ -222,13 +222,13 @@ func (src *Pkgsrc) ListVersions(category string, re regex.Pattern, repl string, keys := make(map[string]int) for _, name := range names { if m, pkgbase, versionStr := match2(name, `^(\D+)(\d+)$`); m { - version, _ := strconv.Atoi(versionStr) + version := toInt(versionStr, 0) if pkgbase == "postgresql" && version < 60 { version = 10 * version } if pkgbase == "go" { - major, _ := strconv.Atoi(versionStr[:1]) - minor, _ := strconv.Atoi(versionStr[1:]) + major := toInt(versionStr[:1], 0) + minor := toInt(versionStr[1:], 0) version = 100*major + minor } keys[name] = version @@ -236,6 +236,8 @@ func (src *Pkgsrc) ListVersions(category string, re regex.Pattern, repl string, } sort.SliceStable(names, func(i, j int) bool { + // TODO: Check if this Less implementation is really transitive. + // examples: ps ps5 ps10 ps96 pq px if keyI, keyJ := keys[names[i]], keys[names[j]]; keyI != 0 && keyJ != 0 { return keyI < keyJ } @@ -275,7 +277,7 @@ func (src *Pkgsrc) loadTools() { toolFiles := []string{"defaults.mk"} { - toc := G.Pkgsrc.File("mk/tools/bsd.tools.mk") + toc := src.File("mk/tools/bsd.tools.mk") mklines := LoadMk(toc, MustSucceed|NotEmpty) for _, mkline := range mklines.mklines { if mkline.IsInclude() { @@ -307,7 +309,7 @@ func (src *Pkgsrc) loadTools() { } for _, basename := range toolFiles { - mklines := G.Pkgsrc.LoadMk("mk/tools/"+basename, MustSucceed|NotEmpty) + mklines := src.LoadMk("mk/tools/"+basename, MustSucceed|NotEmpty) mklines.ForEach(func(mkline MkLine) { tools.ParseToolLine(mkline, true, !mklines.indentation.IsConditional()) }) @@ -315,7 +317,7 @@ func (src *Pkgsrc) loadTools() { for _, relativeName := range [...]string{"mk/bsd.prefs.mk", "mk/bsd.pkg.mk"} { - mklines := G.Pkgsrc.LoadMk(relativeName, MustSucceed|NotEmpty) + mklines := src.LoadMk(relativeName, MustSucceed|NotEmpty) mklines.ForEach(func(mkline MkLine) { if mkline.IsVarassign() { varname := mkline.Varname() @@ -324,8 +326,9 @@ func (src *Pkgsrc) loadTools() { tools.ParseToolLine(mkline, true, !mklines.indentation.IsConditional()) case "_BUILD_DEFS": + // TODO: Compare with src.loadDefaultBuildDefs; is it redundant? for _, buildDefsVar := range mkline.Fields() { - src.AddBuildDefs(buildDefsVar) + src.addBuildDefs(buildDefsVar) } } } @@ -384,7 +387,8 @@ func (src *Pkgsrc) loadUntypedVars() { return nil } - _ = filepath.Walk(src.File("mk"), handleFile) + err := filepath.Walk(src.File("mk"), handleFile) + G.AssertNil(err, "Walk error in pkgsrc infrastructure") } func (src *Pkgsrc) parseSuggestedUpdates(lines Lines) []SuggestedUpdate { @@ -397,6 +401,8 @@ func (src *Pkgsrc) parseSuggestedUpdates(lines Lines) []SuggestedUpdate { for _, line := range lines.Lines { text := line.Text + // TODO: Replace this state transition scheme with explicit code, + // hoping that the code will be easier to understand. if state == 0 && text == "Suggested package updates" { state = 1 } else if state == 1 && text == "" { @@ -423,8 +429,8 @@ func (src *Pkgsrc) parseSuggestedUpdates(lines Lines) []SuggestedUpdate { } func (src *Pkgsrc) loadSuggestedUpdates() { - src.suggestedUpdates = src.parseSuggestedUpdates(Load(G.Pkgsrc.File("doc/TODO"), MustSucceed)) - src.suggestedWipUpdates = src.parseSuggestedUpdates(Load(G.Pkgsrc.File("wip/TODO"), NotEmpty)) + src.suggestedUpdates = src.parseSuggestedUpdates(Load(src.File("doc/TODO"), MustSucceed)) + src.suggestedWipUpdates = src.parseSuggestedUpdates(Load(src.File("wip/TODO"), NotEmpty)) } func (src *Pkgsrc) loadDocChangesFromFile(filename string) []*Change { @@ -461,7 +467,7 @@ func (src *Pkgsrc) loadDocChangesFromFile(filename string) []*Change { } year := "" - if m, yyyy := match1(filename, `-(\d+)$`); m && yyyy >= "2018" { + if m, yyyy := match1(filename, `-(\d+)$`); m && yyyy >= "2018" { // TODO: Why 2018? year = yyyy } @@ -497,7 +503,7 @@ func (src *Pkgsrc) loadDocChangesFromFile(filename string) []*Change { return changes } -func (src *Pkgsrc) GetSuggestedPackageUpdates() []SuggestedUpdate { +func (src *Pkgsrc) SuggestedUpdates() []SuggestedUpdate { if G.Wip { return src.suggestedWipUpdates } else { @@ -515,7 +521,7 @@ func (src *Pkgsrc) loadDocChanges() { var filenames []string for _, file := range files { filename := file.Name() - if matches(filename, `^CHANGES-20\d\d$`) && filename >= "CHANGES-2011" { + if matches(filename, `^CHANGES-20\d\d$`) && filename >= "CHANGES-2011" { // TODO: Why 2011? filenames = append(filenames, filename) } } @@ -531,10 +537,10 @@ func (src *Pkgsrc) loadDocChanges() { } func (src *Pkgsrc) loadUserDefinedVars() { - mklines := G.Pkgsrc.LoadMk("mk/defaults/mk.conf", MustSucceed|NotEmpty) + mklines := src.LoadMk("mk/defaults/mk.conf", MustSucceed|NotEmpty) for _, mkline := range mklines.mklines { - if mkline.IsVarassign() { + if mkline.IsVarassign() { // TODO: What about mkline.IsCommentedVarassign? src.UserDefinedVars.Define(mkline.Varname(), mkline) } } @@ -739,6 +745,7 @@ func (src *Pkgsrc) ReadDir(dirName string) []os.FileInfo { // Example: // NewPkgsrc("/usr/pkgsrc").File("distfiles") => "/usr/pkgsrc/distfiles" func (src *Pkgsrc) File(relativeName string) string { + // TODO: Package.File resolves variables, Pkgsrc.File doesn't. They should behave the same. return cleanpath(src.topdir + "/" + relativeName) } @@ -750,7 +757,7 @@ func (src *Pkgsrc) ToRel(filename string) string { return relpath(src.topdir, filename) } -func (src *Pkgsrc) AddBuildDefs(varnames ...string) { +func (src *Pkgsrc) addBuildDefs(varnames ...string) { for _, varname := range varnames { src.buildDefs[varname] = true } @@ -766,6 +773,7 @@ func (src *Pkgsrc) loadMasterSites() { for _, mkline := range mklines.mklines { if mkline.IsVarassign() { varname := mkline.Varname() + // TODO: Give a plausible reason for the MASTER_SITE_BACKUP exception. if hasPrefix(varname, "MASTER_SITE_") && varname != "MASTER_SITE_BACKUP" { for _, url := range mkline.ValueFields(mkline.Value()) { if matches(url, `^(?:http://|https://|ftp://)`) { @@ -773,17 +781,17 @@ func (src *Pkgsrc) loadMasterSites() { } } - // TODO: register variable type, to avoid redundant - // definitions in vardefs.go. + // TODO: register variable type, to avoid redundant definitions in vardefs.go. } } } // Explicitly allowed, although not defined in mk/fetch/sites.mk. + // TODO: Document where this definition comes from and why it is good. src.registerMasterSite("MASTER_SITE_LOCAL", "ftp://ftp.NetBSD.org/pub/pkgsrc/distfiles/LOCAL_PORTS/") if trace.Tracing { - trace.Stepf("Loaded %d MASTER_SITE_* URLs.", len(G.Pkgsrc.MasterSiteURLToVar)) + trace.Stepf("Loaded %d MASTER_SITE_* URLs.", len(src.MasterSiteURLToVar)) } } @@ -804,7 +812,7 @@ func (src *Pkgsrc) loadPkgOptions() { if m, name, description := match2(line.Text, `^([-0-9a-z_+]+)(?:[\t ]+(.*))?$`); m { src.PkgOptions[name] = description } else { - line.Fatalf("Unknown line format: %s", line.Text) + line.Errorf("Invalid line format: %s", line.Text) } } } @@ -844,6 +852,10 @@ func (src *Pkgsrc) VariableType(varname string) (vartype *Vartype) { } } + return src.guessVariableType(varname) +} + +func (src *Pkgsrc) guessVariableType(varname string) (vartype *Vartype) { allowAll := []ACLEntry{{"*", aclpAll}} allowRuntime := []ACLEntry{{"*", aclpAllRuntime}} @@ -854,6 +866,7 @@ func (src *Pkgsrc) VariableType(varname string) (vartype *Vartype) { case hasSuffix(varbase, "DIRS"): gtype = &Vartype{lkShell, BtPathmask, allowRuntime, true} case hasSuffix(varbase, "DIR") && !hasSuffix(varbase, "DESTDIR"), hasSuffix(varname, "_HOME"): + // TODO: hasSuffix(varbase, "BASE") gtype = &Vartype{lkNone, BtPathname, allowRuntime, true} case hasSuffix(varbase, "FILES"): gtype = &Vartype{lkShell, BtPathmask, allowRuntime, true} @@ -878,6 +891,7 @@ func (src *Pkgsrc) VariableType(varname string) (vartype *Vartype) { case hasSuffix(varname, "_LDFLAGS"): gtype = &Vartype{lkShell, BtLdFlag, allowRuntime, true} case hasSuffix(varbase, "_MK"): + // TODO: Add BtGuard for inclusion guards, since these variables may only be checked using defined(). gtype = &Vartype{lkNone, BtUnknown, allowAll, true} } @@ -900,7 +914,7 @@ func (src *Pkgsrc) VariableType(varname string) (vartype *Vartype) { return gtype } -// Change is a change entry from the `doc/CHANGES-*` files. +// Change describes a modification to a single package, from the doc/CHANGES-* files. type Change struct { Line Line Action string @@ -910,7 +924,7 @@ type Change struct { Date string } -// SuggestedUpdate is from the `doc/TODO` file. +// SuggestedUpdate describes a desired package update, from the doc/TODO file. type SuggestedUpdate struct { Line Line Pkgname string diff --git a/pkgtools/pkglint/files/pkgsrc_test.go b/pkgtools/pkglint/files/pkgsrc_test.go index 6a45ab064d4..5baa0071db5 100644 --- a/pkgtools/pkglint/files/pkgsrc_test.go +++ b/pkgtools/pkglint/files/pkgsrc_test.go @@ -51,7 +51,7 @@ func (s *Suite) Test_Pkgsrc_parseSuggestedUpdates(c *check.C) { func (s *Suite) Test_Pkgsrc_checkToplevelUnusedLicenses(c *check.C) { t := s.Init(c) - t.SetupPkgsrc() + t.SetUpPkgsrc() t.CreateFileLines("mk/misc/category.mk") t.CreateFileLines("licenses/2-clause-bsd") t.CreateFileLines("licenses/gnu-gpl-v3") @@ -68,13 +68,13 @@ func (s *Suite) Test_Pkgsrc_checkToplevelUnusedLicenses(c *check.C) { "", ".include \"../mk/misc/category.mk\"") - t.SetupPackage("category/package", + t.SetUpPackage("category/package", "LICENSE=\t2-clause-bsd") G.Main("pkglint", "-r", "-Cglobal", t.File(".")) t.CheckOutputLines( - "WARN: ~/licenses/gnu-gpl-v2: This license seems to be unused.", // Added by Tester.SetupPkgsrc + "WARN: ~/licenses/gnu-gpl-v2: This license seems to be unused.", // Added by Tester.SetUpPkgsrc "WARN: ~/licenses/gnu-gpl-v3: This license seems to be unused.", "0 errors and 2 warnings found.") } @@ -124,7 +124,7 @@ func (s *Suite) Test_Pkgsrc_loadTools(c *check.C) { t.DisableTracing() t.CheckOutputLines( - "TRACE: + (*Tools).Trace(\"Pkgsrc\")", + "TRACE: + (*Tools).Trace()", "TRACE: 1 tool bzcat:::Nowhere", "TRACE: 1 tool bzip2:::Nowhere", "TRACE: 1 tool chown:CHOWN::Nowhere", @@ -139,15 +139,15 @@ func (s *Suite) Test_Pkgsrc_loadTools(c *check.C) { "TRACE: 1 tool strip:::Nowhere", "TRACE: 1 tool test:TEST:var:AfterPrefsMk", "TRACE: 1 tool true:TRUE:var:AfterPrefsMk", - "TRACE: - (*Tools).Trace(\"Pkgsrc\")") + "TRACE: - (*Tools).Trace()") } // As a side-benefit, loadTools also loads the _BUILD_DEFS. func (s *Suite) Test_Pkgsrc_loadTools__BUILD_DEFS(c *check.C) { t := s.Init(c) - t.SetupTool("echo", "ECHO", AtRunTime) - pkg := t.SetupPackage("category/package", + t.SetUpTool("echo", "ECHO", AtRunTime) + pkg := t.SetUpPackage("category/package", "pre-configure:", "\t@${ECHO} ${PKG_SYSCONFDIR} ${VARBASE}") t.CreateFileLines("mk/bsd.pkg.mk", @@ -184,13 +184,20 @@ func (s *Suite) Test_Pkgsrc_loadDocChangesFromFile(c *check.C) { changes := G.Pkgsrc.loadDocChangesFromFile(t.File("doc/CHANGES-2018")) c.Assert(len(changes), equals, 7) - c.Check(*changes[0], equals, Change{changes[0].Line, "Added", "category/package", "1.0", "author1", "2015-01-01"}) - c.Check(*changes[1], equals, Change{changes[1].Line, "Updated", "category/package", "1.5", "author2", "2018-01-02"}) - c.Check(*changes[2], equals, Change{changes[2].Line, "Renamed", "category/package", "", "author3", "2018-01-03"}) - c.Check(*changes[3], equals, Change{changes[3].Line, "Moved", "category/package", "", "author4", "2018-01-04"}) - c.Check(*changes[4], equals, Change{changes[4].Line, "Removed", "category/package", "", "author5", "2018-01-09"}) - c.Check(*changes[5], equals, Change{changes[5].Line, "Removed", "category/package", "", "author6", "2018-01-06"}) - c.Check(*changes[6], equals, Change{changes[6].Line, "Downgraded", "category/package", "1.2", "author7", "2018-01-07"}) + c.Check(*changes[0], equals, Change{changes[0].Line, + "Added", "category/package", "1.0", "author1", "2015-01-01"}) + c.Check(*changes[1], equals, Change{changes[1].Line, + "Updated", "category/package", "1.5", "author2", "2018-01-02"}) + c.Check(*changes[2], equals, Change{changes[2].Line, + "Renamed", "category/package", "", "author3", "2018-01-03"}) + c.Check(*changes[3], equals, Change{changes[3].Line, + "Moved", "category/package", "", "author4", "2018-01-04"}) + c.Check(*changes[4], equals, Change{changes[4].Line, + "Removed", "category/package", "", "author5", "2018-01-09"}) + c.Check(*changes[5], equals, Change{changes[5].Line, + "Removed", "category/package", "", "author6", "2018-01-06"}) + c.Check(*changes[6], equals, Change{changes[6].Line, + "Downgraded", "category/package", "1.2", "author7", "2018-01-07"}) t.CheckOutputLines( "WARN: ~/doc/CHANGES-2018:1: Year 2015 for category/package does not match the filename ~/doc/CHANGES-2018.", @@ -201,7 +208,7 @@ func (s *Suite) Test_Pkgsrc_loadDocChangesFromFile(c *check.C) { func (s *Suite) Test_Pkgsrc_loadDocChanges__not_found(c *check.C) { t := s.Init(c) - t.SetupPkgsrc() + t.SetUpPkgsrc() t.Remove("doc/CHANGES-2018") t.Remove("doc/TODO") t.Remove("doc") @@ -222,9 +229,9 @@ func (s *Suite) Test_Pkgsrc_loadDocChangesFromFile__not_found(c *check.C) { func (s *Suite) Test_Pkgsrc_loadDocChangesFromFile__wip(c *check.C) { t := s.Init(c) - pkg := t.SetupPackage("wip/package", + pkg := t.SetUpPackage("wip/package", "DISTNAME=\tpackage-1.11", - "CATEGORIES=\tlocal") + "CATEGORIES=\tlocal") // To avoid "Invalid category wip". t.CreateFileLines("wip/TODO", RcsID, "", @@ -236,14 +243,15 @@ func (s *Suite) Test_Pkgsrc_loadDocChangesFromFile__wip(c *check.C) { G.Check(pkg) t.CheckOutputLines( - "WARN: ~/wip/package/Makefile:3: This package should be updated to 1.13 ([cool new features]).") + "WARN: ~/wip/package/Makefile:3: " + + "This package should be updated to 1.13 ([cool new features]).") } func (s *Suite) Test_Pkgsrc__deprecated(c *check.C) { t := s.Init(c) - t.SetupTool("echo", "ECHO", AtRunTime) - t.SetupVartypes() + t.SetUpTool("echo", "ECHO", AtRunTime) + t.SetUpVartypes() G.Pkgsrc.initDeprecatedVars() mklines := t.NewMkLines("Makefile", MkRcsID, @@ -254,9 +262,12 @@ func (s *Suite) Test_Pkgsrc__deprecated(c *check.C) { mklines.Check() t.CheckOutputLines( - "WARN: Makefile:2: Definition of USE_PERL5 is deprecated. Use USE_TOOLS+=perl or USE_TOOLS+=perl:run instead.", - "WARN: Makefile:3: Definition of SUBST_POSTCMD.class is deprecated. Has been removed, as it seemed unused.", - "WARN: Makefile:4: Use of \"PKG_JVM\" is deprecated. Use PKG_DEFAULT_JVM instead.") + "WARN: Makefile:2: Definition of USE_PERL5 is deprecated. "+ + "Use USE_TOOLS+=perl or USE_TOOLS+=perl:run instead.", + "WARN: Makefile:3: Definition of SUBST_POSTCMD.class is deprecated. "+ + "Has been removed, as it seemed unused.", + "WARN: Makefile:4: Use of \"PKG_JVM\" is deprecated. "+ + "Use PKG_DEFAULT_JVM instead.") } func (s *Suite) Test_Pkgsrc_ListVersions__no_basedir(c *check.C) { @@ -313,7 +324,7 @@ func (s *Suite) Test_Pkgsrc__caching(c *check.C) { c.Check(cached, equals, "../../lang/python27") } -func (s *Suite) Test_Pkgsrc_Latest__multi(c *check.C) { +func (s *Suite) Test_Pkgsrc_Latest__multiple_candidates(c *check.C) { t := s.Init(c) t.CreateFileLines("lang/Makefile") @@ -417,9 +428,12 @@ func (s *Suite) Test_Pkgsrc_loadPkgOptions(c *check.C) { "===== Merge conflict", ">>>>> Merge conflict") - t.ExpectFatal( - G.Pkgsrc.loadPkgOptions, - "FATAL: ~/mk/defaults/options.description:2: Unknown line format: <<<<< Merge conflict") + G.Pkgsrc.loadPkgOptions() + + t.CheckOutputLines( + "ERROR: ~/mk/defaults/options.description:2: Invalid line format: <<<<< Merge conflict", + "ERROR: ~/mk/defaults/options.description:3: Invalid line format: ===== Merge conflict", + "ERROR: ~/mk/defaults/options.description:4: Invalid line format: >>>>> Merge conflict") } func (s *Suite) Test_Pkgsrc_loadTools__no_tools_found(c *check.C) { @@ -447,9 +461,9 @@ func (s *Suite) Test_Pkgsrc_loadTools__no_tools_found(c *check.C) { func (s *Suite) Test_Pkgsrc_VariableType(c *check.C) { t := s.Init(c) - t.SetupVartypes() + t.SetUpVartypes() - checkType := func(varname string, vartype string) { + test := func(varname string, vartype string) { actualType := G.Pkgsrc.VariableType(varname) if vartype == "" { c.Check(actualType, check.IsNil) @@ -460,24 +474,24 @@ func (s *Suite) Test_Pkgsrc_VariableType(c *check.C) { } } - checkType("_PERL5_PACKLIST_AWK_STRIP_DESTDIR", "") - checkType("SOME_DIR", "Pathname (guessed)") - checkType("SOMEDIR", "Pathname (guessed)") - checkType("SEARCHPATHS", "List of Pathname (guessed)") - checkType("MYPACKAGE_USER", "UserGroupName (guessed)") - checkType("MYPACKAGE_GROUP", "UserGroupName (guessed)") - checkType("MY_CMD_ENV", "List of ShellWord (guessed)") - checkType("MY_CMD_ARGS", "List of ShellWord (guessed)") - checkType("MY_CMD_CFLAGS", "List of CFlag (guessed)") - checkType("MY_CMD_LDFLAGS", "List of LdFlag (guessed)") - checkType("PLIST.abcde", "Yes") + test("_PERL5_PACKLIST_AWK_STRIP_DESTDIR", "") + test("SOME_DIR", "Pathname (guessed)") + test("SOMEDIR", "Pathname (guessed)") + test("SEARCHPATHS", "List of Pathname (guessed)") + test("MYPACKAGE_USER", "UserGroupName (guessed)") + test("MYPACKAGE_GROUP", "UserGroupName (guessed)") + test("MY_CMD_ENV", "List of ShellWord (guessed)") + test("MY_CMD_ARGS", "List of ShellWord (guessed)") + test("MY_CMD_CFLAGS", "List of CFlag (guessed)") + test("MY_CMD_LDFLAGS", "List of LdFlag (guessed)") + test("PLIST.abcde", "Yes") } // Guessing the variable type works for both plain and parameterized variable names. func (s *Suite) Test_Pkgsrc_VariableType__varparam(c *check.C) { t := s.Init(c) - t.SetupVartypes() + t.SetUpVartypes() t1 := G.Pkgsrc.VariableType("FONT_DIRS") @@ -500,7 +514,7 @@ func (s *Suite) Test_Pkgsrc_VariableType__from_mk(c *check.C) { // but it is a known variable since the pkgsrc infrastructure uses // it. But still, its type is unknown. - t.SetupPkgsrc() + t.SetUpPkgsrc() t.CreateFileLines("mk/sys-vars.mk", MkRcsID, "", @@ -508,7 +522,7 @@ func (s *Suite) Test_Pkgsrc_VariableType__from_mk(c *check.C) { "CPPPATH?=\tcpp", "OSNAME.Linux?=\tLinux") - pkg := t.SetupPackage("category/package", + pkg := t.SetUpPackage("category/package", "PKGSRC_MAKE_ENV+=\tCPP=${CPPPATH:Q}", "PKGSRC_UNKNOWN_ENV+=\tCPP=${ABCPATH:Q}", "OSNAME.SunOS=\t\t${OSNAME.Other}") diff --git a/pkgtools/pkglint/files/pkgver/vercmp.go b/pkgtools/pkglint/files/pkgver/vercmp.go index e48a2df2e36..ae5843b5309 100644 --- a/pkgtools/pkglint/files/pkgver/vercmp.go +++ b/pkgtools/pkglint/files/pkgver/vercmp.go @@ -31,7 +31,7 @@ func Compare(left, right string) int { m := imax(len(lv.v), len(rv.v)) for i := 0; i < m; i++ { - if c := icmp(lv.Place(i), rv.Place(i)); c != 0 { + if c := icmp(lv.Field(i), rv.Field(i)); c != 0 { return c } } @@ -82,7 +82,7 @@ func (v *version) Add(i int) { v.v = append(v.v, i) } -func (v *version) Place(i int) int { +func (v *version) Field(i int) int { if i < len(v.v) { return v.v[i] } diff --git a/pkgtools/pkglint/files/pkgver/vercmp_test.go b/pkgtools/pkglint/files/pkgver/vercmp_test.go index d0c8df07b9c..05c2cb0fbda 100644 --- a/pkgtools/pkglint/files/pkgver/vercmp_test.go +++ b/pkgtools/pkglint/files/pkgver/vercmp_test.go @@ -13,19 +13,32 @@ func Test(t *testing.T) { } func (s *Suite) Test_newVersion(c *check.C) { - c.Check(newVersion("5.0"), check.DeepEquals, &version{[]int{5, 0, 0}, 0}) - c.Check(newVersion("5.0nb5"), check.DeepEquals, &version{[]int{5, 0, 0}, 5}) - c.Check(newVersion("0.0.1-SNAPSHOT"), check.DeepEquals, &version{[]int{0, 0, 0, 0, 1, 19, 14, 1, 16, 19, 8, 15, 20}, 0}) - c.Check(newVersion("1.0alpha3"), check.DeepEquals, &version{[]int{1, 0, 0, -3, 3}, 0}) - c.Check(newVersion("1_0alpha3"), check.DeepEquals, &version{[]int{1, 0, 0, -3, 3}, 0}) - c.Check(newVersion("2.5beta"), check.DeepEquals, &version{[]int{2, 0, 5, -2}, 0}) - c.Check(newVersion("20151110"), check.DeepEquals, &version{[]int{20151110}, 0}) - c.Check(newVersion("0"), check.DeepEquals, &version{[]int{0}, 0}) - c.Check(newVersion("nb1"), check.DeepEquals, &version{nil, 1}) - c.Check(newVersion("1.0.1a"), check.DeepEquals, &version{[]int{1, 0, 0, 0, 1, 1}, 0}) - c.Check(newVersion("1.0.1z"), check.DeepEquals, &version{[]int{1, 0, 0, 0, 1, 26}, 0}) - c.Check(newVersion("0pre20160620"), check.DeepEquals, &version{[]int{0, -1, 20160620}, 0}) - c.Check(newVersion("3.5.DEV1710"), check.DeepEquals, &version{[]int{3, 0, 5, 0, 4, 5, 22, 1710}, 0}) + c.Check(newVersion("5.0"), check.DeepEquals, + &version{[]int{5, 0, 0}, 0}) + c.Check(newVersion("5.0nb5"), check.DeepEquals, + &version{[]int{5, 0, 0}, 5}) + c.Check(newVersion("0.0.1-SNAPSHOT"), check.DeepEquals, + &version{[]int{0, 0, 0, 0, 1, 19, 14, 1, 16, 19, 8, 15, 20}, 0}) + c.Check(newVersion("1.0alpha3"), check.DeepEquals, + &version{[]int{1, 0, 0, -3, 3}, 0}) + c.Check(newVersion("1_0alpha3"), check.DeepEquals, + &version{[]int{1, 0, 0, -3, 3}, 0}) + c.Check(newVersion("2.5beta"), check.DeepEquals, + &version{[]int{2, 0, 5, -2}, 0}) + c.Check(newVersion("20151110"), check.DeepEquals, + &version{[]int{20151110}, 0}) + c.Check(newVersion("0"), check.DeepEquals, + &version{[]int{0}, 0}) + c.Check(newVersion("nb1"), check.DeepEquals, + &version{nil, 1}) + c.Check(newVersion("1.0.1a"), check.DeepEquals, + &version{[]int{1, 0, 0, 0, 1, 1}, 0}) + c.Check(newVersion("1.0.1z"), check.DeepEquals, + &version{[]int{1, 0, 0, 0, 1, 26}, 0}) + c.Check(newVersion("0pre20160620"), check.DeepEquals, + &version{[]int{0, -1, 20160620}, 0}) + c.Check(newVersion("3.5.DEV1710"), check.DeepEquals, + &version{[]int{3, 0, 5, 0, 4, 5, 22, 1710}, 0}) } func (s *Suite) Test_Compare(c *check.C) { @@ -53,23 +66,26 @@ func (s *Suite) Test_Compare(c *check.C) { {"20151110"}, } - checkVersion := func(i int, iversion string, j int, jversion string) { - actual := Compare(iversion, jversion) - switch { - case i < j && !(actual < 0): - c.Check([]interface{}{i, iversion, j, jversion, actual}, check.DeepEquals, []interface{}{i, iversion, j, jversion, "<0"}) - case i == j && !(actual == 0): - c.Check([]interface{}{i, iversion, j, jversion, actual}, check.DeepEquals, []interface{}{i, iversion, j, jversion, "==0"}) - case i > j && !(actual > 0): - c.Check([]interface{}{i, iversion, j, jversion, actual}, check.DeepEquals, []interface{}{i, iversion, j, jversion, ">0"}) + op := func(cmp int) string { + return [...]string{"<0", "==0", ">0"}[cmp+1] + } + + test := func(i int, iVersion string, j int, jVersion string) { + actual := icmp(Compare(iVersion, jVersion), 0) + expected := icmp(i, j) + if actual != expected { + c.Check( + []interface{}{i, iVersion, j, jVersion, op(actual)}, + check.DeepEquals, + []interface{}{i, iVersion, j, jVersion, op(expected)}) } } - for i, iversions := range versions { - for j, jversions := range versions { - for _, iversion := range iversions { - for _, jversion := range jversions { - checkVersion(i, iversion, j, jversion) + for i, iVersions := range versions { + for j, jVersions := range versions { + for _, iVersion := range iVersions { + for _, jVersion := range jVersions { + test(i, iVersion, j, jVersion) } } } diff --git a/pkgtools/pkglint/files/plist.go b/pkgtools/pkglint/files/plist.go index 8ad153b463b..3a9c1cdbbb6 100644 --- a/pkgtools/pkglint/files/plist.go +++ b/pkgtools/pkglint/files/plist.go @@ -18,13 +18,13 @@ func CheckLinesPlist(lines Lines) { lines.Lines[0].Warnf("PLIST files shouldn't be empty.") G.Explain( "One reason for empty PLISTs is that this is a newly created package", - "and that the author didn't run \"bmake print-PLIST\" after installing", - "the files.", + sprintf("and that the author didn't run %q after installing the files.", bmake("print-PLIST")), "", - "Another reason, common for Perl packages, is that the final PLIST is automatically generated.", - "Since the source PLIST is not used at all, it can be removed.", + "For most Perl packages, the final PLIST is generated automatically.", + "Since the source PLIST is not used at all, it can be removed for these packages.", "", - "Meta packages also don't need a PLIST file.") + "Meta packages also don't need a PLIST file", + "since their only purpose is to declare dependencies.") } ck := PlistChecker{ @@ -60,7 +60,7 @@ func (ck *PlistChecker) Check(plainLines Lines) { } for _, pline := range plines { - ck.checkline(pline) + ck.checkLine(pline) pline.CheckTrailingWhitespace() } CheckLinesTrailingEmptyLines(plainLines) @@ -84,6 +84,7 @@ func (ck *PlistChecker) NewLines(lines Lines) []*PlistLine { if m, cond, rest := match2(text, `^(?:\$\{(PLIST\.[\w-.]+)\})+(.*)`); m { condition, text = cond, rest } + // TODO: Support multiple conditions per line. } plines[i] = &PlistLine{line, condition, text} } @@ -105,39 +106,42 @@ func (ck *PlistChecker) collectFilesAndDirs(plines []*PlistLine) { ck.allDirs[dir] = pline } case first == '@': + // TODO: Check if this directive is still used, + // or if it has been removed during a pkg_install re-implementation. if m, dirname := match1(text, `^@exec \$\{MKDIR\} %D/(.*)$`); m { for dir := dirname; dir != "."; dir = path.Dir(dir) { ck.allDirs[dir] = pline } } } - } } } -func (ck *PlistChecker) checkline(pline *PlistLine) { +func (ck *PlistChecker) checkLine(pline *PlistLine) { text := pline.text - if hasAlnumPrefix(text) { - ck.checkpath(pline) - } else if m, cmd, arg := match2(text, `^(?:\$\{[\w.]+\})?@([a-z-]+)[\t ]*(.*)`); m { - pline.CheckDirective(cmd, arg) - } else if hasPrefix(text, "$") { - ck.checkpath(pline) - } else if text == "" { + + if text == "" { fix := pline.Autofix() fix.Warnf("PLISTs should not contain empty lines.") fix.Delete() fix.Apply() + + } else if textproc.AlnumU.Contains(text[0]) || text[0] == '$' { + ck.checkPath(pline) + + } else if m, cmd, arg := match2(text, `^@([a-z-]+)[\t ]*(.*)`); m { + pline.CheckDirective(cmd, arg) + } else { - pline.Warnf("Unknown line type: %s", pline.Line.Text) + pline.Warnf("Invalid line type: %s", pline.Line.Text) } } -func (ck *PlistChecker) checkpath(pline *PlistLine) { +func (ck *PlistChecker) checkPath(pline *PlistLine) { text := pline.text - sdirname, basename := path.Split(text) - dirname := strings.TrimSuffix(sdirname, "/") + dirSlash, basename := path.Split(text) + dirname := strings.TrimSuffix(dirSlash, "/") ck.checkSorted(pline) ck.checkDuplicate(pline) @@ -145,42 +149,41 @@ func (ck *PlistChecker) checkpath(pline *PlistLine) { if contains(basename, "${IMAKE_MANNEWSUFFIX}") { pline.warnImakeMannewsuffix() } + if hasPrefix(text, "${PKGMANDIR}/") { fix := pline.Autofix() - fix.Notef("PLIST files should mention \"man/\" instead of \"${PKGMANDIR}\".") + fix.Notef("PLIST files should use \"man/\" instead of \"${PKGMANDIR}\".") fix.Explain( "The pkgsrc infrastructure takes care of replacing the correct value", "when generating the actual PLIST for the package.") fix.Replace("${PKGMANDIR}/", "man/") fix.Apply() + // Since the autofix only applies to the Line, the PlistLine needs to be updated manually. pline.text = strings.Replace(pline.text, "${PKGMANDIR}/", "man/", 1) } - topdir := "" - if firstSlash := strings.IndexByte(text, '/'); firstSlash != -1 { - topdir = text[:firstSlash] - } + topdir := strings.SplitN(text, "/", 2)[0] switch topdir { case "bin": - ck.checkpathBin(pline, dirname, basename) + ck.checkPathBin(pline, dirname, basename) case "doc": pline.Errorf("Documentation must be installed under share/doc, not doc.") case "etc": - ck.checkpathEtc(pline, dirname, basename) + ck.checkPathEtc(pline, dirname, basename) case "info": - ck.checkpathInfo(pline, dirname, basename) + ck.checkPathInfo(pline, dirname, basename) case "lib": - ck.checkpathLib(pline, dirname, basename) + ck.checkPathLib(pline, dirname, basename) case "man": - ck.checkpathMan(pline) + ck.checkPathMan(pline) case "share": - ck.checkpathShare(pline) + ck.checkPathShare(pline) } if contains(text, "${PKGLOCALEDIR}") && G.Pkg != nil && !G.Pkg.vars.Defined("USE_PKGLOCALEDIR") { - pline.Warnf("PLIST contains ${PKGLOCALEDIR}, but USE_PKGLOCALEDIR was not found.") + pline.Warnf("PLIST contains ${PKGLOCALEDIR}, but USE_PKGLOCALEDIR is not set in the package Makefile.") } if contains(text, "/CVS/") { @@ -192,8 +195,8 @@ func (ck *PlistChecker) checkpath(pline *PlistLine) { if hasSuffix(text, "/perllocal.pod") { pline.Warnf("perllocal.pod files should not be in the PLIST.") G.Explain( - "This file is handled automatically by the INSTALL/DEINSTALL scripts,", - "since its contents changes frequently.") + "This file is handled automatically by the INSTALL/DEINSTALL scripts", + "since its contents depends on more than one package.") } if contains(text, ".egg-info/") { pline.Warnf("Include \"../../lang/python/egg.mk\" instead of listing .egg-info files directly.") @@ -207,7 +210,7 @@ func (ck *PlistChecker) checkSorted(pline *PlistLine) { pline.Warnf("%q should be sorted before %q.", text, ck.lastFname) G.Explain( "The files in the PLIST should be sorted alphabetically.", - "To fix this, run \"pkglint -F PLIST\".") + "This allows human readers to quickly see whether a file is included or not.") } } ck.lastFname = text @@ -231,7 +234,7 @@ func (ck *PlistChecker) checkDuplicate(pline *PlistLine) { fix.Apply() } -func (ck *PlistChecker) checkpathBin(pline *PlistLine, dirname, basename string) { +func (ck *PlistChecker) checkPathBin(pline *PlistLine, dirname, basename string) { if contains(dirname, "/") { pline.Warnf("The bin/ directory should not have subdirectories.") G.Explain( @@ -243,17 +246,20 @@ func (ck *PlistChecker) checkpathBin(pline *PlistLine, dirname, basename string) } } -func (ck *PlistChecker) checkpathEtc(pline *PlistLine, dirname, basename string) { +func (ck *PlistChecker) checkPathEtc(pline *PlistLine, dirname, basename string) { if hasPrefix(pline.text, "etc/rc.d/") { - pline.Errorf("RCD_SCRIPTS must not be registered in the PLIST. Please use the RCD_SCRIPTS framework.") + pline.Errorf("RCD_SCRIPTS must not be registered in the PLIST.") + pline.Explain( + "Please use the RCD_SCRIPTS framework, which is described in mk/pkginstall/bsd.pkginstall.mk.") return } - pline.Errorf("Configuration files must not be registered in the PLIST. " + + pline.Errorf("Configuration files must not be registered in the PLIST.") + pline.Explain( "Please use the CONF_FILES framework, which is described in mk/pkginstall/bsd.pkginstall.mk.") } -func (ck *PlistChecker) checkpathInfo(pline *PlistLine, dirname, basename string) { +func (ck *PlistChecker) checkPathInfo(pline *PlistLine, dirname, basename string) { if pline.text == "info/dir" { pline.Errorf("\"info/dir\" must not be listed. Use install-info to add/remove an entry.") return @@ -264,7 +270,7 @@ func (ck *PlistChecker) checkpathInfo(pline *PlistLine, dirname, basename string } } -func (ck *PlistChecker) checkpathLib(pline *PlistLine, dirname, basename string) { +func (ck *PlistChecker) checkPathLib(pline *PlistLine, dirname, basename string) { switch { case G.Pkg != nil && G.Pkg.EffectivePkgbase != "" && hasPrefix(pline.text, "lib/"+G.Pkg.EffectivePkgbase+"/"): return @@ -294,7 +300,7 @@ func (ck *PlistChecker) checkpathLib(pline *PlistLine, dirname, basename string) } } -func (ck *PlistChecker) checkpathMan(pline *PlistLine) { +func (ck *PlistChecker) checkPathMan(pline *PlistLine) { m, catOrMan, section, manpage, ext, gz := match5(pline.text, `^man/(cat|man)(\w+)/(.*?)\.(\w+)(\.gz)?$`) if !m { // maybe: line.Warnf("Invalid filename %q for manual page.", text) @@ -332,7 +338,7 @@ func (ck *PlistChecker) checkpathMan(pline *PlistLine) { } } -func (ck *PlistChecker) checkpathShare(pline *PlistLine) { +func (ck *PlistChecker) checkPathShare(pline *PlistLine) { text := pline.text switch { case hasPrefix(text, "share/icons/") && G.Pkg != nil: @@ -379,9 +385,6 @@ func (ck *PlistChecker) checkpathShare(pline *PlistLine) { G.Explain( "To fix this, add INFO_FILES=yes to the package Makefile.") - case hasPrefix(text, "share/locale/") && hasSuffix(text, ".mo"): - // Fine. - case hasPrefix(text, "share/man/"): pline.Warnf("Man pages should be installed into man/, not share/man/.") } @@ -415,7 +418,7 @@ func (pline *PlistLine) CheckDirective(cmd, arg string) { } case "comment": - // Nothing to do. + // Nothing to check. case "dirrm": pline.Warnf("@dirrm is obsolete. Please remove this line.") @@ -459,7 +462,7 @@ type plistLineSorter struct { header []*PlistLine // Does not take part in sorting middle []*PlistLine // Only this part is sorted footer []*PlistLine // Does not take part in sorting, typically contains @exec or @pkgdir - unsortable Line // Some lines so difficult to sort that only humans can do that + unsortable Line // Some lines are so difficult to sort that only humans can do that changed bool // Whether the sorting actually changed something autofixed bool // Whether the newly sorted file has been written to disk } diff --git a/pkgtools/pkglint/files/plist_test.go b/pkgtools/pkglint/files/plist_test.go index 12bcfdc2f98..f8ae466414c 100644 --- a/pkgtools/pkglint/files/plist_test.go +++ b/pkgtools/pkglint/files/plist_test.go @@ -22,7 +22,7 @@ func (s *Suite) Test_CheckLinesPlist(c *check.C) { "sbin/clockctl", "share/icons/gnome/delete-icon", "share/icons/hicolor/icon1.png", - "share/icons/hicolor/icon2.png", // No additional warning + "share/icons/hicolor/icon2.png", // No additional error for hicolor-icon-theme. "share/tzinfo", "share/tzinfo") @@ -31,9 +31,8 @@ func (s *Suite) Test_CheckLinesPlist(c *check.C) { t.CheckOutputLines( "ERROR: PLIST:1: Expected \"@comment $"+"NetBSD$\".", "WARN: PLIST:1: The bin/ directory should not have subdirectories.", - "ERROR: PLIST:3: Configuration files must not be registered in the PLIST. "+ - "Please use the CONF_FILES framework, which is described in mk/pkginstall/bsd.pkginstall.mk.", - "ERROR: PLIST:4: RCD_SCRIPTS must not be registered in the PLIST. Please use the RCD_SCRIPTS framework.", + "ERROR: PLIST:3: Configuration files must not be registered in the PLIST.", + "ERROR: PLIST:4: RCD_SCRIPTS must not be registered in the PLIST.", "ERROR: PLIST:6: \"info/dir\" must not be listed. Use install-info to add/remove an entry.", "WARN: PLIST:8: Redundant library found. The libtool library is in line 9.", "WARN: PLIST:9: \"lib/libc.la\" should be sorted before \"lib/libc.so.6\".", @@ -67,7 +66,7 @@ func (s *Suite) Test_CheckLinesPlist__common_end(c *check.C) { t.CreateFileLines("PLIST.common", PlistRcsID, "bin/common") - lines := t.SetupFileLines("PLIST.common_end", + lines := t.SetUpFileLines("PLIST.common_end", PlistRcsID, "sbin/common_end") @@ -93,7 +92,7 @@ func (s *Suite) Test_CheckLinesPlist__condition(c *check.C) { func (s *Suite) Test_CheckLinesPlist__sorting(c *check.C) { t := s.Init(c) - t.SetupCommandLine("-Wplist-sort") + t.SetUpCommandLine("-Wplist-sort") lines := t.NewLines("PLIST", PlistRcsID, "@comment Do not remove", @@ -109,11 +108,19 @@ func (s *Suite) Test_CheckLinesPlist__sorting(c *check.C) { "WARN: PLIST:6: \"bin/cat\" should be sorted before \"bin/otherprogram\".") } +func (s *Suite) Test_CheckLinesPlist__sort_common(c *check.C) { + t := s.Init(c) + + // TODO: Examine what happens if there is a PLIST.common to be sorted. + + t.CheckOutputEmpty() +} + func (s *Suite) Test_plistLineSorter_Sort(c *check.C) { t := s.Init(c) - t.SetupCommandLine("--autofix") - lines := t.SetupFileLines("PLIST", + t.SetUpCommandLine("--autofix") + lines := t.SetUpFileLines("PLIST", PlistRcsID, "@comment Do not remove", "A", @@ -165,7 +172,35 @@ func (s *Suite) Test_plistLineSorter_Sort(c *check.C) { "@exec echo \"after lib/after.la\"") // The footer starts here } -func (s *Suite) Test_PlistChecker_checkpathMan__gz(c *check.C) { +func (s *Suite) Test_PlistChecker_checkLine(c *check.C) { + t := s.Init(c) + + lines := t.NewLines("PLIST", + PlistRcsID, + "bin/program", + "${PLIST.var}bin/conditional-program", + "${PLIST.linux}${PLIST.arm}bin/arm-linux-only", + "${PLIST.linux}${PLIST.arm-64}@exec echo 'This is Linux/arm64'", + "${PLIST.ocaml-opt}share/ocaml", + "${PLIST.ocaml-opt}@exec echo 'This is OCaml'", + "${PLIST.ocaml-opt}@exec echo 'This is OCaml'", + "${PYSITELIB:S,lib,share}/modifiers don't work in PLISTs", + "${PLIST.empty}", + "", + "$prefix/bin", + "<<<<<<<<< merge conflict") + + CheckLinesPlist(lines) + + t.CheckOutputLines( + "WARN: PLIST:3: \"bin/conditional-program\" should be sorted before \"bin/program\".", + "WARN: PLIST:4: \"bin/arm-linux-only\" should be sorted before \"bin/conditional-program\".", + "WARN: PLIST:10: PLISTs should not contain empty lines.", + "WARN: PLIST:11: PLISTs should not contain empty lines.", + "WARN: PLIST:13: Invalid line type: <<<<<<<<< merge conflict") +} + +func (s *Suite) Test_PlistChecker_checkPathMan__gz(c *check.C) { t := s.Init(c) G.Pkg = NewPackage(t.File("category/pkgbase")) @@ -179,7 +214,7 @@ func (s *Suite) Test_PlistChecker_checkpathMan__gz(c *check.C) { "NOTE: PLIST:2: The .gz extension is unnecessary for manual pages.") } -func (s *Suite) Test_PlistChecker_checkpath__PKGMANDIR(c *check.C) { +func (s *Suite) Test_PlistChecker_checkPath__PKGMANDIR(c *check.C) { t := s.Init(c) lines := t.NewLines("PLIST", @@ -189,10 +224,10 @@ func (s *Suite) Test_PlistChecker_checkpath__PKGMANDIR(c *check.C) { CheckLinesPlist(lines) t.CheckOutputLines( - "NOTE: PLIST:2: PLIST files should mention \"man/\" instead of \"${PKGMANDIR}\".") + "NOTE: PLIST:2: PLIST files should use \"man/\" instead of \"${PKGMANDIR}\".") } -func (s *Suite) Test_PlistChecker_checkpath__python_egg(c *check.C) { +func (s *Suite) Test_PlistChecker_checkPath__python_egg(c *check.C) { t := s.Init(c) lines := t.NewLines("PLIST", @@ -208,7 +243,7 @@ func (s *Suite) Test_PlistChecker_checkpath__python_egg(c *check.C) { func (s *Suite) Test_PlistChecker__autofix(c *check.C) { t := s.Init(c) - lines := t.SetupFileLines("PLIST", + lines := t.SetUpFileLines("PLIST", PlistRcsID, "lib/libvirt/connection-driver/libvirt_driver_storage.la", "${PLIST.hal}lib/libvirt/connection-driver/libvirt_driver_nodedev.la", @@ -226,6 +261,7 @@ func (s *Suite) Test_PlistChecker__autofix(c *check.C) { "share/locale/zh_TW/LC_MESSAGES/libvirt.mo", "share/locale/zu/LC_MESSAGES/libvirt.mo", "@pkgdir share/examples/libvirt/nwfilter", + // Directives may contain arbitrary horizontal whitespace. "@pkgdir etc/libvirt/qemu/networks/autostart", "@pkgdir etc/logrotate.d", "@pkgdir etc/sasl2") @@ -237,9 +273,9 @@ func (s *Suite) Test_PlistChecker__autofix(c *check.C) { "should be sorted before \"lib/libvirt/connection-driver/libvirt_driver_storage.la\".", "WARN: ~/PLIST:4: \"lib/libvirt/connection-driver/libvirt_driver_libxl.la\" "+ "should be sorted before \"lib/libvirt/connection-driver/libvirt_driver_nodedev.la\".", - "NOTE: ~/PLIST:6: PLIST files should mention \"man/\" instead of \"${PKGMANDIR}\".") + "NOTE: ~/PLIST:6: PLIST files should use \"man/\" instead of \"${PKGMANDIR}\".") - t.SetupCommandLine("-Wall", "--autofix") + t.SetUpCommandLine("-Wall", "--autofix") CheckLinesPlist(lines) t.CheckOutputLines( @@ -275,7 +311,7 @@ func (s *Suite) Test_PlistChecker__autofix(c *check.C) { func (s *Suite) Test_PlistChecker__remove_same_entries(c *check.C) { t := s.Init(c) - lines := t.SetupFileLines("PLIST", + lines := t.SetUpFileLines("PLIST", PlistRcsID, "${PLIST.option1}bin/true", "bin/true", @@ -294,7 +330,7 @@ func (s *Suite) Test_PlistChecker__remove_same_entries(c *check.C) { "WARN: ~/PLIST:6: \"bin/false\" should be sorted before \"bin/true\".", "ERROR: ~/PLIST:8: Duplicate filename \"bin/true\", already appeared in line 3.") - t.SetupCommandLine("-Wall", "--autofix") + t.SetUpCommandLine("-Wall", "--autofix") CheckLinesPlist(lines) @@ -317,9 +353,9 @@ func (s *Suite) Test_PlistChecker__remove_same_entries(c *check.C) { func (s *Suite) Test_PlistChecker__autofix_with_only(c *check.C) { t := s.Init(c) - t.SetupCommandLine("-Wall", "--autofix", "--only", "matches nothing") + t.SetUpCommandLine("-Wall", "--autofix", "--only", "matches nothing") - lines := t.SetupFileLines("PLIST", + lines := t.SetUpFileLines("PLIST", PlistRcsID, "sbin/program", "bin/program") @@ -336,7 +372,7 @@ func (s *Suite) Test_PlistChecker__autofix_with_only(c *check.C) { func (s *Suite) Test_PlistChecker__exec_MKDIR(c *check.C) { t := s.Init(c) - lines := t.SetupFileLines("PLIST", + lines := t.SetUpFileLines("PLIST", PlistRcsID, "bin/program", "@exec ${MKDIR} %D/share/mk/subdir") @@ -349,7 +385,7 @@ func (s *Suite) Test_PlistChecker__exec_MKDIR(c *check.C) { func (s *Suite) Test_PlistChecker__empty_line(c *check.C) { t := s.Init(c) - lines := t.SetupFileLines("PLIST", + lines := t.SetUpFileLines("PLIST", PlistRcsID, "", "bin/program") @@ -359,7 +395,7 @@ func (s *Suite) Test_PlistChecker__empty_line(c *check.C) { t.CheckOutputLines( "WARN: ~/PLIST:2: PLISTs should not contain empty lines.") - t.SetupCommandLine("-Wall", "--autofix") + t.SetUpCommandLine("-Wall", "--autofix") CheckLinesPlist(lines) @@ -370,25 +406,31 @@ func (s *Suite) Test_PlistChecker__empty_line(c *check.C) { "bin/program") } -func (s *Suite) Test_PlistChecker__unknown_line_type(c *check.C) { +func (s *Suite) Test_PlistChecker__invalid_line_type(c *check.C) { t := s.Init(c) - lines := t.SetupFileLines("PLIST", + lines := t.SetUpFileLines("PLIST", PlistRcsID, - "---unknown", - "+++unknown") + "---invalid", + "+++invalid", + "<<<<<<<< merge conflict", + "======== merge conflict", + ">>>>>>>> merge conflict") CheckLinesPlist(lines) t.CheckOutputLines( - "WARN: ~/PLIST:2: Unknown line type: ---unknown", - "WARN: ~/PLIST:3: Unknown line type: +++unknown") + "WARN: ~/PLIST:2: Invalid line type: ---invalid", + "WARN: ~/PLIST:3: Invalid line type: +++invalid", + "WARN: ~/PLIST:4: Invalid line type: <<<<<<<< merge conflict", + "WARN: ~/PLIST:5: Invalid line type: ======== merge conflict", + "WARN: ~/PLIST:6: Invalid line type: >>>>>>>> merge conflict") } func (s *Suite) Test_PlistChecker__doc(c *check.C) { t := s.Init(c) - lines := t.SetupFileLines("PLIST", + lines := t.SetUpFileLines("PLIST", PlistRcsID, "doc/html/index.html") @@ -401,7 +443,7 @@ func (s *Suite) Test_PlistChecker__doc(c *check.C) { func (s *Suite) Test_PlistChecker__PKGLOCALEDIR(c *check.C) { t := s.Init(c) - lines := t.SetupFileLines("PLIST", + lines := t.SetUpFileLines("PLIST", PlistRcsID, "${PKGLOCALEDIR}/file") G.Pkg = NewPackage(t.File("category/package")) @@ -409,18 +451,17 @@ func (s *Suite) Test_PlistChecker__PKGLOCALEDIR(c *check.C) { CheckLinesPlist(lines) t.CheckOutputLines( - "WARN: ~/PLIST:2: PLIST contains ${PKGLOCALEDIR}, but USE_PKGLOCALEDIR was not found.") + "WARN: ~/PLIST:2: PLIST contains ${PKGLOCALEDIR}, but USE_PKGLOCALEDIR is not set in the package Makefile.") } -func (s *Suite) Test_PlistChecker__unwanted_entries(c *check.C) { +func (s *Suite) Test_PlistChecker_checkPath__unwanted_entries(c *check.C) { t := s.Init(c) - lines := t.SetupFileLines("PLIST", + lines := t.SetUpFileLines("PLIST", PlistRcsID, "share/perllocal.pod", "share/pkgbase/CVS/Entries", "share/pkgbase/Makefile.orig") - G.Pkg = NewPackage(t.File("category/package")) CheckLinesPlist(lines) @@ -430,10 +471,10 @@ func (s *Suite) Test_PlistChecker__unwanted_entries(c *check.C) { "WARN: ~/PLIST:4: .orig files should not be in the PLIST.") } -func (s *Suite) Test_PlistChecker_checkpathInfo(c *check.C) { +func (s *Suite) Test_PlistChecker_checkPathInfo(c *check.C) { t := s.Init(c) - lines := t.SetupFileLines("PLIST", + lines := t.SetUpFileLines("PLIST", PlistRcsID, "info/gmake.1.info") G.Pkg = NewPackage(t.File("category/package")) @@ -444,10 +485,10 @@ func (s *Suite) Test_PlistChecker_checkpathInfo(c *check.C) { "WARN: ~/PLIST:2: Packages that install info files should set INFO_FILES in the Makefile.") } -func (s *Suite) Test_PlistChecker_checkpathLib(c *check.C) { +func (s *Suite) Test_PlistChecker_checkPathLib(c *check.C) { t := s.Init(c) - lines := t.SetupFileLines("PLIST", + lines := t.SetUpFileLines("PLIST", PlistRcsID, "lib/charset.alias", "lib/liberty-1.0.la", @@ -461,13 +502,14 @@ func (s *Suite) Test_PlistChecker_checkpathLib(c *check.C) { t.CheckOutputLines( "ERROR: ~/PLIST:2: Only the libiconv package may install lib/charset.alias.", "WARN: ~/PLIST:3: Packages that install libtool libraries should define USE_LIBTOOL.", - "ERROR: ~/PLIST:4: \"lib/locale\" must not be listed. Use ${PKGLOCALEDIR}/locale and set USE_PKGLOCALEDIR instead.") + "ERROR: ~/PLIST:4: \"lib/locale\" must not be listed. "+ + "Use ${PKGLOCALEDIR}/locale and set USE_PKGLOCALEDIR instead.") } -func (s *Suite) Test_PlistChecker_checkpathMan(c *check.C) { +func (s *Suite) Test_PlistChecker_checkPathMan(c *check.C) { t := s.Init(c) - lines := t.SetupFileLines("PLIST", + lines := t.SetUpFileLines("PLIST", PlistRcsID, "man/man1/program.8", "man/manx/program.x") @@ -479,10 +521,10 @@ func (s *Suite) Test_PlistChecker_checkpathMan(c *check.C) { "WARN: ~/PLIST:3: Unknown section \"x\" for manual page.") } -func (s *Suite) Test_PlistChecker_checkpathShare(c *check.C) { +func (s *Suite) Test_PlistChecker_checkPathShare(c *check.C) { t := s.Init(c) - lines := t.SetupFileLines("PLIST", + lines := t.SetUpFileLines("PLIST", PlistRcsID, "share/doc/html/package/index.html", "share/doc/package/index.html", @@ -506,7 +548,7 @@ func (s *Suite) Test_PlistChecker_checkpathShare(c *check.C) { func (s *Suite) Test_PlistLine_CheckTrailingWhitespace(c *check.C) { t := s.Init(c) - lines := t.SetupFileLines("PLIST", + lines := t.SetUpFileLines("PLIST", PlistRcsID, "bin/program \t") @@ -519,7 +561,7 @@ func (s *Suite) Test_PlistLine_CheckTrailingWhitespace(c *check.C) { func (s *Suite) Test_PlistLine_CheckDirective(c *check.C) { t := s.Init(c) - lines := t.SetupFileLines("PLIST", + lines := t.SetUpFileLines("PLIST", PlistRcsID, "@unexec rmdir %D/bin", "@exec ldconfig", @@ -543,8 +585,8 @@ func (s *Suite) Test_PlistLine_CheckDirective(c *check.C) { func (s *Suite) Test_plistLineSorter__unsortable(c *check.C) { t := s.Init(c) - t.SetupCommandLine("-Wall", "--show-autofix") - lines := t.SetupFileLines("PLIST", + t.SetUpCommandLine("-Wall", "--show-autofix") + lines := t.SetUpFileLines("PLIST", PlistRcsID, "bin/program${OPSYS}", "@exec true", diff --git a/pkgtools/pkglint/files/regex/regex.go b/pkgtools/pkglint/files/regex/regex.go index bca3db70cc3..97f40acf6ac 100644 --- a/pkgtools/pkglint/files/regex/regex.go +++ b/pkgtools/pkglint/files/regex/regex.go @@ -122,7 +122,11 @@ func (r *Registry) ReplaceFirst(s string, re Pattern, replacement string) ([]str replaced := s[:m[0]] + replacement + s[m[1]:] mm := make([]string, len(m)/2) for i := 0; i < len(m); i += 2 { - mm[i/2] = s[max0(m[i]):max0(m[i+1])] + if m[i] < 0 { + mm[i/2] = "" + } else { + mm[i/2] = s[m[i]:m[i+1]] + } } return mm, replaced } @@ -146,10 +150,3 @@ func (r *Registry) matchn(s string, re Pattern, n int) []string { } return nil } - -func max0(a int) int { - if a >= 0 { - return a - } - return 0 -} diff --git a/pkgtools/pkglint/files/shell.go b/pkgtools/pkglint/files/shell.go index 93b20771071..33441eb33c1 100644 --- a/pkgtools/pkglint/files/shell.go +++ b/pkgtools/pkglint/files/shell.go @@ -23,8 +23,8 @@ func NewShellLine(mkline MkLine) *ShellLine { return &ShellLine{mkline} } -var shellcommandsContextType = &Vartype{lkNone, BtShellCommands, []ACLEntry{{"*", aclpAllRuntime}}, false} -var shellwordVuc = &VarUseContext{shellcommandsContextType, vucTimeUnknown, vucQuotPlain, false} +var shellCommandsType = &Vartype{lkNone, BtShellCommands, []ACLEntry{{"*", aclpAllRuntime}}, false} +var shellWordVuc = &VarUseContext{shellCommandsType, vucTimeUnknown, vucQuotPlain, false} func (shline *ShellLine) CheckWord(token string, checkQuoting bool, time ToolTime) { if trace.Tracing { @@ -41,13 +41,14 @@ func (shline *ShellLine) CheckWord(token string, checkQuoting bool, time ToolTim // to the MkLineChecker. Examples for these are ${VAR:Mpattern} or $@. p := NewMkParser(nil, token, false) if varuse := p.VarUse(); varuse != nil && p.EOF() { - MkLineChecker{shline.mkline}.CheckVaruse(varuse, shellwordVuc) + MkLineChecker{shline.mkline}.CheckVaruse(varuse, shellWordVuc) return } if matches(token, `\$\{PREFIX\}/man(?:$|/)`) { line.Warnf("Please use ${PKGMANDIR} instead of \"man\".") } + if contains(token, "etc/rc.d") { line.Warnf("Please use the RCD_SCRIPTS mechanism to install rc.d scripts automatically to ${RCD_SCRIPTS_EXAMPLEDIR}.") } @@ -75,6 +76,7 @@ outer: case atom.Quoting == shqBackt || atom.Quoting == shqDquotBackt: backtCommand := shline.unescapeBackticks(&atoms, quoting) if backtCommand != "" { + // TODO: Wrap the setE into a struct. setE := true shline.CheckShellCommand(backtCommand, &setE, time) } @@ -87,7 +89,7 @@ outer: case quoting == shqPlain: switch { case atom.Type == shtShVarUse: - shline.checkShVarUse(atom, checkQuoting) + shline.checkShVarUsePlain(atom, checkQuoting) case atom.Type == shtSubshell: line.Warnf("Invoking subshells via $(...) is not portable enough.") @@ -113,11 +115,12 @@ outer: } if trimHspace(tok.Rest()) != "" { - line.Warnf("Internal pkglint error in ShellLine.CheckWord at %q (quoting=%s), rest: %s", token, quoting, tok.Rest()) + line.Warnf("Internal pkglint error in ShellLine.CheckWord at %q (quoting=%s), rest: %s", + token, quoting, tok.Rest()) } } -func (shline *ShellLine) checkShVarUse(atom *ShAtom, checkQuoting bool) { +func (shline *ShellLine) checkShVarUsePlain(atom *ShAtom, checkQuoting bool) { line := shline.mkline.Line shVarname := atom.ShVarname() @@ -143,6 +146,8 @@ func (shline *ShellLine) checkShVarUse(atom *ShAtom, checkQuoting bool) { if shVarname == "?" { line.Warnf("The $? shell variable is often not available in \"set -e\" mode.") + // TODO: Explain how to properly fix this warning. + // TODO: Make sure the warning is only shown when applicable. } } @@ -151,6 +156,7 @@ func (shline *ShellLine) checkVaruseToken(atoms *[]*ShAtom, quoting ShQuoting) b if varuse == nil { return false } + *atoms = (*atoms)[1:] varname := varuse.varname @@ -171,17 +177,15 @@ func (shline *ShellLine) checkVaruseToken(atoms *[]*ShAtom, quoting ShQuoting) b // This is ok as long as these variables don't have embedded [$\\"'`]. case quoting == shqDquot && varuse.IsQ(): - shline.mkline.Warnf("Please don't use the :Q operator in double quotes.") + shline.mkline.Warnf("The :Q modifier should not be used inside double quotes.") G.Explain( - "Either remove the :Q or the double quotes.", + "To fix this warning, either remove the :Q or the double quotes.", "In most cases, it is more appropriate to remove the double quotes.") } - if varname != "@" { - vucstate := quoting.ToVarUseContext() - vuc := VarUseContext{shellcommandsContextType, vucTimeUnknown, vucstate, true} - MkLineChecker{shline.mkline}.CheckVaruse(varuse, &vuc) - } + vuc := VarUseContext{shellCommandsType, vucTimeUnknown, quoting.ToVarUseContext(), true} + MkLineChecker{shline.mkline}.CheckVaruse(varuse, &vuc) + return true } @@ -266,6 +270,8 @@ func (shline *ShellLine) CheckShellCommandLine(shelltext string) { line := shline.mkline.Line + // TODO: Add sed and mv in addition to ${SED} and ${MV}. + // TODO: Now that a shell command parser is available, be more precise in the condition. if contains(shelltext, "${SED}") && contains(shelltext, "${MV}") { line.Notef("Please use the SUBST framework instead of ${SED} and ${MV}.") G.Explain( @@ -308,11 +314,12 @@ func (shline *ShellLine) CheckShellCommandLine(shelltext string) { func (shline *ShellLine) CheckShellCommand(shellcmd string, pSetE *bool, time ToolTime) { if trace.Tracing { - defer trace.Call()() + defer trace.Call0()() } line := shline.mkline.Line program, err := parseShellProgram(line, shellcmd) + // FIXME: This code is duplicated in checkWordQuoting. if err != nil && contains(shellcmd, "$$(") { // Hack until the shell parser can handle subshells. line.Warnf("Invoking subshells via $(...) is not portable enough.") return @@ -329,6 +336,7 @@ func (shline *ShellLine) CheckShellCommand(shellcmd string, pSetE *bool, time To walker.Callback.SimpleCommand = func(command *MkShSimpleCommand) { scc := NewSimpleCommandChecker(shline, command, time) scc.Check() + // TODO: Implement getopt parsing for StrCommand. if scc.strcmd.Name == "set" && scc.strcmd.AnyArgMatches(`^-.*e`) { *pSetE = true } @@ -344,7 +352,7 @@ func (shline *ShellLine) CheckShellCommand(shellcmd string, pSetE *bool, time To walker.Callback.Word = func(word *ShToken) { // TODO: Try to replace false with true here; it had been set to false // TODO: in 2016 for no apparent reason. - spc.checkWord(word, false, time) + spc.shline.CheckWord(word.MkText, false, time) } walker.Walk(program) @@ -389,12 +397,12 @@ func (shline *ShellLine) checkHiddenAndSuppress(hiddenAndSuppress, rest string) default: shline.mkline.Warnf("The shell command %q should not be hidden.", cmd) G.Explain( - "Hidden shell commands do not appear on the terminal or in the log", - "file when they are executed.", - "When they fail, the error message", - "cannot be assigned to the command, which is very difficult to debug.", + "Hidden shell commands do not appear on the terminal", + "or in the log file when they are executed.", + "When they fail, the error message cannot be related to the command,", + "which makes debugging more difficult.", "", - "It is better to insert ${RUN} at the beginning of the whole command line", + "It is better to insert ${RUN} at the beginning of the whole command line.", "This will hide the command by default but shows it when PKG_DEBUG_LEVEL is set.") } } @@ -436,7 +444,7 @@ func (scc *SimpleCommandChecker) Check() { func (scc *SimpleCommandChecker) checkCommandStart() { if trace.Tracing { - defer trace.Call()() + defer trace.Call0()() } shellword := scc.strcmd.Name @@ -444,6 +452,8 @@ func (scc *SimpleCommandChecker) checkCommandStart() { switch { case shellword == "${RUN}" || shellword == "": + // FIXME: ${RUN} must never appear as a simple command. + // It should always be trimmed before passing the shell program to the SimpleCommandChecker. break case scc.handleForbiddenCommand(): break @@ -451,7 +461,7 @@ func (scc *SimpleCommandChecker) checkCommandStart() { break case scc.handleCommandVariable(): break - case matches(shellword, `^(?::|break|cd|continue|eval|exec|exit|export|read|set|shift|umask|unset)$`): + case scc.handleShellBuiltin(): break case hasPrefix(shellword, "./"): // All commands from the current directory are fine. break @@ -475,7 +485,7 @@ func (scc *SimpleCommandChecker) checkCommandStart() { // and whether the package has added it to USE_TOOLS. func (scc *SimpleCommandChecker) handleTool() bool { if trace.Tracing { - defer trace.Call()() + defer trace.Call0()() } command := scc.strcmd.Name @@ -495,17 +505,16 @@ func (scc *SimpleCommandChecker) handleTool() bool { func (scc *SimpleCommandChecker) handleForbiddenCommand() bool { if trace.Tracing { - defer trace.Call()() + defer trace.Call0()() } shellword := scc.strcmd.Name switch path.Base(shellword) { - case "ktrace", "mktexlsr", "strace", "texconfig", "truss": + case "mktexlsr", "texconfig": scc.shline.mkline.Errorf("%q must not be used in Makefiles.", shellword) G.Explain( - "This command must appear in INSTALL scripts, not in the package", - "Makefile, so that the package also works if it is installed as a", - "binary package via pkg_add.") + "This command may only appear in INSTALL scripts, not in the package Makefile,", + "so that the package also works if it is installed as a binary package.") return true } return false @@ -513,7 +522,7 @@ func (scc *SimpleCommandChecker) handleForbiddenCommand() bool { func (scc *SimpleCommandChecker) handleCommandVariable() bool { if trace.Tracing { - defer trace.Call()() + defer trace.Call0()() } shellword := scc.strcmd.Name @@ -546,11 +555,22 @@ func (scc *SimpleCommandChecker) handleCommandVariable() bool { return false } +func (scc *SimpleCommandChecker) handleShellBuiltin() bool { + switch scc.strcmd.Name { + case ":", "break", "cd", "continue", "eval", "exec", "exit", "export", "read", "set", "shift", "umask", "unset": + return true + } + return false +} + func (scc *SimpleCommandChecker) handleComment() bool { if trace.Tracing { - defer trace.Call()() + defer trace.Call0()() } + // FIXME: Research and explain how pkglint can ever interpret + // a shell comment as a simple command. That just doesn't fit. + shellword := scc.strcmd.Name if trace.Tracing { defer trace.Call1(shellword)() @@ -565,17 +585,21 @@ func (scc *SimpleCommandChecker) handleComment() bool { if semicolon { scc.shline.mkline.Warnf("A shell comment should not contain semicolons.") + // TODO: Explain. + // TODO: Check whether the existing warnings are useful. } + if multiline { scc.shline.mkline.Warnf("A shell comment does not stop at the end of line.") } if semicolon || multiline { G.Explain( - "When you split a shell command into multiple lines that are", + "When a shell command is split into multiple lines that are", "continued with a backslash, they will nevertheless be converted to", "a single line before the shell sees them.", - "That means that even if it _looks_ like that the comment only spans", + "", + "This means that even if it looks as if the comment only spanned", "one line in the Makefile, in fact it spans until the end of the whole", "shell command.", "", @@ -591,7 +615,7 @@ func (scc *SimpleCommandChecker) handleComment() bool { func (scc *SimpleCommandChecker) checkRegexReplace() { if trace.Tracing { - defer trace.Call()() + defer trace.Call0()() } cmdname := scc.strcmd.Name @@ -613,7 +637,7 @@ func (scc *SimpleCommandChecker) checkRegexReplace() { func (scc *SimpleCommandChecker) checkAutoMkdirs() { if trace.Tracing { - defer trace.Call()() + defer trace.Call0()() } cmdname := scc.strcmd.Name @@ -623,12 +647,14 @@ func (scc *SimpleCommandChecker) checkAutoMkdirs() { case cmdname == "${INSTALL}" && scc.strcmd.HasOption("-d"): cmdname = "${INSTALL} -d" case matches(cmdname, `^\$\{INSTALL_.*_DIR\}$`): + // TODO: Replace regex with proper VarUse. break default: return } for _, arg := range scc.strcmd.Args { + // TODO: Replace regex with proper VarUse. if !contains(arg, "$$") && !matches(arg, `\$\{[_.]*[a-z]`) { if m, dirname := match1(arg, `^(?:\$\{DESTDIR\})?\$\{PREFIX(?:|:Q)\}/(.*)`); m { if G.Pkg != nil && G.Pkg.Plist.Dirs[dirname] { @@ -662,7 +688,7 @@ func (scc *SimpleCommandChecker) checkAutoMkdirs() { func (scc *SimpleCommandChecker) checkInstallMulti() { if trace.Tracing { - defer trace.Call()() + defer trace.Call0()() } cmd := scc.strcmd @@ -691,21 +717,26 @@ func (scc *SimpleCommandChecker) checkInstallMulti() { func (scc *SimpleCommandChecker) checkPaxPe() { if trace.Tracing { - defer trace.Call()() + defer trace.Call0()() } if (scc.strcmd.Name == "${PAX}" || scc.strcmd.Name == "pax") && scc.strcmd.HasOption("-pe") { scc.shline.mkline.Warnf("Please use the -pp option to pax(1) instead of -pe.") G.Explain( - "The -pe option tells pax to preserve the ownership of the files,", - "which means that the installed files will belong to the user that", - "has built the package.") + "The -pe option tells pax to preserve the ownership of the files.", + "", + "When extracting distfiles as root user, this means that whatever numeric uid was", + "used by the upstream package will also appear in the filesystem during the build.", + "", + "The {pre,do,post}-install targets are usually run as root.", + "When pax -pe is used in these targets, this means that the installed files will", + "belong to the user that has built the package.") } } func (scc *SimpleCommandChecker) checkEchoN() { if trace.Tracing { - defer trace.Call()() + defer trace.Call0()() } if scc.strcmd.Name == "${ECHO}" && scc.strcmd.HasOption("-n") { @@ -719,7 +750,7 @@ type ShellProgramChecker struct { func (spc *ShellProgramChecker) checkConditionalCd(list *MkShList) { if trace.Tracing { - defer trace.Call()() + defer trace.Call0()() } getSimple := func(list *MkShList) *MkShSimpleCommand { @@ -743,6 +774,8 @@ func (spc *ShellProgramChecker) checkConditionalCd(list *MkShList) { } } + // TODO: It might be worth reversing the logic, like this: + // walker.Callback.Simple = { if inside if.cond || loop.cond { ... } } walker := NewMkShWalker() walker.Callback.If = func(ifClause *MkShIf) { for _, cond := range ifClause.Conds { @@ -769,17 +802,9 @@ func (spc *ShellProgramChecker) checkConditionalCd(list *MkShList) { walker.Walk(list) } -func (spc *ShellProgramChecker) checkWord(word *ShToken, checkQuoting bool, time ToolTime) { - if trace.Tracing { - defer trace.Call(word.MkText)() - } - - spc.shline.CheckWord(word.MkText, checkQuoting, time) -} - func (spc *ShellProgramChecker) checkPipeExitcode(line Line, pipeline *MkShPipeline) { if trace.Tracing { - defer trace.Call()() + defer trace.Call0()() } canFail := func() (bool, string) { @@ -886,7 +911,7 @@ func (spc *ShellProgramChecker) canFail(cmd *MkShCommand) bool { func (spc *ShellProgramChecker) checkSetE(list *MkShList, index int, andor *MkShAndOr) { if trace.Tracing { - defer trace.Call()() + defer trace.Call0()() } command := list.AndOrs[index-1].Pipes[0].Cmds[0] @@ -922,7 +947,7 @@ func (spc *ShellProgramChecker) checkSetE(list *MkShList, index int, andor *MkSh // Some shell commands should not be used in the install phase. func (shline *ShellLine) checkInstallCommand(shellcmd string) { if trace.Tracing { - defer trace.Call()() + defer trace.Call0()() } if G.Mk == nil || !matches(G.Mk.target, `^(?:pre|do|post)-install$`) { @@ -944,6 +969,7 @@ func (shline *ShellLine) checkInstallCommand(shellcmd string) { case "sed", "${SED}", "tr", "${TR}": + // TODO: Pkglint should not complain when sed and tr are used to transform filenames. line.Warnf("The shell command %q should not be used in the install phase.", shellcmd) G.Explain( "In the install phase, the only thing that should be done is to", @@ -953,23 +979,27 @@ func (shline *ShellLine) checkInstallCommand(shellcmd string) { case "cp", "${CP}": line.Warnf("${CP} should not be used to install files.") G.Explain( - "The ${CP} command is highly platform dependent and cannot overwrite", - "read-only files.", + "The ${CP} command is highly platform dependent and cannot overwrite read-only files.", "Please use ${PAX} instead.", "", - "For example, instead of", + "For example, instead of:", "\t${CP} -R ${WRKSRC}/* ${PREFIX}/foodir", - "you should use", + "use:", "\tcd ${WRKSRC} && ${PAX} -wr * ${PREFIX}/foodir") } } // Example: "word1 word2;;;" => "word1", "word2", ";;", ";" +// +// TODO: Document what this function should be used for. func splitIntoShellTokens(line Line, text string) (tokens []string, rest string) { if trace.Tracing { defer trace.Call(line, text)() } + // TODO: Check whether this function is used correctly by all callers. + // It may be better to use a proper shell parser instead of this tokenizer. + word := "" rest = text p := NewShTokenizer(line, text, false) @@ -978,7 +1008,7 @@ func splitIntoShellTokens(line Line, text string) (tokens []string, rest string) tokens = append(tokens, word) word = "" } - rest = p.mkp.Rest() + rest = p.parser.Rest() } q := shqPlain @@ -1011,6 +1041,10 @@ func splitIntoShellTokens(line Line, text string) (tokens []string, rest string) // Compare devel/bmake/files/str.c, function brk_string. // // TODO: Move to mkline.go or mkparser.go. +// +// TODO: Document what this function should be used for. +// +// TODO: Compare with brk_string from devel/bmake, especially for backticks. func splitIntoMkWords(line Line, text string) (words []string, rest string) { if trace.Tracing { defer trace.Call(line, text)() @@ -1031,5 +1065,5 @@ func splitIntoMkWords(line Line, text string) (words []string, rest string) { words = append(words, word) word = "" } - return words, word + p.mkp.Rest() + return words, word + p.parser.Rest() } diff --git a/pkgtools/pkglint/files/shell_test.go b/pkgtools/pkglint/files/shell_test.go index 5e1e625d237..8cde7d14565 100644 --- a/pkgtools/pkglint/files/shell_test.go +++ b/pkgtools/pkglint/files/shell_test.go @@ -146,13 +146,14 @@ func (s *Suite) Test_splitIntoShellTokens__redirect(c *check.C) { func (s *Suite) Test_ShellLine_CheckShellCommandLine(c *check.C) { t := s.Init(c) - t.SetupVartypes() - t.SetupTool("awk", "AWK", AtRunTime) - t.SetupTool("cp", "CP", AtRunTime) - t.SetupTool("mkdir", "MKDIR", AtRunTime) // This is actually "mkdir -p". - t.SetupTool("unzip", "UNZIP_CMD", AtRunTime) - - checkShellCommandLine := func(shellCommand string) { + t.SetUpVartypes() + t.SetUpTool("awk", "AWK", AtRunTime) + t.SetUpTool("cp", "CP", AtRunTime) + t.SetUpTool("echo", "", AtRunTime) + t.SetUpTool("mkdir", "MKDIR", AtRunTime) // This is actually "mkdir -p". + t.SetUpTool("unzip", "UNZIP_CMD", AtRunTime) + + test := func(shellCommand string) { G.Mk = t.NewMkLines("filename", "\t"+shellCommand) shline := NewShellLine(G.Mk.mklines[0]) @@ -162,84 +163,84 @@ func (s *Suite) Test_ShellLine_CheckShellCommandLine(c *check.C) { }) } - checkShellCommandLine("@# Comment") + test("@# Comment") t.CheckOutputEmpty() - checkShellCommandLine("uname=`uname`; echo $$uname; echo; ${PREFIX}/bin/command") + test("uname=`uname`; echo $$uname; echo; ${PREFIX}/bin/command") t.CheckOutputLines( "WARN: filename:1: Unknown shell command \"uname\".", - "WARN: filename:1: Please switch to \"set -e\" mode before using a semicolon (after \"uname=`uname`\") to separate commands.", - "WARN: filename:1: Unknown shell command \"echo\".", - "WARN: filename:1: Unknown shell command \"echo\".") + "WARN: filename:1: Please switch to \"set -e\" mode "+ + "before using a semicolon (after \"uname=`uname`\") to separate commands.") - t.SetupTool("echo", "", AtRunTime) - t.SetupVartypes() + t.SetUpTool("echo", "", AtRunTime) + t.SetUpVartypes() - checkShellCommandLine("echo ${PKGNAME:Q}") // vucQuotPlain + test("echo ${PKGNAME:Q}") // vucQuotPlain t.CheckOutputLines( - "WARN: filename:1: PKGNAME may not be used in this file; it would be ok in Makefile, Makefile.*, *.mk.", + "WARN: filename:1: PKGNAME may not be used in this file; "+ + "it would be ok in Makefile, Makefile.*, *.mk.", "NOTE: filename:1: The :Q operator isn't necessary for ${PKGNAME} here.") - checkShellCommandLine("echo \"${CFLAGS:Q}\"") // vucQuotDquot + test("echo \"${CFLAGS:Q}\"") // vucQuotDquot t.CheckOutputLines( - "WARN: filename:1: Please don't use the :Q operator in double quotes.", + "WARN: filename:1: The :Q modifier should not be used inside double quotes.", "WARN: filename:1: CFLAGS may not be used in this file; "+ "it would be ok in Makefile, Makefile.common, options.mk, *.mk.", "WARN: filename:1: Please use ${CFLAGS:M*:Q} instead of ${CFLAGS:Q} "+ "and make sure the variable appears outside of any quoting characters.") - checkShellCommandLine("echo '${COMMENT:Q}'") // vucQuotSquot + test("echo '${COMMENT:Q}'") // vucQuotSquot t.CheckOutputLines( "WARN: filename:1: COMMENT may not be used in any file; it is a write-only variable.", "WARN: filename:1: Please move ${COMMENT:Q} outside of any quoting characters.") - checkShellCommandLine("echo target=$@ exitcode=$$? '$$' \"\\$$\"") + test("echo target=$@ exitcode=$$? '$$' \"\\$$\"") t.CheckOutputLines( "WARN: filename:1: Please use \"${.TARGET}\" instead of \"$@\".", "WARN: filename:1: The $? shell variable is often not available in \"set -e\" mode.") - checkShellCommandLine("echo $$@") + test("echo $$@") t.CheckOutputLines( "WARN: filename:1: The $@ shell variable should only be used in double quotes.") - checkShellCommandLine("echo \"$$\"") // As seen by make(1); the shell sees: echo "$" + test("echo \"$$\"") // As seen by make(1); the shell sees: echo "$" // No warning about a possibly missed variable name. // This occurs only rarely, and typically as part of a regular expression // where it is used intentionally. t.CheckOutputEmpty() - checkShellCommandLine("echo \"\\n\"") + test("echo \"\\n\"") t.CheckOutputEmpty() - checkShellCommandLine("${RUN} for f in *.c; do echo $${f%.c}; done") + test("${RUN} for f in *.c; do echo $${f%.c}; done") t.CheckOutputEmpty() - checkShellCommandLine("${RUN} set +x; echo $${variable+set}") + test("${RUN} set +x; echo $${variable+set}") t.CheckOutputEmpty() // Based on mail/thunderbird/Makefile, rev. 1.159 - checkShellCommandLine("${RUN} subdir=\"`unzip -c \"$$e\" install.rdf | awk '/re/ { print \"hello\" }'`\"") + test("${RUN} subdir=\"`unzip -c \"$$e\" install.rdf | awk '/re/ { print \"hello\" }'`\"") t.CheckOutputLines( "WARN: filename:1: Double quotes inside backticks inside double quotes are error prone.", "WARN: filename:1: The exitcode of \"unzip\" at the left of the | operator is ignored.") // From mail/thunderbird/Makefile, rev. 1.159 - checkShellCommandLine("" + + test("" + "${RUN} for e in ${XPI_FILES}; do " + " subdir=\"`${UNZIP_CMD} -c \"$$e\" install.rdf | " + - "" + "awk '/^ <em:id>/ {sub(\".*<em:id>\",\"\");sub(\"</em:id>.*\",\"\");print;exit;}'`\" && " + + "" + "awk '/.../ {print;exit;}'`\" && " + " ${MKDIR} \"${WRKDIR}/extensions/$$subdir\" && " + " cd \"${WRKDIR}/extensions/$$subdir\" && " + " ${UNZIP_CMD} -aqo $$e; " + @@ -251,29 +252,31 @@ func (s *Suite) Test_ShellLine_CheckShellCommandLine(c *check.C) { "WARN: filename:1: The exitcode of \"${UNZIP_CMD}\" at the left of the | operator is ignored.") // From x11/wxGTK28/Makefile - checkShellCommandLine("" + + test("" + "set -e; cd ${WRKSRC}/locale; " + "for lang in *.po; do " + " [ \"$${lang}\" = \"wxstd.po\" ] && continue; " + " ${TOOLS_PATH.msgfmt} -c -o \"$${lang%.po}.mo\" \"$${lang}\"; " + "done") + // TODO: Why is TOOLS_PATH.msgfmt not recognized? + // At least, the warning should be more specific, mentioning USE_TOOLS. t.CheckOutputLines( "WARN: filename:1: WRKSRC may not be used in this file; it would be ok in Makefile, Makefile.*, *.mk.", "WARN: filename:1: Unknown shell command \"[\".", "WARN: filename:1: Unknown shell command \"${TOOLS_PATH.msgfmt}\".") - checkShellCommandLine("@cp from to") + test("@cp from to") t.CheckOutputLines( "WARN: filename:1: The shell command \"cp\" should not be hidden.") - checkShellCommandLine("-cp from to") + test("-cp from to") t.CheckOutputLines( "WARN: filename:1: Using a leading \"-\" to suppress errors is deprecated.") - checkShellCommandLine("-${MKDIR} deeply/nested/subdir") + test("-${MKDIR} deeply/nested/subdir") t.CheckOutputLines( "WARN: filename:1: Using a leading \"-\" to suppress errors is deprecated.") @@ -282,7 +285,10 @@ func (s *Suite) Test_ShellLine_CheckShellCommandLine(c *check.C) { G.Pkg.Plist.Dirs["share/pkgbase"] = true // A directory that is found in the PLIST. - checkShellCommandLine("${RUN} ${INSTALL_DATA_DIR} share/pkgbase ${PREFIX}/share/pkgbase") + test("${RUN} ${INSTALL_DATA_DIR} share/pkgbase ${PREFIX}/share/pkgbase") + + // TODO: Add a test for using this command inside a conditional; + // the note should not appear then. t.CheckOutputLines( "NOTE: filename:1: You can use AUTO_MKDIRS=yes or \"INSTALLATION_DIRS+= share/pkgbase\" "+ @@ -290,7 +296,7 @@ func (s *Suite) Test_ShellLine_CheckShellCommandLine(c *check.C) { "WARN: filename:1: The INSTALL_*_DIR commands can only handle one directory at a time.") // A directory that is not found in the PLIST. - checkShellCommandLine("${RUN} ${INSTALL_DATA_DIR} ${PREFIX}/share/other") + test("${RUN} ${INSTALL_DATA_DIR} ${PREFIX}/share/other") t.CheckOutputLines( "NOTE: filename:1: You can use \"INSTALLATION_DIRS+= share/other\" instead of \"${INSTALL_DATA_DIR}\".") @@ -298,15 +304,16 @@ func (s *Suite) Test_ShellLine_CheckShellCommandLine(c *check.C) { G.Pkg = nil // See PR 46570, item "1. It does not" - checkShellCommandLine("for x in 1 2 3; do echo \"$$x\" || exit 1; done") + test("for x in 1 2 3; do echo \"$$x\" || exit 1; done") t.CheckOutputEmpty() // No warning about missing error checking. } +// TODO: Document in detail that strip is not a regular tool. func (s *Suite) Test_ShellLine_CheckShellCommandLine__strip(c *check.C) { t := s.Init(c) - checkShellCommandLine := func(shellCommand string) { + test := func(shellCommand string) { G.Mk = t.NewMkLines("filename", "\t"+shellCommand) @@ -316,15 +323,15 @@ func (s *Suite) Test_ShellLine_CheckShellCommandLine__strip(c *check.C) { }) } - checkShellCommandLine("${STRIP} executable") + test("${STRIP} executable") t.CheckOutputLines( "WARN: filename:1: Unknown shell command \"${STRIP}\".", "WARN: filename:1: STRIP is used but not defined.") - t.SetupVartypes() + t.SetUpVartypes() - checkShellCommandLine("${STRIP} executable") + test("${STRIP} executable") t.CheckOutputEmpty() } @@ -332,8 +339,8 @@ func (s *Suite) Test_ShellLine_CheckShellCommandLine__strip(c *check.C) { func (s *Suite) Test_ShellLine_CheckShellCommandLine__nofix(c *check.C) { t := s.Init(c) - t.SetupVartypes() - t.SetupTool("echo", "", AtRunTime) + t.SetUpVartypes() + t.SetUpTool("echo", "", AtRunTime) G.Mk = t.NewMkLines("Makefile", "\techo ${PKGNAME:Q}") shline := NewShellLine(G.Mk.mklines[0]) @@ -347,9 +354,9 @@ func (s *Suite) Test_ShellLine_CheckShellCommandLine__nofix(c *check.C) { func (s *Suite) Test_ShellLine_CheckShellCommandLine__show_autofix(c *check.C) { t := s.Init(c) - t.SetupCommandLine("-Wall", "--show-autofix") - t.SetupVartypes() - t.SetupTool("echo", "", AtRunTime) + t.SetUpCommandLine("-Wall", "--show-autofix") + t.SetUpVartypes() + t.SetUpTool("echo", "", AtRunTime) G.Mk = t.NewMkLines("Makefile", "\techo ${PKGNAME:Q}") shline := NewShellLine(G.Mk.mklines[0]) @@ -361,15 +368,34 @@ func (s *Suite) Test_ShellLine_CheckShellCommandLine__show_autofix(c *check.C) { "AUTOFIX: Makefile:1: Replacing \"${PKGNAME:Q}\" with \"${PKGNAME}\".") } +func (s *Suite) Test_ShellLine_CheckShellCommandLine__autofix(c *check.C) { + t := s.Init(c) + + t.SetUpCommandLine("-Wall", "--autofix") + t.SetUpVartypes() + t.SetUpTool("echo", "", AtRunTime) + G.Mk = t.NewMkLines("Makefile", + "\techo ${PKGNAME:Q}") + shline := NewShellLine(G.Mk.mklines[0]) + + shline.CheckShellCommandLine("echo ${PKGNAME:Q}") + + t.CheckOutputLines( + "AUTOFIX: Makefile:1: Replacing \"${PKGNAME:Q}\" with \"${PKGNAME}\".") + + // TODO: There should be a general way of testing a code in the three modes: + // default, --show-autofix, --autofix. +} + func (s *Suite) Test_ShellProgramChecker_checkPipeExitcode(c *check.C) { t := s.Init(c) - t.SetupVartypes() - t.SetupTool("cat", "", AtRunTime) - t.SetupTool("echo", "", AtRunTime) - t.SetupTool("printf", "", AtRunTime) - t.SetupTool("sed", "", AtRunTime) - t.SetupTool("right-side", "", AtRunTime) + t.SetUpVartypes() + t.SetUpTool("cat", "", AtRunTime) + t.SetUpTool("echo", "", AtRunTime) + t.SetUpTool("printf", "", AtRunTime) + t.SetUpTool("sed", "", AtRunTime) + t.SetUpTool("right-side", "", AtRunTime) G.Mk = t.NewMkLines("Makefile", "\t echo | right-side", "\t sed s,s,s, | right-side", @@ -398,26 +424,11 @@ func (s *Suite) Test_ShellProgramChecker_checkPipeExitcode(c *check.C) { "WARN: Makefile:11: The exitcode of the command at the left of the | operator is ignored.") } -func (s *Suite) Test_ShellLine_CheckShellCommandLine__autofix(c *check.C) { - t := s.Init(c) - - t.SetupCommandLine("-Wall", "--autofix") - t.SetupVartypes() - t.SetupTool("echo", "", AtRunTime) - G.Mk = t.NewMkLines("Makefile", - "\techo ${PKGNAME:Q}") - shline := NewShellLine(G.Mk.mklines[0]) - - shline.CheckShellCommandLine("echo ${PKGNAME:Q}") - - t.CheckOutputLines( - "AUTOFIX: Makefile:1: Replacing \"${PKGNAME:Q}\" with \"${PKGNAME}\".") -} - +// TODO: Document the exact purpose of this test, or split it into useful tests. func (s *Suite) Test_ShellLine_CheckShellCommandLine__implementation(c *check.C) { t := s.Init(c) - t.SetupVartypes() + t.SetUpVartypes() G.Mk = t.NewMkLines("filename", "# dummy") shline := NewShellLine(G.Mk.mklines[0]) @@ -445,8 +456,8 @@ func (s *Suite) Test_ShellLine_CheckShellCommandLine__implementation(c *check.C) func (s *Suite) Test_ShellLine_CheckShellCommandLine__dollar_without_variable(c *check.C) { t := s.Init(c) - t.SetupVartypes() - t.SetupTool("pax", "", AtRunTime) + t.SetUpVartypes() + t.SetUpTool("pax", "", AtRunTime) G.Mk = t.NewMkLines("filename", "# dummy") shline := NewShellLine(G.Mk.mklines[0]) @@ -459,68 +470,69 @@ func (s *Suite) Test_ShellLine_CheckShellCommandLine__dollar_without_variable(c func (s *Suite) Test_ShellLine_CheckWord(c *check.C) { t := s.Init(c) - t.SetupVartypes() + t.SetUpVartypes() - checkWord := func(shellWord string, checkQuoting bool) { + test := func(shellWord string, checkQuoting bool) { shline := t.NewShellLine("dummy.mk", 1, "\t echo "+shellWord) shline.CheckWord(shellWord, checkQuoting, RunTime) } - checkWord("${${list}}", false) - - checkWord("${${list}}", false) + test("${${list}}", false) // No warning for the outer variable since it is completely indirect. // The inner variable ${list} must still be defined, though. t.CheckOutputLines( - "WARN: dummy.mk:1: list is used but not defined.", "WARN: dummy.mk:1: list is used but not defined.") - checkWord("${SED_FILE.${id}}", false) + test("${SED_FILE.${id}}", false) // No warning for variables that are partly indirect. + // TODO: Why not? t.CheckOutputLines( "WARN: dummy.mk:1: id is used but not defined.") + // TODO: Since $@ refers to ${.TARGET} and not sh.argv, there is no point in checking for quotes. + // TODO: Having the same tests for $$@ would be much more interesting. + // The unquoted $@ takes a different code path in pkglint than the quoted $@. - checkWord("$@", false) + test("$@", false) t.CheckOutputLines( "WARN: dummy.mk:1: Please use \"${.TARGET}\" instead of \"$@\".") // When $@ appears as part of a shell token, it takes another code path in pkglint. - checkWord("-$@-", false) + test("-$@-", false) t.CheckOutputLines( "WARN: dummy.mk:1: Please use \"${.TARGET}\" instead of \"$@\".") // The unquoted $@ takes a different code path in pkglint than the quoted $@. - checkWord("\"$@\"", false) + test("\"$@\"", false) t.CheckOutputLines( "WARN: dummy.mk:1: Please use \"${.TARGET}\" instead of \"$@\".") - checkWord("${COMMENT:Q}", true) + test("${COMMENT:Q}", true) t.CheckOutputLines( "WARN: dummy.mk:1: COMMENT may not be used in any file; it is a write-only variable.") - checkWord("\"${DISTINFO_FILE:Q}\"", true) + test("\"${DISTINFO_FILE:Q}\"", true) t.CheckOutputLines( "NOTE: dummy.mk:1: The :Q operator isn't necessary for ${DISTINFO_FILE} here.") - checkWord("embed${DISTINFO_FILE:Q}ded", true) + test("embed${DISTINFO_FILE:Q}ded", true) t.CheckOutputLines( "NOTE: dummy.mk:1: The :Q operator isn't necessary for ${DISTINFO_FILE} here.") - checkWord("s,\\.,,", true) + test("s,\\.,,", true) t.CheckOutputEmpty() - checkWord("\"s,\\.,,\"", true) + test("\"s,\\.,,\"", true) t.CheckOutputEmpty() } @@ -538,7 +550,7 @@ func (s *Suite) Test_ShellLine_CheckWord__dollar_without_variable(c *check.C) { func (s *Suite) Test_ShellLine_CheckWord__backslash_plus(c *check.C) { t := s.Init(c) - t.SetupTool("find", "FIND", AtRunTime) + t.SetUpTool("find", "FIND", AtRunTime) shline := t.NewShellLine("filename", 1, "\tfind . -exec rm -rf {} \\+") shline.CheckShellCommandLine(shline.mkline.ShellCommand()) @@ -568,11 +580,9 @@ func (s *Suite) Test_ShellLine_CheckWord__dquot_dollar(c *check.C) { shline.CheckWord(shline.mkline.ShellCommand(), false, RunTime) - // FIXME: Should be parsed correctly. Make passes the dollar through (probably), - // and the shell parser should complain about the unfinished string literal. - t.CheckOutputLines( - "WARN: filename:1: Internal pkglint error in ShTokenizer.ShAtom at \"$\" (quoting=d).", - "WARN: filename:1: Internal pkglint error in ShellLine.CheckWord at \"\\\"$\" (quoting=d), rest: $") + // FIXME: Make consumes the dollar silently. + // This could be worth another pkglint warning. + t.CheckOutputEmpty() } func (s *Suite) Test_ShellLine_CheckWord__dollar_subshell(c *check.C) { @@ -589,7 +599,7 @@ func (s *Suite) Test_ShellLine_CheckWord__dollar_subshell(c *check.C) { func (s *Suite) Test_ShellLine_CheckWord__PKGMANDIR(c *check.C) { t := s.Init(c) - t.SetupVartypes() + t.SetUpVartypes() G.Mk = t.NewMkLines("chat/ircII/Makefile", MkRcsID, "CONFIGURE_ARGS+=--mandir=${DESTDIR}${PREFIX}/man", @@ -618,10 +628,9 @@ func (s *Suite) Test_ShellLine_unescapeBackticks__unfinished(c *check.C) { // Breakpoint in ShellLine.unescapeBackticks mklines.Check() - // FIXME: Mention the unfinished backquote. t.CheckOutputLines( - "WARN: filename.mk:4: Pkglint ShellLine.CheckShellCommand: parse error at []string{\"\"}", - "WARN: filename.mk:5: Pkglint ShellLine.CheckShellCommand: parse error at []string{\"echo\"}") + "WARN: filename.mk:4: Pkglint ShellLine.CheckShellCommand: splitIntoShellTokens couldn't parse \"`${VAR}\"", + "WARN: filename.mk:5: Pkglint ShellLine.CheckShellCommand: splitIntoShellTokens couldn't parse \"`${VAR}\"") } func (s *Suite) Test_ShellLine_unescapeBackticks__unfinished_direct(c *check.C) { @@ -675,20 +684,20 @@ func (s *Suite) Test_ShellLine_variableNeedsQuoting(c *check.C) { func (s *Suite) Test_ShellLine_variableNeedsQuoting__integration(c *check.C) { t := s.Init(c) - t.SetupVartypes() - t.SetupTool("cp", "", AtRunTime) + t.SetUpVartypes() + t.SetUpTool("cp", "", AtRunTime) mklines := t.NewMkLines("filename.mk", MkRcsID, "", // It's a bit silly to use shell variables in CONFIGURE_ARGS, - // but currently that's the only way to run ShellLine.variableNeedsQuoting. + // but as of January 2019 that's the only way to run ShellLine.variableNeedsQuoting. "CONFIGURE_ARGS+=\t; cp $$dir $$\\# $$target", "pre-configure:", "\tcp $$dir $$\\# $$target") mklines.Check() - // Quoting check is currently disabled for real shell commands. + // As of January 2019, the quoting check is disabled for real shell commands. // See ShellLine.CheckShellCommand, spc.checkWord. t.CheckOutputLines( "WARN: filename.mk:3: Unquoted shell variable \"target\".") @@ -697,7 +706,7 @@ func (s *Suite) Test_ShellLine_variableNeedsQuoting__integration(c *check.C) { func (s *Suite) Test_ShellLine_CheckShellCommandLine__echo(c *check.C) { t := s.Init(c) - echo := t.SetupTool("echo", "ECHO", AtRunTime) + echo := t.SetUpTool("echo", "ECHO", AtRunTime) echo.MustUseVarForm = true G.Mk = t.NewMkLines("filename", "# dummy") @@ -716,11 +725,11 @@ func (s *Suite) Test_ShellLine_CheckShellCommandLine__echo(c *check.C) { func (s *Suite) Test_ShellLine_CheckShellCommandLine__shell_variables(c *check.C) { t := s.Init(c) - t.SetupVartypes() - t.SetupTool("install", "INSTALL", AtRunTime) - t.SetupTool("cp", "CP", AtRunTime) - t.SetupTool("mv", "MV", AtRunTime) - t.SetupTool("sed", "SED", AtRunTime) + t.SetUpVartypes() + t.SetUpTool("install", "INSTALL", AtRunTime) + t.SetUpTool("cp", "CP", AtRunTime) + t.SetUpTool("mv", "MV", AtRunTime) + t.SetUpTool("sed", "SED", AtRunTime) text := "\tfor f in *.pl; do ${SED} s,@PREFIX@,${PREFIX}, < $f > $f.tmp && ${MV} $f.tmp $f; done" shline := t.NewShellLine("Makefile", 3, text) @@ -774,12 +783,18 @@ func (s *Suite) Test_splitIntoMkWords(c *check.C) { words, rest := splitIntoShellTokens(dummyLine, url) // Doesn't really make sense - c.Check(words, check.DeepEquals, []string{"http://registry.gimp.org/file/fix-ca.c?action=download", "&", "id=9884", "&", "file="}) + c.Check(words, check.DeepEquals, []string{ + "http://registry.gimp.org/file/fix-ca.c?action=download", + "&", + "id=9884", + "&", + "file="}) c.Check(rest, equals, "") words, rest = splitIntoMkWords(dummyLine, url) - c.Check(words, check.DeepEquals, []string{"http://registry.gimp.org/file/fix-ca.c?action=download&id=9884&file="}) + c.Check(words, check.DeepEquals, []string{ + "http://registry.gimp.org/file/fix-ca.c?action=download&id=9884&file="}) c.Check(rest, equals, "") words, rest = splitIntoMkWords(dummyLine, "a b \"c c c\" d;;d;; \"e\"''`` 'rest") @@ -791,9 +806,9 @@ func (s *Suite) Test_splitIntoMkWords(c *check.C) { func (s *Suite) Test_ShellLine_CheckShellCommandLine__sed_and_mv(c *check.C) { t := s.Init(c) - t.SetupVartypes() - t.SetupTool("sed", "SED", AtRunTime) - t.SetupTool("mv", "MV", AtRunTime) + t.SetUpVartypes() + t.SetUpTool("sed", "SED", AtRunTime) + t.SetUpTool("mv", "MV", AtRunTime) shline := t.NewShellLine("Makefile", 85, "\t${RUN} ${SED} 's,#,// comment:,g' filename > filename.tmp; ${MV} filename.tmp filename") shline.CheckShellCommandLine(shline.mkline.ShellCommand()) @@ -816,7 +831,7 @@ func (s *Suite) Test_ShellLine_CheckShellCommandLine__subshell(c *check.C) { func (s *Suite) Test_ShellLine_CheckShellCommandLine__install_dir(c *check.C) { t := s.Init(c) - t.SetupVartypes() + t.SetUpVartypes() shline := t.NewShellLine("Makefile", 85, "\t${RUN} ${INSTALL_DATA_DIR} ${DESTDIR}${PREFIX}/dir1 ${DESTDIR}${PREFIX}/dir2") shline.CheckShellCommandLine(shline.mkline.ShellCommand()) @@ -843,7 +858,7 @@ func (s *Suite) Test_ShellLine_CheckShellCommandLine__install_dir(c *check.C) { func (s *Suite) Test_ShellLine_CheckShellCommandLine__install_option_d(c *check.C) { t := s.Init(c) - t.SetupVartypes() + t.SetUpVartypes() shline := t.NewShellLine("Makefile", 85, "\t${RUN} ${INSTALL} -d ${DESTDIR}${PREFIX}/dir1 ${DESTDIR}${PREFIX}/dir2") shline.CheckShellCommandLine(shline.mkline.ShellCommand()) @@ -856,7 +871,7 @@ func (s *Suite) Test_ShellLine_CheckShellCommandLine__install_option_d(c *check. func (s *Suite) Test_ShellLine__shell_comment_with_line_continuation(c *check.C) { t := s.Init(c) - mklines := t.SetupFileMkLines("Makefile", + mklines := t.SetUpFileMkLines("Makefile", MkRcsID, "pre-install:", "\t"+"# comment\\", @@ -871,8 +886,8 @@ func (s *Suite) Test_ShellLine__shell_comment_with_line_continuation(c *check.C) func (s *Suite) Test_ShellLine_checkWordQuoting(c *check.C) { t := s.Init(c) - t.SetupVartypes() - t.SetupTool("grep", "GREP", AtRunTime) + t.SetUpVartypes() + t.SetUpTool("grep", "GREP", AtRunTime) test := func(lineno int, input string) { shline := t.NewShellLine("module.mk", lineno, "\t"+input) @@ -979,7 +994,7 @@ func (s *Suite) Test_ShellLine_unescapeBackticks(c *check.C) { func (s *Suite) Test_ShellLine_unescapeBackticks__dquotBacktDquot(c *check.C) { t := s.Init(c) - t.SetupTool("echo", "", AtRunTime) + t.SetUpTool("echo", "", AtRunTime) mkline := t.NewMkLine("dummy.mk", 13, "\t var=\"`echo \"\"`\"") MkLineChecker{mkline}.Check() @@ -991,7 +1006,7 @@ func (s *Suite) Test_ShellLine_unescapeBackticks__dquotBacktDquot(c *check.C) { func (s *Suite) Test_ShellLine__variable_outside_quotes(c *check.C) { t := s.Init(c) - t.SetupVartypes() + t.SetUpVartypes() mklines := t.NewMkLines("dummy.mk", MkRcsID, "GZIP=\t${ECHO} $$comment") @@ -1007,8 +1022,8 @@ func (s *Suite) Test_ShellLine__variable_outside_quotes(c *check.C) { func (s *Suite) Test_ShellLine_CheckShellCommand__cd_inside_if(c *check.C) { t := s.Init(c) - t.SetupVartypes() - t.SetupTool("echo", "ECHO", AtRunTime) + t.SetUpVartypes() + t.SetUpTool("echo", "ECHO", AtRunTime) mklines := t.NewMkLines("Makefile", MkRcsID, "", @@ -1024,9 +1039,9 @@ func (s *Suite) Test_ShellLine_CheckShellCommand__cd_inside_if(c *check.C) { func (s *Suite) Test_ShellLine_CheckShellCommand__negated_pipe(c *check.C) { t := s.Init(c) - t.SetupVartypes() - t.SetupTool("echo", "ECHO", AtRunTime) - t.SetupTool("test", "TEST", AtRunTime) + t.SetUpVartypes() + t.SetUpTool("echo", "ECHO", AtRunTime) + t.SetUpTool("test", "TEST", AtRunTime) mklines := t.NewMkLines("Makefile", MkRcsID, "", @@ -1042,8 +1057,8 @@ func (s *Suite) Test_ShellLine_CheckShellCommand__negated_pipe(c *check.C) { func (s *Suite) Test_ShellLine_CheckShellCommand__subshell(c *check.C) { t := s.Init(c) - t.SetupTool("echo", "ECHO", AtRunTime) - t.SetupTool("expr", "EXPR", AtRunTime) + t.SetUpTool("echo", "ECHO", AtRunTime) + t.SetUpTool("expr", "EXPR", AtRunTime) mklines := t.NewMkLines("Makefile", MkRcsID, "", @@ -1070,7 +1085,7 @@ func (s *Suite) Test_ShellLine_CheckShellCommand__subshell(c *check.C) { func (s *Suite) Test_ShellLine_CheckShellCommand__case_patterns_from_variable(c *check.C) { t := s.Init(c) - t.SetupVartypes() + t.SetUpVartypes() mklines := t.NewMkLines("Makefile", MkRcsID, "", @@ -1088,8 +1103,8 @@ func (s *Suite) Test_ShellLine_CheckShellCommand__case_patterns_from_variable(c func (s *Suite) Test_ShellLine_checkHiddenAndSuppress(c *check.C) { t := s.Init(c) - t.SetupTool("echo", "ECHO", AtRunTime) - t.SetupTool("ls", "LS", AtRunTime) + t.SetUpTool("echo", "ECHO", AtRunTime) + t.SetUpTool("ls", "LS", AtRunTime) mklines := t.NewMkLines("Makefile", MkRcsID, "", @@ -1099,6 +1114,7 @@ func (s *Suite) Test_ShellLine_checkHiddenAndSuppress(c *check.C) { mklines.Check() + // No warning about the hidden ls since the target name starts with "show-". t.CheckOutputEmpty() } @@ -1108,23 +1124,20 @@ func (s *Suite) Test_SimpleCommandChecker_handleForbiddenCommand(c *check.C) { mklines := t.NewMkLines("Makefile", MkRcsID, "", - "\t${RUN} ktrace; mktexlsr; strace; texconfig; truss") + "\t${RUN} mktexlsr; texconfig") mklines.Check() t.CheckOutputLines( - "ERROR: Makefile:3: \"ktrace\" must not be used in Makefiles.", "ERROR: Makefile:3: \"mktexlsr\" must not be used in Makefiles.", - "ERROR: Makefile:3: \"strace\" must not be used in Makefiles.", - "ERROR: Makefile:3: \"texconfig\" must not be used in Makefiles.", - "ERROR: Makefile:3: \"truss\" must not be used in Makefiles.") + "ERROR: Makefile:3: \"texconfig\" must not be used in Makefiles.") } func (s *Suite) Test_SimpleCommandChecker_handleCommandVariable(c *check.C) { t := s.Init(c) - t.SetupTool("perl", "PERL5", AtRunTime) - t.SetupTool("perl6", "PERL6", Nowhere) + t.SetUpTool("perl", "PERL5", AtRunTime) + t.SetUpTool("perl6", "PERL6", Nowhere) mklines := t.NewMkLines("Makefile", MkRcsID, "", @@ -1148,11 +1161,11 @@ func (s *Suite) Test_SimpleCommandChecker_handleCommandVariable(c *check.C) { // .mk file includes the file defining the command. // FIXME: This paragraph sounds wrong. All commands from included files should be valid. // -// The variable must not be called *_CMD, or another code path is taken. +// The PYTHON_BIN variable below must not be called *_CMD, or another code path is taken. func (s *Suite) Test_SimpleCommandChecker_handleCommandVariable__from_package(c *check.C) { t := s.Init(c) - pkg := t.SetupPackage("category/package", + pkg := t.SetUpPackage("category/package", "post-install:", "\t${PYTHON_BIN}", "", @@ -1180,8 +1193,8 @@ func (s *Suite) Test_SimpleCommandChecker_handleComment(c *check.C) { func (s *Suite) Test_SimpleCommandChecker_checkPaxPe(c *check.C) { t := s.Init(c) - t.SetupVartypes() - t.SetupTool("pax", "PAX", AtRunTime) + t.SetUpVartypes() + t.SetUpTool("pax", "PAX", AtRunTime) mklines := t.NewMkLines("Makefile", MkRcsID, "", @@ -1199,8 +1212,8 @@ func (s *Suite) Test_SimpleCommandChecker_checkPaxPe(c *check.C) { func (s *Suite) Test_SimpleCommandChecker_checkEchoN(c *check.C) { t := s.Init(c) - t.SetupTool("echo", "ECHO", AtRunTime) - t.SetupTool("echo -n", "ECHO_N", AtRunTime) + t.SetUpTool("echo", "ECHO", AtRunTime) + t.SetUpTool("echo -n", "ECHO_N", AtRunTime) mklines := t.NewMkLines("Makefile", MkRcsID, "", @@ -1218,30 +1231,29 @@ func (s *Suite) Test_SimpleCommandChecker_checkEchoN(c *check.C) { func (s *Suite) Test_ShellProgramChecker_checkConditionalCd(c *check.C) { t := s.Init(c) - t.SetupTool("ls", "LS", AtRunTime) - t.SetupTool("printf", "PRINTF", AtRunTime) - t.SetupTool("tr", "TR", AtRunTime) + t.SetUpTool("ls", "LS", AtRunTime) + t.SetUpTool("printf", "PRINTF", AtRunTime) mklines := t.NewMkLines("Makefile", MkRcsID, "pre-configure:", "\t${RUN} while cd ..; do printf .; done", - // TODO: "\t${RUN} if ls | tr -d $$; then :; fi", - "\t${RUN} if ls | tr -d shell$$; then :; fi") + "\t${RUN} if cd ..; then printf .; fi", + "\t${RUN} ! cd ..", + "\t${RUN} if cd .. && cd ..; then printf .; fi") // For code coverage mklines.Check() - // FIXME: Fix the parse error. t.CheckOutputLines( "ERROR: Makefile:3: The Solaris /bin/sh cannot handle \"cd\" inside conditionals.", - "WARN: Internal pkglint error in ShTokenizer.ShAtom at \"$$\" (quoting=plain).", - "WARN: Makefile:4: The exitcode of \"ls\" at the left of the | operator is ignored.") + "ERROR: Makefile:4: The Solaris /bin/sh cannot handle \"cd\" inside conditionals.", + "WARN: Makefile:5: The Solaris /bin/sh does not support negation of shell commands.") } func (s *Suite) Test_SimpleCommandChecker_checkRegexReplace(c *check.C) { t := s.Init(c) - t.SetupTool("pax", "PAX", AtRunTime) - t.SetupTool("sed", "SED", AtRunTime) + t.SetUpTool("pax", "PAX", AtRunTime) + t.SetUpTool("sed", "SED", AtRunTime) mklines := t.NewMkLines("Makefile", MkRcsID, "pre-configure:", @@ -1258,6 +1270,8 @@ func (s *Suite) Test_SimpleCommandChecker_checkRegexReplace(c *check.C) { // FIXME: warn for "sed -e". // TODO: don't warn for "pax .orig". // TODO: don't warn for "s,a,b,g". + // TODO: Merge the code with BtSedCommands. + // TODO: Finally, remove the G.Testing from the main code. t.CheckOutputLines( "WARN: Makefile:3: Substitution commands like \"s,.*,,\" should always be quoted.", "WARN: Makefile:5: Substitution commands like \"s,.*,,\" should always be quoted.") @@ -1267,9 +1281,9 @@ func (s *Suite) Test_SimpleCommandChecker_checkRegexReplace(c *check.C) { func (s *Suite) Test_ShellProgramChecker_checkSetE__simple_commands(c *check.C) { t := s.Init(c) - t.SetupTool("echo", "", AtRunTime) - t.SetupTool("rm", "", AtRunTime) - t.SetupTool("touch", "", AtRunTime) + t.SetUpTool("echo", "", AtRunTime) + t.SetUpTool("rm", "", AtRunTime) + t.SetUpTool("touch", "", AtRunTime) mklines := t.NewMkLines("Makefile", MkRcsID, "pre-configure:", @@ -1280,37 +1294,42 @@ func (s *Suite) Test_ShellProgramChecker_checkSetE__simple_commands(c *check.C) mklines.Check() t.CheckOutputLines( - "WARN: Makefile:4: Please switch to \"set -e\" mode before using a semicolon (after \"touch file\") to separate commands.") + "WARN: Makefile:4: Please switch to \"set -e\" mode before using a semicolon " + + "(after \"touch file\") to separate commands.") } func (s *Suite) Test_ShellProgramChecker_checkSetE__compound_commands(c *check.C) { t := s.Init(c) - t.SetupTool("echo", "", AtRunTime) - t.SetupTool("touch", "", AtRunTime) + t.SetUpTool("echo", "", AtRunTime) + t.SetUpTool("touch", "", AtRunTime) mklines := t.NewMkLines("Makefile", MkRcsID, "pre-configure:", "\ttouch file; for f in file; do echo \"$$f\"; done", - "\tfor f in file; do echo \"$$f\"; done; touch file") + "\tfor f in file; do echo \"$$f\"; done; touch file", + "\ttouch 1; touch 2; touch 3; touch 4") mklines.Check() t.CheckOutputLines( - "WARN: Makefile:3: Please switch to \"set -e\" mode before using a semicolon (after \"touch file\") to separate commands.") + "WARN: Makefile:3: Please switch to \"set -e\" mode before using a semicolon "+ + "(after \"touch file\") to separate commands.", + "WARN: Makefile:5: Please switch to \"set -e\" mode before using a semicolon "+ + "(after \"touch 1\") to separate commands.") } func (s *Suite) Test_ShellProgramChecker_canFail(c *check.C) { t := s.Init(c) - t.SetupVartypes() - t.SetupTool("echo", "", AtRunTime) - t.SetupTool("env", "", AtRunTime) - t.SetupTool("grep", "GREP", AtRunTime) - t.SetupTool("sed", "", AtRunTime) - t.SetupTool("touch", "", AtRunTime) - t.SetupTool("tr", "tr", AtRunTime) - t.SetupTool("true", "TRUE", AtRunTime) + t.SetUpVartypes() + t.SetUpTool("echo", "", AtRunTime) + t.SetUpTool("env", "", AtRunTime) + t.SetUpTool("grep", "GREP", AtRunTime) + t.SetUpTool("sed", "", AtRunTime) + t.SetUpTool("touch", "", AtRunTime) + t.SetUpTool("tr", "tr", AtRunTime) + t.SetUpTool("true", "TRUE", AtRunTime) mklines := t.NewMkLines("Makefile", MkRcsID, "pre-configure:", @@ -1333,9 +1352,14 @@ func (s *Suite) Test_ShellProgramChecker_canFail(c *check.C) { mklines.Check() t.CheckOutputLines( - "WARN: Makefile:3: Please switch to \"set -e\" mode before using a semicolon (after \"socklen=`${GREP} 'expr' ${WRKSRC}/config.h`\") to separate commands.", - "WARN: Makefile:6: Please switch to \"set -e\" mode before using a semicolon (after \"${FAIL_MSG} \\\"Failure\\\"\") to separate commands.", - "WARN: Makefile:7: Please switch to \"set -e\" mode before using a semicolon (after \"set -x\") to separate commands.", - "WARN: Makefile:12: Please switch to \"set -e\" mode before using a semicolon (after \"touch file\") to separate commands.", - "WARN: Makefile:14: Please switch to \"set -e\" mode before using a semicolon (after \"echo 'logging'\") to separate commands.") + "WARN: Makefile:3: Please switch to \"set -e\" mode before using a semicolon "+ + "(after \"socklen=`${GREP} 'expr' ${WRKSRC}/config.h`\") to separate commands.", + "WARN: Makefile:6: Please switch to \"set -e\" mode before using a semicolon "+ + "(after \"${FAIL_MSG} \\\"Failure\\\"\") to separate commands.", + "WARN: Makefile:7: Please switch to \"set -e\" mode before using a semicolon "+ + "(after \"set -x\") to separate commands.", + "WARN: Makefile:12: Please switch to \"set -e\" mode before using a semicolon "+ + "(after \"touch file\") to separate commands.", + "WARN: Makefile:14: Please switch to \"set -e\" mode before using a semicolon "+ + "(after \"echo 'logging'\") to separate commands.") } diff --git a/pkgtools/pkglint/files/shtokenizer.go b/pkgtools/pkglint/files/shtokenizer.go index 7d6422e7111..6b6a161f7e6 100644 --- a/pkgtools/pkglint/files/shtokenizer.go +++ b/pkgtools/pkglint/files/shtokenizer.go @@ -3,14 +3,13 @@ package pkglint import "netbsd.org/pkglint/textproc" type ShTokenizer struct { - parser *Parser - mkp *MkParser + parser *MkParser } func NewShTokenizer(line Line, text string, emitWarnings bool) *ShTokenizer { - p := NewParser(line, text, emitWarnings) - mkp := MkParser{p} - return &ShTokenizer{p, &mkp} + // TODO: Switching to NewMkParser is nontrivial since emitWarnings must equal (line != nil). + p := MkParser{line, textproc.NewLexer(text), emitWarnings} + return &ShTokenizer{&p} } // ShAtom parses a basic building block of a shell program. @@ -26,10 +25,12 @@ func (p *ShTokenizer) ShAtom(quoting ShQuoting) *ShAtom { lexer := p.parser.lexer mark := lexer.Mark() - if varuse := p.mkp.VarUse(); varuse != nil { + if varuse := p.parser.VarUse(); varuse != nil { return &ShAtom{shtVaruse, lexer.Since(mark), quoting, varuse} } + // TODO: Most probably there is a more elegant way than the large switch block below. + var atom *ShAtom switch quoting { case shqPlain: @@ -60,9 +61,12 @@ func (p *ShTokenizer) ShAtom(quoting ShQuoting) *ShAtom { if atom == nil { lexer.Reset(mark) - if hasPrefix(lexer.Rest(), "${") { + switch { + case hasPrefix(lexer.Rest(), "${"): p.parser.Line.Warnf("Unclosed Make variable starting at %q.", shorten(lexer.Rest(), 20)) - } else { + case hasPrefix(lexer.Rest(), "$${"): + p.parser.Line.Warnf("Unclosed shell variable starting at %q.", shorten(lexer.Rest(), 20)) + default: p.parser.Line.Warnf("Internal pkglint error in ShTokenizer.ShAtom at %q (quoting=%s).", lexer.Rest(), quoting) } } @@ -295,6 +299,10 @@ loop: break case matches(lexer.Rest(), `^\$\$[^!#(*\-0-9?@A-Z_a-z{]`): lexer.NextString("$$") + case lexer.Rest() == "$$": + lexer.Skip(2) + case lexer.Rest() == "$": + lexer.Skip(1) default: break loop } @@ -354,7 +362,7 @@ func (p *ShTokenizer) shOperator(q ShQuoting) *ShAtom { case lexer.SkipString("||"), lexer.SkipString("&&"), lexer.SkipString(";;"), - lexer.SkipByte('\n'), + lexer.NextBytesFunc(func(b byte) bool { return b == '\n' }) != "", lexer.SkipByte(';'), lexer.SkipByte('('), lexer.SkipByte(')'), @@ -413,15 +421,17 @@ func (p *ShTokenizer) ShToken() *ShToken { return NewShToken(atom.MkText, atom) } -nextAtom: - mark := lexer.Mark() - atom := peek() - if atom != nil && (atom.Type.IsWord() || atom.Quoting != shqPlain) { - skip() - atoms = append(atoms, atom) - goto nextAtom + for { + mark := lexer.Mark() + atom := peek() + if atom != nil && (atom.Type.IsWord() || atom.Quoting != shqPlain) { + skip() + atoms = append(atoms, atom) + continue + } + lexer.Reset(mark) + break } - lexer.Reset(mark) G.Assertf(len(atoms) > 0, "ShTokenizer.ShToken") return NewShToken(lexer.Since(initialMark), atoms...) diff --git a/pkgtools/pkglint/files/shtokenizer_test.go b/pkgtools/pkglint/files/shtokenizer_test.go index 2460345b2b2..5dd9ec5ea16 100644 --- a/pkgtools/pkglint/files/shtokenizer_test.go +++ b/pkgtools/pkglint/files/shtokenizer_test.go @@ -8,9 +8,9 @@ import ( func (s *Suite) Test_ShTokenizer_ShAtom(c *check.C) { t := s.Init(c) - // checkRest ensures that the given string is parsed to the expected + // testRest ensures that the given string is parsed to the expected // atoms, and returns the remaining text. - checkRest := func(s string, expected ...*ShAtom) string { + testRest := func(s string, expected ...*ShAtom) string { p := NewShTokenizer(dummyLine, s, false) q := shqPlain for _, exp := range expected { @@ -23,7 +23,7 @@ func (s *Suite) Test_ShTokenizer_ShAtom(c *check.C) { // test ensures that the given string is parsed to the expected // atoms, and that the text is completely consumed by the parser. test := func(str string, expected ...*ShAtom) { - rest := checkRest(str, expected...) + rest := testRest(str, expected...) c.Check(rest, equals, "") t.CheckOutputEmpty() } @@ -69,7 +69,7 @@ func (s *Suite) Test_ShTokenizer_ShAtom(c *check.C) { // Ignore unused functions; useful for deleting some of the tests during debugging. use := func(args ...interface{}) {} - use(checkRest, test) + use(testRest, test) use(operator, comment, mkvar, text, whitespace) use(space, semicolon, pipe, subshell) use(backt, dquot, squot, subsh) @@ -94,7 +94,7 @@ func (s *Suite) Test_ShTokenizer_ShAtom(c *check.C) { squot(text("single-quoted")), text("'")) - rest := checkRest("\"" /* none */) + rest := testRest("\"" /* none */) c.Check(rest, equals, "\"") test("$${file%.c}.o", @@ -279,7 +279,7 @@ func (s *Suite) Test_ShTokenizer_ShAtom(c *check.C) { semicolon, shvar("$$-", "-")) - rest = checkRest("COMMENT=\t\\Make $$$$ fast\"", + rest = testRest("COMMENT=\t\\Make $$$$ fast\"", text("COMMENT="), whitespace("\t"), text("\\Make"), @@ -392,7 +392,7 @@ func (s *Suite) Test_ShTokenizer_ShAtom(c *check.C) { } func (s *Suite) Test_ShTokenizer_ShAtom__quoting(c *check.C) { - checkQuotingChange := func(input, expectedOutput string) { + test := func(input, expectedOutput string) { p := NewShTokenizer(dummyLine, input, false) q := shqPlain result := "" @@ -411,18 +411,18 @@ func (s *Suite) Test_ShTokenizer_ShAtom__quoting(c *check.C) { c.Check(p.Rest(), equals, "") } - checkQuotingChange("hello, world", "hello, world") - checkQuotingChange("hello, \"world\"", "hello, \"[d]world\"[plain]") - checkQuotingChange("1 \"\" 2 '' 3 `` 4", "1 \"[d]\"[plain] 2 '[s]'[plain] 3 `[b]`[plain] 4") - checkQuotingChange("\"\"", "\"[d]\"[plain]") - checkQuotingChange("''", "'[s]'[plain]") - checkQuotingChange("``", "`[b]`[plain]") - checkQuotingChange("x\"x`x`x\"x'x\"x'", "x\"[d]x`[db]x`[d]x\"[plain]x'[s]x\"x'[plain]") - checkQuotingChange("x\"x`x'x'x`x\"", "x\"[d]x`[db]x'[dbs]x'[db]x`[d]x\"[plain]") - checkQuotingChange("x\\\"x\\'x\\`x\\\\", "x\\\"x\\'x\\`x\\\\") - checkQuotingChange("x\"x\\\"x\\'x\\`x\\\\", "x\"[d]x\\\"x\\'x\\`x\\\\") - checkQuotingChange("x'x\\\"x\\'x\\`x\\\\", "x'[s]x\\\"x\\'[plain]x\\`x\\\\") - checkQuotingChange("x`x\\\"x\\'x\\`x\\\\", "x`[b]x\\\"x\\'x\\`x\\\\") + test("hello, world", "hello, world") + test("hello, \"world\"", "hello, \"[d]world\"[plain]") + test("1 \"\" 2 '' 3 `` 4", "1 \"[d]\"[plain] 2 '[s]'[plain] 3 `[b]`[plain] 4") + test("\"\"", "\"[d]\"[plain]") + test("''", "'[s]'[plain]") + test("``", "`[b]`[plain]") + test("x\"x`x`x\"x'x\"x'", "x\"[d]x`[db]x`[d]x\"[plain]x'[s]x\"x'[plain]") + test("x\"x`x'x'x`x\"", "x\"[d]x`[db]x'[dbs]x'[db]x`[d]x\"[plain]") + test("x\\\"x\\'x\\`x\\\\", "x\\\"x\\'x\\`x\\\\") + test("x\"x\\\"x\\'x\\`x\\\\", "x\"[d]x\\\"x\\'x\\`x\\\\") + test("x'x\\\"x\\'x\\`x\\\\", "x'[s]x\\\"x\\'[plain]x\\`x\\\\") + test("x`x\\\"x\\'x\\`x\\\\", "x`[b]x\\\"x\\'x\\`x\\\\") } func (s *Suite) Test_ShTokenizer_ShToken(c *check.C) { @@ -437,6 +437,7 @@ func (s *Suite) Test_ShTokenizer_ShToken(c *check.C) { } return p.Rest() } + test := func(str string, expected ...string) { p := NewShTokenizer(dummyLine, str, false) for _, exp := range expected { @@ -445,20 +446,19 @@ func (s *Suite) Test_ShTokenizer_ShToken(c *check.C) { c.Check(p.Rest(), equals, "") t.CheckOutputEmpty() } - checkNil := func(str string) { + + testNil := func(str string) { p := NewShTokenizer(dummyLine, str, false) c.Check(p.ShToken(), check.IsNil) c.Check(p.Rest(), equals, "") t.CheckOutputEmpty() } - checkNil("") - checkNil(" ") + testNil("") + testNil(" ") rest := testRest("\t\t\t\n\n\n\n\t ", - "\n", - "\n", // TODO: Why three separators? One should be enough. What does the grammar say? - "\n") - c.Check(rest, equals, "\n\t ") // TODO: Why is the newline still here? + "\n\n\n\n") + c.Check(rest, equals, "\t ") test("echo", "echo") @@ -545,6 +545,10 @@ func (s *Suite) Test_ShTokenizer_shVarUse(c *check.C) { test("$${\\}", nil, "$${\\}") } +// This test demonstrates that the shell tokenizer is not perfect yet. +// There are still some corner cases that trigger a parse error. +// To get 100% code coverage, they have been found using the fuzzer +// and trimmed down to minimal examples. func (s *Suite) Test_ShTokenizer__examples_from_fuzzing(c *check.C) { t := s.Init(c) @@ -589,20 +593,20 @@ func (s *Suite) Test_ShTokenizer__examples_from_fuzzing(c *check.C) { // Just good that these redundant error messages don't occur every day. t.CheckOutputLines( "WARN: fuzzing.mk:4: Internal pkglint error in ShTokenizer.ShAtom at \"`\" (quoting=bd).", - "WARN: fuzzing.mk:4: Pkglint ShellLine.CheckShellCommand: parse error at []string{\"\"}", + "WARN: fuzzing.mk:4: Pkglint ShellLine.CheckShellCommand: splitIntoShellTokens couldn't parse \"`\\\"`\"", "WARN: fuzzing.mk:5: Internal pkglint error in ShTokenizer.ShAtom at \"$`\" (quoting=bs).", - "WARN: fuzzing.mk:5: Pkglint ShellLine.CheckShellCommand: parse error at []string{\"\"}", + "WARN: fuzzing.mk:5: Pkglint ShellLine.CheckShellCommand: splitIntoShellTokens couldn't parse \"`'$`\"", "WARN: fuzzing.mk:5: Internal pkglint error in MkLine.Tokenize at \"$`\".", - "WARN: fuzzing.mk:6: Pkglint ShellLine.CheckShellCommand: parse error at []string{\"\"}", + "WARN: fuzzing.mk:6: Pkglint ShellLine.CheckShellCommand: splitIntoShellTokens couldn't parse \"\\\"`'`y\"", "WARN: fuzzing.mk:7: Internal pkglint error in ShTokenizer.ShAtom at \"$|\" (quoting=db).", - "WARN: fuzzing.mk:7: Pkglint ShellLine.CheckShellCommand: parse error at []string{\"\"}", + "WARN: fuzzing.mk:7: Pkglint ShellLine.CheckShellCommand: splitIntoShellTokens couldn't parse \"\\\"`$|\"", "WARN: fuzzing.mk:7: Internal pkglint error in MkLine.Tokenize at \"$|\".", "WARN: fuzzing.mk:8: Internal pkglint error in ShTokenizer.ShAtom at \"`\" (quoting=dbd).", - "WARN: fuzzing.mk:8: Pkglint ShellLine.CheckShellCommand: parse error at []string{\"\"}", + "WARN: fuzzing.mk:8: Pkglint ShellLine.CheckShellCommand: splitIntoShellTokens couldn't parse \"\\\"`\\\"`\"", "WARN: fuzzing.mk:9: Invoking subshells via $(...) is not portable enough.", @@ -613,9 +617,14 @@ func (s *Suite) Test_ShTokenizer__examples_from_fuzzing(c *check.C) { "WARN: fuzzing.mk:11: Invoking subshells via $(...) is not portable enough.", "WARN: fuzzing.mk:11: Internal pkglint error in MkLine.Tokenize at \"$)\".", - "WARN: fuzzing.mk:12: Pkglint ShellLine.CheckShellCommand: parse error at []string{\"\"}") + "WARN: fuzzing.mk:12: Pkglint ShellLine.CheckShellCommand: splitIntoShellTokens couldn't parse \"\\\"`# comment\"") } +// In order to get 100% code coverage for the shell tokenizer, a panic() statement has been +// added to each uncovered basic block. After that, this fuzzer quickly found relatively +// small example programs that led to the uncovered code. +// +// This test is not useful as-is. func (s *Suite) Test_ShTokenizer__fuzzing(c *check.C) { t := s.Init(c) @@ -627,7 +636,7 @@ func (s *Suite) Test_ShTokenizer__fuzzing(c *check.C) { for i := 0; i < 1000; i++ { tokenizer := NewShTokenizer(dummyLine, fuzzer.Generate(50), false) tokenizer.ShAtoms() - t.Output() // Discard the output, only react on fatal errors. + t.Output() // Discard the output, only react on panics. } fuzzer.Ok() } diff --git a/pkgtools/pkglint/files/shtypes.go b/pkgtools/pkglint/files/shtypes.go index 59066d2381e..ea73dd44805 100644 --- a/pkgtools/pkglint/files/shtypes.go +++ b/pkgtools/pkglint/files/shtypes.go @@ -88,6 +88,9 @@ const ( shqDquotBacktSquot // e.g. "`'word'`" ) +// String returns a very short identifier for the quoting state. +// In this, d means double quotes, s means single quotes, +// b means backticks and S means subshell. func (q ShQuoting) String() string { return [...]string{ "plain", diff --git a/pkgtools/pkglint/files/substcontext.go b/pkgtools/pkglint/files/substcontext.go index b69c7f673a7..6faba4b7c1e 100644 --- a/pkgtools/pkglint/files/substcontext.go +++ b/pkgtools/pkglint/files/substcontext.go @@ -46,7 +46,7 @@ func (st *SubstContextStats) Or(other SubstContextStats) { func (ctx *SubstContext) Varassign(mkline MkLine) { if trace.Tracing { - trace.Stepf("SubstContext.Varassign %#v %v#", ctx.curr, ctx.inAllBranches) + trace.Stepf("SubstContext.Varassign curr=%v all=%v", ctx.curr, ctx.inAllBranches) } varname := mkline.Varname() @@ -176,7 +176,7 @@ func (ctx *SubstContext) Directive(mkline MkLine) { } if trace.Tracing { - trace.Stepf("+ SubstContext.Directive %#v %v#", ctx.curr, ctx.inAllBranches) + trace.Stepf("+ SubstContext.Directive %v %v", ctx.curr, ctx.inAllBranches) } dir := mkline.Directive() if dir == "if" { @@ -195,7 +195,7 @@ func (ctx *SubstContext) Directive(mkline MkLine) { ctx.curr.Or(ctx.inAllBranches) } if trace.Tracing { - trace.Stepf("- SubstContext.Directive %#v %v#", ctx.curr, ctx.inAllBranches) + trace.Stepf("- SubstContext.Directive %v %v", ctx.curr, ctx.inAllBranches) } } diff --git a/pkgtools/pkglint/files/substcontext_test.go b/pkgtools/pkglint/files/substcontext_test.go index a3ae47c9b0e..4f55c2a69eb 100644 --- a/pkgtools/pkglint/files/substcontext_test.go +++ b/pkgtools/pkglint/files/substcontext_test.go @@ -8,7 +8,7 @@ import ( func (s *Suite) Test_SubstContext__incomplete(c *check.C) { t := s.Init(c) - t.SetupCommandLine("-Wextra") + t.SetUpCommandLine("-Wextra") ctx := NewSubstContext() ctx.Varassign(newSubstLine(t, 10, "PKGNAME=pkgname-1.0")) @@ -38,7 +38,7 @@ func (s *Suite) Test_SubstContext__incomplete(c *check.C) { func (s *Suite) Test_SubstContext__complete(c *check.C) { t := s.Init(c) - t.SetupCommandLine("-Wextra") + t.SetUpCommandLine("-Wextra") ctx := NewSubstContext() ctx.Varassign(newSubstLine(t, 10, "PKGNAME=pkgname-1.0")) @@ -65,6 +65,7 @@ func (s *Suite) Test_SubstContext__OPSYSVARS(c *check.C) { G.Opts.WarnExtra = true ctx := NewSubstContext() + // SUBST_CLASSES is added to OPSYSVARS in mk/bsd.pkg.mk. ctx.Varassign(newSubstLine(t, 11, "SUBST_CLASSES.SunOS+=prefix")) ctx.Varassign(newSubstLine(t, 12, "SUBST_CLASSES.NetBSD+=prefix")) ctx.Varassign(newSubstLine(t, 13, "SUBST_FILES.prefix=Makefile")) @@ -83,7 +84,7 @@ func (s *Suite) Test_SubstContext__OPSYSVARS(c *check.C) { func (s *Suite) Test_SubstContext__no_class(c *check.C) { t := s.Init(c) - t.SetupCommandLine("-Wextra") + t.SetUpCommandLine("-Wextra") ctx := NewSubstContext() ctx.Varassign(newSubstLine(t, 10, "UNRELATED=anything")) @@ -99,7 +100,7 @@ func (s *Suite) Test_SubstContext__no_class(c *check.C) { func (s *Suite) Test_SubstContext__multiple_classes_in_one_line(c *check.C) { t := s.Init(c) - t.SetupCommandLine("-Wextra") + t.SetUpCommandLine("-Wextra") simulateSubstLines(t, "10: SUBST_CLASSES+= one two", @@ -118,7 +119,7 @@ func (s *Suite) Test_SubstContext__multiple_classes_in_one_line(c *check.C) { func (s *Suite) Test_SubstContext__multiple_classes_in_one_block(c *check.C) { t := s.Init(c) - t.SetupCommandLine("-Wextra") + t.SetUpCommandLine("-Wextra") simulateSubstLines(t, "10: SUBST_CLASSES+= one", @@ -142,7 +143,7 @@ func (s *Suite) Test_SubstContext__multiple_classes_in_one_block(c *check.C) { func (s *Suite) Test_SubstContext__directives(c *check.C) { t := s.Init(c) - t.SetupCommandLine("-Wextra") + t.SetUpCommandLine("-Wextra") simulateSubstLines(t, "10: SUBST_CLASSES+= os", @@ -171,7 +172,7 @@ func (s *Suite) Test_SubstContext__directives(c *check.C) { func (s *Suite) Test_SubstContext__missing_transformation_in_one_branch(c *check.C) { t := s.Init(c) - t.SetupCommandLine("-Wextra") + t.SetUpCommandLine("-Wextra") simulateSubstLines(t, "10: SUBST_CLASSES+= os", @@ -197,7 +198,7 @@ func (s *Suite) Test_SubstContext__missing_transformation_in_one_branch(c *check func (s *Suite) Test_SubstContext__nested_conditionals(c *check.C) { t := s.Init(c) - t.SetupCommandLine("-Wextra") + t.SetUpCommandLine("-Wextra") simulateSubstLines(t, "10: SUBST_CLASSES+= os", @@ -225,8 +226,8 @@ func (s *Suite) Test_SubstContext__nested_conditionals(c *check.C) { func (s *Suite) Test_SubstContext__post_patch(c *check.C) { t := s.Init(c) - t.SetupCommandLine("-Wextra,no-space", "--show-autofix") - t.SetupVartypes() + t.SetUpCommandLine("-Wextra,no-space", "--show-autofix") + t.SetUpVartypes() mklines := t.NewMkLines("os.mk", MkRcsID, @@ -246,8 +247,8 @@ func (s *Suite) Test_SubstContext__post_patch(c *check.C) { func (s *Suite) Test_SubstContext__pre_configure_with_NO_CONFIGURE(c *check.C) { t := s.Init(c) - t.SetupCommandLine("-Wall,no-space") - pkg := t.SetupPackage("category/package", + t.SetUpCommandLine("-Wall,no-space") + pkg := t.SetUpPackage("category/package", "SUBST_CLASSES+= os", "SUBST_STAGE.os= pre-configure", "SUBST_FILES.os= guess-os.h", @@ -264,8 +265,8 @@ func (s *Suite) Test_SubstContext__pre_configure_with_NO_CONFIGURE(c *check.C) { func (s *Suite) Test_SubstContext__adjacent(c *check.C) { t := s.Init(c) - t.SetupCommandLine("-Wextra") - t.SetupVartypes() + t.SetUpCommandLine("-Wextra") + t.SetUpVartypes() mklines := t.NewMkLines("os.mk", MkRcsID, @@ -289,8 +290,8 @@ func (s *Suite) Test_SubstContext__adjacent(c *check.C) { func (s *Suite) Test_SubstContext__do_patch(c *check.C) { t := s.Init(c) - t.SetupCommandLine("-Wextra,no-space") - t.SetupVartypes() + t.SetUpCommandLine("-Wextra,no-space") + t.SetUpVartypes() mklines := t.NewMkLines("os.mk", MkRcsID, @@ -312,8 +313,8 @@ func (s *Suite) Test_SubstContext__do_patch(c *check.C) { func (s *Suite) Test_SubstContext__SUBST_VARS_defined_in_block(c *check.C) { t := s.Init(c) - t.SetupCommandLine("-Wextra,no-space") - t.SetupVartypes() + t.SetUpCommandLine("-Wextra,no-space") + t.SetUpVartypes() mklines := t.NewMkLines("os.mk", MkRcsID, @@ -337,8 +338,8 @@ func (s *Suite) Test_SubstContext__SUBST_VARS_defined_in_block(c *check.C) { func (s *Suite) Test_SubstContext__SUBST_VARS_in_next_paragraph(c *check.C) { t := s.Init(c) - t.SetupCommandLine("-Wextra,no-space") - t.SetupVartypes() + t.SetUpCommandLine("-Wextra,no-space") + t.SetUpVartypes() mklines := t.NewMkLines("os.mk", MkRcsID, @@ -360,8 +361,8 @@ func (s *Suite) Test_SubstContext__SUBST_VARS_in_next_paragraph(c *check.C) { func (s *Suite) Test_SubstContext_suggestSubstVars(c *check.C) { t := s.Init(c) - t.SetupVartypes() - t.SetupTool("sh", "SH", AtRunTime) + t.SetUpVartypes() + t.SetUpTool("sh", "SH", AtRunTime) mklines := t.NewMkLines("subst.mk", MkRcsID, @@ -379,6 +380,8 @@ func (s *Suite) Test_SubstContext_suggestSubstVars(c *check.C) { "SUBST_SED.test+=\ts,'@SH@','${SH}',", // Can be replaced, even when the -e is missing. "SUBST_SED.test+=\t-e s,@SH@,${PKGNAME},", // Cannot be replaced since the variable name differs. "SUBST_SED.test+=\t-e s,@SH@,'\"'${SH:Q}'\"',g", // Cannot be replaced since the double quotes are added. + "SUBST_SED.test+=\t-e s", // Just to get 100% code coverage. + "SUBST_SED.test+=\t-e s,@SH@,${SH:Q}", // Just to get 100% code coverage. "# end") mklines.Check() @@ -403,7 +406,7 @@ func simulateSubstLines(t *Tester, texts ...string) { for _, lineText := range texts { var lineno int _, err := fmt.Sscanf(lineText[0:4], "%d: ", &lineno) - G.Assertf(err == nil, "%s", err) + G.AssertNil(err, "") text := lineText[4:] line := newSubstLine(t, lineno, text) diff --git a/pkgtools/pkglint/files/tools.go b/pkgtools/pkglint/files/tools.go index 716a0e7b47f..2bc2a382a45 100644 --- a/pkgtools/pkglint/files/tools.go +++ b/pkgtools/pkglint/files/tools.go @@ -10,14 +10,27 @@ import ( // pkgsrc. // // See `mk/tools/`. -// -// TODO: MustUseVarForm does not really depend on the tool but only depends -// on where the tool is used (load time, run time). This had already been -// modeled wrong in pkglint 4, more than 10 years ago. type Tool struct { - Name string // e.g. "sed", "gzip" - Varname string // e.g. "SED", "GZIP_CMD" - MustUseVarForm bool // True for `echo`, because of many differing implementations. + Name string // e.g. "sed", "gzip" + Varname string // e.g. "SED", "GZIP_CMD" + + // Some of the very simple tools (echo, printf, test) differ in their implementations. + // + // When bmake encounters a "simple" command line, it bypasses the + // call to a shell (see devel/bmake/files/compat.c:/useShell/). + // Therefore, sometimes the shell builtin is run, and sometimes the + // native tool. + // + // In particular, this decision depends on PKG_DEBUG_LEVEL + // since that variable adds a semicolon to the command line, which is + // considered one of the characters that force the commands being + // executed by the shell. As of December 2018, the list of special characters + // is "~#=|^(){};&<>*?[]:$`\\\n". + // + // To work around this tricky situation, pkglint warns when these shell builtins + // are used by their simple names (echo, test) instead of the variable form + // (${ECHO}, ${TEST}). + MustUseVarForm bool Validity Validity } @@ -60,8 +73,8 @@ func (tool *Tool) UsableAtLoadTime(seenPrefs bool) bool { // VAR= ${${TOOL}:sh} # Probably ok; the :sh modifier is evaluated at // # run time. But if VAR should ever be evaluated // # at load time (see the "Not allowed" cases -// # above), it doesn't work. Currently pkglint -// # cannot detect these cases reliably. +// # above), it doesn't work. As of January 2019, +// # pkglint cannot reliably distinguish these cases. // // own-target: // ${TOOL} # Allowed. @@ -79,7 +92,6 @@ func (tool *Tool) UsableAtRunTime() bool { // and remembers whether these tools are defined at all, // and whether they are declared to be used via USE_TOOLS. type Tools struct { - TraceName string // Only for the trace log byName map[string]*Tool // "sed" => tool byVarname map[string]*Tool // "GREP_CMD" => tool fallback *Tools @@ -95,9 +107,8 @@ type Tools struct { SeenPrefs bool } -func NewTools(traceName string) *Tools { +func NewTools() *Tools { return &Tools{ - traceName, make(map[string]*Tool), make(map[string]*Tool), nil, @@ -112,7 +123,7 @@ func NewTools(traceName string) *Tools { // (e.g. "awk") or by its variable (e.g. ${AWK}). func (tr *Tools) Define(name, varname string, mkline MkLine) *Tool { if trace.Tracing { - trace.Stepf("Tools.Define for %s: %q %q in %s", tr.TraceName, name, varname, mkline) + trace.Stepf("Tools.Define: %q %q in %s", name, varname, mkline) } if !tr.IsValidToolName(name) { @@ -163,7 +174,7 @@ func (tr *Tools) merge(target, source *Tool) { func (tr *Tools) Trace() { if trace.Tracing { - defer trace.Call1(tr.TraceName)() + defer trace.Call0()() } else { return } @@ -280,10 +291,10 @@ func (tr *Tools) validity(basename string, useTools bool) Validity { } } -func (tr *Tools) ByVarname(varname string) *Tool { - tool := tr.byVarname[varname] +func (tr *Tools) ByName(name string) *Tool { + tool := tr.byName[name] if tool == nil && tr.fallback != nil { - fallback := tr.fallback.ByVarname(varname) + fallback := tr.fallback.ByName(name) if fallback != nil { return tr.def(fallback.Name, fallback.Varname, fallback.MustUseVarForm, fallback.Validity) } @@ -291,10 +302,10 @@ func (tr *Tools) ByVarname(varname string) *Tool { return tool } -func (tr *Tools) ByName(name string) *Tool { - tool := tr.byName[name] +func (tr *Tools) ByVarname(varname string) *Tool { + tool := tr.byVarname[varname] if tool == nil && tr.fallback != nil { - fallback := tr.fallback.ByName(name) + fallback := tr.fallback.ByVarname(varname) if fallback != nil { return tr.def(fallback.Name, fallback.Varname, fallback.MustUseVarForm, fallback.Validity) } @@ -302,6 +313,8 @@ func (tr *Tools) ByName(name string) *Tool { return tool } +// TODO: Tools.ByCommand (name or ${VARNAME}) + func (tr *Tools) Usable(tool *Tool, time ToolTime) bool { if time == LoadTime { return tool.UsableAtLoadTime(tr.SeenPrefs) diff --git a/pkgtools/pkglint/files/tools_test.go b/pkgtools/pkglint/files/tools_test.go index 101b9b05a46..4ce248090a7 100644 --- a/pkgtools/pkglint/files/tools_test.go +++ b/pkgtools/pkglint/files/tools_test.go @@ -37,8 +37,8 @@ func (s *Suite) Test_Tool_UsableAtRunTime(c *check.C) { func (s *Suite) Test_Tools_ParseToolLine(c *check.C) { t := s.Init(c) - t.SetupTool("tool1", "", Nowhere) - t.SetupVartypes() + t.SetUpTool("tool1", "", Nowhere) + t.SetUpVartypes() t.CreateFileLines("Makefile", MkRcsID, "", @@ -54,7 +54,7 @@ func (s *Suite) Test_Tools_Define__invalid_tool_name(c *check.C) { t := s.Init(c) mkline := t.NewMkLine("dummy.mk", 123, "DUMMY=\tvalue") - reg := NewTools("") + reg := NewTools() reg.Define("tool_name", "", mkline) reg.Define("tool:dependency", "", mkline) @@ -73,7 +73,7 @@ func (s *Suite) Test_Tools_Trace__coverage(c *check.C) { t.DisableTracing() - reg := NewTools("") + reg := NewTools() reg.Trace() t.CheckOutputEmpty() @@ -82,7 +82,7 @@ func (s *Suite) Test_Tools_Trace__coverage(c *check.C) { func (s *Suite) Test_Tools__USE_TOOLS_predefined_sed(c *check.C) { t := s.Init(c) - t.SetupPkgsrc() + t.SetUpPkgsrc() t.CreateFileLines("mk/bsd.prefs.mk", "USE_TOOLS+=\tsed:pkgsrc") t.CreateFileLines("mk/tools/defaults.mk", @@ -112,7 +112,7 @@ func (s *Suite) Test_Tools__add_varname_later(c *check.C) { t := s.Init(c) mkline := t.NewMkLine("dummy.mk", 123, "DUMMY=\tvalue") - tools := NewTools("") + tools := NewTools() tool := tools.Define("tool", "", mkline) c.Check(tool.Name, equals, "tool") @@ -128,7 +128,7 @@ func (s *Suite) Test_Tools__add_varname_later(c *check.C) { func (s *Suite) Test_Tools__load_from_infrastructure(c *check.C) { t := s.Init(c) - tools := NewTools("") + tools := NewTools() tools.ParseToolLine(t.NewMkLine("create.mk", 2, "TOOLS_CREATE+= load"), true, false) tools.ParseToolLine(t.NewMkLine("create.mk", 3, "TOOLS_CREATE+= run"), true, false) @@ -184,7 +184,7 @@ func (s *Suite) Test_Tools__load_from_infrastructure(c *check.C) { func (s *Suite) Test_Tools__package_Makefile(c *check.C) { t := s.Init(c) - t.SetupPkgsrc() + t.SetUpPkgsrc() t.CreateFileLines("mk/tools/defaults.mk", "TOOLS_CREATE+= load", "TOOLS_CREATE+= run", @@ -202,7 +202,7 @@ func (s *Suite) Test_Tools__package_Makefile(c *check.C) { "USE_TOOLS+= run") G.Pkgsrc.LoadInfrastructure() - tools := NewTools("") + tools := NewTools() tools.Fallback(G.Pkgsrc.Tools) load := tools.ByName("load") @@ -236,8 +236,8 @@ func (s *Suite) Test_Tools__package_Makefile(c *check.C) { func (s *Suite) Test_Tools__builtin_mk(c *check.C) { t := s.Init(c) - t.SetupPkgsrc() - t.SetupCommandLine("-Wall,no-space") + t.SetUpPkgsrc() + t.SetUpCommandLine("-Wall,no-space") t.CreateFileLines("mk/tools/defaults.mk", "TOOLS_CREATE+= load", "TOOLS_CREATE+= run", @@ -255,7 +255,7 @@ func (s *Suite) Test_Tools__builtin_mk(c *check.C) { // Tools that are defined by pkgsrc as load-time tools // may be used in any file at load time. - mklines := t.SetupFileMkLines("category/package/builtin.mk", + mklines := t.SetUpFileMkLines("category/package/builtin.mk", MkRcsID, "", "VAR!= ${ECHO} 'too early'", @@ -286,8 +286,8 @@ func (s *Suite) Test_Tools__builtin_mk(c *check.C) { func (s *Suite) Test_Tools__implicit_definition_in_bsd_pkg_mk(c *check.C) { t := s.Init(c) - t.SetupPkgsrc() - t.SetupCommandLine("-Wall,no-space") + t.SetUpPkgsrc() + t.SetUpCommandLine("-Wall,no-space") t.CreateFileLines("mk/tools/defaults.mk", MkRcsID) // None t.CreateFileLines("mk/bsd.prefs.mk", @@ -307,8 +307,8 @@ func (s *Suite) Test_Tools__implicit_definition_in_bsd_pkg_mk(c *check.C) { func (s *Suite) Test_Tools__both_prefs_and_pkg_mk(c *check.C) { t := s.Init(c) - t.SetupPkgsrc() - t.SetupCommandLine("-Wall,no-space") + t.SetUpPkgsrc() + t.SetUpCommandLine("-Wall,no-space") t.CreateFileLines("mk/tools/defaults.mk", MkRcsID) t.CreateFileLines("mk/bsd.prefs.mk", @@ -326,8 +326,8 @@ func (s *Suite) Test_Tools__both_prefs_and_pkg_mk(c *check.C) { func (s *Suite) Test_Tools__tools_having_the_same_variable_name(c *check.C) { t := s.Init(c) - t.SetupPkgsrc() - t.SetupCommandLine("-Wall,no-space") + t.SetUpPkgsrc() + t.SetUpCommandLine("-Wall,no-space") t.CreateFileLines("mk/tools/defaults.mk", "_TOOLS_VARNAME.awk= AWK", "_TOOLS_VARNAME.gawk= AWK", @@ -348,7 +348,7 @@ func (s *Suite) Test_Tools__tools_having_the_same_variable_name(c *check.C) { t.DisableTracing() t.CheckOutputLines( - "TRACE: + (*Tools).Trace(\"Pkgsrc\")", + "TRACE: + (*Tools).Trace()", "TRACE: 1 tool awk:AWK::AfterPrefsMk", "TRACE: 1 tool echo:ECHO:var:AfterPrefsMk", "TRACE: 1 tool echo -n:ECHO_N:var:AfterPrefsMk", @@ -358,9 +358,9 @@ func (s *Suite) Test_Tools__tools_having_the_same_variable_name(c *check.C) { "TRACE: 1 tool sed:SED::AfterPrefsMk", "TRACE: 1 tool test:TEST:var:AfterPrefsMk", "TRACE: 1 tool true:TRUE:var:AfterPrefsMk", - "TRACE: - (*Tools).Trace(\"Pkgsrc\")") + "TRACE: - (*Tools).Trace()") - tools := NewTools("module.mk") + tools := NewTools() tools.Fallback(G.Pkgsrc.Tools) t.EnableTracingToLog() @@ -368,8 +368,8 @@ func (s *Suite) Test_Tools__tools_having_the_same_variable_name(c *check.C) { t.DisableTracing() t.CheckOutputLines( - "TRACE: + (*Tools).Trace(\"module.mk\")", - "TRACE: 1 + (*Tools).Trace(\"Pkgsrc\")", + "TRACE: + (*Tools).Trace()", + "TRACE: 1 + (*Tools).Trace()", "TRACE: 1 2 tool awk:AWK::AfterPrefsMk", "TRACE: 1 2 tool echo:ECHO:var:AfterPrefsMk", "TRACE: 1 2 tool echo -n:ECHO_N:var:AfterPrefsMk", @@ -379,8 +379,8 @@ func (s *Suite) Test_Tools__tools_having_the_same_variable_name(c *check.C) { "TRACE: 1 2 tool sed:SED::AfterPrefsMk", "TRACE: 1 2 tool test:TEST:var:AfterPrefsMk", "TRACE: 1 2 tool true:TRUE:var:AfterPrefsMk", - "TRACE: 1 - (*Tools).Trace(\"Pkgsrc\")", - "TRACE: - (*Tools).Trace(\"module.mk\")") + "TRACE: 1 - (*Tools).Trace()", + "TRACE: - (*Tools).Trace()") } func (s *Suite) Test_ToolTime_String(c *check.C) { @@ -391,7 +391,7 @@ func (s *Suite) Test_ToolTime_String(c *check.C) { func (s *Suite) Test_Tools__var(c *check.C) { t := s.Init(c) - t.SetupPkgsrc() + t.SetUpPkgsrc() t.CreateFileLines("mk/tools/defaults.mk", "TOOLS_CREATE+= ln", "_TOOLS_VARNAME.ln= LN") @@ -418,20 +418,20 @@ func (s *Suite) Test_Tools__var(c *check.C) { // // See also Pkglint.Tool. func (s *Suite) Test_Tools_Fallback__tools_having_the_same_variable_name_realistic(c *check.C) { - nonGnu := NewTools("non-gnu") + nonGnu := NewTools() nonGnu.def("sed", "SED", false, AfterPrefsMk) - gnu := NewTools("gnu") + gnu := NewTools() gnu.def("gsed", "SED", false, Nowhere) - local1 := NewTools("local") + local1 := NewTools() local1.def("sed", "SED", false, AfterPrefsMk) local1.Fallback(gnu) c.Check(local1.ByName("sed").Validity, equals, AfterPrefsMk) c.Check(local1.ByName("gsed").Validity, equals, Nowhere) - local2 := NewTools("local") + local2 := NewTools() local2.def("gsed", "SED", false, Nowhere) local2.Fallback(nonGnu) @@ -453,20 +453,20 @@ func (s *Suite) Test_Tools_Fallback__tools_having_the_same_variable_name_realist // // See also Pkglint.Tool. func (s *Suite) Test_Tools_Fallback__tools_having_the_same_variable_name_unrealistic(c *check.C) { - nonGnu := NewTools("non-gnu") + nonGnu := NewTools() nonGnu.def("sed", "SED", false, Nowhere) - gnu := NewTools("gnu") + gnu := NewTools() gnu.def("gsed", "SED", false, AfterPrefsMk) - local1 := NewTools("local") + local1 := NewTools() local1.def("sed", "SED", false, Nowhere) local1.Fallback(gnu) c.Check(local1.ByName("sed").Validity, equals, Nowhere) c.Check(local1.ByName("gsed").Validity, equals, AfterPrefsMk) - local2 := NewTools("local") + local2 := NewTools() local2.def("gsed", "SED", false, AfterPrefsMk) local2.Fallback(nonGnu) @@ -489,7 +489,7 @@ func (s *Suite) Test_Tools_Fallback__tools_having_the_same_variable_name_unreali func (s *Suite) Test_Tools__cmake(c *check.C) { t := s.Init(c) - t.SetupPackage("category/package", + t.SetUpPackage("category/package", "USE_CMAKE=\tyes", "", "do-test:", diff --git a/pkgtools/pkglint/files/toplevel_test.go b/pkgtools/pkglint/files/toplevel_test.go index 6a52568269d..3e171dd60e7 100644 --- a/pkgtools/pkglint/files/toplevel_test.go +++ b/pkgtools/pkglint/files/toplevel_test.go @@ -19,7 +19,7 @@ func (s *Suite) Test_CheckdirToplevel(c *check.C) { t.CreateFileLines("bbb/Makefile") t.CreateFileLines("ccc/Makefile") t.CreateFileLines("x11/Makefile") - t.SetupVartypes() + t.SetUpVartypes() CheckdirToplevel(t.File(".")) diff --git a/pkgtools/pkglint/files/util.go b/pkgtools/pkglint/files/util.go index 276df4d171b..efc9c8bc27d 100644 --- a/pkgtools/pkglint/files/util.go +++ b/pkgtools/pkglint/files/util.go @@ -110,17 +110,23 @@ func imax(a, b int) int { } func mustMatch(s string, re regex.Pattern) []string { - if m := G.res.Match(s, re); m != nil { - return m + m := G.res.Match(s, re) + if m == nil { + G.Assertf(false, "mustMatch %q %q", s, re) } - panic(sprintf("mustMatch %q %q", s, re)) + return m } func isEmptyDir(filename string) bool { + if hasSuffix(filename, "/CVS") { + return true + } + dirents, err := ioutil.ReadDir(filename) - if err != nil || hasSuffix(filename, "/CVS") { + if err != nil { return true } + for _, dirent := range dirents { name := dirent.Name() if isIgnoredFilename(name) { @@ -158,9 +164,23 @@ func isIgnoredFilename(filename string) bool { return false } +func dirglob(dirname string) []string { + infos, err := ioutil.ReadDir(dirname) + if err != nil { + return nil + } + var filenames []string + for _, info := range infos { + if !(isIgnoredFilename(info.Name())) { + filenames = append(filenames, cleanpath(dirname+"/"+info.Name())) + } + } + return filenames +} + // Checks whether a file is already committed to the CVS repository. func isCommitted(filename string) bool { - lines := loadCvsEntries(filename) + lines := G.loadCvsEntries(filename) if lines == nil { return false } @@ -176,7 +196,7 @@ func isCommitted(filename string) bool { func isLocallyModified(filename string) bool { baseName := path.Base(filename) - lines := loadCvsEntries(filename) + lines := G.loadCvsEntries(filename) if lines == nil { return false } @@ -189,7 +209,7 @@ func isLocallyModified(filename string) bool { return true } - // According to http://cvsman.com/cvs-1.12.12/cvs_19.php, format both timestamps. + // Following http://cvsman.com/cvs-1.12.12/cvs_19.php, format both timestamps. cvsModTime := fields[3] fsModTime := st.ModTime().UTC().Format(time.ANSIC) if trace.Tracing { @@ -202,21 +222,6 @@ func isLocallyModified(filename string) bool { return false } -func loadCvsEntries(filename string) Lines { - dir := path.Dir(filename) - if dir == G.CvsEntriesDir { - return G.CvsEntriesLines - } - - lines := Load(dir+"/CVS/Entries", 0) - if lines == nil { - return nil - } - G.CvsEntriesDir = dir - G.CvsEntriesLines = lines - return lines -} - // Returns the number of columns that a string occupies when printed with // a tabulator size of 8. func tabWidth(s string) int { @@ -244,12 +249,12 @@ func detab(s string) string { } func shorten(s string, maxChars int) string { - chars := 0 + codePoints := 0 for i := range s { - if chars >= maxChars { + if codePoints >= maxChars { return s[:i] + "..." } - chars++ + codePoints++ } return s } @@ -261,6 +266,7 @@ func varnameBase(varname string) string { } return varname } + func varnameCanon(varname string) string { dot := strings.IndexByte(varname, '.') if dot > 0 { @@ -268,6 +274,7 @@ func varnameCanon(varname string) string { } return varname } + func varnameParam(varname string) string { dot := strings.IndexByte(varname, '.') if dot > 0 { @@ -317,25 +324,8 @@ func toInt(s string, def int) int { return def } -func dirglob(dirname string) []string { - fis, err := ioutil.ReadDir(dirname) - if err != nil { - return nil - } - var fnames []string - for _, fi := range fis { - if !(isIgnoredFilename(fi.Name())) { - fnames = append(fnames, cleanpath(dirname+"/"+fi.Name())) - } - } - return fnames -} - -// Emulates make(1)'s :S substitution operator. +// mkopSubst evaluates make(1)'s :S substitution operator. func mkopSubst(s string, left bool, from string, right bool, to string, flags string) string { - if trace.Tracing { - defer trace.Call(s, left, from, right, to, flags)() - } re := regex.Pattern(ifelseStr(left, "^", "") + regexp.QuoteMeta(from) + ifelseStr(right, "$", "")) done := false gflag := contains(flags, "g") @@ -354,10 +344,11 @@ func relpath(from, to string) string { return path.Clean(to[len(from)+1:]) } + // TODO: avoid these filesystem calls as they require IO. absFrom := abspath(from) absTo := abspath(to) rel, err := filepath.Rel(absFrom, absTo) - G.Assertf(err == nil, "relpath %q %q.", from, to) + G.AssertNil(err, "relpath %q %q", from, to) result := filepath.ToSlash(rel) if trace.Tracing { trace.Stepf("relpath from %q to %q = %q", from, to, result) @@ -367,7 +358,7 @@ func relpath(from, to string) string { func abspath(filename string) string { abs, err := filepath.Abs(filename) - G.Assertf(err == nil, "abspath %q.", filename) + G.AssertNil(err, "abspath %q", filename) return filepath.ToSlash(abs) } @@ -383,16 +374,22 @@ func cleanpath(filename string) string { for !lex.EOF() { part := lex.NextBytesFunc(func(b byte) bool { return b != '/' }) parts = append(parts, part) - n := len(parts) - if n >= 5 && parts[n-1] == ".." && parts[n-2] == ".." && parts[n-3] != ".." && parts[n-4] != ".." { - parts = parts[:n-4] - } if lex.SkipByte('/') { for lex.SkipByte('/') || lex.SkipString("./") { } } } + for i := 2; i+3 < len(parts); /* nothing */ { + if parts[i] != ".." && parts[i+1] != ".." && parts[i+2] == ".." && parts[i+3] == ".." { + if i+4 == len(parts) || parts[i+4] != ".." { + parts = append(parts[:i], parts[i+4:]...) + continue + } + } + i++ + } + if len(parts) == 0 { return "." } @@ -408,7 +405,7 @@ func hasAlnumPrefix(s string) bool { return s != "" && textproc.AlnumU.Contains( // Once remembers with which arguments its FirstTime method has been called // and only returns true on each first call. type Once struct { - seen map[uint64]bool + seen map[uint64]struct{} } func (o *Once) FirstTime(what string) bool { @@ -428,9 +425,9 @@ func (o *Once) check(key uint64) bool { return false } if o.seen == nil { - o.seen = make(map[uint64]bool) + o.seen = make(map[uint64]struct{}) } - o.seen[key] = true + o.seen[key] = struct{}{} return true } @@ -438,12 +435,12 @@ func (o *Once) check(key uint64) bool { // in a certain scope, such as a package or a file. type Scope struct { defined map[string]MkLine - fallback map[string]string used map[string]MkLine + fallback map[string]string } func NewScope() Scope { - return Scope{make(map[string]MkLine), make(map[string]string), make(map[string]MkLine)} + return Scope{make(map[string]MkLine), make(map[string]MkLine), make(map[string]string)} } // Define marks the variable and its canonicalized form as defined. @@ -454,6 +451,7 @@ func (s *Scope) Define(varname string, mkline MkLine) { trace.Step2("Defining %q in %s", varname, mkline.String()) } } + varcanon := varnameCanon(varname) if varcanon != varname && s.defined[varcanon] == nil { s.defined[varcanon] = mkline @@ -475,6 +473,7 @@ func (s *Scope) Use(varname string, line MkLine) { trace.Step2("Using %q in %s", varname, line.String()) } } + varcanon := varnameCanon(varname) if varcanon != varname && s.used[varcanon] == nil { s.used[varcanon] = line @@ -501,6 +500,7 @@ func (s *Scope) DefinedSimilar(varname string) bool { } return true } + varcanon := varnameCanon(varname) if s.defined[varcanon] != nil { if trace.Tracing { @@ -526,6 +526,7 @@ func (s *Scope) UsedSimilar(varname string) bool { } // FirstDefinition returns the line in which the variable has been defined first. +// // Having multiple definitions is typical in the branches of "if" statements. func (s *Scope) FirstDefinition(varname string) MkLine { mkline := s.defined[varname] @@ -649,7 +650,7 @@ func naturalLess(str1, str2 string) bool { // but that's deep in the infrastructure and only affects the "nb13" extension.) type RedundantScope struct { vars map[string]*redundantScopeVarinfo - dirLevel int + dirLevel int // The number of enclosing directives (.if, .for). OnIgnore func(old, new MkLine) OnOverwrite func(old, new MkLine) } @@ -675,8 +676,9 @@ func (s *RedundantScope) Handle(mkline MkLine) { value := mkline.Value() valueNovar := mkline.WithoutMakeVariables(value) if op == opAssignEval && value == valueNovar { - op = opAssign // The two operators are effectively the same in this case. + op = /* effectively */ opAssign } + existing, found := s.vars[varname] if !found { if op == opAssignShell || op == opAssignEval { @@ -687,10 +689,12 @@ func (s *RedundantScope) Handle(mkline MkLine) { } s.vars[varname] = &redundantScopeVarinfo{mkline, value} } + } else if existing != nil { if op == opAssign && existing.value == value { - op = opAssignDefault + op = /* effectively */ opAssignDefault } + switch op { case opAssign: if s.OnOverwrite != nil { @@ -733,15 +737,6 @@ func IsPrefs(filename string) bool { return false } -func isalnum(s string) bool { - for _, ch := range []byte(s) { - if !textproc.AlnumU.Contains(ch) { - return false - } - } - return true -} - // FileCache reduces the IO load for commonly loaded files by about 50%, // especially for buildlink3.mk and *.buildlink3.mk files. type FileCache struct { diff --git a/pkgtools/pkglint/files/util_test.go b/pkgtools/pkglint/files/util_test.go index e4d18633fc3..2b475df96c3 100644 --- a/pkgtools/pkglint/files/util_test.go +++ b/pkgtools/pkglint/files/util_test.go @@ -51,10 +51,11 @@ func (s *Suite) Test__regex_ReplaceFirst(c *check.C) { } func (s *Suite) Test_mustMatch(c *check.C) { - c.Check( + t := s.Init(c) + + t.ExpectPanic( func() { mustMatch("aaa", `b`) }, - check.Panics, - "mustMatch \"aaa\" \"b\"") + "Pkglint internal error: mustMatch \"aaa\" \"b\"") } func (s *Suite) Test_shorten(c *check.C) { @@ -74,14 +75,37 @@ func (s *Suite) Test_tabWidth(c *check.C) { func (s *Suite) Test_cleanpath(c *check.C) { c.Check(cleanpath("simple/path"), equals, "simple/path") c.Check(cleanpath("/absolute/path"), equals, "/absolute/path") + + // Single dot components are removed, unless it's the only component of the path. c.Check(cleanpath("./././."), equals, ".") c.Check(cleanpath("./././"), equals, ".") - c.Check(cleanpath("dir/../dir/../dir/../dir/subdir/../../Makefile"), equals, "dir/../dir/../dir/../Makefile") c.Check(cleanpath("dir/multi/././/file"), equals, "dir/multi/file") + c.Check(cleanpath("dir/"), equals, "dir") + + // Components like aa/bb/../.. are removed, but not in the initial part of the path, + // and only if they are not followed by another "..". + c.Check(cleanpath("dir/../dir/../dir/../dir/subdir/../../Makefile"), equals, "dir/../dir/../dir/../Makefile") c.Check(cleanpath("111/222/../../333/444/../../555/666/../../777/888/9"), equals, "111/222/../../777/888/9") - c.Check(cleanpath("1/2/3/../../4/5/6/../../7/8/9/../../../../10"), equals, "1/10") + c.Check(cleanpath("1/2/3/../../4/5/6/../../7/8/9/../../../../10"), equals, "1/2/3/../../4/7/8/9/../../../../10") c.Check(cleanpath("cat/pkg.v1/../../cat/pkg.v2/Makefile"), equals, "cat/pkg.v1/../../cat/pkg.v2/Makefile") - c.Check(cleanpath("dir/"), equals, "dir") + c.Check(cleanpath("aa/../../../../../a/b/c/d"), equals, "aa/../../../../../a/b/c/d") + c.Check(cleanpath("aa/bb/../../../../a/b/c/d"), equals, "aa/bb/../../../../a/b/c/d") + c.Check(cleanpath("aa/bb/cc/../../../a/b/c/d"), equals, "aa/bb/cc/../../../a/b/c/d") + c.Check(cleanpath("aa/bb/cc/dd/../../a/b/c/d"), equals, "aa/bb/a/b/c/d") + c.Check(cleanpath("aa/bb/cc/dd/ee/../a/b/c/d"), equals, "aa/bb/cc/dd/ee/../a/b/c/d") + c.Check(cleanpath("../../../../../a/b/c/d"), equals, "../../../../../a/b/c/d") + c.Check(cleanpath("aa/../../../../a/b/c/d"), equals, "aa/../../../../a/b/c/d") + c.Check(cleanpath("aa/bb/../../../a/b/c/d"), equals, "aa/bb/../../../a/b/c/d") + c.Check(cleanpath("aa/bb/cc/../../a/b/c/d"), equals, "aa/bb/cc/../../a/b/c/d") + c.Check(cleanpath("aa/bb/cc/dd/../a/b/c/d"), equals, "aa/bb/cc/dd/../a/b/c/d") + c.Check(cleanpath("aa/../cc/../../a/b/c/d"), equals, "aa/../cc/../../a/b/c/d") + + // The initial 2 components of the path are typically category/package, when + // pkglint is called from the pkgsrc top-level directory. + // This path serves as the context and therefore is always kept. + c.Check(cleanpath("aa/bb/../../cc/dd/../../ee/ff"), equals, "aa/bb/../../ee/ff") + c.Check(cleanpath("aa/bb/../../cc/dd/../.."), equals, "aa/bb/../..") + c.Check(cleanpath("aa/bb/cc/dd/../.."), equals, "aa/bb") } // Relpath is called so often that handling the most common calls @@ -101,20 +125,42 @@ func (s *Suite) Test_relpath__failure_on_Windows(c *check.C) { if runtime.GOOS == "windows" { t.ExpectPanic( func() { relpath("c:/", "d:/") }, - "Pkglint internal error: relpath \"c:/\" \"d:/\".") + "Pkglint internal error: relpath \"c:/\" \"d:/\": Rel: can't make d:/ relative to c:/") } } -func (s *Suite) Test_abspath__on_Windows(c *check.C) { +func (s *Suite) Test_abspath__failure_on_Windows(c *check.C) { t := s.Init(c) if runtime.GOOS == "windows" { t.ExpectPanic( func() { abspath("file\u0000name") }, - "Pkglint internal error: abspath \"file\\x00name\".") + "Pkglint internal error: abspath \"file\\x00name\": invalid argument") } } +func (s *Suite) Test_fileExists(c *check.C) { + t := s.Init(c) + + t.CreateFileLines("dir/file") + + t.Check(fileExists(t.File("nonexistent")), equals, false) + t.Check(fileExists(t.File("dir")), equals, false) + t.Check(fileExists(t.File("dir/nonexistent")), equals, false) + t.Check(fileExists(t.File("dir/file")), equals, true) +} + +func (s *Suite) Test_dirExists(c *check.C) { + t := s.Init(c) + + t.CreateFileLines("dir/file") + + t.Check(dirExists(t.File("nonexistent")), equals, false) + t.Check(dirExists(t.File("dir")), equals, true) + t.Check(dirExists(t.File("dir/nonexistent")), equals, false) + t.Check(dirExists(t.File("dir/file")), equals, false) +} + func (s *Suite) Test_isEmptyDir__and_getSubdirs(c *check.C) { t := s.Init(c) @@ -141,7 +187,7 @@ func (s *Suite) Test_isEmptyDir__and_getSubdirs(c *check.C) { } } -func (s *Suite) Test_isEmptyDir__empty_subdir(c *check.C) { +func (s *Suite) Test_isEmptyDir(c *check.C) { t := s.Init(c) t.CreateFileLines("CVS/Entries", @@ -150,6 +196,17 @@ func (s *Suite) Test_isEmptyDir__empty_subdir(c *check.C) { "dummy") c.Check(isEmptyDir(t.File(".")), equals, true) + c.Check(isEmptyDir(t.File("CVS")), equals, true) +} + +func (s *Suite) Test_getSubdirs(c *check.C) { + t := s.Init(c) + + t.CreateFileLines("subdir/file") + t.CreateFileLines("empty/file") + c.Check(os.Remove(t.File("empty/file")), check.IsNil) + + c.Check(getSubdirs(t.File(".")), deepEquals, []string{"subdir"}) } func (s *Suite) Test_detab(c *check.C) { @@ -242,6 +299,7 @@ func (s *Suite) Test_isLocallyModified(c *check.C) { modified := t.CreateFileLines("modified") t.CreateFileLines("CVS/Entries", + "//", // Just for code coverage. "/unmodified//"+modTime.Format(time.ANSIC)+"//", "/modified//"+modTime.Format(time.ANSIC)+"//", "/enoent//"+modTime.Format(time.ANSIC)+"//") @@ -251,6 +309,10 @@ func (s *Suite) Test_isLocallyModified(c *check.C) { c.Check(isLocallyModified(t.File("enoent")), equals, true) c.Check(isLocallyModified(t.File("not_mentioned")), equals, false) c.Check(isLocallyModified(t.File("subdir/file")), equals, false) + + t.DisableTracing() + + c.Check(isLocallyModified(t.File("unmodified")), equals, false) } func (s *Suite) Test_Scope_Defined(c *check.C) { @@ -300,12 +362,62 @@ func (s *Suite) Test_Scope_DefineAll(c *check.C) { c.Check(dst.Defined("VAR"), equals, true) } +func (s *Suite) Test_Scope_FirstDefinition(c *check.C) { + t := s.Init(c) + + mkline1 := t.NewMkLine("fname.mk", 3, "VAR=\tvalue") + mkline2 := t.NewMkLine("fname.mk", 3, ".if ${VAR::=value}") + + scope := NewScope() + scope.Define("VAR", mkline1) + scope.Define("SNEAKY", mkline2) + + t.Check(scope.FirstDefinition("VAR"), equals, mkline1) + + // This call returns nil because it's not a variable assignment + // and the calling code typically assumes a variable definition. + // These sneaky variables with implicit definition are an edge + // case that only few people actually know. It's better that way. + t.Check(scope.FirstDefinition("SNEAKY"), check.IsNil) +} + +func (s *Suite) Test_Scope__no_tracing(c *check.C) { + t := s.Init(c) + + scope := NewScope() + scope.Define("VAR.param", t.NewMkLine("fname.mk", 3, "VAR.param=\tvalue")) + t.DisableTracing() + + t.Check(scope.DefinedSimilar("VAR.param"), equals, true) + t.Check(scope.DefinedSimilar("VAR.other"), equals, true) + t.Check(scope.DefinedSimilar("OTHER"), equals, false) +} + func (s *Suite) Test_naturalLess(c *check.C) { + c.Check(naturalLess("", "a"), equals, true) + c.Check(naturalLess("a", ""), equals, false) + + c.Check(naturalLess("a", "b"), equals, true) + c.Check(naturalLess("b", "a"), equals, false) + + // Numbers are always considered smaller than other characters. + c.Check(naturalLess("0", "!"), equals, true) + c.Check(naturalLess("!", "0"), equals, false) + c.Check(naturalLess("0", "a"), equals, true) c.Check(naturalLess("a", "0"), equals, false) + + c.Check(naturalLess("5", "12"), equals, true) + c.Check(naturalLess("12", "5"), equals, false) + + c.Check(naturalLess("5", "7"), equals, true) + c.Check(naturalLess("7", "5"), equals, false) + c.Check(naturalLess("000", "0000"), equals, true) c.Check(naturalLess("0000", "000"), equals, false) + c.Check(naturalLess("000", "000"), equals, false) + c.Check(naturalLess("00011", "000111"), equals, true) c.Check(naturalLess("00011", "00012"), equals, true) } @@ -328,25 +440,6 @@ func (s *Suite) Test_varnameCanon(c *check.C) { c.Check(varnameCanon(".CURDIR"), equals, ".CURDIR") } -func (s *Suite) Test_isalnum(c *check.C) { - c.Check(isalnum(""), equals, true) - c.Check(isalnum("/"), equals, false) - c.Check(isalnum("0"), equals, true) - c.Check(isalnum("9"), equals, true) - c.Check(isalnum(":"), equals, false) - c.Check(isalnum("@"), equals, false) - c.Check(isalnum("A"), equals, true) - c.Check(isalnum("Z"), equals, true) - c.Check(isalnum("["), equals, false) - c.Check(isalnum("_"), equals, true) - c.Check(isalnum("`"), equals, false) - c.Check(isalnum("a"), equals, true) - c.Check(isalnum("z"), equals, true) - c.Check(isalnum("{"), equals, false) - c.Check(isalnum("Hello_world005"), equals, true) - c.Check(isalnum("Hello,world005"), equals, false) -} - func (s *Suite) Test_FileCache(c *check.C) { t := s.Init(c) @@ -415,6 +508,14 @@ func (s *Suite) Test_makeHelp(c *check.C) { c.Check(makeHelp("subst"), equals, confMake+" help topic=subst") } +func (s *Suite) Test_hasAlnumPrefix(c *check.C) { + t := s.Init(c) + + t.Check(hasAlnumPrefix(""), equals, false) + t.Check(hasAlnumPrefix("A"), equals, true) + t.Check(hasAlnumPrefix(","), equals, false) +} + func (s *Suite) Test_Once(c *check.C) { var once Once diff --git a/pkgtools/pkglint/files/vardefs.go b/pkgtools/pkglint/files/vardefs.go index 3784ed9f797..2d4c0f3a328 100644 --- a/pkgtools/pkglint/files/vardefs.go +++ b/pkgtools/pkglint/files/vardefs.go @@ -15,7 +15,7 @@ import ( // See vartypecheck.go for how these types are checked. // InitVartypes initializes the long list of predefined pkgsrc variables. -// After this is done, ${PKGNAME}, ${MAKE_ENV} and all the other variables +// After this is done, PKGNAME, MAKE_ENV and all the other variables // can be used in Makefiles without triggering warnings about typos. func (src *Pkgsrc) InitVartypes() { @@ -58,20 +58,20 @@ func (src *Pkgsrc) InitVartypes() { "*.mk: append, default, use") } - // A user-defined or system-defined variable must not be set by any - // package file. It also must not be used in buildlink3.mk and - // builtin.mk files or at load-time, since the system/user preferences - // may not have been loaded when these files are included. + // sys declares a user-defined or system-defined variable that must not be modified by packages. + // + // It also must not be used in buildlink3.mk and builtin.mk files or at load-time, + // since the system/user preferences may not have been loaded when these files are included. sys := func(varname string, kindOfList KindOfList, checker *BasicType) { acl(varname, kindOfList, checker, "buildlink3.mk:; *: use") } + // usr declares a user-defined variable that must not be modified by packages. usr := func(varname string, kindOfList KindOfList, checker *BasicType) { acl(varname, kindOfList, checker, "buildlink3.mk:; *: use-loadtime, use") } - // sysload defines a system-provided variable that may already be used - // at load time. + // sysload declares a system-provided variable that may already be used at load time. sysload := func(varname string, kindOfList KindOfList, checker *BasicType) { acl(varname, kindOfList, checker, "*: use-loadtime, use") } @@ -79,11 +79,12 @@ func (src *Pkgsrc) InitVartypes() { bl3list := func(varname string, kindOfList KindOfList, checker *BasicType) { acl(varname, kindOfList, checker, "buildlink3.mk, builtin.mk: append; *: use") } + cmdline := func(varname string, kindOfList KindOfList, checker *BasicType) { acl(varname, kindOfList, checker, "buildlink3.mk, builtin.mk:; *: use-loadtime, use") } - languages := enum( + compilerLanguages := enum( func() string { mklines := LoadMk(src.File("mk/compiler.mk"), NotEmpty) languages := make(map[string]bool) @@ -192,7 +193,7 @@ func (src *Pkgsrc) InitVartypes() { "openjdk8 oracle-jdk8 openjdk7 sun-jdk7 sun-jdk6 jdk16 jdk15 kaffe", "_PKG_JVMS.*") - // Last synced with mk/defaults/mk.conf revision 1.269 + // Last synced with mk/defaults/mk.conf revision 1.269 (2017-01-01). usr("USE_CWRAPPERS", lkNone, enum("yes no auto")) usr("ALLOW_VULNERABLE_PACKAGES", lkNone, BtYes) usr("AUDIT_PACKAGES_FLAGS", lkShell, BtShellWord) @@ -297,6 +298,7 @@ func (src *Pkgsrc) InitVartypes() { "Makefile.*, *.mk: default, set, use, use-loadtime; "+ "*: use-loadtime, use") } + usrpkg("ACROREAD_FONTPATH", lkNone, BtPathlist) usrpkg("AMANDA_USER", lkNone, BtUserGroupName) usrpkg("AMANDA_TMP", lkNone, BtPathname) @@ -702,7 +704,7 @@ func (src *Pkgsrc) InitVartypes() { pkg("EMUL_PLATFORMS", lkShell, BtEmulPlatform) usr("EMUL_PREFER", lkShell, BtEmulPlatform) pkg("EMUL_REQD", lkShell, BtDependency) - usr("EMUL_TYPE.*", lkNone, enum("native builtin suse suse-9.1 suse-9.x suse-10.0 suse-10.x")) + usr("EMUL_TYPE.*", lkNone, enum("native builtin suse suse-10.0 suse-12.1 suse-13.1")) sys("ERROR_CAT", lkNone, BtShellCommand) sys("ERROR_MSG", lkNone, BtShellCommand) sys("EXPORT_SYMBOLS_LDFLAGS", lkShell, BtLdFlag) @@ -851,6 +853,7 @@ func (src *Pkgsrc) InitVartypes() { sys("MANMODE", lkNone, BtFileMode) sys("MANOWN", lkNone, BtUserGroupName) pkglist("MASTER_SITES", lkShell, BtFetchURL) + // TODO: Extract the MASTER_SITE_* definitions from mk/fetch/sites.mk instead of listing them here. sys("MASTER_SITE_APACHE", lkShell, BtFetchURL) sys("MASTER_SITE_BACKUP", lkShell, BtFetchURL) sys("MASTER_SITE_CRATESIO", lkShell, BtFetchURL) @@ -963,7 +966,7 @@ func (src *Pkgsrc) InitVartypes() { acl("PGSQL_VERSIONS_ACCEPTED", lkShell, pgsqlVersions, "") usr("PGSQL_VERSION_DEFAULT", lkNone, BtVersion) sys("PG_LIB_EXT", lkNone, enum("dylib so")) - sys("PGSQL_TYPE", lkNone, enum("postgresql81-client postgresql80-client")) + sys("PGSQL_TYPE", lkNone, enumFrom("mk/pgsql.buildlink3.mk", "postgresql11-client", "PGSQL_TYPE")) sys("PGPKGSRCDIR", lkNone, BtPathname) sys("PHASE_MSG", lkNone, BtShellCommand) usr("PHP_VERSION_REQD", lkNone, BtVersion) @@ -1174,7 +1177,7 @@ func (src *Pkgsrc) InitVartypes() { acl("USE_IMAKE", lkNone, BtYes, "Makefile: set") pkg("USE_JAVA", lkNone, enum("run yes build")) pkg("USE_JAVA2", lkNone, enum("YES yes no 1.4 1.5 6 7 8")) - acl("USE_LANGUAGES", lkShell, languages, "Makefile, Makefile.common, options.mk: set, append") + acl("USE_LANGUAGES", lkShell, compilerLanguages, "Makefile, Makefile.common, options.mk: set, append") pkg("USE_LIBTOOL", lkNone, BtYes) pkg("USE_MAKEINFO", lkNone, BtYes) pkg("USE_MSGFMT_PLURALS", lkNone, BtYes) @@ -1227,6 +1230,7 @@ func parseACLEntries(varname string, aclEntries string) []ACLEntry { if aclEntries == "" { return nil } + var result []ACLEntry prevperms := "(first)" for _, arg := range strings.Split(aclEntries, "; ") { @@ -1271,7 +1275,8 @@ func parseACLEntries(varname string, aclEntries string) []ACLEntry { } for _, prev := range result { matched, err := path.Match(prev.glob, glob) - G.Assertf(err == nil && !matched, "Ineffective ACL glob %q for %q.", glob, varname) + G.AssertNil(err, "Invalid ACL pattern %q for %q.", glob, varname) + G.Assertf(!matched, "Unreachable ACL pattern %q for %q.", glob, varname) } result = append(result, ACLEntry{glob, permissions}) } diff --git a/pkgtools/pkglint/files/vardefs_test.go b/pkgtools/pkglint/files/vardefs_test.go index 534622791b6..ccaf44a1dd3 100644 --- a/pkgtools/pkglint/files/vardefs_test.go +++ b/pkgtools/pkglint/files/vardefs_test.go @@ -37,20 +37,26 @@ func (s *Suite) Test_Pkgsrc_InitVartypes__enumFrom(c *check.C) { ". if !empty(USE_LANGUAGES:M${_version_})", "USE_LANGUAGES+= c++", ". endif", + ".endfor", + "", + ".for _version_", // Just for code coverage. + ".endfor", + "", + ".for version in c99 c200x", // Just for code coverage. ".endfor") - t.SetupVartypes() + t.SetUpVartypes() - checkEnumValues := func(varname, values string) { + test := func(varname, values string) { vartype := G.Pkgsrc.VariableType(varname).String() c.Check(vartype, equals, values) } - checkEnumValues("EMACS_VERSIONS_ACCEPTED", "List of enum: emacs29 emacs31 ") - checkEnumValues("PKG_JVM", "enum: jdk16 openjdk7 openjdk8 oracle-jdk8 sun-jdk6 sun-jdk7 ") - checkEnumValues("USE_LANGUAGES", "List of enum: ada c c++ c++03 c++0x c++11 c++14 c99 "+ + test("EMACS_VERSIONS_ACCEPTED", "List of enum: emacs29 emacs31 ") + test("PKG_JVM", "enum: jdk16 openjdk7 openjdk8 oracle-jdk8 sun-jdk6 sun-jdk7 ") + test("USE_LANGUAGES", "List of enum: ada c c++ c++03 c++0x c++11 c++14 c99 "+ "fortran fortran77 gnu++03 gnu++0x gnu++11 gnu++14 java obj-c++ objc ") - checkEnumValues("PKGSRC_COMPILER", "List of enum: ccache distcc f2c g95 gcc ido mipspro-ucode sunpro ") + test("PKGSRC_COMPILER", "List of enum: ccache distcc f2c g95 gcc ido mipspro-ucode sunpro ") } func (s *Suite) Test_Pkgsrc_InitVartypes__enumFromDirs(c *check.C) { @@ -61,14 +67,14 @@ func (s *Suite) Test_Pkgsrc_InitVartypes__enumFromDirs(c *check.C) { t.CreateFileLines("lang/python28/Makefile", MkRcsID) t.CreateFileLines("lang/python33/Makefile", MkRcsID) - t.SetupVartypes() + t.SetUpVartypes() - checkEnumValues := func(varname, values string) { + test := func(varname, values string) { vartype := G.Pkgsrc.VariableType(varname).String() c.Check(vartype, equals, values) } - checkEnumValues("PYPKGPREFIX", "enum: py28 py33 ") + test("PYPKGPREFIX", "enum: py28 py33 ") } func (s *Suite) Test_parseACLEntries(c *check.C) { @@ -88,13 +94,13 @@ func (s *Suite) Test_parseACLEntries(c *check.C) { t.ExpectPanic( func() { parseACLEntries("VARNAME", "*.mk: use; buildlink3.mk: append") }, - "Pkglint internal error: Ineffective ACL glob \"buildlink3.mk\" for \"VARNAME\".") + "Pkglint internal error: Unreachable ACL pattern \"buildlink3.mk\" for \"VARNAME\".") } func (s *Suite) Test_Pkgsrc_InitVartypes__LP64PLATFORMS(c *check.C) { t := s.Init(c) - pkg := t.SetupPackage("category/package", + pkg := t.SetUpPackage("category/package", "BROKEN_ON_PLATFORM=\t${LP64PLATFORMS}") G.Check(pkg) @@ -103,3 +109,18 @@ func (s *Suite) Test_Pkgsrc_InitVartypes__LP64PLATFORMS(c *check.C) { // All PLATFORM variables must be either lkNone or lkSpace. t.CheckOutputEmpty() } + +func (s *Suite) Test_Pkgsrc_InitVartypes__no_tracing(c *check.C) { + t := s.Init(c) + + t.CreateFileLines("editors/emacs/modules.mk", + MkRcsID, + "", + "_EMACS_VERSIONS_ALL= emacs31", + "_EMACS_VERSIONS_ALL+= emacs29") + t.DisableTracing() + + t.SetUpVartypes() // Just for code coverage. + + t.CheckOutputEmpty() +} diff --git a/pkgtools/pkglint/files/vartype.go b/pkgtools/pkglint/files/vartype.go index 53e3b2b4770..c65b6ed5784 100644 --- a/pkgtools/pkglint/files/vartype.go +++ b/pkgtools/pkglint/files/vartype.go @@ -16,10 +16,16 @@ type Vartype struct { type KindOfList uint8 -// TODO: Rename lkNone to Plain, and lkShell to List. const ( - lkNone KindOfList = iota // Plain data type - lkShell // List entries are shell words; used in the :M, :S modifiers. + // lkNone is a plain data type, no list at all. + lkNone KindOfList = iota + + // lkShell is a compound type, consisting of several space-separated elements. + // Elements can have embedded spaces by enclosing them in quotes, like in the shell. + // + // These lists are used in the :M, :S modifiers, in .for loops, + // and as lists of arbitrary things. + lkShell ) type ACLEntry struct { @@ -90,6 +96,7 @@ func (vt *Vartype) Union() ACLPermissions { return permissions } +// AllowedFiles lists the file patterns in which the given permissions are allowed. func (vt *Vartype) AllowedFiles(perms ACLPermissions) string { files := make([]string, 0, len(vt.aclEntries)) for _, aclEntry := range vt.aclEntries { @@ -100,7 +107,7 @@ func (vt *Vartype) AllowedFiles(perms ACLPermissions) string { return strings.Join(files, ", ") } -// IsConsideredList returns whether the type is considered a shell list. +// IsConsideredList returns whether the type is considered a list. // This distinction between "real lists" and "considered a list" makes // the implementation of checklineMkVartype easier. func (vt *Vartype) IsConsideredList() bool { @@ -193,9 +200,11 @@ type BasicType struct { func (bt *BasicType) IsEnum() bool { return hasPrefix(bt.name, "enum: ") } + func (bt *BasicType) HasEnum(value string) bool { return !contains(value, " ") && contains(bt.name, " "+value+" ") } + func (bt *BasicType) AllowedEnums() string { return bt.name[6 : len(bt.name)-1] } @@ -266,7 +275,11 @@ var ( BtYesNoIndirectly = &BasicType{"YesNoIndirectly", (*VartypeCheck).YesNoIndirectly} ) -func init() { // Necessary due to circular dependency +// Necessary due to circular dependencies between the checkers. +// +// The Go compiler is stricter than absolutely necessary for this particular case. +// The following methods are only referred to but not invoked during initialization. +func init() { BtShellCommand.checker = (*VartypeCheck).ShellCommand BtShellCommands.checker = (*VartypeCheck).ShellCommands BtShellWord.checker = (*VartypeCheck).ShellWord diff --git a/pkgtools/pkglint/files/vartype_test.go b/pkgtools/pkglint/files/vartype_test.go index eac49471ca3..678c055861a 100644 --- a/pkgtools/pkglint/files/vartype_test.go +++ b/pkgtools/pkglint/files/vartype_test.go @@ -7,7 +7,7 @@ import ( func (s *Suite) Test_Vartype_EffectivePermissions(c *check.C) { t := s.Init(c) - t.SetupVartypes() + t.SetUpVartypes() if typ := G.Pkgsrc.vartypes["PREFIX"]; c.Check(typ, check.NotNil) { c.Check(typ.basicType.name, equals, "Pathname") @@ -23,11 +23,16 @@ func (s *Suite) Test_Vartype_EffectivePermissions(c *check.C) { } func (s *Suite) Test_BasicType_HasEnum(c *check.C) { - vc := enum("catinstall middle maninstall") + vc := enum("start middle end") - c.Check(vc.HasEnum("catinstall"), equals, true) + c.Check(vc.HasEnum("start"), equals, true) c.Check(vc.HasEnum("middle"), equals, true) - c.Check(vc.HasEnum("maninstall"), equals, true) + c.Check(vc.HasEnum("end"), equals, true) + + c.Check(vc.HasEnum("star"), equals, false) + c.Check(vc.HasEnum("mid"), equals, false) + c.Check(vc.HasEnum("nd"), equals, false) + c.Check(vc.HasEnum("start middle"), equals, false) } func (s *Suite) Test_ACLPermissions_Contains(c *check.C) { @@ -59,7 +64,7 @@ func (s *Suite) Test_ACLPermissions_HumanString(c *check.C) { func (s *Suite) Test_Vartype_IsConsideredList(c *check.C) { t := s.Init(c) - t.SetupVartypes() + t.SetUpVartypes() c.Check(G.Pkgsrc.VariableType("COMMENT").IsConsideredList(), equals, false) c.Check(G.Pkgsrc.VariableType("DEPENDS").IsConsideredList(), equals, true) diff --git a/pkgtools/pkglint/files/vartypecheck.go b/pkgtools/pkglint/files/vartypecheck.go index 3ddb0b98249..b1271ef0843 100644 --- a/pkgtools/pkglint/files/vartypecheck.go +++ b/pkgtools/pkglint/files/vartypecheck.go @@ -6,14 +6,16 @@ import ( "strings" ) +// VartypeCheck groups together the various checks for variables of the different types. type VartypeCheck struct { // Note: if "go vet" or "go test" complains about a "variable with invalid type", update to go1.11.4. // See https://github.com/golang/go/issues/28972. // That doesn't help though since pkglint contains these "more convoluted alias declarations" // mentioned in https://github.com/golang/go/commit/6971090515ba. + // Therefore MkLine is declared as *MkLineImpl here. + // Ideally the "more convoluted cyclic type declaration" should be broken up. - MkLine MkLine - Line Line + MkLine *MkLineImpl // The name of the variable being checked. // @@ -24,13 +26,14 @@ type VartypeCheck struct { Op MkOperator Value string ValueNoVar string - MkComment string - Guessed bool // Whether the type definition is guessed (based on the variable name) or explicitly defined (see vardefs.go). + MkComment string // The comment including the "#". + Guessed bool // Whether the type definition is guessed (based on the variable name) or explicitly defined (see vardefs.go). } -func (cv *VartypeCheck) Errorf(format string, args ...interface{}) { cv.Line.Errorf(format, args...) } -func (cv *VartypeCheck) Warnf(format string, args ...interface{}) { cv.Line.Warnf(format, args...) } -func (cv *VartypeCheck) Notef(format string, args ...interface{}) { cv.Line.Notef(format, args...) } +func (cv *VartypeCheck) Errorf(format string, args ...interface{}) { cv.MkLine.Errorf(format, args...) } +func (cv *VartypeCheck) Warnf(format string, args ...interface{}) { cv.MkLine.Warnf(format, args...) } +func (cv *VartypeCheck) Notef(format string, args ...interface{}) { cv.MkLine.Notef(format, args...) } +func (cv *VartypeCheck) Explain(explanation ...string) { cv.MkLine.Explain(explanation...) } // Autofix returns the autofix instance belonging to the line. // @@ -55,7 +58,7 @@ func (cv *VartypeCheck) Notef(format string, args ...interface{}) { cv.Line.Not // fix.Custom(func(showAutofix, autofix bool) {}) // // fix.Apply() -func (cv *VartypeCheck) Autofix() *Autofix { return cv.Line.Autofix() } +func (cv *VartypeCheck) Autofix() *Autofix { return cv.MkLine.Autofix() } // WithValue returns a new VartypeCheck context by copying all // fields except the value. @@ -93,41 +96,39 @@ func (cv *VartypeCheck) WithVarnameValueMatch(varname, value string) *VartypeChe } const ( - reMachineOpsys = "" + // See mk/platform - "AIX|BSDOS|Bitrig|Cygwin|Darwin|DragonFly|FreeBSD|FreeMiNT|GNUkFreeBSD|" + - "HPUX|Haiku|IRIX|Interix|Linux|Minix|MirBSD|NetBSD|OSF1|OpenBSD|QNX|SCO_SV|SunOS|UnixWare" + machineOpsysValues = "" + // See mk/platform + "AIX BSDOS Bitrig Cygwin Darwin DragonFly FreeBSD FreeMiNT GNUkFreeBSD " + + "HPUX Haiku IRIX Interix Linux Minix MirBSD NetBSD OSF1 OpenBSD QNX SCO_SV SunOS UnixWare" - // See mk/emulator/emulator-vars.mk. - reEmulOpsys = "" + - "bitrig|bsdos|cygwin|darwin|dragonfly|freebsd|" + - "haiku|hpux|interix|irix|linux|mirbsd|netbsd|openbsd|osf1|solaris|sunos" + // See mk/emulator/emulator-vars.mk. + emulOpsysValues = "" + + "bitrig bsdos cygwin darwin dragonfly freebsd " + + "haiku hpux interix irix linux mirbsd netbsd openbsd osf1 solaris sunos" // Hardware architectures having the same name in bsd.own.mk and the GNU world. // These are best-effort guesses, since they depend on the operating system. - reArch = "" + - "aarch64|alpha|amd64|arc|arm|cobalt|convex|dreamcast|i386|" + - "hpcmips|hpcsh|hppa|hppa64|ia64|" + - "m68k|m88k|mips|mips64|mips64el|mipseb|mipsel|mipsn32|mlrisc|" + - "ns32k|pc532|pmax|powerpc|powerpc64|rs6000|s390|sparc|sparc64|vax|x86_64" + archValues = "" + + "aarch64 alpha amd64 arc arm cobalt convex dreamcast i386 " + + "hpcmips hpcsh hppa hppa64 ia64 " + + "m68k m88k mips mips64 mips64el mipseb mipsel mipsn32 mlrisc " + + "ns32k pc532 pmax powerpc powerpc64 rs6000 s390 sparc sparc64 vax x86_64" // See mk/bsd.prefs.mk:/^GNU_ARCH\./ - reMachineArch = "" + - reArch + "|" + - "aarch64eb|amd64|arm26|arm32|coldfire|earm|earmeb|earmhf|earmhfeb|earmv4|earmv4eb|earmv5|" + - "earmv5eb|earmv6|earmv6eb|earmv6hf|earmv6hfeb|earmv7|earmv7eb|earmv7hf|earmv7hfeb|evbarm|" + - "i386|i586|i686|m68000|mips|mips64eb|sh3eb|sh3el" + machineArchValues = "" + + archValues + " " + + "aarch64eb amd64 arm26 arm32 coldfire earm earmeb earmhf earmhfeb earmv4 earmv4eb earmv5 " + + "earmv5eb earmv6 earmv6eb earmv6hf earmv6hfeb earmv7 earmv7eb earmv7hf earmv7hfeb evbarm " + + "i386 i586 i686 m68000 mips mips64eb sh3eb sh3el" // See mk/bsd.prefs.mk:/^GNU_ARCH\./ - reMachineGnuArch = "" + - reArch + "|" + - "aarch64_be|arm|armeb|armv4|armv4eb|armv6|armv6eb|armv7|armv7eb|" + - "i486|m5407|m68010|mips64|mipsel|sh|shle|x86_64" - - reEmulArch = reMachineArch // Just a wild guess. + machineGnuArchValues = "" + + archValues + " " + + "aarch64_be arm armeb armv4 armv4eb armv6 armv6eb armv7 armv7eb " + + "i486 m5407 m68010 mips64 mipsel sh shle x86_64" ) -func enumFromRe(re string) *BasicType { - values := strings.Split(re, "|") +func enumFromValues(spaceSeparated string) *BasicType { + values := strings.Fields(spaceSeparated) sort.Strings(values) seen := make(map[string]bool) var unique []string @@ -141,11 +142,11 @@ func enumFromRe(re string) *BasicType { } var ( - enumMachineOpsys = enumFromRe(reMachineOpsys) - enumMachineArch = enumFromRe(reMachineArch) - enumMachineGnuArch = enumFromRe(reMachineGnuArch) - enumEmulOpsys = enumFromRe(reEmulOpsys) - enumEmulArch = enumFromRe(reEmulArch) + enumMachineOpsys = enumFromValues(machineOpsysValues) + enumMachineArch = enumFromValues(machineArchValues) + enumMachineGnuArch = enumFromValues(machineGnuArchValues) + enumEmulOpsys = enumFromValues(emulOpsysValues) + enumEmulArch = enumFromValues(machineArchValues) // Just a wild guess. enumMachineGnuPlatformOpsys = enumEmulOpsys ) @@ -214,23 +215,30 @@ func (cv *VartypeCheck) CFlag() { } } -// Comment checks for the single-line description of the package. +// Comment checks for the single-line description of a package. +// +// The comment for categories is checked in CheckdirCategory since these +// almost never change. func (cv *VartypeCheck) Comment() { value := cv.Value - if value == "TODO: Short description of the package" { // See pkgtools/url2pkg/files/url2pkg.pl, keyword "COMMENT". + // See pkgtools/url2pkg/files/url2pkg.pl, keyword "COMMENT". + if value == "TODO: Short description of the package" { cv.Errorf("COMMENT must be set.") } + if m, first := match1(value, `^(?i)(a|an)[\t ]`); m { cv.Warnf("COMMENT should not begin with %q.", first) } - if m, isA := match1(value, ` (is a|is an) `); m { + + if m, isA := match1(value, `\b(is an?)\b`); m { cv.Warnf("COMMENT should not contain %q.", isA) G.Explain( "The words \"package is a\" are redundant.", "Since every package comment could start with them,", "it is better to remove this redundancy in all cases.") } + if G.Pkg != nil && G.Pkg.EffectivePkgbase != "" { pkgbase := G.Pkg.EffectivePkgbase if hasPrefix(strings.ToLower(value), strings.ToLower(pkgbase+" ")) { @@ -241,15 +249,19 @@ func (cv *VartypeCheck) Comment() { "provide additional information instead.") } } + if matches(value, `^[a-z]`) && cv.Op == opAssign { cv.Warnf("COMMENT should start with a capital letter.") } + if hasSuffix(value, ".") { cv.Warnf("COMMENT should not end with a period.") } + if len(value) > 70 { cv.Warnf("COMMENT should not be longer than 70 characters.") } + if hasPrefix(value, "\"") && hasSuffix(value, "\"") || hasPrefix(value, "'") && hasSuffix(value, "'") { cv.Warnf("COMMENT should not be enclosed in quotes.") @@ -283,7 +295,7 @@ func (cv *VartypeCheck) ConfFiles() { func (cv *VartypeCheck) Dependency() { value := cv.Value - parser := NewParser(cv.Line, value, false) + parser := NewMkParser(nil, value, false) deppat := parser.Dependency() if deppat != nil && deppat.Wildcard == "" && (parser.Rest() == "{,nb*}" || parser.Rest() == "{,nb[0-9]*}") { cv.Warnf("Dependency patterns of the form pkgbase>=1.0 don't need the \"{,nb*}\" extension.") @@ -294,6 +306,11 @@ func (cv *VartypeCheck) Dependency() { "For dependency patterns using the comparison operators,", "this is not necessary.") + } else if deppat == nil && contains(value, "{") { + // Don't warn about complicated patterns like "{ssh{,6}>=0,openssh>=0}" + // that pkglint doesn't understand as of January 2019. + return + } else if deppat == nil || !parser.EOF() { cv.Warnf("Invalid dependency pattern %q.", value) G.Explain( @@ -375,7 +392,7 @@ func (cv *VartypeCheck) DependencyWithPath() { cv.Warnf("Please use USE_TOOLS+=gmake instead of this dependency.") } - MkLineChecker{cv.MkLine}.CheckVartypeBasic(cv.Varname, BtDependency, cv.Op, pattern, cv.MkComment, cv.Guessed) + cv.WithValue(pattern).Dependency() return } @@ -390,12 +407,21 @@ func (cv *VartypeCheck) DependencyWithPath() { "Examples for valid dependency patterns with path are:", " package-[0-9]*:../../category/package", " package>=3.41:../../category/package", - " package-2.718:../../category/package") + " package-2.718{,nb*}:../../category/package") } func (cv *VartypeCheck) DistSuffix() { - if cv.Value == ".tar.gz" { + if cv.Value == ".tar.gz" && !hasPrefix(cv.MkComment, "#") { cv.Notef("%s is \".tar.gz\" by default, so this definition may be redundant.", cv.Varname) + cv.Explain( + "To check whether the definition is really redundant:", + "", + sprintf("\t1. run %q", bmake("show-var VARNAME="+cv.Varname)), + "\t2. remove this definition", + sprintf("\t3. run %q again", bmake("show-var VARNAME="+cv.Varname)), + "", + "If the values from step 1 and step 3 really differ, add a comment to this line", + "in the Makefile that this (apparently redundant) definition is really needed.") } } @@ -421,32 +447,41 @@ func (cv *VartypeCheck) EmulPlatform() { } } -func (cv *VartypeCheck) Enum(vmap map[string]bool, basicType *BasicType) { +// Enum checks an enumeration for valid values. +// +// The given allowedValues contains all allowed enum values. +// The given basicType is only provided to lazily access the allowed enum values as a sorted list. +func (cv *VartypeCheck) Enum(allowedValues map[string]bool, basicType *BasicType) { if cv.Op == opUseMatch { - if !vmap[cv.Value] && cv.Value == cv.ValueNoVar { + if !allowedValues[cv.Value] && cv.Value == cv.ValueNoVar { canMatch := false - for value := range vmap { + for value := range allowedValues { if ok, err := path.Match(cv.Value, value); err != nil { cv.Warnf("Invalid match pattern %q.", cv.Value) - break + return } else if ok { canMatch = true } } if !canMatch { - cv.Warnf("The pattern %q cannot match any of { %s } for %s.", cv.Value, basicType.AllowedEnums(), cv.Varname) + cv.Warnf("The pattern %q cannot match any of { %s } for %s.", + cv.Value, basicType.AllowedEnums(), cv.Varname) } } return } - if cv.Value == cv.ValueNoVar && !vmap[cv.Value] { - cv.Warnf("%q is not valid for %s. Use one of { %s } instead.", cv.Value, cv.Varname, basicType.AllowedEnums()) + if cv.Value == cv.ValueNoVar && !allowedValues[cv.Value] { + cv.Warnf("%q is not valid for %s. Use one of { %s } instead.", + cv.Value, cv.Varname, basicType.AllowedEnums()) } } func (cv *VartypeCheck) FetchURL() { - MkLineChecker{cv.MkLine}.CheckVartypeBasic(cv.Varname, BtURL, cv.Op, cv.Value, cv.MkComment, cv.Guessed) + + // TODO: Handle leading "-". + + cv.URL() for siteURL, siteName := range G.Pkgsrc.MasterSiteURLToVar { if hasPrefix(cv.Value, siteURL) { @@ -462,6 +497,7 @@ func (cv *VartypeCheck) FetchURL() { } } + // TODO: Replace the regular expression by accessing the MkVarUse. if m, name, subdir := match2(cv.Value, `\$\{(MASTER_SITE_[^:]*).*:=(.*)\}$`); m { if G.Pkgsrc.MasterSiteVarToURL[name] == "" { cv.Errorf("The site %s does not exist.", name) @@ -487,6 +523,9 @@ func (cv *VartypeCheck) Filename() { } func (cv *VartypeCheck) FileMask() { + + // TODO: Decide whether to call this a "mask" or a "pattern", and use only that word everywhere. + switch { case cv.Op == opUseMatch: break @@ -527,7 +566,7 @@ func (cv *VartypeCheck) GccReqd() { } func (cv *VartypeCheck) Homepage() { - MkLineChecker{cv.MkLine}.CheckVartypeBasic(cv.Varname, BtURL, cv.Op, cv.Value, cv.MkComment, cv.Guessed) + cv.URL() if m, wrong, sitename, subdir := match3(cv.Value, `^(\$\{(MASTER_SITE\w+)(?::=([\w\-/]+))?\})`); m { baseURL := G.Pkgsrc.MasterSiteVarToURL[sitename] @@ -538,7 +577,9 @@ func (cv *VartypeCheck) Homepage() { } } } + fixedURL := baseURL + subdir + fix := cv.Autofix() if baseURL != "" { fix.Warnf("HOMEPAGE should not be defined in terms of MASTER_SITEs. Use %s directly.", fixedURL) @@ -557,22 +598,28 @@ func (cv *VartypeCheck) Homepage() { } } +// Identifier checks for valid identifiers in various contexts, limiting the +// valid characters to A-Za-z0-9_. func (cv *VartypeCheck) Identifier() { if cv.Op == opUseMatch { - if cv.Value == cv.ValueNoVar && !matches(cv.Value, `^[\w*?]`) { + if cv.Value == cv.ValueNoVar && !matches(cv.Value, `^[\w*\-?\[\]]+$`) { cv.Warnf("Invalid identifier pattern %q for %s.", cv.Value, cv.Varname) } return } + if cv.Value != cv.ValueNoVar { // TODO: Activate this warning again, or document why it is not useful. //line.logWarning("Identifiers should be given directly.") } + switch { - case matches(cv.ValueNoVar, `^[+\-.0-9A-Z_a-z]+$`): + case matches(cv.ValueNoVar, `^[+\-.\w]+$`): // Fine. + case cv.Value != "" && cv.ValueNoVar == "": // Don't warn here. + default: cv.Warnf("Invalid identifier %q.", cv.Value) } @@ -646,7 +693,7 @@ func (cv *VartypeCheck) MachineGnuPlatform() { cv.Warnf("%q is not a valid platform pattern.", cv.Value) G.Explain( "A platform pattern has the form <OPSYS>-<OS_VERSION>-<MACHINE_ARCH>.", - "Each of these components may be a shell globbing expression.", + "Each of these components may use wildcards.", "", "Examples:", "* NetBSD-[456].*-i386", @@ -667,6 +714,7 @@ func (cv *VartypeCheck) MailAddress() { if strings.EqualFold(domain, "NetBSD.org") && domain != "NetBSD.org" { cv.Warnf("Please write \"NetBSD.org\" instead of %q.", domain) } + if matches(value, `(?i)^(tech-pkg|packages)@NetBSD\.org$`) { cv.Errorf("This mailing list address is obsolete. Use pkgsrc-users@NetBSD.org instead.") } @@ -703,7 +751,8 @@ func (cv *VartypeCheck) Option() { return } - if _, found := G.Pkgsrc.PkgOptions[optname]; !found { // There's a difference between empty and absent here. + // There's a difference between empty and absent here. + if _, found := G.Pkgsrc.PkgOptions[optname]; !found { cv.Warnf("Unknown option %q.", optname) G.Explain( "This option is not documented in the mk/defaults/options.description file.", @@ -758,10 +807,12 @@ func (cv *VartypeCheck) PathMask() { if cv.Op == opUseMatch { return } + if !matches(cv.ValueNoVar, `^[#%*+\-./0-9?@A-Z\[\]_a-z~]*$`) { cv.Warnf("%q is not a valid pathname mask.", cv.Value) } - LineChecker{cv.Line}.CheckAbsolutePathname(cv.Value) + + LineChecker{cv.MkLine.Line}.CheckAbsolutePathname(cv.Value) } // Pathname checks for pathnames. @@ -773,10 +824,12 @@ func (cv *VartypeCheck) Pathname() { if cv.Op == opUseMatch { return } + if !matches(cv.ValueNoVar, `^[#\-0-9A-Za-z._~+%/]*$`) { cv.Warnf("%q is not a valid pathname.", cv.Value) } - LineChecker{cv.Line}.CheckAbsolutePathname(cv.Value) + + LineChecker{cv.MkLine.Line}.CheckAbsolutePathname(cv.Value) } func (cv *VartypeCheck) Perl5Packlist() { @@ -809,6 +862,7 @@ func (cv *VartypeCheck) Pkgname() { func (cv *VartypeCheck) PkgOptionsVar() { cv.VariableName() + // TODO: Replace regex with proper VarUse. if matches(cv.Value, `\$\{PKGBASE[:\}]`) { cv.Errorf("PKGBASE must not be used in PKG_OPTIONS_VAR.") G.Explain( @@ -824,6 +878,7 @@ func (cv *VartypeCheck) PkgOptionsVar() { } // PkgPath checks a directory name relative to the top-level pkgsrc directory. +// // Despite its name, it is more similar to RelativePkgDir than to RelativePkgPath. func (cv *VartypeCheck) PkgPath() { pkgsrcdir := relpath(path.Dir(cv.MkLine.Filename), G.Pkgsrc.File(".")) @@ -834,7 +889,7 @@ func (cv *VartypeCheck) PkgRevision() { if !matches(cv.Value, `^[1-9]\d*$`) { cv.Warnf("%s must be a positive integer number.", cv.Varname) } - if cv.Line.Basename != "Makefile" { + if cv.MkLine.Basename != "Makefile" { cv.Errorf("%s only makes sense directly in the package Makefile.", cv.Varname) G.Explain( "Usually, different packages using the same Makefile.common have", @@ -935,15 +990,14 @@ func (cv *VartypeCheck) Restricted() { } func (cv *VartypeCheck) SedCommands() { - tokens, rest := splitIntoShellTokens(cv.Line, cv.Value) + tokens, rest := splitIntoShellTokens(cv.MkLine.Line, cv.Value) if rest != "" { - if contains(cv.Line.Text, "#") { + if contains(cv.MkLine.Text, "#") { cv.Errorf("Invalid shell words %q in sed commands.", rest) G.Explain( "When sed commands have embedded \"#\" characters, they need to be", "escaped with a backslash, otherwise make(1) will interpret them as a", - "comment, no matter if they occur in single or double quotes or", - "whatever.") + "comment, even if they occur in single or double quotes or whatever.") } return } @@ -980,7 +1034,7 @@ func (cv *VartypeCheck) SedCommands() { case token == "-n": // Don't print lines per default. - case i == 0 && matches(token, `^(["']?)(?:\d*|/.*/)s.+["']?$`): + case matches(token, `^["']?(\d+|/.*/)?s`): cv.Notef("Please always use \"-e\" in sed commands, even if there is only one substitution.") default: @@ -1013,6 +1067,8 @@ func (cv *VartypeCheck) Stage() { } // Tool checks for tool names like "awk", "m4:pkgsrc", "digest:bootstrap". +// +// TODO: Distinguish between Tool and ToolDependency. func (cv *VartypeCheck) Tool() { if cv.Varname == "TOOLS_NOOP" && cv.Op == opAssignAppend { // no warning for package-defined tool definitions @@ -1025,10 +1081,11 @@ func (cv *VartypeCheck) Tool() { switch tooldep { case "", "bootstrap", "build", "pkgsrc", "run", "test": default: - cv.Errorf("Unknown tool dependency %q. Use one of \"bootstrap\", \"build\", \"pkgsrc\", \"run\" or \"test\".", tooldep) + cv.Errorf("Invalid tool dependency %q. Use one of \"bootstrap\", \"build\", \"pkgsrc\", \"run\" or \"test\".", tooldep) } + } else if cv.Op != opUseMatch && cv.Value == cv.ValueNoVar { - cv.Errorf("Malformed tool dependency: %q.", cv.Value) + cv.Errorf("Invalid tool dependency %q.", cv.Value) G.Explain( "A tool dependency typically looks like \"sed\" or \"sed:run\".") } @@ -1089,8 +1146,8 @@ func (cv *VartypeCheck) VariableName() { "parameterized part, following the dot.", "", "Examples:", - "\t* PKGNAME", - "\t* PKG_OPTIONS.gnuchess") + "* PKGNAME", + "* PKG_OPTIONS.gtk+-2.0") } } @@ -1129,18 +1186,21 @@ func (cv *VartypeCheck) WrapperReorder() { func (cv *VartypeCheck) WrapperTransform() { cmd := cv.Value - if hasPrefix(cmd, "rm:-") || - matches(cmd, `^(R|l|rpath):([^:]+):(.+)$`) || - matches(cmd, `^'?(opt|rename|rm-optarg|rmdir):.*$`) || - cmd == "-e" || - matches(cmd, `^["']?s[|:,]`) { - return + switch { + case hasPrefix(cmd, "rm:-"), + matches(cmd, `^(R|l|rpath):([^:]+):(.+)$`), + matches(cmd, `^'?(opt|rename|rm-optarg|rmdir):.*$`), + cmd == "-e", + matches(cmd, `^["']?s[|:,]`): + break + + default: + cv.Warnf("Unknown wrapper transform command %q.", cmd) } - cv.Warnf("Unknown wrapper transform command %q.", cmd) } func (cv *VartypeCheck) WrkdirSubdirectory() { - MkLineChecker{cv.MkLine}.CheckVartypeBasic(cv.Varname, BtPathname, cv.Op, cv.Value, cv.MkComment, cv.Guessed) + cv.Pathname() } // WrksrcSubdirectory checks a directory relative to ${WRKSRC}, @@ -1150,16 +1210,33 @@ func (cv *VartypeCheck) WrksrcSubdirectory() { if rest == "" { rest = "." } - cv.Notef("You can use %q instead of %q.", rest, cv.Value) - G.Explain( + + fix := cv.Autofix() + fix.Notef("You can use %q instead of %q.", rest, cv.Value) + fix.Explain( "These directories are interpreted relative to ${WRKSRC}.") + fix.Replace(cv.Value, rest) + fix.Apply() - } else if cv.Value != "" && cv.ValueNoVar == "" { + } else if cv.ValueNoVar == "" { // The value of another variable } else if !matches(cv.ValueNoVar, `^(?:\.|[0-9A-Za-z_@][-0-9A-Za-z_@./+]*)$`) { cv.Warnf("%q is not a valid subdirectory of ${WRKSRC}.", cv.Value) + cv.Explain( + "WRKSRC should be defined so that there is no need to do anything", + "outside of this directory.", + "", + "Example:", + "", + "\tWRKSRC=\t${WRKDIR}", + "\tCONFIGURE_DIRS=\t${WRKSRC}/lib ${WRKSRC}/src", + "\tBUILD_DIRS=\t${WRKSRC}/lib ${WRKSRC}/src ${WRKSRC}/cmd", + "", + seeGuide("Directories used during the build process", "build.builddirs")) } + + // TODO: Check for ${WRKSRC}/.. or a simple .., like in checkTextWrksrcDotDot. } func (cv *VartypeCheck) Yes() { diff --git a/pkgtools/pkglint/files/vartypecheck_test.go b/pkgtools/pkglint/files/vartypecheck_test.go index 9e339328a5e..7197298c186 100644 --- a/pkgtools/pkglint/files/vartypecheck_test.go +++ b/pkgtools/pkglint/files/vartypecheck_test.go @@ -5,28 +5,46 @@ import ( ) func (s *Suite) Test_VartypeCheck_AwkCommand(c *check.C) { - vt := NewVartypeCheckTester(s.Init(c), (*VartypeCheck).AwkCommand) + t := s.Init(c) + vt := NewVartypeCheckTester(t, (*VartypeCheck).AwkCommand) vt.Varname("PLIST_AWK") vt.Op(opAssignAppend) vt.Values( "{print $0}", "{print $$0}") + t.DisableTracing() + vt.Values( + "{print $0}", + "{print $$0}") + + // TODO: In this particular context of AWK programs, $$0 is not a shell variable. + // The warning should be adjusted to reflect this. vt.Output( - "WARN: filename:1: $0 is ambiguous. Use ${0} if you mean a Make variable or $$0 if you mean a shell variable.") + "WARN: filename:1: $0 is ambiguous. "+ + "Use ${0} if you mean a Make variable or $$0 if you mean a shell variable.", + "WARN: filename:3: $0 is ambiguous. "+ + "Use ${0} if you mean a Make variable or $$0 if you mean a shell variable.") } func (s *Suite) Test_VartypeCheck_BasicRegularExpression(c *check.C) { - vt := NewVartypeCheckTester(s.Init(c), (*VartypeCheck).BasicRegularExpression) + t := s.Init(c) + vt := NewVartypeCheckTester(t, (*VartypeCheck).BasicRegularExpression) vt.Varname("REPLACE_FILES.pl") vt.Values( ".*\\.pl$", ".*\\.pl$$") + t.DisableTracing() + vt.Values( + ".*\\.pl$", + ".*\\.pl$$") vt.Output( - "WARN: filename:1: Internal pkglint error in MkLine.Tokenize at \"$\".") + "WARN: filename:1: Internal pkglint error in MkLine.Tokenize at \"$\".", + "WARN: filename:3: Internal pkglint error in MkLine.Tokenize at \"$\".") + } func (s *Suite) Test_VartypeCheck_BuildlinkDepmethod(c *check.C) { @@ -36,7 +54,8 @@ func (s *Suite) Test_VartypeCheck_BuildlinkDepmethod(c *check.C) { vt.Op(opAssignDefault) vt.Values( "full", - "unknown") + "unknown", + "${BUILDLINK_DEPMETHOD.kernel}") vt.Output( "WARN: filename:2: Invalid dependency method \"unknown\". Valid methods are \"build\" or \"full\".") @@ -66,7 +85,7 @@ func (s *Suite) Test_VartypeCheck_Category(c *check.C) { func (s *Suite) Test_VartypeCheck_CFlag(c *check.C) { vt := NewVartypeCheckTester(s.Init(c), (*VartypeCheck).CFlag) - vt.tester.SetupTool("pkg-config", "", AtRunTime) + vt.tester.SetUpTool("pkg-config", "", AtRunTime) vt.Varname("CFLAGS") vt.Op(opAssignAppend) @@ -76,12 +95,18 @@ func (s *Suite) Test_VartypeCheck_CFlag(c *check.C) { "target:sparc64", "-std=c99", "-XX:+PrintClassHistogramAfterFullGC", - "`pkg-config pidgin --cflags`") + "`pkg-config pidgin --cflags`", + "-c99", + "-c", + "-no-integrated-as", + "-pthread", + "`pkg-config`_plus") vt.Output( "WARN: filename:2: Compiler flag \"/W3\" should start with a hyphen.", "WARN: filename:3: Compiler flag \"target:sparc64\" should start with a hyphen.", - "WARN: filename:5: Unknown compiler flag \"-XX:+PrintClassHistogramAfterFullGC\".") + "WARN: filename:5: Unknown compiler flag \"-XX:+PrintClassHistogramAfterFullGC\".", + "WARN: filename:11: Compiler flag \"`pkg-config`_plus\" should start with a hyphen.") vt.Op(opUseMatch) vt.Values( @@ -108,7 +133,9 @@ func (s *Suite) Test_VartypeCheck_Comment(c *check.C) { "Package is a great package", "Package is an awesome package", "The Big New Package is a great package", - "Converter converts between measurement units") + "Converter converts between measurement units", + "\"Official\" office suite", + "'SQL injection fuzzer") vt.Output( "ERROR: filename:2: COMMENT must be set.", @@ -169,7 +196,12 @@ func (s *Suite) Test_VartypeCheck_Dependency(c *check.C) { "postgresql8[0-35-9]-${module}-[0-9]*", "ncurses-${NC_VERS}{,nb*}", "{ssh{,6}-[0-9]*,openssh-[0-9]*}", - "gnome-control-center>=2.20.1{,nb*}") + "gnome-control-center>=2.20.1{,nb*}", + "gnome-control-center>=2.20.1{,nb[0-9]*}", + "package-1.0|garbage", + "${_EMACS_CONFLICTS.${_EMACS_FLAVOR}}", + "package>=1.0:../../category/package", + "package-1.0>=1.0.3") vt.Output( "WARN: filename:1: Invalid dependency pattern \"Perl\".", @@ -180,18 +212,24 @@ func (s *Suite) Test_VartypeCheck_Dependency(c *check.C) { "WARN: filename:10: Please use \"5.22{,nb*}\" instead of \"5.22\" as the version pattern.", "WARN: filename:11: Please use \"5.*\" instead of \"5*\" as the version pattern.", "WARN: filename:12: The version pattern \"2.0-[0-9]*\" should not contain a hyphen.", - "WARN: filename:20: The version pattern \"[0-9]*,openssh-[0-9]*}\" should not contain a hyphen.", // XXX - "WARN: filename:21: Dependency patterns of the form pkgbase>=1.0 don't need the \"{,nb*}\" extension.") + "WARN: filename:21: Dependency patterns of the form pkgbase>=1.0 don't need the \"{,nb*}\" extension.", + "WARN: filename:22: Dependency patterns of the form pkgbase>=1.0 don't need the \"{,nb*}\" extension.", + "WARN: filename:23: Invalid dependency pattern \"package-1.0|garbage\".", + // TODO: Mention that the path should be removed. + "WARN: filename:25: Invalid dependency pattern \"package>=1.0:../../category/package\".", + // TODO: Mention that version numbers in a pkgbase must be appended directly, without hyphen. + "WARN: filename:26: Invalid dependency pattern \"package-1.0>=1.0.3\".") } func (s *Suite) Test_VartypeCheck_DependencyWithPath(c *check.C) { t := s.Init(c) vt := NewVartypeCheckTester(t, (*VartypeCheck).DependencyWithPath) - t.CreateFileLines("x11/alacarte/Makefile") t.CreateFileLines("category/package/Makefile") t.CreateFileLines("devel/gettext/Makefile") t.CreateFileLines("devel/gmake/Makefile") + t.CreateFileLines("devel/py-module/Makefile") + t.CreateFileLines("x11/alacarte/Makefile") G.Pkg = NewPackage(t.File("category/package")) vt.Varname("DEPENDS") @@ -211,7 +249,8 @@ func (s *Suite) Test_VartypeCheck_DependencyWithPath(c *check.C) { "broken>:../../x11/alacarte", "gtk2+>=2.16:../../x11/alacarte", "gettext-[0-9]*:../../devel/gettext", - "gmake-[0-9]*:../../devel/gmake") + "gmake-[0-9]*:../../devel/gmake", + "${PYPKGPREFIX}-module>=0:../../devel/py-module") vt.Output( "WARN: ~/category/package/filename.mk:1: Invalid dependency pattern with path \"Perl\".", @@ -237,7 +276,8 @@ func (s *Suite) Test_VartypeCheck_DistSuffix(c *check.C) { vt.Varname("EXTRACT_SUFX") vt.Values( ".tar.gz", - ".tar.bz2") + ".tar.bz2", + ".tar.gz # overrides a definition from a Makefile.common") vt.Output( "NOTE: filename:1: EXTRACT_SUFX is \".tar.gz\" by default, so this definition may be redundant.") @@ -259,12 +299,17 @@ func (s *Suite) Test_VartypeCheck_EmulPlatform(c *check.C) { "interix irix linux mirbsd netbsd openbsd osf1 solaris sunos "+ "} instead.", "WARN: filename:2: \"8087\" is not valid for the hardware architecture part of EMUL_PLATFORM. "+ - "Use one of "+ - "{ aarch64 aarch64eb alpha amd64 arc arm arm26 arm32 cobalt coldfire convex dreamcast "+ - "earm earmeb earmhf earmhfeb earmv4 earmv4eb earmv5 earmv5eb earmv6 earmv6eb earmv6hf "+ - "earmv6hfeb earmv7 earmv7eb earmv7hf earmv7hfeb evbarm hpcmips hpcsh hppa hppa64 "+ - "i386 i586 i686 ia64 m68000 m68k m88k mips mips64 mips64eb mips64el mipseb mipsel mipsn32 "+ - "mlrisc ns32k pc532 pmax powerpc powerpc64 rs6000 s390 sh3eb sh3el sparc sparc64 vax x86_64 "+ + "Use one of { "+ + "aarch64 aarch64eb alpha amd64 arc arm arm26 arm32 "+ + "cobalt coldfire convex dreamcast "+ + "earm earmeb earmhf earmhfeb earmv4 earmv4eb earmv5 earmv5eb "+ + "earmv6 earmv6eb earmv6hf earmv6hfeb "+ + "earmv7 earmv7eb earmv7hf earmv7hfeb evbarm "+ + "hpcmips hpcsh hppa hppa64 "+ + "i386 i586 i686 ia64 m68000 m68k m88k "+ + "mips mips64 mips64eb mips64el mipseb mipsel mipsn32 "+ + "mlrisc ns32k pc532 pmax powerpc powerpc64 rs6000 "+ + "s390 sh3eb sh3el sparc sparc64 vax x86_64 "+ "} instead.", "WARN: filename:3: \"${LINUX}\" is not a valid emulation platform.") } @@ -283,14 +328,14 @@ func (s *Suite) Test_VartypeCheck_Enum(c *check.C) { vt.Output( "WARN: filename:3: The pattern \"sun-jdk*\" cannot match any of { jdk1 jdk2 jdk4 } for JDK.", - "WARN: filename:5: Invalid match pattern \"[\".", - "WARN: filename:5: The pattern \"[\" cannot match any of { jdk1 jdk2 jdk4 } for JDK.") + "WARN: filename:5: Invalid match pattern \"[\".") } func (s *Suite) Test_VartypeCheck_Enum__use_match(c *check.C) { t := s.Init(c) - t.SetupVartypes() + t.SetUpVartypes() + t.SetUpCommandLine("-Wall", "--explain") mklines := t.NewMkLines("module.mk", MkRcsID, @@ -306,15 +351,29 @@ func (s *Suite) Test_VartypeCheck_Enum__use_match(c *check.C) { t.CheckOutputLines( "NOTE: module.mk:3: MACHINE_ARCH should be compared using == instead of matching against \":Mi386\".", - "WARN: module.mk:5: Use ${PKGSRC_COMPILER:Mclang} instead of the == operator.") + "", + "\tThis variable has a single value, not a list of values. Therefore it", + "\tfeels strange to apply list operators like :M and :N onto it. A more", + "\tdirect approach is to use the == and != operators.", + "", + "\tAn entirely different case is when the pattern contains wildcards", + "\tlike ^, *, $. In such a case, using the :M or :N modifiers is useful", + "\tand preferred.", + "", + "WARN: module.mk:5: Use ${PKGSRC_COMPILER:Mclang} instead of the == operator.", + "", + "\tThe PKGSRC_COMPILER can be a list of chained compilers, e.g. \"ccache", + "\tdistcc clang\". Therefore, comparing it using == or != leads to wrong", + "\tresults in these cases.", + "") } func (s *Suite) Test_VartypeCheck_FetchURL(c *check.C) { t := s.Init(c) vt := NewVartypeCheckTester(t, (*VartypeCheck).FetchURL) - t.SetupMasterSite("MASTER_SITE_GNU", "http://ftp.gnu.org/pub/gnu/") - t.SetupMasterSite("MASTER_SITE_GITHUB", "https://github.com/") + t.SetUpMasterSite("MASTER_SITE_GNU", "http://ftp.gnu.org/pub/gnu/") + t.SetUpMasterSite("MASTER_SITE_GITHUB", "https://github.com/") vt.Varname("MASTER_SITES") vt.Values( @@ -327,7 +386,8 @@ func (s *Suite) Test_VartypeCheck_FetchURL(c *check.C) { "WARN: filename:1: Please use ${MASTER_SITE_GITHUB:=example/} "+ "instead of \"https://github.com/example/project/\" "+ "and run \""+confMake+" help topic=github\" for further tips.", - "WARN: filename:2: Please use ${MASTER_SITE_GNU:=bison} instead of \"http://ftp.gnu.org/pub/gnu/bison\".", + "WARN: filename:2: Please use ${MASTER_SITE_GNU:=bison} "+ + "instead of \"http://ftp.gnu.org/pub/gnu/bison\".", "ERROR: filename:3: The subdirectory in MASTER_SITE_GNU must end with a slash.", "ERROR: filename:4: The site MASTER_SITE_INVALID does not exist.") @@ -439,39 +499,75 @@ func (s *Suite) Test_VartypeCheck_Homepage(c *check.C) { vt.Varname("HOMEPAGE") vt.Values( + "http://www.pkgsrc.org/", + "https://www.pkgsrc.org/", "${MASTER_SITES}") vt.Output( - "WARN: filename:1: HOMEPAGE should not be defined in terms of MASTER_SITEs.") + "WARN: filename:3: HOMEPAGE should not be defined in terms of MASTER_SITEs.") G.Pkg = NewPackage(t.File("category/package")) - G.Pkg.vars.Define("MASTER_SITES", t.NewMkLine(G.Pkg.File("Makefile"), 5, "MASTER_SITES=\thttps://cdn.NetBSD.org/pub/pkgsrc/distfiles/")) + + vt.Values( + "${MASTER_SITES}") + + // When this assignment occurs while checking a package, but the package + // doesn't define MASTER_SITES, that variable cannot be expanded, which means + // the warning cannot refer to its value. + vt.Output( + "WARN: filename:4: HOMEPAGE should not be defined in terms of MASTER_SITEs.") + + delete(G.Pkg.vars.defined, "MASTER_SITES") + G.Pkg.vars.Define("MASTER_SITES", t.NewMkLine(G.Pkg.File("Makefile"), 5, + "MASTER_SITES=\thttps://cdn.NetBSD.org/pub/pkgsrc/distfiles/")) vt.Values( "${MASTER_SITES}") vt.Output( - "WARN: filename:2: HOMEPAGE should not be defined in terms of MASTER_SITEs. Use https://cdn.NetBSD.org/pub/pkgsrc/distfiles/ directly.") + "WARN: filename:5: HOMEPAGE should not be defined in terms of MASTER_SITEs. " + + "Use https://cdn.NetBSD.org/pub/pkgsrc/distfiles/ directly.") + + delete(G.Pkg.vars.defined, "MASTER_SITES") + G.Pkg.vars.Define("MASTER_SITES", t.NewMkLine(G.Pkg.File("Makefile"), 5, + "MASTER_SITES=\t${MASTER_SITE_GITHUB}")) + + vt.Values( + "${MASTER_SITES}") + + // When MASTER_SITES itself makes use of another variable, pkglint doesn't + // resolve that variable and just outputs the simple variant of this warning. + vt.Output( + "WARN: filename:6: HOMEPAGE should not be defined in terms of MASTER_SITEs.") + } func (s *Suite) Test_VartypeCheck_Identifier(c *check.C) { t := s.Init(c) vt := NewVartypeCheckTester(t, (*VartypeCheck).Identifier) - vt.Varname("SUBST_CLASSES") + vt.Varname("MYSQL_CHARSET") vt.Values( "${OTHER_VAR}", "identifiers cannot contain spaces", - "id/cannot/contain/slashes") + "id/cannot/contain/slashes", + "id-${OTHER_VAR}", + "") + + vt.Output( + "WARN: filename:2: Invalid identifier \"identifiers cannot contain spaces\".", + "WARN: filename:3: Invalid identifier \"id/cannot/contain/slashes\".", + "WARN: filename:5: Invalid identifier \"\".") + vt.Op(opUseMatch) vt.Values( "[A-Z]", + "[A-Z.]", + "${PKG_OPTIONS:Moption}", "A*B") vt.Output( - "WARN: filename:2: Invalid identifier \"identifiers cannot contain spaces\".", - "WARN: filename:3: Invalid identifier \"id/cannot/contain/slashes\".", - "WARN: filename:11: Invalid identifier pattern \"[A-Z]\" for SUBST_CLASSES.") + "WARN: filename:12: Invalid identifier pattern \"[A-Z.]\" for MYSQL_CHARSET.") } func (s *Suite) Test_VartypeCheck_Integer(c *check.C) { @@ -493,7 +589,7 @@ func (s *Suite) Test_VartypeCheck_Integer(c *check.C) { func (s *Suite) Test_VartypeCheck_LdFlag(c *check.C) { vt := NewVartypeCheckTester(s.Init(c), (*VartypeCheck).LdFlag) - vt.tester.SetupTool("pkg-config", "", AtRunTime) + vt.tester.SetUpTool("pkg-config", "", AtRunTime) vt.Varname("LDFLAGS") vt.Op(opAssignAppend) @@ -508,7 +604,8 @@ func (s *Suite) Test_VartypeCheck_LdFlag(c *check.C) { "-static", "-static-something", "${LDFLAGS.NetBSD}", - "-l${LIBNCURSES}") + "-l${LIBNCURSES}", + "`pkg-config`_plus") vt.Op(opUseMatch) vt.Values( "anything") @@ -516,12 +613,13 @@ func (s *Suite) Test_VartypeCheck_LdFlag(c *check.C) { vt.Output( "WARN: filename:4: Unknown linker flag \"-unknown\".", "WARN: filename:5: Linker flag \"no-hyphen\" should start with a hyphen.", - "WARN: filename:6: Please use \"${COMPILER_RPATH_FLAG}\" instead of \"-Wl,--rpath\".") + "WARN: filename:6: Please use \"${COMPILER_RPATH_FLAG}\" instead of \"-Wl,--rpath\".", + "WARN: filename:12: Linker flag \"`pkg-config`_plus\" should start with a hyphen.") } func (s *Suite) Test_VartypeCheck_License(c *check.C) { t := s.Init(c) - t.SetupPkgsrc() // Adds the gnu-gpl-v2 and 2-clause-bsd licenses + t.SetUpPkgsrc() // Adds the gnu-gpl-v2 and 2-clause-bsd licenses G.Mk = t.NewMkLines("perl5.mk", MkRcsID, @@ -562,7 +660,8 @@ func (s *Suite) Test_VartypeCheck_MachineGnuPlatform(c *check.C) { "Cygwin-*-amd64", "x86_64-*", "*-*-*-*", - "${OTHER_VAR}") + "${OTHER_VAR}", + "x86_64-pc") // Just for code coverage. vt.Output( "WARN: filename:2: The pattern \"Cygwin\" cannot match any of "+ @@ -575,7 +674,59 @@ func (s *Suite) Test_VartypeCheck_MachineGnuPlatform(c *check.C) { "{ bitrig bsdos cygwin darwin dragonfly freebsd haiku hpux interix irix linux mirbsd "+ "netbsd openbsd osf1 solaris sunos } "+ "for the operating system part of MACHINE_GNU_PLATFORM.", - "WARN: filename:4: \"*-*-*-*\" is not a valid platform pattern.") + "WARN: filename:4: \"*-*-*-*\" is not a valid platform pattern.", + "WARN: filename:6: \"x86_64-pc\" is not a valid platform pattern.") +} + +func (s *Suite) Test_VartypeCheck_MachinePlatformPattern(c *check.C) { + vt := NewVartypeCheckTester(s.Init(c), (*VartypeCheck).MachinePlatformPattern) + + vt.Varname("ONLY_FOR_PLATFORM") + vt.Op(opUseMatch) + vt.Values( + "linux-i386", + "nextbsd-5.0-8087", + "netbsd-7.0-l*", + "NetBSD-1.6.2-i386", + "FreeBSD*", + "FreeBSD-*", + "${LINUX}", + "NetBSD-[0-1]*-*") + + vt.Output( + "WARN: filename:1: \"linux-i386\" is not a valid platform pattern.", + "WARN: filename:2: The pattern \"nextbsd\" cannot match any of "+ + "{ AIX BSDOS Bitrig Cygwin Darwin DragonFly FreeBSD FreeMiNT GNUkFreeBSD HPUX Haiku "+ + "IRIX Interix Linux Minix MirBSD NetBSD OSF1 OpenBSD QNX SCO_SV SunOS UnixWare "+ + "} for the operating system part of ONLY_FOR_PLATFORM.", + "WARN: filename:2: The pattern \"8087\" cannot match any of "+ + "{ aarch64 aarch64eb alpha amd64 arc arm arm26 arm32 "+ + "cobalt coldfire convex dreamcast "+ + "earm earmeb earmhf earmhfeb earmv4 earmv4eb "+ + "earmv5 earmv5eb earmv6 earmv6eb earmv6hf "+ + "earmv6hfeb earmv7 earmv7eb earmv7hf earmv7hfeb evbarm hpcmips hpcsh hppa hppa64 "+ + "i386 i586 i686 ia64 m68000 m68k m88k "+ + "mips mips64 mips64eb mips64el mipseb mipsel mipsn32 "+ + "mlrisc ns32k pc532 pmax powerpc powerpc64 "+ + "rs6000 s390 sh3eb sh3el sparc sparc64 vax x86_64 "+ + "} for the hardware architecture part of ONLY_FOR_PLATFORM.", + "WARN: filename:3: The pattern \"netbsd\" cannot match any of "+ + "{ AIX BSDOS Bitrig Cygwin Darwin DragonFly FreeBSD FreeMiNT GNUkFreeBSD HPUX Haiku "+ + "IRIX Interix Linux Minix MirBSD NetBSD OSF1 OpenBSD QNX SCO_SV SunOS UnixWare "+ + "} for the operating system part of ONLY_FOR_PLATFORM.", + "WARN: filename:3: The pattern \"l*\" cannot match any of "+ + "{ aarch64 aarch64eb alpha amd64 arc arm arm26 arm32 "+ + "cobalt coldfire convex dreamcast "+ + "earm earmeb earmhf earmhfeb earmv4 earmv4eb "+ + "earmv5 earmv5eb earmv6 earmv6eb earmv6hf "+ + "earmv6hfeb earmv7 earmv7eb earmv7hf earmv7hfeb evbarm hpcmips hpcsh hppa hppa64 "+ + "i386 i586 i686 ia64 m68000 m68k m88k "+ + "mips mips64 mips64eb mips64el mipseb mipsel mipsn32 "+ + "mlrisc ns32k pc532 pmax powerpc powerpc64 "+ + "rs6000 s390 sh3eb sh3el sparc sparc64 vax x86_64 "+ + "} for the hardware architecture part of ONLY_FOR_PLATFORM.", + "WARN: filename:5: \"FreeBSD*\" is not a valid platform pattern.", + "WARN: filename:8: Please use \"[0-1].*\" instead of \"[0-1]*\" as the version pattern.") } func (s *Suite) Test_VartypeCheck_MailAddress(c *check.C) { @@ -701,10 +852,13 @@ func (s *Suite) Test_VartypeCheck_Perms(c *check.C) { "root", "${ROOT_USER}", "ROOT_USER", - "${REAL_ROOT_USER}") + "${REAL_ROOT_USER}", + "${ROOT_GROUP}", + "${REAL_ROOT_GROUP}") vt.Output( - "ERROR: filename:2: ROOT_USER must not be used in permission definitions. Use REAL_ROOT_USER instead.") + "ERROR: filename:2: ROOT_USER must not be used in permission definitions. Use REAL_ROOT_USER instead.", + "ERROR: filename:5: ROOT_GROUP must not be used in permission definitions. Use REAL_ROOT_GROUP instead.") } func (s *Suite) Test_VartypeCheck_Pkgname(c *check.C) { @@ -724,6 +878,12 @@ func (s *Suite) Test_VartypeCheck_Pkgname(c *check.C) { vt.Output( "WARN: filename:8: \"pkgbase-z1\" is not a valid package name.") + + vt.Op(opUseMatch) + vt.Values( + "pkgbase-[0-9]*") + + vt.OutputEmpty() } func (s *Suite) Test_VartypeCheck_PkgOptionsVar(c *check.C) { @@ -737,7 +897,8 @@ func (s *Suite) Test_VartypeCheck_PkgOptionsVar(c *check.C) { vt.Output( "ERROR: filename:1: PKGBASE must not be used in PKG_OPTIONS_VAR.", - "ERROR: filename:3: PKG_OPTIONS_VAR must be of the form \"PKG_OPTIONS.*\", not \"PKG_OPTS.mc\".") + "ERROR: filename:3: PKG_OPTIONS_VAR must be "+ + "of the form \"PKG_OPTIONS.*\", not \"PKG_OPTS.mc\".") } func (s *Suite) Test_VartypeCheck_PkgPath(c *check.C) { @@ -779,49 +940,6 @@ func (s *Suite) Test_VartypeCheck_PkgRevision(c *check.C) { vt.OutputEmpty() } -func (s *Suite) Test_VartypeCheck_MachinePlatformPattern(c *check.C) { - vt := NewVartypeCheckTester(s.Init(c), (*VartypeCheck).MachinePlatformPattern) - - vt.Varname("ONLY_FOR_PLATFORM") - vt.Op(opUseMatch) - vt.Values( - "linux-i386", - "nextbsd-5.0-8087", - "netbsd-7.0-l*", - "NetBSD-1.6.2-i386", - "FreeBSD*", - "FreeBSD-*", - "${LINUX}", - "NetBSD-[0-1]*-*") - - vt.Output( - "WARN: filename:1: \"linux-i386\" is not a valid platform pattern.", - "WARN: filename:2: The pattern \"nextbsd\" cannot match any of "+ - "{ AIX BSDOS Bitrig Cygwin Darwin DragonFly FreeBSD FreeMiNT GNUkFreeBSD HPUX Haiku "+ - "IRIX Interix Linux Minix MirBSD NetBSD OSF1 OpenBSD QNX SCO_SV SunOS UnixWare "+ - "} for the operating system part of ONLY_FOR_PLATFORM.", - "WARN: filename:2: The pattern \"8087\" cannot match any of "+ - "{ aarch64 aarch64eb alpha amd64 arc arm arm26 arm32 cobalt coldfire convex dreamcast "+ - "earm earmeb earmhf earmhfeb earmv4 earmv4eb earmv5 earmv5eb earmv6 earmv6eb earmv6hf "+ - "earmv6hfeb earmv7 earmv7eb earmv7hf earmv7hfeb evbarm hpcmips hpcsh hppa hppa64 "+ - "i386 i586 i686 ia64 m68000 m68k m88k mips mips64 mips64eb mips64el mipseb mipsel mipsn32 "+ - "mlrisc ns32k pc532 pmax powerpc powerpc64 rs6000 s390 sh3eb sh3el sparc sparc64 vax x86_64 "+ - "} for the hardware architecture part of ONLY_FOR_PLATFORM.", - "WARN: filename:3: The pattern \"netbsd\" cannot match any of "+ - "{ AIX BSDOS Bitrig Cygwin Darwin DragonFly FreeBSD FreeMiNT GNUkFreeBSD HPUX Haiku "+ - "IRIX Interix Linux Minix MirBSD NetBSD OSF1 OpenBSD QNX SCO_SV SunOS UnixWare "+ - "} for the operating system part of ONLY_FOR_PLATFORM.", - "WARN: filename:3: The pattern \"l*\" cannot match any of "+ - "{ aarch64 aarch64eb alpha amd64 arc arm arm26 arm32 cobalt coldfire convex dreamcast "+ - "earm earmeb earmhf earmhfeb earmv4 earmv4eb earmv5 earmv5eb earmv6 earmv6eb earmv6hf "+ - "earmv6hfeb earmv7 earmv7eb earmv7hf earmv7hfeb evbarm hpcmips hpcsh hppa hppa64 "+ - "i386 i586 i686 ia64 m68000 m68k m88k mips mips64 mips64eb mips64el mipseb mipsel mipsn32 "+ - "mlrisc ns32k pc532 pmax powerpc powerpc64 rs6000 s390 sh3eb sh3el sparc sparc64 vax x86_64 "+ - "} for the hardware architecture part of ONLY_FOR_PLATFORM.", - "WARN: filename:5: \"FreeBSD*\" is not a valid platform pattern.", - "WARN: filename:8: Please use \"[0-1].*\" instead of \"[0-1]*\" as the version pattern.") -} - func (s *Suite) Test_VartypeCheck_PythonDependency(c *check.C) { vt := NewVartypeCheckTester(s.Init(c), (*VartypeCheck).PythonDependency) @@ -893,20 +1011,25 @@ func (s *Suite) Test_VartypeCheck_SedCommands(c *check.C) { "-n", "-e 1d", "1d", - "-e") + "-e", + "-i s,from,to,", + "-e s,$${unclosedShellVar") // Just for code coverage. vt.Output( "NOTE: filename:1: Please always use \"-e\" in sed commands, even if there is only one substitution.", "NOTE: filename:2: Each sed command should appear in an assignment of its own.", - "WARN: filename:3: The # character starts a comment.", + "WARN: filename:3: The # character starts a Makefile comment.", "ERROR: filename:3: Invalid shell words \"\\\"s,\" in sed commands.", "WARN: filename:8: Unknown sed command \"1d\".", - "ERROR: filename:9: The -e option to sed requires an argument.") + "ERROR: filename:9: The -e option to sed requires an argument.", + "WARN: filename:10: Unknown sed command \"-i\".", + "NOTE: filename:10: Please always use \"-e\" in sed commands, even if there is only one substitution.", + "WARN: filename:11: Unclosed shell variable starting at \"$${unclosedShellVar\".") } func (s *Suite) Test_VartypeCheck_ShellCommand(c *check.C) { t := s.Init(c) - t.SetupVartypes() + t.SetUpVartypes() vt := NewVartypeCheckTester(t, (*VartypeCheck).ShellCommand) vt.Varname("INSTALL_CMD") @@ -921,8 +1044,8 @@ func (s *Suite) Test_VartypeCheck_ShellCommand(c *check.C) { func (s *Suite) Test_VartypeCheck_ShellCommands(c *check.C) { t := s.Init(c) - t.SetupVartypes() - t.SetupTool("echo", "ECHO", AtRunTime) + t.SetUpVartypes() + t.SetUpTool("echo", "ECHO", AtRunTime) vt := NewVartypeCheckTester(t, (*VartypeCheck).ShellCommands) vt.Varname("GENERATE_PLIST") @@ -952,9 +1075,9 @@ func (s *Suite) Test_VartypeCheck_Tool(c *check.C) { t := s.Init(c) vt := NewVartypeCheckTester(t, (*VartypeCheck).Tool) - t.SetupTool("tool1", "", AtRunTime) - t.SetupTool("tool2", "", AtRunTime) - t.SetupTool("tool3", "", AtRunTime) + t.SetUpTool("tool1", "", AtRunTime) + t.SetUpTool("tool2", "", AtRunTime) + t.SetUpTool("tool3", "", AtRunTime) vt.Varname("USE_TOOLS") vt.Op(opAssignAppend) @@ -966,9 +1089,9 @@ func (s *Suite) Test_VartypeCheck_Tool(c *check.C) { "unknown") vt.Output( - "ERROR: filename:2: Unknown tool dependency \"unknown\". "+ + "ERROR: filename:2: Invalid tool dependency \"unknown\". "+ "Use one of \"bootstrap\", \"build\", \"pkgsrc\", \"run\" or \"test\".", - "ERROR: filename:4: Malformed tool dependency: \"mal:formed:tool\".", + "ERROR: filename:4: Invalid tool dependency \"mal:formed:tool\".", "ERROR: filename:5: Unknown tool \"unknown\".") vt.Varname("USE_TOOLS.NetBSD") @@ -978,7 +1101,7 @@ func (s *Suite) Test_VartypeCheck_Tool(c *check.C) { "tool2:unknown") vt.Output( - "ERROR: filename:12: Unknown tool dependency \"unknown\". " + + "ERROR: filename:12: Invalid tool dependency \"unknown\". " + "Use one of \"bootstrap\", \"build\", \"pkgsrc\", \"run\" or \"test\".") vt.Varname("TOOLS_NOOP") @@ -986,6 +1109,22 @@ func (s *Suite) Test_VartypeCheck_Tool(c *check.C) { vt.Values( "gmake:run") + vt.Varname("TOOLS_NOOP") + vt.Op(opAssign) // TODO: In a Makefile, this should be equivalent to opAssignAppend. + vt.Values( + "gmake:run") + + vt.Output( + "ERROR: filename:31: Unknown tool \"gmake\".") + + vt.Varname("USE_TOOLS") + vt.Op(opUseMatch) + vt.Values( + "tool1", + "tool1:build", + "tool1:*", + "${t}:build") + vt.OutputEmpty() } @@ -997,20 +1136,39 @@ func (s *Suite) Test_VartypeCheck_URL(c *check.C) { vt.Values( "# none", "${OTHER_VAR}", + "https://www.NetBSD.org/", "https://www.netbsd.org/", - "mailto:someone@example.org", - "httpxs://www.example.org", "https://www.example.org", + "ftp://example.org/pub/", + "gopher://example.org/", + + "", + "ftp://example.org/<", + "gopher://example.org/<", + "http://example.org/<", + "https://example.org/<", "https://www.example.org/path with spaces", + "httpxs://www.example.org", + "mailto:someone@example.org", "string with spaces") vt.Output( - "WARN: filename:3: Please write NetBSD.org instead of www.netbsd.org.", - "WARN: filename:4: \"mailto:someone@example.org\" is not a valid URL.", - "WARN: filename:5: \"httpxs://www.example.org\" is not a valid URL. Only ftp, gopher, http, and https URLs are allowed here.", - "NOTE: filename:6: For consistency, please add a trailing slash to \"https://www.example.org\".", - "WARN: filename:7: \"https://www.example.org/path with spaces\" is not a valid URL.", - "WARN: filename:8: \"string with spaces\" is not a valid URL.") + "WARN: filename:4: Please write NetBSD.org instead of www.netbsd.org.", + "NOTE: filename:5: For consistency, please add a trailing slash to \"https://www.example.org\".", + "WARN: filename:8: \"\" is not a valid URL.", + "WARN: filename:9: \"ftp://example.org/<\" is not a valid URL.", + "WARN: filename:10: \"gopher://example.org/<\" is not a valid URL.", + "WARN: filename:11: \"http://example.org/<\" is not a valid URL.", + "WARN: filename:12: \"https://example.org/<\" is not a valid URL.", + "WARN: filename:13: \"https://www.example.org/path with spaces\" is not a valid URL.", + "WARN: filename:14: \"httpxs://www.example.org\" is not a valid URL. Only ftp, gopher, http, and https URLs are allowed here.", + "WARN: filename:15: \"mailto:someone@example.org\" is not a valid URL.", + "WARN: filename:16: \"string with spaces\" is not a valid URL.") + + // Yes, even in 2019, some pkgsrc-wip packages really use a gopher HOMEPAGE. + vt.Values( + "gopher://bitreich.org/1/scm/geomyidae") + vt.OutputEmpty() } func (s *Suite) Test_VartypeCheck_UserGroupName(c *check.C) { @@ -1054,7 +1212,8 @@ func (s *Suite) Test_VartypeCheck_Version(c *check.C) { "1.2.3.4.5.6", "4.1nb17", "4.1-SNAPSHOT", - "4pre7") + "4pre7", + "${VER}") vt.Output( "WARN: filename:4: Invalid version number \"4.1-SNAPSHOT\".") @@ -1091,7 +1250,7 @@ func (s *Suite) Test_VartypeCheck_WrapperReorder(c *check.C) { func (s *Suite) Test_VartypeCheck_WrapperTransform(c *check.C) { vt := NewVartypeCheckTester(s.Init(c), (*VartypeCheck).WrapperTransform) - vt.Varname("WRAPPER_TRANSFORM") + vt.Varname("WRAPPER_TRANSFORM_CMDS") vt.Op(opAssignAppend) vt.Values( "rm:-O3", @@ -1101,7 +1260,8 @@ func (s *Suite) Test_VartypeCheck_WrapperTransform(c *check.C) { "rmdir:/usr/include", "rpath:/usr/lib:/usr/pkg/lib", "rpath:/usr/lib", - "unknown") + "unknown", + "-e 's,-Wall,-Wall -Wextra,'") vt.Output( "WARN: filename:7: Unknown wrapper transform command \"rpath:/usr/lib\".", "WARN: filename:8: Unknown wrapper transform command \"unknown\".") @@ -1118,15 +1278,20 @@ func (s *Suite) Test_VartypeCheck_WrksrcSubdirectory(c *check.C) { "${WRKSRC}/.", "${WRKSRC}/subdir", "${CONFIGURE_DIRS}", - "${WRKSRC}/directory with spaces", - "directory with spaces") + "${WRKSRC}/directory with spaces", // This is a list of 3 directories. + "directory with spaces", // This is a list of 3 directories. + "../other", + "${WRKDIR}/sub", + "${SRCDIR}/sub") vt.Output( "NOTE: filename:1: You can use \".\" instead of \"${WRKSRC}\".", "NOTE: filename:2: You can use \".\" instead of \"${WRKSRC}/\".", "NOTE: filename:3: You can use \".\" instead of \"${WRKSRC}/.\".", "NOTE: filename:4: You can use \"subdir\" instead of \"${WRKSRC}/subdir\".", - "NOTE: filename:6: You can use \"directory with spaces\" instead of \"${WRKSRC}/directory with spaces\".", - "WARN: filename:7: \"directory with spaces\" is not a valid subdirectory of ${WRKSRC}.") + "NOTE: filename:6: You can use \"directory\" instead of \"${WRKSRC}/directory\".", + "WARN: filename:8: \"../other\" is not a valid subdirectory of ${WRKSRC}.", + "WARN: filename:9: \"${WRKDIR}/sub\" is not a valid subdirectory of ${WRKSRC}.", + "WARN: filename:10: \"${SRCDIR}/sub\" is not a valid subdirectory of ${WRKSRC}.") } func (s *Suite) Test_VartypeCheck_Yes(c *check.C) { @@ -1199,6 +1364,11 @@ type VartypeCheckTester struct { // NewVartypeCheckTester starts the test with a filename of "filename", at line 1, // with "=" as the operator. The variable has to be initialized explicitly. func NewVartypeCheckTester(t *Tester, checker func(cv *VartypeCheck)) *VartypeCheckTester { + + // This is necessary to know whether the variable name is a list type + // since in such a case each value is split into the list elements. + t.SetUpVartypes() + return &VartypeCheckTester{ t, checker, @@ -1260,9 +1430,23 @@ func (vt *VartypeCheckTester) Values(values ...string) { effectiveValue = mkline.Value() } - valueNovar := mkline.WithoutMakeVariables(effectiveValue) - vc := VartypeCheck{mkline, mkline.Line, varname, op, effectiveValue, valueNovar, comment, false} - vt.checker(&vc) + vartype := G.Pkgsrc.VariableType(varname) + + // See MkLineChecker.checkVartype. + var lineValues []string + if vartype == nil || vartype.kindOfList == lkNone { + lineValues = []string{effectiveValue} + } else { + var rest string + lineValues, rest = splitIntoMkWords(mkline.Line, effectiveValue) + vt.tester.Check(rest, equals, "") + } + + for _, lineValue := range lineValues { + valueNovar := mkline.WithoutMakeVariables(lineValue) + vc := VartypeCheck{mkline, varname, op, lineValue, valueNovar, comment, false} + vt.checker(&vc) + } vt.lineno++ } |