Allow packages to define their own tools using TOOLS_CREATE.

Suggested by @jperkin for lang/rust.

6 files changed, 88 insertions, 49 deletions

diff --git a/pkgtools/pkglint/files/globaldata.go b/pkgtools/pkglint/files/globaldata.go
index 4bf6699bfb6..6a7ae1e8cc8 100644
--- a/pkgtools/pkglint/files/globaldata.go
+++ b/pkgtools/pkglint/files/globaldata.go
@@ -138,23 +138,7 @@ func (gd *GlobalData) loadTools() {
 		fname := G.globalData.Pkgsrcdir + "/mk/tools/" + basename
 		lines := LoadExistingLines(fname, true)
 		for _, line := range lines {
-			if m, varname, _, _, _, value, _ := MatchVarassign(line.Text); m {
-				if varname == "TOOLS_CREATE" && (value == "[" || matches(value, `^?[-\w.]+$`)) {
-					reg.Register(value)
-
-				} else if m, toolname := match1(varname, `^_TOOLS_VARNAME\.([-\w.]+|\[)$`); m {
-					reg.RegisterVarname(toolname, value)
-
-				} else if m, toolname := match1(varname, `^(?:TOOLS_PATH|_TOOLS_DEPMETHOD)\.([-\w.]+|\[)$`); m {
-					reg.Register(toolname)
-
-				} else if m, toolname := match1(varname, `_TOOLS\.(.*)`); m {
-					reg.Register(toolname)
-					for _, tool := range splitOnSpace(value) {
-						reg.Register(tool)
-					}
-				}
-			}
+			reg.ParseToolLine(line)
 		}
 	}
 
@@ -580,3 +564,23 @@ func (tr *ToolRegistry) Trace() {
 		traceStep("tool %+v", tr.byName[toolname])
 	}
 }
+
+func (tr *ToolRegistry) ParseToolLine(line *Line) {
+	if m, varname, _, _, _, value, _ := MatchVarassign(line.Text); m {
+		if varname == "TOOLS_CREATE" && (value == "[" || matches(value, `^?[-\w.]+$`)) {
+			tr.Register(value)
+
+		} else if m, toolname := match1(varname, `^_TOOLS_VARNAME\.([-\w.]+|\[)$`); m {
+			tr.RegisterVarname(toolname, value)
+
+		} else if m, toolname := match1(varname, `^(?:TOOLS_PATH|_TOOLS_DEPMETHOD)\.([-\w.]+|\[)$`); m {
+			tr.Register(toolname)
+
+		} else if m, toolname := match1(varname, `_TOOLS\.(.*)`); m {
+			tr.Register(toolname)
+			for _, tool := range splitOnSpace(value) {
+				tr.Register(tool)
+			}
+		}
+	}
+}
diff --git a/pkgtools/pkglint/files/mklines.go b/pkgtools/pkglint/files/mklines.go
index 2ef80f29ad5..946b4bcbd87 100644
--- a/pkgtools/pkglint/files/mklines.go
+++ b/pkgtools/pkglint/files/mklines.go
@@ -16,6 +16,7 @@ type MkLines struct {
 	buildDefs      map[string]bool    // Variables that are registered in BUILD_DEFS, to ensure that all user-defined variables are added to it.
 	plistVars      map[string]bool    // Variables that are registered in PLIST_VARS, to ensure that all user-defined variables are added to it.
 	tools          map[string]bool    // Set of tools that are declared to be used.
+	toolRegistry   ToolRegistry       // Tools defined in file scope.
 	SeenBsdPrefsMk bool
 	indentation    Indentation // Indentation depth of preprocessing directives
 }
@@ -42,6 +43,7 @@ func NewMkLines(lines []*Line) *MkLines {
 		make(map[string]bool),
 		make(map[string]bool),
 		tools,
+		NewToolRegistry(),
 		false,
 		Indentation{}}
 }
@@ -202,6 +204,8 @@ func (mklines *MkLines) determineDefinedVariables() {
 				defineVar(mkline, osvar)
 			}
 		}
+
+		mklines.toolRegistry.ParseToolLine(mkline.Line)
 	}
 }
 
diff --git a/pkgtools/pkglint/files/mklines_test.go b/pkgtools/pkglint/files/mklines_test.go
index e86fc71e2fc..b4411fd76c3 100644
--- a/pkgtools/pkglint/files/mklines_test.go
+++ b/pkgtools/pkglint/files/mklines_test.go
@@ -386,3 +386,30 @@ func (s *Suite) Test_MkLines_DetermineUsedVariables__nested(c *check.C) {
 	c.Check(mklines.varuse["outer."], equals, mkline)
 	c.Check(mklines.varuse["outer.*"], equals, mkline)
 }
+
+func (s *Suite) Test_MkLines_PrivateTool_Undefined(c *check.C) {
+	G.globalData.InitVartypes()
+	s.UseCommandLine(c, "-Wall")
+	mklines := s.NewMkLines("fname",
+		mkrcsid,
+		"",
+		"\tmd5sum filename")
+
+	mklines.Check()
+
+	c.Check(s.Output(), equals, "WARN: fname:3: Unknown shell command \"md5sum\".\n")
+}
+
+func (s *Suite) Test_MkLines_PrivateTool_Defined(c *check.C) {
+	G.globalData.InitVartypes()
+	s.UseCommandLine(c, "-Wall")
+	mklines := s.NewMkLines("fname",
+		mkrcsid,
+		"TOOLS_CREATE+=\tmd5sum",
+		"",
+		"\tmd5sum filename")
+
+	mklines.Check()
+
+	c.Check(s.Output(), equals, "")
+}
diff --git a/pkgtools/pkglint/files/shell.go b/pkgtools/pkglint/files/shell.go
index 0ed87fc1598..2401f2ecab0 100644
--- a/pkgtools/pkglint/files/shell.go
+++ b/pkgtools/pkglint/files/shell.go
@@ -474,12 +474,15 @@ func (scc *SimpleCommandChecker) handleTool() bool {
 	}
 
 	shellword := scc.strcmd.Name
-	tool := G.globalData.Tools.byName[shellword]
+	tool, localTool := G.globalData.Tools.byName[shellword], false
+	if tool == nil && G.Mk != nil {
+		tool, localTool = G.Mk.toolRegistry.byName[shellword], true
+	}
 	if tool == nil {
 		return false
 	}
 
-	if !G.Mk.tools[shellword] && !G.Mk.tools["g"+shellword] {
+	if !localTool && !G.Mk.tools[shellword] && !G.Mk.tools["g"+shellword] {
 		scc.shline.line.Warn1("The %q tool is used but not added to USE_TOOLS.", shellword)
 	}
 
diff --git a/pkgtools/pkglint/files/vardefs.go b/pkgtools/pkglint/files/vardefs.go
index a24689c5f4c..da1c8f36c20 100644
--- a/pkgtools/pkglint/files/vardefs.go
+++ b/pkgtools/pkglint/files/vardefs.go
@@ -17,6 +17,35 @@ import (
 // Last synced with mk/defaults/mk.conf revision 1.118
 
 func (gd *GlobalData) InitVartypes() {
+
+	// A package-defined variable may be set in all Makefiles except buildlink3.mk and builtin.mk.
+	pkg := func(varname string, kindOfList KindOfList, checker *BasicType) {
+		acl(varname, kindOfList, checker, "Makefile: set, use; buildlink3.mk, builtin.mk:; Makefile.*, *.mk: default, set, use")
+	}
+
+	// A package-defined list may be appended to in all Makefiles except buildlink3.mk and builtin.mk.
+	// Simple assignment (instead of appending) is only allowed in Makefile and Makefile.common.
+	pkglist := func(varname string, kindOfList KindOfList, checker *BasicType) {
+		acl(varname, kindOfList, checker, "Makefile, Makefile.common, options.mk: append, default, set, use; buildlink3.mk, builtin.mk:; *.mk: append, default, use")
+	}
+
+	// A user-defined or system-defined variable must not be set by any
+	// package file. It also must not be used in buildlink3.mk and
+	// builtin.mk files or at load-time, since the system/user preferences
+	// may not have been loaded when these files are included.
+	sys := func(varname string, kindOfList KindOfList, checker *BasicType) {
+		acl(varname, kindOfList, checker, "buildlink3.mk:; *: use")
+	}
+	usr := func(varname string, kindOfList KindOfList, checker *BasicType) {
+		acl(varname, kindOfList, checker, "buildlink3.mk:; *: use-loadtime, use")
+	}
+	bl3list := func(varname string, kindOfList KindOfList, checker *BasicType) {
+		acl(varname, kindOfList, checker, "buildlink3.mk, builtin.mk: append")
+	}
+	cmdline := func(varname string, kindOfList KindOfList, checker *BasicType) {
+		acl(varname, kindOfList, checker, "buildlink3.mk, builtin.mk:; *: use-loadtime, use")
+	}
+
 	usr("ALLOW_VULNERABLE_PACKAGES", lkNone, BtYes)
 	usr("MANINSTALL", lkShell, enum("maninstall catinstall"))
 	usr("MANZ", lkNone, BtYes)
@@ -688,7 +717,7 @@ func (gd *GlobalData) InitVartypes() {
 	sys("TOOLS_ALIASES", lkShell, BtFilename)
 	sys("TOOLS_BROKEN", lkShell, BtTool)
 	sys("TOOLS_CMD.*", lkNone, BtPathname)
-	sys("TOOLS_CREATE", lkShell, BtTool)
+	acl("TOOLS_CREATE", lkShell, BtTool, "Makefile, Makefile.common, options.mk: append")
 	acl("TOOLS_DEPENDS.*", lkSpace, BtDependencyWithPath, "buildlink3.mk:; Makefile, Makefile.*: set, default; *: use")
 	sys("TOOLS_GNU_MISSING", lkShell, BtTool)
 	sys("TOOLS_NOOP", lkShell, BtTool)
@@ -841,31 +870,3 @@ func parseAclEntries(varname string, aclentries string) []AclEntry {
 	}
 	return result
 }
-
-// A package-defined variable may be set in all Makefiles except buildlink3.mk and builtin.mk.
-func pkg(varname string, kindOfList KindOfList, checker *BasicType) {
-	acl(varname, kindOfList, checker, "Makefile: set, use; buildlink3.mk, builtin.mk:; Makefile.*, *.mk: default, set, use")
-}
-
-// A package-defined list may be appended to in all Makefiles except buildlink3.mk and builtin.mk.
-// Simple assignment (instead of appending) is only allowed in Makefile and Makefile.common.
-func pkglist(varname string, kindOfList KindOfList, checker *BasicType) {
-	acl(varname, kindOfList, checker, "Makefile, Makefile.common, options.mk: append, default, set, use; buildlink3.mk, builtin.mk:; *.mk: append, default, use")
-}
-
-// A user-defined or system-defined variable must not be set by any
-// package file. It also must not be used in buildlink3.mk and
-// builtin.mk files or at load-time, since the system/user preferences
-// may not have been loaded when these files are included.
-func sys(varname string, kindOfList KindOfList, checker *BasicType) {
-	acl(varname, kindOfList, checker, "buildlink3.mk:; *: use")
-}
-func usr(varname string, kindOfList KindOfList, checker *BasicType) {
-	acl(varname, kindOfList, checker, "buildlink3.mk:; *: use-loadtime, use")
-}
-func bl3list(varname string, kindOfList KindOfList, checker *BasicType) {
-	acl(varname, kindOfList, checker, "buildlink3.mk, builtin.mk: append")
-}
-func cmdline(varname string, kindOfList KindOfList, checker *BasicType) {
-	acl(varname, kindOfList, checker, "buildlink3.mk, builtin.mk:; *: use-loadtime, use")
-}
diff --git a/pkgtools/pkglint/files/vartypecheck.go b/pkgtools/pkglint/files/vartypecheck.go
index 16fefc09152..64e81c2625f 100644
--- a/pkgtools/pkglint/files/vartypecheck.go
+++ b/pkgtools/pkglint/files/vartypecheck.go
@@ -894,7 +894,7 @@ func (cv *VartypeCheck) Tool() {
 		// no warning for package-defined tool definitions
 
 	} else if m, toolname, tooldep := match2(cv.Value, `^([-\w]+|\[)(?::(\w+))?$`); m {
-		if G.globalData.Tools.byName[toolname] == nil {
+		if G.globalData.Tools.byName[toolname] == nil && (G.Mk == nil || G.Mk.toolRegistry.byName[toolname] == nil) {
 			cv.Line.Error1("Unknown tool %q.", toolname)
 		}
 		switch tooldep {