Add pattern tree parser infra and connect with cacheset and apt list

This adds a transformation from parse tree into a CacheFilter and
connects it with cachesets and the apt list command.

7 files changed, 182 insertions, 9 deletions

diff --git a/apt-pkg/cachefilter-patterns.cc b/apt-pkg/cachefilter-patterns.cc
index 3c958ebae..ea4cf3cd8 100644
--- a/apt-pkg/cachefilter-patterns.cc
+++ b/apt-pkg/cachefilter-patterns.cc
@@ -201,5 +201,56 @@ bool PatternTreeParser::PatternNode::matches(APT::StringView name, int min, int
    return true;
 }
 
+std::unique_ptr<APT::CacheFilter::Matcher> PatternParser::aPattern(std::unique_ptr<PatternTreeParser::Node> &nodeP)
+{
+   assert(nodeP != nullptr);
+   auto node = dynamic_cast<PatternTreeParser::PatternNode *>(nodeP.get());
+   if (node == nullptr)
+      nodeP->error("Expected a pattern");
+
+   node->error(rstrprintf("Unrecognized pattern '%s'", node->term.to_string().c_str()));
+
+   return nullptr;
+}
+
+std::string PatternParser::aWord(std::unique_ptr<PatternTreeParser::Node> &nodeP)
+{
+   assert(nodeP != nullptr);
+   auto node = dynamic_cast<PatternTreeParser::WordNode *>(nodeP.get());
+   if (node == nullptr)
+      nodeP->error("Expected a word");
+   return node->word.to_string();
+}
+
 } // namespace Internal
+
+// The bridge into the public world
+std::unique_ptr<APT::CacheFilter::Matcher> APT::CacheFilter::ParsePattern(APT::StringView pattern, pkgCacheFile *file)
+{
+   if (file != nullptr && !file->BuildDepCache())
+      return nullptr;
+
+   try
+   {
+      auto top = APT::Internal::PatternTreeParser(pattern).parseTop();
+      APT::Internal::PatternParser parser{file};
+      return parser.aPattern(top);
+   }
+   catch (APT::Internal::PatternTreeParser::Error &e)
+   {
+      std::stringstream ss;
+      ss << "input:" << e.location.start << "-" << e.location.end << ": error: " << e.message << "\n";
+      ss << pattern.to_string() << "\n";
+      for (size_t i = 0; i < e.location.start; i++)
+	 ss << " ";
+      for (size_t i = e.location.start; i < e.location.end; i++)
+	 ss << "^";
+
+      ss << "\n";
+
+      _error->Error("%s", ss.str().c_str());
+      return nullptr;
+   }
+}
+
 } // namespace APT
diff --git a/apt-pkg/cachefilter-patterns.h b/apt-pkg/cachefilter-patterns.h
index dfbcd66a5..3097ba94b 100644
--- a/apt-pkg/cachefilter-patterns.h
+++ b/apt-pkg/cachefilter-patterns.h
@@ -98,6 +98,21 @@ struct PatternTreeParser
    std::unique_ptr<Node> parseQuotedWord();
 };
 
+/**
+ * \brief PatternParser parses the given sentence into a parse tree.
+ *
+ * The parse tree consists of nodes:
+ *  - Word nodes which contains words or quoted words
+ *  - Patterns, which represent ?foo and ?foo(...) patterns
+ */
+struct PatternParser
+{
+   pkgCacheFile *file;
+
+   std::unique_ptr<APT::CacheFilter::Matcher> aPattern(std::unique_ptr<PatternTreeParser::Node> &nodeP);
+   std::string aWord(std::unique_ptr<PatternTreeParser::Node> &nodeP);
+};
+
 } // namespace Internal
 } // namespace APT
 #endif
diff --git a/apt-pkg/cachefilter.h b/apt-pkg/cachefilter.h
index 8a6c01341..3c6e1559d 100644
--- a/apt-pkg/cachefilter.h
+++ b/apt-pkg/cachefilter.h
@@ -7,7 +7,9 @@
 #define APT_CACHEFILTER_H
 // Include Files							/*{{{*/
 #include <apt-pkg/pkgcache.h>
+#include <apt-pkg/string_view.h>
 
+#include <memory>
 #include <string>
 #include <vector>
 
@@ -145,6 +147,8 @@ public:
 };
 									/*}}}*/
 
+/// \brief Parse a pattern, return nullptr or pattern
+std::unique_ptr<APT::CacheFilter::Matcher> ParsePattern(APT::StringView pattern, pkgCacheFile *file);
 }
 }
 #endif
diff --git a/apt-pkg/cacheset.cc b/apt-pkg/cacheset.cc
index 789727266..dd55edb4e 100644
--- a/apt-pkg/cacheset.cc
+++ b/apt-pkg/cacheset.cc
@@ -46,6 +46,7 @@ bool CacheSetHelper::PackageFrom(enum PkgSelector const select, PackageContainer
 	case FNMATCH: return PackageFromFnmatch(pci, Cache, pattern);
 	case PACKAGENAME: return PackageFromPackageName(pci, Cache, pattern);
 	case STRING: return PackageFromString(pci, Cache, pattern);
+	case PATTERN: return PackageFromPattern(pci, Cache, pattern);
 	}
 	return false;
 }
@@ -281,13 +282,33 @@ bool CacheSetHelper::PackageFromPackageName(PackageContainerInterface * const pc
 	pci->insert(Pkg);
 	return true;
 }
+
+bool CacheSetHelper::PackageFromPattern(PackageContainerInterface *const pci, pkgCacheFile &Cache, std::string const &pattern)
+{
+   if (pattern.size() < 1 || pattern[0] != '?')
+      return false;
+
+   auto compiledPattern = APT::CacheFilter::ParsePattern(pattern, &Cache);
+   if (!compiledPattern)
+      return false;
+
+   for (pkgCache::PkgIterator Pkg = Cache->PkgBegin(); Pkg.end() == false; ++Pkg)
+   {
+      if ((*compiledPattern)(Pkg) == false)
+	 continue;
+
+      pci->insert(Pkg);
+   }
+   return true;
+}
 									/*}}}*/
 // PackageFromString - Return all packages matching a specific string	/*{{{*/
 bool CacheSetHelper::PackageFromString(PackageContainerInterface * const pci, pkgCacheFile &Cache, std::string const &str) {
 	bool found = true;
 	_error->PushToStack();
 
-	if (PackageFrom(CacheSetHelper::PACKAGENAME, pci, Cache, str) == false &&
+	if (PackageFrom(CacheSetHelper::PATTERN, pci, Cache, str) == false &&
+	     PackageFrom(CacheSetHelper::PACKAGENAME, pci, Cache, str) == false &&
 		 PackageFrom(CacheSetHelper::TASK, pci, Cache, str) == false &&
 		 // FIXME: hm, hm, regexp/fnmatch incompatible?
 		 PackageFrom(CacheSetHelper::FNMATCH, pci, Cache, str) == false &&
@@ -686,6 +707,7 @@ void CacheSetHelper::canNotFindPackage(enum PkgSelector const select,
 	case FNMATCH: canNotFindFnmatch(pci, Cache, pattern); break;
 	case PACKAGENAME: canNotFindPackage(pci, Cache, pattern); break;
 	case STRING: canNotFindPackage(pci, Cache, pattern); break;
+	case PATTERN: canNotFindPackage(pci, Cache, pattern); break;
 	case UNKNOWN: break;
 	}
 }
@@ -822,6 +844,7 @@ void CacheSetHelper::showPackageSelection(pkgCache::PkgIterator const &pkg, enum
 	case REGEX: showRegExSelection(pkg, pattern); break;
 	case TASK: showTaskSelection(pkg, pattern); break;
 	case FNMATCH: showFnmatchSelection(pkg, pattern); break;
+	case PATTERN: showPatternSelection(pkg, pattern); break;
 	case PACKAGENAME: /* no surprises here */ break;
 	case STRING: /* handled by the special cases */ break;
 	case UNKNOWN: break;
@@ -842,6 +865,12 @@ void CacheSetHelper::showFnmatchSelection(pkgCache::PkgIterator const &/*pkg*/,
                                          std::string const &/*pattern*/) {
 }
 									/*}}}*/
+// showPatternSelection							/*{{{*/
+void CacheSetHelper::showPatternSelection(pkgCache::PkgIterator const & /*pkg*/,
+					  std::string const & /*pattern*/)
+{
+}
+									/*}}}*/
 									/*}}}*/
 // showVersionSelection							/*{{{*/
 void CacheSetHelper::showVersionSelection(pkgCache::PkgIterator const &Pkg,
diff --git a/apt-pkg/cacheset.h b/apt-pkg/cacheset.h
index 489fb6220..6023b861d 100644
--- a/apt-pkg/cacheset.h
+++ b/apt-pkg/cacheset.h
@@ -52,7 +52,7 @@ public:									/*{{{*/
 		GlobalError::MsgType ErrorType = GlobalError::ERROR);
 	virtual ~CacheSetHelper();
 
-	enum PkgSelector { UNKNOWN, REGEX, TASK, FNMATCH, PACKAGENAME, STRING };
+	enum PkgSelector { UNKNOWN, REGEX, TASK, FNMATCH, PACKAGENAME, STRING, PATTERN };
 
 	virtual bool PackageFrom(enum PkgSelector const select, PackageContainerInterface * const pci, pkgCacheFile &Cache, std::string const &pattern);
 
@@ -172,10 +172,12 @@ protected:
 	bool PackageFromFnmatch(PackageContainerInterface * const pci, pkgCacheFile &Cache, std::string pattern);
 	bool PackageFromPackageName(PackageContainerInterface * const pci, pkgCacheFile &Cache, std::string pattern);
 	bool PackageFromString(PackageContainerInterface * const pci, pkgCacheFile &Cache, std::string const &pattern);
+	bool PackageFromPattern(PackageContainerInterface * const pci, pkgCacheFile &Cache, std::string const &pattern);
 private:
 	void showTaskSelection(pkgCache::PkgIterator const &pkg, std::string const &pattern);
 	void showRegExSelection(pkgCache::PkgIterator const &pkg, std::string const &pattern);
 	void showFnmatchSelection(pkgCache::PkgIterator const &pkg, std::string const &pattern);
+	void showPatternSelection(pkgCache::PkgIterator const &pkg, std::string const &pattern);
 	void canNotFindTask(PackageContainerInterface *const pci, pkgCacheFile &Cache, std::string pattern);
 	void canNotFindRegEx(PackageContainerInterface *const pci, pkgCacheFile &Cache, std::string pattern);
 	void canNotFindFnmatch(PackageContainerInterface *const pci, pkgCacheFile &Cache, std::string pattern);
@@ -739,6 +741,8 @@ public:
 			       std::string pkg, CacheSetHelper::VerSelector const fallback, CacheSetHelper &helper,
 			       bool const onlyFromName = false);
 
+	static bool FromPattern(VersionContainerInterface *const vci, pkgCacheFile &Cache,
+				std::string pkg, CacheSetHelper::VerSelector const fallback, CacheSetHelper &helper);
 
 	static bool FromPackage(VersionContainerInterface * const vci, pkgCacheFile &Cache,
 				pkgCache::PkgIterator const &P, CacheSetHelper::VerSelector const fallback,
diff --git a/apt-private/private-list.cc b/apt-private/private-list.cc
index 7c8c89777..6071129a7 100644
--- a/apt-private/private-list.cc
+++ b/apt-private/private-list.cc
@@ -39,17 +39,26 @@ struct PackageSortAlphabetic						/*{{{*/
 
 class PackageNameMatcher : public Matcher
 {
+   pkgCacheFile &cacheFile;
   public:
-   explicit PackageNameMatcher(const char **patterns)
+   explicit PackageNameMatcher(pkgCacheFile &cacheFile, const char **patterns)
+   : cacheFile(cacheFile)
    {
       for(int i=0; patterns[i] != NULL; ++i)
       {
          std::string pattern = patterns[i];
-         APT::CacheFilter::PackageMatcher *cachefilter = NULL;
-         if(_config->FindB("APT::Cmd::Use-Regexp", false) == true)
+         APT::CacheFilter::Matcher *cachefilter = NULL;
+         if (pattern.size() > 0 && pattern[0] == '?')
+            cachefilter = APT::CacheFilter::ParsePattern(pattern, &cacheFile).release();
+         else if(_config->FindB("APT::Cmd::Use-Regexp", false) == true)
             cachefilter = new APT::CacheFilter::PackageNameMatchesRegEx(pattern);
          else
             cachefilter = new APT::CacheFilter::PackageNameMatchesFnmatch(pattern);
+
+         if (cachefilter == nullptr) {
+            return;
+            filters.clear();
+         }
          filters.push_back(cachefilter);
       }
    }
@@ -62,7 +71,7 @@ class PackageNameMatcher : public Matcher
    {
       for(J=filters.begin(); J != filters.end(); ++J)
       {
-         APT::CacheFilter::PackageMatcher *cachefilter = *J;
+         APT::CacheFilter::Matcher *cachefilter = *J;
          if((*cachefilter)(P)) 
             return true;
       }
@@ -70,8 +79,8 @@ class PackageNameMatcher : public Matcher
    }
 
 private:
-   std::vector<APT::CacheFilter::PackageMatcher*> filters;   
-   std::vector<APT::CacheFilter::PackageMatcher*>::const_iterator J;
+   std::vector<APT::CacheFilter::Matcher*> filters;
+   std::vector<APT::CacheFilter::Matcher*>::const_iterator J;
    #undef PackageMatcher
 };
 									/*}}}*/
@@ -111,7 +120,7 @@ bool DoList(CommandLine &Cmd)
    if (_config->FindB("APT::Cmd::List-Include-Summary", false) == true)
       format += "\n  ${Description}\n";
 
-   PackageNameMatcher matcher(patterns);
+   PackageNameMatcher matcher(CacheFile, patterns);
    LocalitySortedVersionSet bag;
    OpTextProgress progress(*_config);
    progress.OverallProgress(0,
diff --git a/test/integration/test-apt-patterns b/test/integration/test-apt-patterns
new file mode 100755
index 000000000..c33fa1bf8
--- /dev/null
+++ b/test/integration/test-apt-patterns
@@ -0,0 +1,61 @@
+#!/bin/sh
+TESTDIR="$(readlink -f "$(dirname "$0")")"
+. "$TESTDIR/framework"
+
+setupenvironment
+configarchitecture 'i386' 'amd64'
+
+insertpackage 'unstable' 'available' 'all' '1.0'
+
+insertinstalledpackage 'manual1' 'i386' '1.0' 'Depends: automatic1'
+insertinstalledpackage 'manual2' 'i386' '1.0'
+
+insertinstalledpackage 'automatic1' 'i386' '1.0'
+insertinstalledpackage 'automatic2' 'i386' '1.0'
+
+insertinstalledpackage 'essential' 'i386' '1.0' 'Essential: yes'
+insertinstalledpackage 'conf-only' 'i386' '1.0' '' '' 'deinstall ok config-files'
+insertinstalledpackage 'broken' 'i386' '1.0' 'Depends: does-not-exist'
+
+insertinstalledpackage 'not-obsolete' 'i386' '1.0'
+insertpackage 'unstable' 'not-obsolete' 'all' '2.0'
+
+insertpackage 'unstable' 'foreign' 'amd64' '2.0'
+
+setupaptarchive
+
+testsuccess aptmark auto automatic1 automatic2
+
+msgmsg "Check that commands understand patterns"
+
+testfailureequal "E: input:0-14: error: Unrecognized pattern '?not-a-pattern'
+   ?not-a-pattern
+   ^^^^^^^^^^^^^^
+N: Unable to locate package ?not-a-pattern
+N: Couldn't find any package by glob '?not-a-pattern'
+E: Regex compilation error - Invalid preceding regular expression
+N: Couldn't find any package by regex '?not-a-pattern'
+E: input:0-14: error: Unrecognized pattern '?not-a-pattern'
+   ?not-a-pattern
+   ^^^^^^^^^^^^^^
+N: Unable to locate package ?not-a-pattern
+N: Couldn't find any package by glob '?not-a-pattern'
+E: Regex compilation error - Invalid preceding regular expression
+N: Couldn't find any package by regex '?not-a-pattern'
+E: No packages found" apt show '?not-a-pattern'
+
+testfailureequal "Listing...
+E: input:0-14: error: Unrecognized pattern '?not-a-pattern'
+   ?not-a-pattern
+   ^^^^^^^^^^^^^^" apt list '?not-a-pattern'
+
+testfailureequal "Reading package lists...
+Building dependency tree...
+Reading state information...
+E: input:0-14: error: Unrecognized pattern '?not-a-pattern'
+   ?not-a-pattern
+   ^^^^^^^^^^^^^^
+E: Unable to locate package ?not-a-pattern
+E: Couldn't find any package by glob '?not-a-pattern'
+E: Regex compilation error - Invalid preceding regular expression
+E: Couldn't find any package by regex '?not-a-pattern'" apt install -s '?not-a-pattern'