Search in all available description translations

When multiple translations of package descriptions are available,
perform search in all of them. It allows using search patterns in
any of the configured languages.

Previously, only the first available translation was searched. As
the result, patterns in e.g. English never matched packages which
had their descriptions translated into local language.

Closes: #490000

4 files changed, 228 insertions, 49 deletions

diff --git a/apt-pkg/cacheiterators.h b/apt-pkg/cacheiterators.h
index fbd4bcb8d..5c3ad9bbb 100644
--- a/apt-pkg/cacheiterators.h
+++ b/apt-pkg/cacheiterators.h
@@ -220,6 +220,7 @@ class pkgCache::VerIterator : public Iterator<Version, VerIterator> {
 	inline PkgIterator ParentPkg() const {return PkgIterator(*Owner,Owner->PkgP + S->ParentPkg);}
 
 	inline DescIterator DescriptionList() const;
+	DescIterator TranslatedDescriptionForLanguage(APT::StringView lang) const;
 	DescIterator TranslatedDescription() const;
 	inline DepIterator DependsList() const;
 	inline PrvIterator ProvidesList() const;
diff --git a/apt-pkg/pkgcache.cc b/apt-pkg/pkgcache.cc
index ba9a39f13..160c58273 100644
--- a/apt-pkg/pkgcache.cc
+++ b/apt-pkg/pkgcache.cc
@@ -992,6 +992,23 @@ string pkgCache::PkgFileIterator::RelStr()				/*{{{*/
    return Res;
 }
 									/*}}}*/
+// VerIterator::TranslatedDescriptionForLanguage - Return a DescIter for language/*{{{*/
+// ---------------------------------------------------------------------
+/* return a DescIter for the specified language
+ */
+pkgCache::DescIterator pkgCache::VerIterator::TranslatedDescriptionForLanguage(StringView lang) const
+{
+   for (pkgCache::DescIterator Desc = DescriptionList(); Desc.end() == false; ++Desc)
+      if (lang == Desc.LanguageCode())
+         return Desc;
+
+   if (lang == "en")
+      return TranslatedDescriptionForLanguage("");
+
+   return DescIterator();
+}
+
+									/*}}}*/
 // VerIterator::TranslatedDescription - Return the a DescIter for locale/*{{{*/
 // ---------------------------------------------------------------------
 /* return a DescIter for the current locale or the default if none is
@@ -1003,30 +1020,15 @@ pkgCache::DescIterator pkgCache::VerIterator::TranslatedDescription() const
    for (std::vector<string>::const_iterator l = lang.begin();
 	l != lang.end(); ++l)
    {
-      pkgCache::DescIterator Desc = DescriptionList();
-      for (; Desc.end() == false; ++Desc)
-	 if (*l == Desc.LanguageCode())
-	    break;
-      if (Desc.end() == true)
-      {
-	 if (*l == "en")
-	 {
-	    Desc = DescriptionList();
-	    for (; Desc.end() == false; ++Desc)
-	       if (strcmp(Desc.LanguageCode(), "") == 0)
-		  break;
-	    if (Desc.end() == true)
-	       continue;
-	 }
-	 else
-	    continue;
-      }
-      return Desc;
+      pkgCache::DescIterator Desc = TranslatedDescriptionForLanguage(*l);
+      if (Desc.IsGood())
+         return Desc;
    }
-   for (pkgCache::DescIterator Desc = DescriptionList();
-	Desc.end() == false; ++Desc)
-      if (strcmp(Desc.LanguageCode(), "") == 0)
-	 return Desc;
+
+   pkgCache::DescIterator Desc = TranslatedDescriptionForLanguage("");
+   if (Desc.IsGood())
+      return Desc;
+
    return DescriptionList();
 }
 
diff --git a/apt-private/private-search.cc b/apt-private/private-search.cc
index de1b19758..b3f9469ac 100644
--- a/apt-private/private-search.cc
+++ b/apt-private/private-search.cc
@@ -1,6 +1,7 @@
 // Includes								/*{{{*/
 #include <config.h>
 
+#include <apt-pkg/aptconfiguration.h>
 #include <apt-pkg/cachefile.h>
 #include <apt-pkg/cacheset.h>
 #include <apt-pkg/cmndline.h>
@@ -24,11 +25,30 @@
 #include <sstream>
 #include <string>
 #include <utility>
+#include <vector>
 #include <string.h>
 
 #include <apti18n.h>
 									/*}}}*/
 
+static std::vector<pkgCache::DescIterator> const TranslatedDescriptionsList(pkgCache::VerIterator const &V) /*{{{*/
+{
+   std::vector<pkgCache::DescIterator> Descriptions;
+
+   for (std::string const &lang: APT::Configuration::getLanguages())
+   {
+      pkgCache::DescIterator Desc = V.TranslatedDescriptionForLanguage(lang);
+      if (Desc.IsGood())
+         Descriptions.push_back(Desc);
+   }
+
+   if (Descriptions.empty())
+      Descriptions.push_back(V.TranslatedDescription());
+
+   return Descriptions;
+}
+
+									/*}}}*/
 static bool FullTextSearch(CommandLine &CmdL)				/*{{{*/
 {
 
@@ -94,27 +114,49 @@ static bool FullTextSearch(CommandLine &CmdL)				/*{{{*/
       if (PkgsDone[P->ID] == true)
 	 continue;
 
-      char const * const PkgName = P.Name();
-      pkgCache::DescIterator Desc = V.TranslatedDescription();
-      std::string LongDesc = "";
-      if (Desc.end() == false)
+      std::vector<std::string> PkgDescriptions;
+      if (not NamesOnly)
       {
-	 pkgRecords::Parser &parser = records.Lookup(Desc.FileList());
-	 LongDesc = parser.LongDesc();
+         for (auto &Desc: TranslatedDescriptionsList(V))
+         {
+            pkgRecords::Parser &parser = records.Lookup(Desc.FileList());
+            PkgDescriptions.push_back(parser.LongDesc());
+         }
       }
 
       bool all_found = true;
+
+      char const * const PkgName = P.Name();
+      std::vector<bool> SkipDescription(PkgDescriptions.size(), false);
       for (std::vector<regex_t>::const_iterator pattern = Patterns.begin();
-	    pattern != Patterns.end(); ++pattern)
+           pattern != Patterns.end(); ++pattern)
       {
-	 if (regexec(&(*pattern), PkgName, 0, 0, 0) == 0)
-	    continue;
-	 else if (NamesOnly == false && regexec(&(*pattern), LongDesc.c_str(), 0, 0, 0) == 0)
-	    continue;
-	 // search patterns are AND, so one failing fails all
-	 all_found = false;
-	 break;
+         if (regexec(&(*pattern), PkgName, 0, 0, 0) == 0)
+            continue;
+         else if (not NamesOnly)
+         {
+            bool found = false;
+
+            for (std::vector<std::string>::size_type i = 0; i < PkgDescriptions.size(); ++i)
+            {
+               if (not SkipDescription[i])
+               {
+                  if (regexec(&(*pattern), PkgDescriptions[i].c_str(), 0, 0, 0) == 0)
+                     found = true;
+                  else
+                     SkipDescription[i] = true;
+               }
+            }
+
+            if (found)
+               continue;
+         }
+
+         // search patterns are AND, so one failing fails all
+         all_found = false;
+         break;
       }
+
       if (all_found == true)
       {
 	 PkgsDone[P->ID] = true;
@@ -290,19 +332,43 @@ static bool Search(CommandLine &CmdL)
    // Iterate over all the version records and check them
    for (ExDescFile *J = DFList; J->Df != 0; ++J)
    {
-      pkgRecords::Parser &P = Recs.Lookup(pkgCache::DescFileIterator(*Cache,J->Df));
       size_t const PatternOffset = J->ID * NumPatterns;
-
-      if (NamesOnly == false)
+      if (not NamesOnly)
       {
-	 std::string const LongDesc = P.LongDesc();
-	 for (unsigned I = 0; I < NumPatterns; ++I)
-	 {
-	    if (PatternMatch[PatternOffset + I] == true)
-	       continue;
-	    else if (regexec(&Patterns[I],LongDesc.c_str(),0,0,0) == 0)
-	       PatternMatch[PatternOffset + I] = true;
-	 }
+         std::vector<std::string> PkgDescriptions;
+         for (auto &Desc: TranslatedDescriptionsList(J->V))
+         {
+            pkgRecords::Parser &parser = Recs.Lookup(Desc.FileList());
+            PkgDescriptions.push_back(parser.LongDesc());
+         }
+
+         std::vector<bool> SkipDescription(PkgDescriptions.size(), false);
+         for (unsigned I = 0; I < NumPatterns; ++I)
+         {
+            if (PatternMatch[PatternOffset + I])
+               continue;
+            else
+            {
+               bool found = false;
+
+               for (std::vector<std::string>::size_type k = 0; k < PkgDescriptions.size(); ++k)
+               {
+                  if (not SkipDescription[k])
+                  {
+                     if (regexec(&Patterns[I], PkgDescriptions[k].c_str(), 0, 0, 0) == 0)
+                     {
+                        found = true;
+                        PatternMatch[PatternOffset + I] = true;
+                     }
+                     else
+                        SkipDescription[k] = true;
+                  }
+               }
+
+               if (not found)
+                  break;
+            }
+         }
       }
 
       bool matchedAll = true;
@@ -325,7 +391,10 @@ static bool Search(CommandLine &CmdL)
 	    DisplayRecordV1(CacheFile, Recs, J->V, Vf, Start, Length, std::cout);
 	 }
 	 else
-	    printf("%s - %s\n",P.Name().c_str(),P.ShortDesc().c_str());
+	 {
+	    pkgRecords::Parser &P = Recs.Lookup(pkgCache::DescFileIterator(*Cache, J->Df));
+	    printf("%s - %s\n", P.Name().c_str(), P.ShortDesc().c_str());
+	 }
       }
    }
    
diff --git a/test/integration/test-bug-490000-search-in-all-translations b/test/integration/test-bug-490000-search-in-all-translations
new file mode 100755
index 000000000..3a034bc49
--- /dev/null
+++ b/test/integration/test-bug-490000-search-in-all-translations
@@ -0,0 +1,107 @@
+#!/bin/sh
+
+set -e
+
+TESTDIR="$(readlink -f "$(dirname "$0")")"
+. "${TESTDIR}/framework"
+
+setupenvironment
+
+ARCH='amd64'
+DIST='unstable'
+
+PKG_NAME='foo'
+PKG_VERSION='1.0'
+
+SHORT_DESCRIPTION_EN='have you fooed today?'
+LONG_DESCRIPTION_EN="${SHORT_DESCRIPTION_EN}
+ Where there's foo, there's fire."
+
+SHORT_DESCRIPTION_ZZ='bar alter ego'
+LONG_DESCRIPTION_ZZ="${SHORT_DESCRIPTION_ZZ}
+ He who foos last foos best."
+
+configure_languages()
+{
+	{
+		echo '#clear Acquire::Languages;'
+		echo 'Acquire::Languages {'
+		for language in "$@"
+		do
+			echo "  \"${language}\";"
+		done
+		echo '};'
+	} > rootdir/etc/apt/apt.conf.d/languages.conf
+}
+
+new_translation_record()
+{
+	echo "Package: ${1:?Package name expected}"
+	echo "Description-md5: ${2:?Description-md5 expected}"
+	echo "Description-${3:?Language code expected}: ${4:?Package description expected}"
+	echo
+}
+
+str_md5sum()
+{
+	echo -n "${1:?String expected}" | md5sum | cut -d ' ' -f 1
+}
+
+configarchitecture "${ARCH}"
+
+insertpackage "${DIST}" "${PKG_NAME}" "${ARCH}" "${PKG_VERSION}" '' '' "${LONG_DESCRIPTION_EN}"
+# English translation was already added by insertpackage above
+new_translation_record "${PKG_NAME}" "$(str_md5sum "${LONG_DESCRIPTION_EN}")" 'zz' "${LONG_DESCRIPTION_ZZ}" > "aptarchive/dists/${DIST}/main/i18n/Translation-zz"
+
+configure_languages en zz
+setupaptarchive
+
+# ===========================
+#
+#            Tests
+#
+# ===========================
+
+# ----------[ apt ]----------
+
+# Test that all translations are searched, but the short
+# description is in the first configured language
+
+configure_languages en zz
+testequal "${PKG_NAME}/${DIST} ${PKG_VERSION} ${ARCH}
+  ${SHORT_DESCRIPTION_EN}
+" apt -qq search alter ego
+
+configure_languages zz en
+testequal "${PKG_NAME}/${DIST} ${PKG_VERSION} ${ARCH}
+  ${SHORT_DESCRIPTION_ZZ}
+" apt -qq search you today
+
+# Search in configured languages only
+configure_languages zz
+testempty apt -qq search where fire
+
+# Patterns are AND-ed i.e. all must match against a single
+# description translation
+configure_languages en zz
+testempty apt -qq search there best
+
+# -------[ apt-cache ]-------
+
+# Test that all translations are searched, but the short
+# description is in the first configured language
+
+configure_languages en zz
+testequal "${PKG_NAME} - ${SHORT_DESCRIPTION_EN}" aptcache search alter ego
+
+configure_languages zz en
+testequal "${PKG_NAME} - ${SHORT_DESCRIPTION_ZZ}" aptcache search you today
+
+# Search in configured languages only
+configure_languages zz
+testempty aptcache search where fire
+
+# Patterns are AND-ed i.e. all must match against a single
+# description translation
+configure_languages en zz
+testempty aptcache search there best