Updated parser for Tag: field, and deal with debtags now not merging sources anymore

5 files changed, 133 insertions, 41 deletions

diff --git a/CMakeLists.txt b/CMakeLists.txt
index f60ddc9..1ca236a 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -3,7 +3,7 @@ include( FindDoxygen )
 
 add_custom_target( unit )
 
-set( EPT_VERSION "1.0.12" )
+set( EPT_VERSION "1.0.12.1" )
 
 # Get the soversion from libapt-pkg to include in our own
 execute_process(
diff --git a/debian/changelog b/debian/changelog
index 5aaf37b..f9905f1 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,12 @@
+libept (1.0.12.1) unstable; urgency=medium
+
+  * Redone parsing of Tag fields, to cope with multiline fields
+  * Get tag data from /var/lib/debtags/package-tags and
+    /var/lib/debtags/vocabulary if they are available. Closes: #762347.
+  * Updated standards-version, no changes required.
+
+ -- Enrico Zini <enrico@debian.org>  Mon, 06 Oct 2014 14:36:56 +0200
+
 libept (1.0.12) unstable; urgency=low
 
   * Fixes a case of sizeof(struct stat) disagreeing between libwibble and
diff --git a/debian/control b/debian/control
index b942020..bf83705 100644
--- a/debian/control
+++ b/debian/control
@@ -10,7 +10,7 @@ Build-Depends:
   libwibble-dev (>= 1.1), libwibble-dev (<< 2.0),
   libtagcoll2-dev (>= 2.0.6), libtagcoll2-dev (<< 2.1),
   libxapian-dev (>= 1.0.5)
-Standards-Version: 3.9.4.0
+Standards-Version: 3.9.6.0
 Vcs-Git: git://git.debian.org/git/debtags/libept.git
 Vcs-Browser: http://git.debian.org/?p=debtags/libept.git
 
diff --git a/ept/apt/packagerecord.cc b/ept/apt/packagerecord.cc
index f842bac..69832f2 100644
--- a/ept/apt/packagerecord.cc
+++ b/ept/apt/packagerecord.cc
@@ -66,50 +66,130 @@ std::string PackageRecord::parseLongDescription(const std::string& def, const st
 	}
 }
 
+namespace {
+
+struct Tagparser
+{
+    set<string>& res;
+    string cur_tag;
+    bool has_braces;
+
+    Tagparser(set<string>& res) : res(res) {}
+
+    void expand_tag_braces(const std::string& tag)
+    {
+        size_t begin = tag.find('{');
+        if (begin != string::npos)
+        {
+            string prefix(tag, 0, begin);
+            ++begin;
+            size_t end;
+            while ((end = tag.find(',', begin)) != string::npos)
+            {
+                res.insert(prefix + tag.substr(begin, end-begin));
+                begin = end + 1;
+            }
+            res.insert(prefix + tag.substr(begin, tag.size() - 1 - begin));
+        }
+    }
+
+    void reset_tag()
+    {
+        cur_tag.clear();
+        has_braces = false;
+    }
+
+    void have_tag()
+    {
+        if (has_braces)
+            expand_tag_braces(cur_tag);
+        else
+            res.insert(cur_tag);
+        reset_tag();
+    }
+
+    void parse(const std::string& s)
+    {
+        enum State {
+            SEP,
+            TAG,
+            BRACES,
+        } state = SEP;
+
+        reset_tag();
+
+        // Tokenize, dealing with braces
+        for (string::const_iterator c = s.begin(); c != s.end();)
+        {
+            switch (state)
+            {
+                case SEP:
+                    switch (*c)
+                    {
+                        case ' ':
+                        case '\t':
+                        case '\n':
+                        case ',':
+                            ++c;
+                            break;
+                        default:
+                            state = TAG;
+                            break;
+                    }
+                    break;
+                case TAG:
+                    switch (*c)
+                    {
+                        case ' ':
+                        case '\t':
+                        case '\n':
+                        case ',':
+                            ++c;
+                            have_tag();
+                            state = SEP;
+                            break;
+                        case '{':
+                            cur_tag += *c;
+                            ++c;
+                            has_braces = true;
+                            state = BRACES;
+                            break;
+                        default:
+                            cur_tag += *c;
+                            ++c;
+                            break;
+                    }
+                    break;
+                case BRACES:
+                    cur_tag += *c;
+                    ++c;
+                    switch (*c)
+                    {
+                        case '}':
+                            state = TAG;
+                            break;
+                    }
+                    break;
+            }
+        }
+        if (!cur_tag.empty())
+            have_tag();
+    }
+};
+
+}
+
 std::set<std::string> PackageRecord::parseTags(const std::set<std::string>& def, const std::string& str) const
 {
-	if (str == string())
-		return def;
+    if (str == string())
+        return def;
 
-	set<string> res;
+    set<string> res;
 
-	size_t pos = 0;
-	while (pos < str.size())
-	{
-		string tag;
-		size_t i = str.find(", ", pos);
-		if (i == string::npos)
-			tag = str.substr(pos);
-		else
-			tag = str.substr(pos, i-pos);
-
-		// Check if we need curly brace expansion
-		if (tag[tag.size() - 1] == '}')
-		{
-			size_t begin = tag.find('{');
-			if (begin != string::npos)
-			{
-				string prefix(tag, 0, begin);
-				++begin;
-				size_t end;
-				while ((end = tag.find(',', begin)) != string::npos)
-				{
-					res.insert(prefix + tag.substr(begin, end-begin));
-					begin = end + 1;
-				}
-				res.insert(prefix + tag.substr(begin, tag.size() - 1 - begin));
-			}
-		} else {
-			res.insert(tag);
-		}
-
-		if (i == string::npos)
-			break;
-		else
-			pos = i + 2;
-	}
+    Tagparser parser(res);
+    parser.parse(str);
 
-	return res;
+    return res;
 }
 
 }
diff --git a/ept/debtags/maint/sourcedir.cc b/ept/debtags/maint/sourcedir.cc
index cd1f1f9..859455a 100644
--- a/ept/debtags/maint/sourcedir.cc
+++ b/ept/debtags/maint/sourcedir.cc
@@ -30,6 +30,9 @@ SourceDir::FileType SourceDir::fileType(const std::string& name)
 	// extension)
 	if (name.size() <= 4) return SKIP;
 
+    if (name == "package-tags") return TAG;
+    if (name == "vocabulary") return VOC;
+
 	// Only look at .voc and .tag files
 	std::string ext(name, name.size() - 4);
 	if (ext == ".voc")