split up generator and parsing logic in Archive

also introduce a singleton-like ArchiveDiffer class.  the idea
is to eventually minimize the number of db queries and speed up
the archive<->database sync.

1 files changed, 68 insertions, 5 deletions

diff --git a/patchtracker/SourceArchive.py b/patchtracker/SourceArchive.py
index a0aab1b..886019a 100755
--- a/patchtracker/SourceArchive.py
+++ b/patchtracker/SourceArchive.py
@@ -4,32 +4,64 @@ from fnmatch import fnmatch
 from gzip import GzipFile
 from debian_bundle import deb822
 from operator import itemgetter
+import difflib
 
-import patchtracker.Conf as Conf
 from DiffGzHandler import DiffGzHandler
 
 class Archive:
-  root = None
-  distsdir = None
+  def __init__(self, dir, suitefilter=None, pkgfilter=None):
+    self.root = None
+    self.distsdir = None
+    self.pkgs = {}
 
-  def __init__(self, dir):
     self.root = dir
     self.distsdir = os.sep.join([dir, "dists"])
+    for s in self.parsesuites(filter=suitefilter):
+      print "suite: ",s
+      self.addsuite(s)
+      for c in self.parsecomponents(s):
+        print "\tcomponent:",c
+        self.addcomponent(s,c)
+        for p in self.parsesourcepackages(s,c,filter=pkgfilter):
+          print "\t\tpackage:",p
+          self.addsourcepackage(s,c,p)
+
+  def addsuite(self, suite):
+    self.pkgs[suite] = {}
 
   def suites(self, filter=None):
+    return sorted(self.pkgs.iterkeys())
+
+  def parsesuites(self, filter=None):
     slist = os.listdir(self.distsdir)
     if filter:
       slist = set(filter).intersection(set(slist))
     for s in sorted(slist):
       yield s
 
+  def addcomponent(self, suite, component):
+    self.pkgs[suite][component] = {}
+
   def components(self, suite):
+    return sorted(self.pkgs[suite].iterkeys())
+
+  def parsecomponents(self, suite):
     f = file(self.distsdir + os.sep + os.sep.join([suite, "Release"]))
     rel = deb822.Release(f)
     for comp in rel['Components'].split(' '):
       yield comp
 
-  def sourcepackages(self, suite, component, filter=None):
+  def addsourcepackage(self, suite, component, package):
+    self.pkgs[suite][component][package.name] = package
+
+  def sourcepackages(self, suite, component, filter=None, namesonly=False):
+    if not namesonly:
+      return sorted(self.pkgs[suite][component].itervalues())
+    else:
+      namer = lambda x: x.name
+      return sorted(map(namer, self.pkgs[suite][component].itervalues()))
+
+  def parsesourcepackages(self, suite, component, filter=None):
     sfile=os.sep.join([self.distsdir,suite,component,"/source/Sources.gz"])
     k = itemgetter('Package')
     fdict = {}
@@ -45,6 +77,17 @@ class Archive:
   def __str__(self):
     return "Archive rooted at "+self.root
 
+class ArchiveDiffer():
+  def diffsuites(self,a,b):
+    return difflib.ndiff(a.suites(), b.suites())
+
+  def diffcomponents(self,a,b, suite):
+    return difflib.ndiff(a.components(suite), b.components(suite))
+
+  def diffsourcepackages(self,a,b, suite, component):
+    return difflib.ndiff(a.sourcepackages(suite, component, namesonly=True),
+                         b.sourcepackages(suite, component, namesonly=True))
+
 # XXX maybe there's a better place/way for this
 def getidx(letter):
   name = str(letter)
@@ -109,3 +152,23 @@ class SourcePackageIndex:
   def indices(self):
     for k,v in sorted(self.pkgs.iteritems()):
       yield (k,v)
+
+# basic module tests
+if __name__ == "__main__":
+  import Conf
+  import sys
+  import copy
+  a = Archive(sys.argv[1], suitefilter=["etch"])
+  b = copy.deepcopy(a)
+  b.addsuite("lenny")
+  b.addcomponent("etch", "something")
+  class x:
+    def __init__(self,name):
+      self.name=name
+  b.addsourcepackage("etch","main",x("foo"))
+  print "suite diff"
+  print "\n".join( ArchiveDiffer().diffsuites(a,b) )
+  print "component diff"
+  print "\n".join( ArchiveDiffer().diffcomponents(a,b,"etch") )
+  print "package diff"
+  print "\n".join( ArchiveDiffer().diffsourcepackages(a,b,"etch","main") )