initial work-in-progress code

8 files changed, 407 insertions, 0 deletions

diff --git a/README b/README
new file mode 100644
index 0000000..cffbb16
--- /dev/null
+++ b/README
@@ -0,0 +1,46 @@
+welcome to the README file.
+
+--------------------
+setup instructions
+--------------------
+
+required software:
+
+python
+python-cheetah
+python-pygments
+python-debian
+(optional) reprepro
+something providing an httpd
+
+first, you need a partial source archive, with the .dsc, diff.gz, Release,
+and Sources.gz files available.  there's a sample config in the subdir 
+./reprepro, if you don't have one handy.  currently it just fetches a
+few packages for testing, and it also needlessly fetches the orig.tar.gz
+files too, just because i don't know reprepro very well.
+
+so, you have your source archive available, and installed somewhere.
+next set up a web server, and configure it to serve up the "application" from
+wherever, using standard Alias or VirtualHost/DocumentRoot stuff.
+
+next, edit the main python script, gen-patch-info.py.  there's a Conf
+class which contains all the configuration stuff that you will need to
+modify to get the script to run and generate pages as it should.
+
+--------------------
+overall design/scheme
+--------------------
+
+currently the plan is to generate static information based on the info
+available in a standard source archive (thus hopefully eliminating the need for 
+some kind of complicated tracking infrastructure).  The in-between stage
+may involve a lightweight database (i.e. sqlite) for quick/efficient handling
+of source package metadata, but in the current scheme it is all handled
+on the fs directly or within classes.  the database shouldn't be necessary
+in the "final output", which is currently static.  of course the right is
+reserved for later additions that include dynamic content and/or
+interaction (posting comments to a patch, fetching other data from external 
+sources)
+
+the webpages are generated from cheetah based templates in ./templates.  if the content is later dynamically generated the templates may still be useful.
+
diff --git a/gen-patch-info.py b/gen-patch-info.py
new file mode 100755
index 0000000..74d819f
--- /dev/null
+++ b/gen-patch-info.py
@@ -0,0 +1,184 @@
+#!/usr/bin/python
+
+import os
+import errno
+import string
+import tempfile
+from fnmatch import fnmatch
+from gzip import GzipFile
+from debian_bundle import deb822
+from Cheetah.Template import Template
+
+class Conf:
+  archive_root = '/scratch/debian-archive/debian'
+  output_dir = '/scratch/patches'
+  template_dir  = './templates'
+  static_dir  = './static'
+  root_url = '/patches'
+
+class Archive:
+  root = None
+  distsdir = None
+
+  def __init__(self, dir):
+    self.root = dir
+    self.distsdir = os.sep.join([dir, "dists"])
+
+  def suites(self):
+    for s in os.listdir(self.distsdir):
+      yield s
+
+  def components(self, suite):
+    f = file(self.distsdir + os.sep + os.sep.join([suite, "Release"]))
+    rel = deb822.Release(f)
+    for comp in rel['Components'].split(' '):
+      yield comp
+
+  def sourcepackages(self, suite, component):
+    sfile=os.sep.join([self.distsdir,suite,component,"/source/Sources.gz"])
+    for ent in deb822.Sources.iter_paragraphs(GzipFile(sfile)):
+      yield SourcePackage(ent)
+
+  def __str__(self):
+    return "Archive rooted at "+self.root
+
+class PageWriter:
+  def __init__(self, filename, template):
+    try:
+      os.makedirs(os.path.dirname(filename))
+    except OSError, e:
+      if e.errno != errno.EEXIST:
+        raise e
+    o = file(filename, "w")
+    o.write(str(template))
+    o.close()
+
+class SourcePackage:
+  name = None
+  format = None
+  diffgz = None
+  loc = None
+  type = "Native"
+  version = None
+  seriespatches = []
+  directpatches = []
+  # todo
+  vcs = {}
+
+  def __init__(self, info):
+    self.name = info['Package']  
+    self.format = info['Format']  
+    self.loc = info['Directory']  
+    self.version = info['Version']  
+
+    for f in info['Files']:
+      if fnmatch(f['name'], '*.diff.gz'):
+        self.diffgz=f
+        self.type = "Debian-diff"
+
+  def idx(self):
+    name = str(self)
+    if len(name) < 4 or name[0:3] != "lib":
+      return name[0]
+    else:
+      return name[0:4]
+
+  def __str__(self):
+    return self.name
+
+class OurTemplate(Template):
+  def __init__(self, file):
+    Template.__init__(self, file=file, searchList={"conf":Conf})
+
+class PackageVersTemplate(OurTemplate):
+  src = None
+  suite = None
+
+  def __init__(self, srcpkg, suite):
+    self.src = srcpkg
+    self.suite = suite
+    tpl=os.sep.join([Conf.template_dir, "package-vers.tmpl"])
+    OurTemplate.__init__(self, file=tpl)
+
+class PackageVersWriter(PageWriter):
+  def __init__(self, template):
+    dst = os.sep.join([Conf.output_dir, "packages", template.src.name, template.src.version, "index.html"])
+    PageWriter.__init__(self, dst, template)
+
+    (tfd, tfn) = tempfile.mkstemp()
+    os.close(tfd)
+    #os.system("filterdiff -z -p 1 -x 'debian/*' blah > farsar")
+
+class SourcePackageIndex:
+  pkgs = {}
+
+  def ins(self, srcpkg, rel):
+    idx = srcpkg.idx()
+    if not self.pkgs.has_key(idx):
+      self.pkgs[idx] = {}
+    if not self.pkgs[idx].has_key(srcpkg.name):
+      self.pkgs[idx][srcpkg.name] = {}
+    if not self.pkgs[idx][srcpkg.name].has_key(rel):
+      self.pkgs[idx][srcpkg.name][rel] = srcpkg
+
+  def indices(self):
+    for k,v in self.pkgs.iteritems():
+      yield (k,v)
+
+class FrontPageTemplate(OurTemplate):
+  allindex = None
+  relindices = []
+
+  def __init__(self, allindex, release_indices=[]):
+    tpl = os.sep.join([Conf.template_dir, "frontpage.tmpl"])
+    OurTemplate.__init__(self, file=tpl)
+    self.allindex = allindex
+    self.relindices = release_indices
+
+class FrontPageWriter(PageWriter):
+  def __init__(self, template):
+    dest = os.sep.join([Conf.output_dir, "index.html"])
+    PageWriter.__init__(self, dest, template)
+
+class LetterTocTemplate(OurTemplate):
+  idx = None
+  pkgs = None
+  dists = None
+
+  def releases(self):
+    return dists
+
+  def __init__(self, letter, collection):
+    self.pkgs = collection
+    self.idx = letter
+    self.dists = {}
+    for name,packagelist in collection.iteritems():
+      for d in packagelist.iterkeys():
+        self.dists[d] = True
+    tpl = os.sep.join([Conf.template_dir, "letter-toc.tmpl"])
+    OurTemplate.__init__(self, file=tpl)
+
+class LetterTocWriter(PageWriter):
+
+  def __init__(self, template):
+    dest = os.sep.join([Conf.output_dir, "index", template.idx, "index.html"])
+    PageWriter.__init__(self, dest, template)
+
+if __name__ == '__main__':
+  a = Archive(Conf.archive_root)
+  print a
+  master_index = SourcePackageIndex()
+  for s in a.suites():
+    print "suite: ",s
+    for c in a.components(s):
+      print "\tcomponent:",c
+      for p in a.sourcepackages(s, c):
+        print "\t\tpackage:",p
+        print "\t\tdiff:",p.diffgz
+        PackageVersWriter(PackageVersTemplate(p, s))
+        master_index.ins(p,s)
+
+  os.system("cp -a "+Conf.static_dir+"/* "+Conf.output_dir)
+  FrontPageWriter(FrontPageTemplate(master_index))
+  for letter,stuff in master_index.indices():
+    LetterTocWriter(LetterTocTemplate(letter,stuff))
diff --git a/reprepro/conf/distributions b/reprepro/conf/distributions
new file mode 100644
index 0000000..8abb935
--- /dev/null
+++ b/reprepro/conf/distributions
@@ -0,0 +1,14 @@
+Codename: etch
+Architectures: source
+Components: main
+Update: - debian
+
+Codename: lenny
+Architectures: source
+Components: main
+Update: - debian
+
+Codename: sid
+Architectures: source
+Components: main
+Update: - debian
diff --git a/reprepro/conf/updates b/reprepro/conf/updates
new file mode 100644
index 0000000..9cab2e3
--- /dev/null
+++ b/reprepro/conf/updates
@@ -0,0 +1,3 @@
+Name: debian
+Method: http://ftp.se.debian.org/debian
+FilterFormula: package (== php5) | package (== libapache2-mod-auth-pam) | package (== dbconfig-common)
diff --git a/static/css/patches.css b/static/css/patches.css
new file mode 100644
index 0000000..a055320
--- /dev/null
+++ b/static/css/patches.css
@@ -0,0 +1,27 @@
+table.summary {
+  border: 1px solid;
+}
+
+table.summary th {
+  text-align: left;
+}
+
+table.patchlisting {
+  border-spacing: 0px;
+}
+
+table.patchlisting td, table.patchlisting th {
+  border: 1px solid;
+}
+
+table.packagelisting {
+  border-spacing: 0px;
+}
+
+table.packagelisting td, table.packagelisting th {
+  border: 1px solid;
+}
+
+td, th {
+  padding: 10px;
+}
diff --git a/templates/frontpage.tmpl b/templates/frontpage.tmpl
new file mode 100644
index 0000000..6bf9b34
--- /dev/null
+++ b/templates/frontpage.tmpl
@@ -0,0 +1,17 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" 
+    "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html>
+ <head>
+  <title> Debian Project patch tracking system </title>
+  <link rel="stylesheet" type="text/css" href="$conf.root_url/css/patches.css"/>
+ </head>
+<body>
+ <h1>Debian Project patch tracking system</h1>
+  <h2>Browse patches by package name</h2>
+#for $k in $allindex.pkgs.iterkeys()
+ #if $allindex.pkgs[$k]
+ <a href="index/$k">$k</a>
+ #end if
+#end for
+ </body>
+</html>
diff --git a/templates/letter-toc.tmpl b/templates/letter-toc.tmpl
new file mode 100644
index 0000000..625bded
--- /dev/null
+++ b/templates/letter-toc.tmpl
@@ -0,0 +1,32 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" 
+    "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html>
+ <head>
+  <title>Debian Project patch tracking system</title>
+  <link rel="stylesheet" type="text/css" href="$conf.root_url/css/patches.css"/>
+ </head>
+<body>
+<h1>Debian Project patch tracking system</h1>
+<h2>Packaging patches by index - $idx</h2>
+<table class="packagelisting">
+<tr>
+<th>package</th>
+#for $d in $dists.iterkeys()
+ <th>$d</th>
+#end for
+</tr>
+#for $p in $pkgs
+<tr>
+ <td>
+  <a href="http://packages.debian.org/$p">$p</a>
+ </td>
+ #for $d in $dists.iterkeys()
+  <td>
+   <a href="/patches/packages/$p/$pkgs[$p][$d].version">$pkgs[$p][$d].version</a>
+  </td>
+ #end for
+</tr>
+#end for
+</table>
+</body>
+</html>
diff --git a/templates/package-vers.tmpl b/templates/package-vers.tmpl
new file mode 100755
index 0000000..ac08db2
--- /dev/null
+++ b/templates/package-vers.tmpl
@@ -0,0 +1,84 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" 
+    "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html>
+ <head>
+  <title>debian specific patch information for $src.name / $suite</title>
+  <link rel="stylesheet" type="text/css" href="$conf.root_url/css/patches.css"/>
+ </head>
+
+ <body>
+  <h1>debian specific patch information for $src.name / $suite</h1>
+  <h2> Summary </h2>
+  <table class="summary">
+   <tr>
+    <th>Package Version</th>
+    <td>$src.version</td>
+   </tr>
+   <tr>
+    <th>Package Type</th>
+    <td>$src.type</td>
+   </tr>
+   <tr>
+    <th>Source Package Format</th>
+    <td>$src.format</td>
+   </tr>
+#if $src.diffgz
+   <tr>
+    <th>Diff.gz</th>
+    <td>$src.loc / $src.diffgz['name']</td>
+   </tr>
+#end if
+  </table>
+
+#if $src.diffgz
+  <h2> Entire "Debian diff" Information </h2>
+  <table class="patchlisting">
+   <tr>
+    <th>Diff file</th>
+    <td>
+     <a href="$conf.root_url/$src.loc/$src.diffgz['name']">
+      $src.diffgz['name']
+     </a>
+    </td>
+   </tr>
+   <tr>
+    <th>Size</th><td>$src.diffgz['size']</td>
+   </tr>
+   <tr>
+    <th>MD5sum</th><td>$src.diffgz['md5sum']</td>
+   </tr>
+  </table>
+#end if
+
+#if $src.directpatches
+  <h2> Non-packaging "direct" style patches </h2>
+  <table class="patchlisting">
+   <tr>
+    <th>file</th>
+    <th>inserted</th>
+    <th>deleted</th>
+    <th>modified</th>
+    <th>download</th>
+   </tr>
+  </table>
+#end if
+
+  <h2> "series" style patches </h2>
+  <table class="patchlisting">
+   <tr>
+    <th>patch</th>
+    <th>summary</th>
+    <th>view</th>
+    <th>raw</th>
+   </tr>
+#for $p in $src.seriespatches
+   <tr>
+    <td>$p</td>
+    <td>info</td>
+    <td><a href="link">view</a></td>
+    <td><a href="link">download</a></td>
+   </tr>
+#end for
+  </table>
+ </body>
+</html>