diff options
Diffstat (limited to 'patchtracker')
| -rw-r--r-- | patchtracker/CacheObject.py | 63 | ||||
| -rw-r--r-- | patchtracker/ComplexQueries.py | 29 | ||||
| -rw-r--r-- | patchtracker/DB.py | 193 | ||||
| -rw-r--r-- | patchtracker/DebTarHandler.py | 27 | ||||
| -rw-r--r-- | patchtracker/DiffGzHandler.py | 74 | ||||
| -rw-r--r-- | patchtracker/Patch.py | 75 | ||||
| -rw-r--r-- | patchtracker/PtsIndex.py | 4 | ||||
| -rwxr-xr-x | patchtracker/ReqHandler.py | 222 | ||||
| -rwxr-xr-x | patchtracker/SourceArchive.py | 79 | ||||
| -rwxr-xr-x | patchtracker/Templates.py | 103 | ||||
| -rw-r--r-- | patchtracker/admin.py | 7 | ||||
| -rw-r--r-- | patchtracker/models.py | 89 | ||||
| -rw-r--r-- | patchtracker/tests.py | 23 | ||||
| -rw-r--r-- | patchtracker/urls.py | 18 | ||||
| -rw-r--r-- | patchtracker/views.py | 75 |
15 files changed, 345 insertions, 736 deletions
diff --git a/patchtracker/CacheObject.py b/patchtracker/CacheObject.py deleted file mode 100644 index c37c3b5..0000000 --- a/patchtracker/CacheObject.py +++ /dev/null @@ -1,63 +0,0 @@ -import errno -import gzip -import md5 -import os - -import Conf - -class CacheMissException (Exception): - pass - -class CacheObject: - """ A CacheObject is a compressed on-disk version of a serialized object. - """ - def __init__ (self, key=None, keys=None): - self.obj = None - #print "CacheObject: key=%s keys=%s"%(key, keys) - if not key and not keys: - raise "CacheObject needs at least one of key, keys" - checksum = md5.md5() - if key: - checksum.update(key) - if keys: - for k in keys: - checksum.update(str(k)) - self.path = os.path.sep.join([Conf.cachedir, checksum.hexdigest()]) - - def get(self): - if not self.obj: - try: - if Conf.cachecompress: - self.obj = gzip.GzipFile(self.path).read() - else: - self.obj = file(self.path).read() - except IOError, e: - if e.errno != errno.ENOENT: - raise e - else: - #print "CacheObject: cache miss" - raise CacheMissException("Object not present in cache") - #print "CacheObject: cache hit" - return self.obj - - def put(self, obj): - self.obj = obj - #print "CacheObject: cache put" - try: - if Conf.cachecompress: - gzip.GzipFile(self.path, "wb").write(str(self.obj)) - else: - file(self.path, "wb").write(str(self.obj)) - except Exception, e: - os.unlink(self.path) - raise e - -if __name__ == '__main__': - co = CacheObject( key="magic" ) - try: - print "going to try to read an object before it exists" - print "first line:", co.get().split()[0] - except CacheMissException: - print "file was missing as expected. now let's try to put it and fetch it" - co.put(file("/etc/passwd").read()) - print "first line:", co.get().split()[0] diff --git a/patchtracker/ComplexQueries.py b/patchtracker/ComplexQueries.py new file mode 100644 index 0000000..be7e30f --- /dev/null +++ b/patchtracker/ComplexQueries.py @@ -0,0 +1,29 @@ +import models + +class PackageIndexQuery: + def __init__( self, queryobj ): + self.dists = [d.name for d in models.RepositorySuite.objects.all()] + self.index = {} + self.packages = [] + self.queryobj = queryobj + + for mapping in queryobj: + self._addPackageMapping(mapping.package, mapping.suite) + + def _addPackageMapping( self, package, suite ): + if package.name not in self.index: + self.index[package.name] = [None] * len(self.dists) + self.packages.append(package) + self.index[package.name][self.dists.index(suite.name)] = package + + def __iter__( self ): + for pkg in self.packages: + yield { 'package':pkg, 'entries':self.index[pkg.name] } + +class PackageIndex (PackageIndexQuery): + def __init__( self, index ): + mappings = models.SourcePackageMapping.objects.order_by('package__name').filter( package__name=index ) + if not mappings: + mappings = models.SourcePackageMapping.objects.order_by('package__name').filter( package__name__startswith=index ) + + PackageIndexQuery.__init__(self, mappings) diff --git a/patchtracker/DB.py b/patchtracker/DB.py deleted file mode 100644 index 0bb0966..0000000 --- a/patchtracker/DB.py +++ /dev/null @@ -1,193 +0,0 @@ -from pysqlite2 import dbapi2 as sqlite -import Conf -import os -import errno - -import patchtracker.SourceArchive as SourceArchive - -def srcpkg_factory(cursor, row): - d = {} - for idx, col in enumerate(cursor.description): - d[col[0]] = row[idx] - - info = {} - info['Files'] = [] - if d['diffgz_name']: - diffgz = {'name':d['diffgz_name'], 'size':d['diffgz_size'], - 'md5sum':d['diffgz_md5sum'] } - info['Files'].append(diffgz) - if d['debtar_name']: - debtar = {'name':d['debtar_name'], 'size':d['debtar_size'], - 'md5sum':d['debtar_md5sum'] } - info['Files'].append(debtar) - - colmap = {'name':'Package','version':'Version','format':'Format', - 'loc':'Directory','maintainer':'Maintainer','uploaders':'Uploaders'} - - for col,field in colmap.iteritems(): - info[field] = d[col] - return SourceArchive.SourcePackage(info) - -def srcpkg_collection_factory(cursor, row): - d = {} - rest = {} - for idx, col in enumerate(cursor.description): - if not d.has_key(col[0]): - d[col[0]] = row[idx] - else: - rest[col[0]] = row[idx] - - info = {} - info['Files'] = [] - if d['diffgz_name']: - diffgz = {'name':d['diffgz_name'], 'size':d['diffgz_size'], - 'md5sum':d['diffgz_md5sum'] } - info['Files'].append(diffgz) - if d['debtar_name']: - debtar = {'name':d['debtar_name'], 'size':d['debtar_size'], - 'md5sum':d['debtar_md5sum'] } - info['Files'].append(debtar) - - colmap = {'name':'Package','version':'Version','format':'Format', - 'loc':'Directory','maintainer':'Maintainer','uploaders':'Uploaders'} - - for col,field in colmap.iteritems(): - info[field] = d[col] - return (SourceArchive.SourcePackage(info), rest) - - -class PatchTrackerDB: - def __init__(self, dbname=Conf.database): - self.db = sqlite.connect(dbname) - cursor = self.db.cursor() - cursor.execute("SELECT * FROM sqlite_master WHERE name='packages'") - if not cursor.fetchone(): - print "repopulating empty database..." - nextcmd = "" - for l in file(Conf.sqlschema).readlines(): - if len(l.strip()): - nextcmd += " " + l.strip() - else: - cursor.execute(nextcmd) - nextcmd = "" - - def setFactory(self, factory): - self.db.row_factory = factory - - def saveSourcePackage(self, srcpkg): - cursor = self.db.cursor() - #print "creating new record for",srcpkg - q = "INSERT OR IGNORE INTO packages \ - (name,format,loc,version,diffgz_name,diffgz_size,diffgz_md5sum,debtar_name,debtar_size,debtar_md5sum,maintainer,uploaders) \ - VALUES (?,?,?,?,?,?,?,?,?,?,?,?)" - cursor.execute(q, (srcpkg.name,srcpkg.format,srcpkg.loc, - srcpkg.version,srcpkg.diffgz_name,srcpkg.diffgz_size, - srcpkg.diffgz_md5sum, - srcpkg.debtar_name,srcpkg.debtar_size, - srcpkg.debtar_md5sum, - srcpkg.maintainers,srcpkg.uploaders)) - - def saveSuite(self, suite): - q = "INSERT INTO suites (name) VALUES (?)" - cursor = self.db.cursor() - cursor.execute(q, (suite,)) - - def saveComponent(self, component): - q = "INSERT INTO components (name) VALUES (?)" - cursor = self.db.cursor() - cursor.execute(q, (component,)) - - def findCollection(self, package="%", version=None, email=None): - oldfactory = self.db.row_factory - self.db.row_factory = srcpkg_collection_factory - cursor = self.db.cursor() - toc = SourceArchive.SourcePackageIndex() - q = "SELECT * FROM packages AS p,package_rel_map AS m,suites AS s \ - WHERE p.name LIKE ? AND p.id = m.package_id AND m.suite_id = s.id" - qargs = (package,) - if version: - q += " AND p.version = ?" - qargs += (version,) - if email: - q += " AND ( p.maintainer like ? OR p.uploaders like ? )" - qargs += ("%%%s%%"%(email),"%%%s%%"%(email)) - - cursor.execute(q, qargs) - # use srcpkg_factory to fetch sourcepackages, once per suite - for srcpkg,rest in cursor.fetchall(): - toc.ins(srcpkg, rest["name"]) - self.db.row_factory = oldfactory - return toc - - def findLetterToc(self, letter): - return self.findCollection(package=letter+"%").getletter(letter) - - def findIndices(self): - indices = [] - cursor = self.db.cursor() - q1 = "SELECT DISTINCT SUBSTR(name,1,1) FROM packages \ - WHERE name NOT LIKE 'lib%'" - q2 = "SELECT DISTINCT SUBSTR(name,1,4) FROM packages \ - WHERE name LIKE 'lib%'" - - for q in [q1, q2]: - cursor.execute(q) - for idx in cursor.fetchall(): - indices.append(idx[0]) - - indices.sort() - return indices - - def relateSourcePackage(self, name, version, suite, component): - q = "INSERT INTO package_rel_map \ - (package_id,suite_id,component_id,marked) \ - VALUES ((SELECT id FROM packages WHERE name=? AND version=?), \ - (SELECT id FROM suites WHERE name=?), \ - (SELECT id FROM components WHERE name=?), 1)" - cursor = self.db.cursor() - cursor.execute(q, (name, version, suite, component)) - - def findDiffGz(self, pkgname, version): - q = "SELECT diffgz_name,loc FROM packages WHERE name=? AND version=?" - cursor = self.db.cursor() - cursor.execute(q, (pkgname, version)) - try: - diffgz,loc = cursor.fetchone() - return os.sep.join([Conf.archive_root, loc, diffgz]) - except: - return None - - def findDebTar(self, pkgname, version): - q = "SELECT debtar_name,loc FROM packages WHERE name=? AND version=?" - cursor = self.db.cursor() - cursor.execute(q, (pkgname, version)) - try: - debtar,loc = cursor.fetchone() - return os.sep.join([Conf.archive_root, loc, debtar]) - except: - return None - - def prune(self): - q = "DELETE FROM package_rel_map WHERE marked != 1" - cursor = self.db.cursor() - cursor.execute(q) - - def unmark(self): - q = "UPDATE package_rel_map SET marked=0" - cursor = self.db.cursor() - cursor.execute(q) - - def finalize(self): - self.db.commit() - - def __del__(self): - self.finalize() - - -if __name__ == "__main__": - print "patch tracker db testing" - d = PatchTrackerDB(dbname="foo.db") - sp = SourceArchive.SourcePackage({'Package':'foo','Format':'blah','Directory':'/foo','Version':'1.2.3.4','Files':[]}) - d.saveSourcePackage(sp) - d.finalize() - diff --git a/patchtracker/DebTarHandler.py b/patchtracker/DebTarHandler.py deleted file mode 100644 index 311e39b..0000000 --- a/patchtracker/DebTarHandler.py +++ /dev/null @@ -1,27 +0,0 @@ -import os -import stat -import sys - -import Patch - -class DebTarHandler: - diff = None - def __init__(self,fname): - self.tarfile = fname - self.size = os.stat(fname)[stat.ST_SIZE] - - def series(self): - return Patch.Quilt30PatchSeries(self.tarfile) - -if __name__ == "__main__": - print "DebTarHandler testing" - try: - dh = DebTarHandler(sys.argv[1]) - except IndexError: - print "usage: %s <diffgz>"%(sys.argv[0]) - sys.exit(1) - - print "series:" - for f,p in dh.series(): - print f - print p diff --git a/patchtracker/DiffGzHandler.py b/patchtracker/DiffGzHandler.py deleted file mode 100644 index 84be827..0000000 --- a/patchtracker/DiffGzHandler.py +++ /dev/null @@ -1,74 +0,0 @@ -import tempfile -import sys -import os -import stat - -from Patch import Patch, PatchSeries - -class DiffGzException(Exception): - pass - -class DiffGzHandler: - diff = None - def __init__(self,fname): - self.diff = fname - self.size = os.stat(fname)[stat.ST_SIZE] - - def filterdiff(self, include=None, exclude=None): - cmd = ["filterdiff","-z","-p","1"] - if include: - cmd += [ "-i", include] - elif exclude: - cmd += [ "-x", exclude] - else: - raise Exception("DiffGzHandler.filterdiff called w/o include/exclude") - i,o,e=os.popen3(cmd+[self.diff]) - i.close() - p = Patch(o) - err = e.read() - if len(err): - raise DiffGzException("filterdiff gave errors: "+err) - return p - - def debiandir(self): - return self.filterdiff(include='debian/*') - - def nondebiandir(self): - return self.filterdiff(exclude='debian/*') - - def series(self): - patches = None - embedded = self.filterdiff(include='debian/patches*') - - # XXX *cough* cache *cough* - if embedded.lines(): - td = tempfile.mkdtemp() - i,o,e=os.popen3("patch -d %s -p3"%(td)) - i.write(str(embedded)) - i.close() - err = e.read() - if len(err): - raise Exception("unable to extract series patches:\n"+err) - patches = PatchSeries(td) - os.system("rm -rf %s"%(td)) - - return patches - -if __name__ == "__main__": - print "DiffGzHandler testing" - try: - dh = DiffGzHandler(sys.argv[1]) - except IndexError: - print "usage: %s <diffgz>"%(sys.argv[0]) - sys.exit(1) - - print "debian dir:" - print dh.debiandir().diffstat() - print "nondebian dir:" - print dh.nondebiandir().diffstat() - print "series:" - s = dh.series() - print s - for name,patch in s.iterpatches(): - print "patch:",name - print patch.diffstat() diff --git a/patchtracker/Patch.py b/patchtracker/Patch.py index 6533910..c872d1c 100644 --- a/patchtracker/Patch.py +++ b/patchtracker/Patch.py @@ -1,7 +1,12 @@ -import sys -import os import errno +import os +import stat +import sys +import tempfile from glob import glob +import pygments +import pygments.lexers +import pygments.formatters import tarfile class Diffstat: @@ -39,6 +44,14 @@ class Patch: def diffstat(self): return Diffstat(self) + def highlight(self): + for enc in ['utf-8', 'latin-1']: + try: + return pygments.highlight(str(self).decode(enc), pygments.lexers.DiffLexer(), + pygments.formatters.HtmlFormatter(style='colorful', noclasses=True, encoding=enc, nobackground=True)) + except UnicodeDecodeError: + pass + class GenericPatchSeries (list): def blank(self): self.names = [] @@ -164,6 +177,64 @@ class Quilt30PatchSeries (GenericPatchSeries): for name in self.names: self.patches[name] = Patch(self.tarfh.extractfile("debian/patches/"+name)) +class DebTarHandler: + diff = None + def __init__(self,fname): + self.tarfile = fname + self.size = os.stat(fname)[stat.ST_SIZE] + + def series(self): + return Quilt30PatchSeries(self.tarfile) + +class DiffGzException(Exception): + pass + +class DiffGzHandler: + diff = None + def __init__(self,fname): + self.diff = fname + self.size = os.stat(fname)[stat.ST_SIZE] + + def filterdiff(self, include=None, exclude=None): + cmd = ["filterdiff","-z","-p","1"] + if include: + cmd += [ "-i", include] + elif exclude: + cmd += [ "-x", exclude] + else: + raise Exception("DiffGzHandler.filterdiff called w/o include/exclude") + i,o,e=os.popen3(cmd+[self.diff]) + i.close() + p = Patch(o) + err = e.read() + if len(err): + raise DiffGzException("filterdiff gave errors: "+err) + return p + + def debiandir(self): + return self.filterdiff(include='debian/*') + + def nondebiandir(self): + return self.filterdiff(exclude='debian/*') + + def series(self): + patches = None + embedded = self.filterdiff(include='debian/patches*') + + # XXX *cough* cache *cough* + if embedded.lines(): + td = tempfile.mkdtemp() + i,o,e=os.popen3("patch -d %s -p3"%(td)) + i.write(str(embedded)) + i.close() + err = e.read() + if len(err): + raise Exception("unable to extract series patches:\n"+err) + patches = PatchSeries(td) + os.system("rm -rf %s"%(td)) + + return patches + if __name__ == "__main__": print "Patch.py testing" try: diff --git a/patchtracker/PtsIndex.py b/patchtracker/PtsIndex.py index 08a6e68..831b4b6 100644 --- a/patchtracker/PtsIndex.py +++ b/patchtracker/PtsIndex.py @@ -1,7 +1,7 @@ import os import Conf -from DiffGzHandler import DiffGzHandler +import Patch class PtsIndexPackageInfo ( dict ): def __init__ (self, srcpkg): @@ -11,7 +11,7 @@ class PtsIndexPackageInfo ( dict ): if srcpkg.diffgz_name: diffgz = os.sep.join([Conf.archive_root,srcpkg.loc,srcpkg.diffgz_name]) - dh = DiffGzHandler( diffgz ) + dh = Patch.DiffGzHandler( diffgz ) ser = dh.series() if ser: self['series-patches'] = len(ser) diff --git a/patchtracker/ReqHandler.py b/patchtracker/ReqHandler.py deleted file mode 100755 index 90fc75a..0000000 --- a/patchtracker/ReqHandler.py +++ /dev/null @@ -1,222 +0,0 @@ -# -*- coding: utf-8 -*- - -import cgi -import os -import sys - -import patchtracker.Conf as Conf -from patchtracker.Templates import ErrorTemplate, PatchTemplate, PackageVersTemplate, LetterTocTemplate, FrontPageTemplate, SearchResultsTemplate -from patchtracker.DiffGzHandler import DiffGzHandler, DiffGzException -from patchtracker.DebTarHandler import DebTarHandler -from patchtracker.CacheObject import CacheObject, CacheMissException -import patchtracker.DB as DB -from patchtracker.DB import PatchTrackerDB -import pygments -from pygments.lexers import DiffLexer -from pygments.formatters import HtmlFormatter -import patchtracker.SourceArchive as SourceArchive - -class ReqHandlerException(Exception): - def __init__(self, msg, code="500 Oh noes"): - Exception.__init__(self, msg) - self.status=code - -class Cmd: - def __init__(self): - self.content_type = 'text/html' - self.status = "200 OK" - -class ErrorCmd(Cmd): - def __init__(self, msg, code="500 Oh noes"): - Cmd.__init__(self) - self.status = code - self.msg = msg - - def output(self): - return str(ErrorTemplate(self.msg)) - -class PatchCmd(Cmd): - def __init__(self, args): - Cmd.__init__(self) - self.db = PatchTrackerDB() - self.patchtype,mode,pkgname,version = args[0:4] - self.parsemode(mode) - dh = self.make_diffhandler(pkgname,version) - if self.patchtype == "series": - self.patchname = os.sep.join(args[4:]) - self.content = dh.series().fetch(self.patchname) - elif self.patchtype == "debianonly": - self.patchname = "debian-dir only changes" - self.content = dh.debiandir() - elif self.patchtype == "nondebian": - self.patchname = "direct (non packaging) changes" - self.content = dh.nondebiandir() - elif self.patchtype == "misc": - self.patchname = os.sep.join(args[4:]) - self.content = dh.filterdiff(include=self.patchname) - else: - raise ReqHandlerException("unhandled patch type '%s'"%(self.patchtype)) - self.pkgname = pkgname - self.version = version - - def parsemode(self, mode): - if mode == "view" or mode == "dl": - self.mode = mode - else: - raise ReqHandlerException("unhandled display mode '%s'"%(mode)) - if mode == "dl": - self.content_type = "text/x-diff" - - # XXX this is kinda ugly... - def make_diffhandler(self, pkgname, vers): - dfile = self.db.findDiffGz(pkgname,vers) - if dfile: - return DiffGzHandler(dfile) - else: - dfile = self.db.findDebTar(pkgname, vers) - if dfile: - return DebTarHandler(dfile) - else: - raise ReqHandlerException("can not find diff file for %s / %s"%(pkgname,vers)) - - def output(self): - if self.mode == "dl": - return str(self.content) - else: - return str(PatchTemplate(pkg=self.pkgname,vers=self.version, - patch=self.content,name=self.patchname, - patchtype=self.patchtype)) - -class PackageCmd(Cmd): - def __init__(self, args): - Cmd.__init__(self) - db = PatchTrackerDB() - self.name = args[0] - if len(args) > 1: - version = args[1] - else: - version = None - self.toc = db.findCollection(package=self.name, version=version) - - # if there's no match, try with a wildcard match - if not self.toc.size(): - # ... but don't allow pathologically short names - if len(self.name) < 3: - raise ReqHandlerException("search terms must be 3 or more letters...") - else: - self.toc = db.findCollection(package="%"+self.name+"%", version=version) - - plist = self.toc.getletter(self.name) - if not plist or len(plist) == 0: - raise ReqHandlerException("can't find any package named or containing '%s'"%self.name, code="404 ENOPKG kthxbye") - - def output(self): - p = self.toc.getpackage(self.name) - # if there is no match, or if multiple versions were returned - if not p or len(set(map(lambda x: x.version, p.values()))) > 1: - querydesc = "package name contains" - return str(SearchResultsTemplate(self.name, querydesc, self.toc)) - else: - return str(PackageVersTemplate(p.popitem()[1])) - -class IndexCmd(Cmd): - def __init__(self, args): - Cmd.__init__(self) - if len(args) < 1 or not len(args[0]): - raise ReqHandlerException("please provide a letter on which to index") - else: - self.db = PatchTrackerDB() - self.letter = args[0] - self.toc = self.db.findLetterToc(self.letter) - - def output(self): - return str(LetterTocTemplate(self.letter, self.toc)) - -class MaintCmd(Cmd): - def __init__(self, args): - Cmd.__init__(self) - if len(args) < 1 or not len(args[0]): - raise ReqHandlerException("please provide a email address on which to index") - else: - self.db = PatchTrackerDB() - self.email = args[0] - self.toc = self.db.findCollection(email=self.email) - - def output(self): - return str(SearchResultsTemplate(self.email, "maintainer email", self.toc)) - -class JumpCmd(Cmd): - def __init__(self, env): - Cmd.__init__(self) - form = cgi.FieldStorage(fp=env['wsgi.input'],environ=env) - self.name = form.getfirst("package") - self.uri = "%s/package/%s"%(Conf.root_url, self.name) - self.status = "302 Try this other place kthx" - - def output(self): - return "" - -class FrontPageCmd(Cmd): - def __init__(self): - Cmd.__init__(self) - self.db = PatchTrackerDB() - self.index = self.db.findIndices() - - def output(self): - return str(FrontPageTemplate(self.index)) - -class CmdHandler: - def __init__(self, env): - self.headers = [] - uri = Conf.root_url+env['PATH_INFO'] - self.cacheobj = None - #print "Accept:",env['HTTP_ACCEPT'] - - args = uri[len(Conf.root_url)+1:].split("/") - cmdarg = args[0] - cacheable = False - if cmdarg == "patch": - self.cmd = PatchCmd(args[1:]) - cacheable = True - elif cmdarg == "package": - self.cmd = PackageCmd(args[1:]) - if len(args[1:]) > 1: - cacheable = True - elif cmdarg == "index": - self.cmd = IndexCmd(args[1:]) - elif cmdarg == "jump": - self.cmd = JumpCmd(env) - self.headers.append( ('Location', self.cmd.uri) ) - elif cmdarg == "email": - self.cmd = MaintCmd(args[1:]) - elif not len(cmdarg): - self.cmd = FrontPageCmd() - else: - self.cmd = ErrorCmd("invalid command/location '%s'"%(cmdarg), "404 Not found") - - if Conf.caching and cacheable: - self.cacheobj = CacheObject(key=uri) - - self.headers.append( ('Content-type', self.cmd.content_type) ) - self.status = self.cmd.status - - def output(self): - result = None - try: - if self.cacheobj: - result = self.cacheobj.get() - else: - result = self.cmd.output() - except CacheMissException: - result = self.cmd.output() - self.cacheobj.put(result) - except DiffGzException, e: - return ErrorCmd(str(e), "500 Oh Noez!!1!").output() - - return result - -if __name__ == '__main__': - fake_env = { 'PATH_INFO': sys.argv[1] } - cmdh = CmdHandler(fake_env) - print "Status: %s\nHeaders: %s"%(cmdh.status, cmdh.headers) - print cmdh.output() diff --git a/patchtracker/SourceArchive.py b/patchtracker/SourceArchive.py index 0cdeb10..ce3459a 100755 --- a/patchtracker/SourceArchive.py +++ b/patchtracker/SourceArchive.py @@ -5,6 +5,8 @@ from gzip import GzipFile from debian_bundle import deb822 import difflib +import models + class Archive: def __init__(self, dir, suitefilter=None, pkgfilter=None): self.root = None @@ -65,7 +67,33 @@ class Archive: if filter and not fdict.has_key(ent['Package']): continue else: - yield SourcePackage(ent) + pkg = models.SourcePackage() + # some other defaults + pkg.name = ent['Package'] + pkg.format = ent['Format'] + pkg.loc = ent['Directory'] + pkg.version = ent['Version'] + try: + pkg.maintainers = unicode(ent['Maintainer']) + except UnicodeDecodeError: + pkg.maintainers = unicode(ent['Maintainer'], 'latin-1') + if ent.has_key('Uploaders'): + try: + pkg.uploaders = unicode(ent['Uploaders']) + except UnicodeDecodeError: + pkg.uploaders = unicode(ent['Uploaders'], 'latin-1') + + for f in ent['Files']: + if fnmatch(f['name'], '*.diff.gz'): + pkg.diffgz_name=f['name'] + pkg.diffgz_size=f['size'] + pkg.diffgz_md5sum=f['md5sum'] + elif fnmatch(f['name'], '*.debian.tar.*'): + pkg.debtar_name=f['name'] + pkg.debtar_size=f['size'] + pkg.debtar_md5sum=f['md5sum'] + + yield pkg def __str__(self): return "Archive rooted at "+self.root @@ -89,55 +117,6 @@ def getidx(letter): else: return name[0:4] -class SourcePackage: - def __init__(self, info): - # attributes for debian .diff.gz source packages - self.diffgz_name = None - self.diffgz_size = None - self.diffgz_md5sum = None - self.diffgz = None - - # attributes for debian .debian.tar.gz source packages - self.debtar_name = None - self.debtar_size = None - self.debtar_md5sum = None - self.debtar = None - - # some other defaults - self.type = "Native" - self.name = info['Package'] - self.format = info['Format'] - self.loc = info['Directory'] - self.version = info['Version'] - self.uploaders = None - - try: - self.maintainers = unicode(info['Maintainer']) - except UnicodeDecodeError: - self.maintainers = unicode(info['Maintainer'], 'latin-1') - if info.has_key('Uploaders'): - try: - self.uploaders = unicode(info['Uploaders']) - except UnicodeDecodeError: - self.uploaders = unicode(info['Uploaders'], 'latin-1') - - self.idx = getidx(self) - - for f in info['Files']: - if fnmatch(f['name'], '*.diff.gz'): - self.diffgz_name=f['name'] - self.diffgz_size=f['size'] - self.diffgz_md5sum=f['md5sum'] - self.type = "Debian-diff" - elif fnmatch(f['name'], '*.debian.tar.*'): - self.debtar_name=f['name'] - self.debtar_size=f['size'] - self.debtar_md5sum=f['md5sum'] - self.type = "Debian-diff" - - def __str__(self): - return self.name - class SourcePackageIndex: def __init__(self): self.pkgs = {} diff --git a/patchtracker/Templates.py b/patchtracker/Templates.py deleted file mode 100755 index 42140a8..0000000 --- a/patchtracker/Templates.py +++ /dev/null @@ -1,103 +0,0 @@ -from patchtracker import Conf, DB -from patchtracker.DiffGzHandler import DiffGzHandler -from patchtracker.DebTarHandler import DebTarHandler -from patchtracker.SourceArchive import ReleaseList - -from Cheetah.Template import Template -from Cheetah.Compiler import Compiler -import os, errno -import re - -class OurTemplate(Template): - def __init__(self, file, searchList=None): - ourSearchList={"conf":Conf, "crumbs":[]} - if searchList: - for k,v in searchList.iteritems(): - ourSearchList[k]=v - self.escape_name = self._escape_name - self.wrappable_version = self._wrappable_version - self.link = self._link - Template.__init__(self, file=file, searchList=ourSearchList) - - def _link(self, where, name): - return str("<a href=\"%s\">%s</a>"%(where,name)) - - def _escape_name(self, name): - return re.sub("([^a-zA-Z0-9-])", (lambda x: "_%d"%(ord(x.group(1)))), name) - - def _wrappable_version(self, version): - """ output a version that can be broken up by a web browser using - the unicode zero-whitespace-break character (​) """ - return '​'.join([version[i:i+12] for i in range(0,len(version),12)]) - -class PackageVersTemplate(OurTemplate): - def __init__(self, srcpkg): - self.src = srcpkg - tpl=os.sep.join([Conf.template_dir, "package_vers.tmpl"]) - sl = {} - # XXX c/p from PatchCmd.make_diffhandler - db = DB.PatchTrackerDB() - dfile = db.findDiffGz(srcpkg.name,srcpkg.version) - if dfile: - sl['diffhandler'] = DiffGzHandler(dfile) - else: - dfile = db.findDebTar(srcpkg.name, srcpkg.version) - if dfile: - sl['diffhandler'] = DebTarHandler(dfile) - sl['crumbs'] = [("package/"+srcpkg.name,srcpkg.name), - ("package/"+srcpkg.name+"/"+srcpkg.version, - srcpkg.name+"/"+srcpkg.version)] - OurTemplate.__init__(self, file=tpl, searchList=sl) - -class FrontPageTemplate(OurTemplate): - def __init__(self, indices): - tpl = os.sep.join([Conf.template_dir, "frontpage.tmpl"]) - OurTemplate.__init__(self, file=tpl) - self.indices = indices - -class LetterTocTemplate(OurTemplate): - def __init__(self, letter, collection): - self.pkgs = collection - self.idx = letter - self.dists = ReleaseList("debian") - sl = {} - sl['crumbs'] = [("index/"+letter,"index for "+letter)] - tpl = os.sep.join([Conf.template_dir, "letter_toc.tmpl"]) - OurTemplate.__init__(self, file=tpl, searchList=sl) - -class SearchResultsTemplate(OurTemplate): - def __init__(self, search, searchtype, collection): - self.pkgs = {} - self.idx = search - self.searchtype = searchtype - self.dists = ReleaseList("debian") - for idx in collection.indices(): - for name,packagelist in collection.getletter(idx).iteritems(): - self.pkgs[name] = packagelist - tpl = os.sep.join([Conf.template_dir, "searchresults.tmpl"]) - OurTemplate.__init__(self, file=tpl) - - -class ErrorTemplate(OurTemplate): - def __init__(self, msg): - tpl = os.sep.join([Conf.template_dir, "cgi_error.tmpl"]) - OurTemplate.__init__(self, file=tpl, searchList={'error':msg}) - -class PatchTemplate(OurTemplate): - def __init__(self, pkg=None, vers=None, name=None, patch=None,patchtype=None): - tpl = os.sep.join([Conf.template_dir, "patch_view.tmpl"]) - sl = {'package':pkg, 'version':vers, 'name':name, - 'patch':patch, 'patchtype':patchtype} - sl['crumbs'] = [("package/"+pkg,pkg), - ("package/"+pkg+"/"+vers, pkg+"/"+vers)] - if patchtype == "debianonly": - sl['crumbs'].append( ("patch/%s/view/%s/%s"%(patchtype,pkg,vers), - "./debian-dir only patch") ) - elif patchtype == "series": - sl['crumbs'].append(("patch/%s/view/%s/%s/%s"%(patchtype,pkg,vers,name), - "series patch "+name) ) - elif patchtype == "misc": - sl['crumbs'].append(("patch/%s/view/%s/%s/%s"%(patchtype,pkg,vers,name), - "direct patch of "+name) ) - OurTemplate.__init__(self, file=tpl, searchList=sl) - diff --git a/patchtracker/admin.py b/patchtracker/admin.py new file mode 100644 index 0000000..0fbd418 --- /dev/null +++ b/patchtracker/admin.py @@ -0,0 +1,7 @@ +import models +import django.contrib.admin + +django.contrib.admin.site.register(models.SourcePackage) +django.contrib.admin.site.register(models.RepositorySuite) +django.contrib.admin.site.register(models.RepositoryComponent) +django.contrib.admin.site.register(models.SourcePackageMapping) diff --git a/patchtracker/models.py b/patchtracker/models.py new file mode 100644 index 0000000..1758609 --- /dev/null +++ b/patchtracker/models.py @@ -0,0 +1,89 @@ +import os +import re + +import Conf +import Patch + +from django.db import models + +class SourcePackage (models.Model): + name = models.CharField(max_length=1024) + version = models.CharField(max_length=1024) + format = models.CharField(max_length=64) + loc = models.CharField(max_length=4096) + diffgz_name = models.CharField(max_length=4096, null=True, blank=True) + diffgz_size = models.IntegerField(null=True, blank=True) + diffgz_md5sum = models.CharField(max_length=32, null=True, blank=True) + debtar_name = models.CharField(max_length=4096, null=True, blank=True) + debtar_size = models.IntegerField(null=True, blank=True) + debtar_md5sum = models.CharField(max_length=32, null=True, blank=True) + maintainer = models.CharField(max_length=1024) + uploaders = models.CharField(max_length=4096, null=True, blank=True) + + def type( self ): + """ Source package "type". Not incredibly useful, i guess... """ + if self.diffgz_name or self.debtar_name: + return "Debian-diff" + else: + return "Native" + + def diffhandler( self ): + """ Return a diffhandler object too the caller """ + if self.diffgz_name: + dgz = os.sep.join([Conf.archive_root, self.loc, self.diffgz_name]) + return Patch.DiffGzHandler(dgz) + elif self.debtar_name: + dtar = os.sep.join([Conf.archive_root, self.loc, self.debtar_name]) + return Patch.DebTarHandler(dtar) + + def anchor_name(self): + """ output a version of the package name suitable for use as an + anchor name (i.e. <a name="foo">) """ + return re.sub("([^a-zA-Z0-9-])", (lambda x: "_%d"%(ord(x.group(1)))), self.name) + + def wrappable_version(self): + """ output a version that can be broken up by a web browser using + the unicode zero-whitespace-break character (​) """ + return '​'.join([self.version[i:i+12] for i in range(0,len(self.version),12)]) + + def __unicode__( self ): + return "%s (%s)"%(self.name, self.version) + + class Meta: + managed = False + db_table = 'packages' + unique_together = (('name','version'),) + +class RepositorySuite (models.Model): + name = models.CharField(max_length=1024, unique=True) + + def __unicode__( self ): + return unicode(self.name) + + class Meta: + managed = False + db_table = 'suites' + +class RepositoryComponent (models.Model): + name = models.CharField(max_length=1024, unique=True) + + def __unicode__( self ): + return unicode(self.name) + + class Meta: + managed = False + db_table = 'components' + +class SourcePackageMapping (models.Model): + rowid = models.AutoField(primary_key=True) + package = models.ForeignKey(SourcePackage) + suite = models.ForeignKey(RepositorySuite) + component = models.ForeignKey(RepositoryComponent) + + def __unicode__( self ): + return "%s / %s / %s"%(unicode(self.package),unicode(self.suite),unicode(self.component)) + + class Meta: + managed = False + db_table = 'package_rel_map' + unique_together = (('package','suite','component'),) diff --git a/patchtracker/tests.py b/patchtracker/tests.py new file mode 100644 index 0000000..2247054 --- /dev/null +++ b/patchtracker/tests.py @@ -0,0 +1,23 @@ +""" +This file demonstrates two different styles of tests (one doctest and one +unittest). These will both pass when you run "manage.py test". + +Replace these with more appropriate tests for your application. +""" + +from django.test import TestCase + +class SimpleTest(TestCase): + def test_basic_addition(self): + """ + Tests that 1 + 1 always equals 2. + """ + self.failUnlessEqual(1 + 1, 2) + +__test__ = {"doctest": """ +Another way to test that 1 + 1 is equal to 2. + +>>> 1 + 1 == 2 +True +"""} + diff --git a/patchtracker/urls.py b/patchtracker/urls.py new file mode 100644 index 0000000..b174272 --- /dev/null +++ b/patchtracker/urls.py @@ -0,0 +1,18 @@ +from django.conf.urls.defaults import * +import views + +urlpatterns = patterns('', + url(r'^package/(?P<package>\w+)/(?P<version>[^/]+)$', + views.package_vers), + url(r'^package/(?P<index>\w+)$', + views.display_toc), + url(r'^index/(?P<index>[^/]+)$', + views.display_toc), + url(r'^email/(?P<maintainer>.*)$', + views.maintainer_search), + url(r'^patch/(?P<patchType>(debianonly|misc|nondebian|series))/view/(?P<package>\w+)/(?P<version>[^/]+)/(?P<patchName>.*)$', + views.display_patch), + url(r'^patch/(?P<patchType>(debianonly|misc|nondebian|series))/dl/(?P<package>\w+)/(?P<version>[^/]+)/(?P<patchName>.*)$', + views.download_patch), + url(r'^$', views.frontpage), + ) diff --git a/patchtracker/views.py b/patchtracker/views.py new file mode 100644 index 0000000..0a316eb --- /dev/null +++ b/patchtracker/views.py @@ -0,0 +1,75 @@ +import os + +import django.db.models +import django.http +import django.shortcuts +import django.template + +import ComplexQueries +import Conf +import models + +def package_vers(request, package, version): + pkg = models.SourcePackage.objects.get( name=package, version=version ) + ctx = django.template.RequestContext(request) + tmpl = 'package_vers.html' + extra = { 'pkg':pkg, 'ctx':ctx, 'conf':Conf } + return django.shortcuts.render_to_response(tmpl, extra, context_instance=ctx) + +def display_toc(request, index): + packageIndex = ComplexQueries.PackageIndex(index) + tmpl = 'searchresults.html' + searchDescription = "Packages by index" + extra = { 'index':index, 'packageIndex':packageIndex, 'conf':Conf, + 'searchDescription':searchDescription, 'searchKey':index } + return django.shortcuts.render_to_response(tmpl, extra) + +def maintainer_search(request, maintainer): + mappings = models.SourcePackageMapping.objects.order_by('package__name').filter( django.db.models.Q(package__maintainer__contains=maintainer) | django.db.models.Q(package__uploaders__contains=maintainer ) ) + packageIndex = ComplexQueries.PackageIndexQuery( mappings ) + tmpl = 'searchresults.html' + searchDescription = "Maintainer email" + extra = { 'index':maintainer, 'packageIndex':packageIndex, 'conf':Conf, + 'searchDescription':searchDescription, 'searchKey':maintainer } + return django.shortcuts.render_to_response(tmpl, extra) + +def display_patch(request, patchType, package, version, patchName): + pkg = models.SourcePackage.objects.get( name=package, version=version ) + ctx = django.template.RequestContext(request) + tmpl = 'patch_view.html' + + if patchType == "debianonly": + patch = pkg.diffhandler().debiandir() + patchTitle = "debian-dir only changes" + elif patchType == "misc": + patch = pkg.diffhandler().filterdiff(include=patchName) + patchTitle = patchName + elif patchType == "nondebian": + patch = pkg.diffhandler().nondebiandir() + patchTitle = "direct (non-packaging) changes" + elif patchType == "series": + patch = pkg.diffhandler().series().fetch(patchName) + patchTitle = patchName + + extra = { 'pkg':pkg, 'patch':patch, 'patchType':patchType, 'conf':Conf, + 'patchName':patchName, 'patchTitle':patchTitle } + return django.shortcuts.render_to_response(tmpl, extra, context_instance=ctx) + +def download_patch(request, patchType, package, version, patchName): + pkg = models.SourcePackage.objects.get( name=package, version=version ) + if patchType == "debianonly": + patch = pkg.diffhandler().debiandir() + elif patchType == "misc": + patch = pkg.diffhandler().filterdiff(include=patchName) + elif patchType == "nondebian": + patch = pkg.diffhandler().nondebiandir() + elif patchType == "series": + patch = pkg.diffhandler().series().fetch(patchName) + return django.http.HttpResponse(patch, mimetype="text/plain") + +def frontpage(request): + nonlibs = [idx for idx in models.SourcePackage.objects.filter(~django.db.models.Q(name__startswith='lib')).extra(select={'index' : "SUBSTR(name, 1, 1)"}).distinct().values_list('index', flat=True)] + libs = [idx for idx in models.SourcePackage.objects.filter(name__startswith='lib').extra(select={'index' : "SUBSTR(name, 1, 4)"}).distinct().values_list('index', flat=True)] + indices = sorted(nonlibs + libs) + extra = { 'indices':indices, 'conf':Conf } + return django.shortcuts.render_to_response('frontpage.html', extra) |