initial implementation of selective output caching

the PackageCmd and PatchCmd classes are now cached on their
first request.

note that this is incomplete, as the PackageCmd is also currently
used to print non-static information as well (when a package name
doesn't find an exact match and it does a query).

3 files changed, 101 insertions, 2 deletions

diff --git a/patchtracker/CacheObject.py b/patchtracker/CacheObject.py
new file mode 100644
index 0000000..c37c3b5
--- /dev/null
+++ b/patchtracker/CacheObject.py
@@ -0,0 +1,63 @@
+import errno
+import gzip
+import md5
+import os
+
+import Conf
+
+class CacheMissException (Exception):
+  pass
+ 
+class CacheObject:
+  """ A CacheObject is a compressed on-disk version of a serialized object.
+  """
+  def __init__ (self, key=None, keys=None):
+    self.obj = None
+    #print "CacheObject: key=%s keys=%s"%(key, keys)
+    if not key and not keys:
+      raise "CacheObject needs at least one of key, keys"
+    checksum = md5.md5()
+    if key:
+      checksum.update(key)
+    if keys:
+      for k in keys:
+        checksum.update(str(k))
+    self.path = os.path.sep.join([Conf.cachedir, checksum.hexdigest()])
+
+  def get(self):
+    if not self.obj:
+      try:
+        if Conf.cachecompress:
+          self.obj = gzip.GzipFile(self.path).read()
+        else:
+          self.obj = file(self.path).read()
+      except IOError, e:
+        if e.errno != errno.ENOENT:
+          raise e
+        else:
+          #print "CacheObject: cache miss"
+          raise CacheMissException("Object not present in cache")
+    #print "CacheObject: cache hit"
+    return self.obj
+
+  def put(self, obj):
+    self.obj = obj
+    #print "CacheObject: cache put"
+    try:
+      if Conf.cachecompress:
+        gzip.GzipFile(self.path, "wb").write(str(self.obj))
+      else:
+        file(self.path, "wb").write(str(self.obj))
+    except Exception, e:
+      os.unlink(self.path)
+      raise e
+
+if __name__ == '__main__':
+  co = CacheObject( key="magic" )
+  try:
+    print "going to try to read an object before it exists"
+    print "first line:", co.get().split()[0]
+  except CacheMissException:
+    print "file was missing as expected.  now let's try to put it and fetch it"
+    co.put(file("/etc/passwd").read())
+    print "first line:", co.get().split()[0]
diff --git a/patchtracker/Conf.py b/patchtracker/Conf.py
index 954001e..448f595 100755
--- a/patchtracker/Conf.py
+++ b/patchtracker/Conf.py
@@ -22,6 +22,18 @@ pts_index_file = "pts-index.json.gz"
     by the PTS system
 """
 
+caching = True
+""" Should caching be enabled? """
+
+cachedir = "cache"
+""" The location of the output cache directory.  This directory contains
+    cached output from system commands or other nontrivial calculations
+    for faster and less resource intensive re-use
+"""
+
+cachecompress = False
+""" Should compression of cache data be enabled? """
+
 try:
   from localconfig import *
 except ImportError:
diff --git a/patchtracker/ReqHandler.py b/patchtracker/ReqHandler.py
index a87fd68..59e0f53 100755
--- a/patchtracker/ReqHandler.py
+++ b/patchtracker/ReqHandler.py
@@ -7,6 +7,7 @@ import sys
 import patchtracker.Conf as Conf
 from patchtracker.Templates import ErrorTemplate, PatchTemplate, PackageVersTemplate, LetterTocTemplate, FrontPageTemplate, SearchResultsTemplate
 from patchtracker.DiffGzHandler import DiffGzHandler, DiffGzException
+from patchtracker.CacheObject import CacheObject, CacheMissException
 import patchtracker.DB as DB
 from patchtracker.DB import PatchTrackerDB
 import pygments
@@ -158,14 +159,19 @@ class FrontPageCmd(Cmd):
 class CmdHandler:
   def __init__(self, env):
     self.headers = []
-    uri=Conf.root_url+env['PATH_INFO']
+    uri = Conf.root_url+env['PATH_INFO']
+    self.cacheobj = None
+    #print "Accept:",env['HTTP_ACCEPT']
 
     args = uri[len(Conf.root_url)+1:].split("/")
     cmdarg = args[0]
+    cacheable = False
     if cmdarg == "patch":
       self.cmd = PatchCmd(args[1:])
+      cacheable = True
     elif cmdarg == "package":
       self.cmd = PackageCmd(args[1:]) 
+      cacheable = True
     elif cmdarg == "index":
       self.cmd = IndexCmd(args[1:]) 
     elif cmdarg == "jump":
@@ -178,11 +184,29 @@ class CmdHandler:
     else:
       self.cmd = ErrorCmd("invalid command/location '%s'"%(cmdarg), "404 Not found")
 
+    if Conf.caching and cacheable:
+      self.cacheobj = CacheObject(key=uri)
+
     self.headers.append( ('Content-type', self.cmd.content_type) )
     self.status = self.cmd.status
 
   def output(self):
+    result = None
     try:
-      return self.cmd.output()
+      if self.cacheobj:
+        result = self.cacheobj.get()
+      else:
+        result = self.cmd.output()
+    except CacheMissException:
+      result = self.cmd.output()
+      self.cacheobj.put(result)
     except DiffGzException, e:
       return ErrorCmd(str(e), "500 Oh Noez!!1!").output()
+
+    return result
+
+if __name__ == '__main__':
+  fake_env = { 'PATH_INFO': sys.argv[1] }
+  cmdh = CmdHandler(fake_env)
+  print "Status: %s\nHeaders: %s"%(cmdh.status, cmdh.headers)
+  print cmdh.output()