dashboard: lots of caching to avoid datastore queries

reorganize benchmark computation so that it is
incremental.  if it times out, it doesn't lose the
pieces it already computed, so that next time it
has a fighting chance to finish.

R=agl1, agl
CC=golang-dev
http://codereview.appspot.com/216046

2 files changed, 143 insertions, 118 deletions

diff --git a/misc/dashboard/godashboard/gobuild.py b/misc/dashboard/godashboard/gobuild.py
index c10d92dbd..dcea6a4ea 100644
--- a/misc/dashboard/godashboard/gobuild.py
+++ b/misc/dashboard/godashboard/gobuild.py
@@ -6,6 +6,7 @@
 # by AppEngine.
 
 from google.appengine.api import memcache
+from google.appengine.runtime import DeadlineExceededError
 from google.appengine.ext import db
 from google.appengine.ext import webapp
 from google.appengine.ext.webapp import template
@@ -76,6 +77,15 @@ def builderInfo(b):
         note = f[2]
     return {'name': b, 'goos': goos, 'goarch': goarch, 'note': note}
 
+def builderset():
+    q = Commit.all()
+    q.order('-__key__')
+    results = q.fetch(N)
+    builders = set()
+    for c in results:
+        builders.update(set(parseBuild(build)['builder'] for build in c.builds))
+    return builders
+    
 class MainPage(webapp.RequestHandler):
     def get(self):
         self.response.headers['Content-Type'] = 'text/html; charset=utf-8'
@@ -108,30 +118,32 @@ class MainPage(webapp.RequestHandler):
 class GetHighwater(webapp.RequestHandler):
     def get(self):
         builder = self.request.get('builder')
-
-        hw = Highwater.get_by_key_name('hw-%s' % builder)
-        if hw is None:
-            # If no highwater has been recorded for this builder,
-            # we go back N+1 commits and return that.
-            q = Commit.all()
-            q.order('-__key__')
-            c = q.fetch(N+1)[-1]
-            self.response.set_status(200)
-            self.response.out.write(c.node)
-            return
-
-        # if the proposed hw is too old, bump it forward
-        node = hw.commit
-        found = False
-        q = Commit.all()
-        q.order('-__key__')
-        recent = q.fetch(N+1)
-        for c in recent:
-            if c.node == node:
-                found = True
-                break
-        if not found:
-            node = recent[-1].node
+        
+        key = 'hw-%s' % builder
+        node = memcache.get(key)
+        if node is None:
+            hw = Highwater.get_by_key_name('hw-%s' % builder)
+            if hw is None:
+                # If no highwater has been recorded for this builder,
+                # we go back N+1 commits and return that.
+                q = Commit.all()
+                q.order('-__key__')
+                c = q.fetch(N+1)[-1]
+                node = c.node
+            else:
+                # if the proposed hw is too old, bump it forward
+                node = hw.commit
+                found = False
+                q = Commit.all()
+                q.order('-__key__')
+                recent = q.fetch(N+1)
+                for c in recent:
+                    if c.node == node:
+                        found = True
+                        break
+                if not found:
+                    node = recent[-1].node
+            memcache.set(key, node, 3600)
         self.response.set_status(200)
         self.response.out.write(node)
 
@@ -166,7 +178,9 @@ class SetHighwater(webapp.RequestHandler):
         if not found:
             c = recent[-1]
 
-        hw = Highwater(key_name = 'hw-%s' % builder)
+        key = 'hw-%s' % builder
+        memcache.delete(key)
+        hw = Highwater(key_name = key)
         hw.commit = c.node
         hw.put()
 
@@ -270,6 +284,8 @@ class Build(webapp.RequestHandler):
             hw = Highwater(key_name = 'hw-%s' % builder)
         hw.commit = node
         hw.put()
+        memcache.delete('hw')
+        memcache.delete('bench')
 
         self.response.set_status(200)
 
@@ -295,95 +311,91 @@ class Benchmarks(webapp.RequestHandler):
             return self.json()
         self.response.set_status(200)
         self.response.headers['Content-Type'] = 'text/html; charset=utf-8'
-        q = Commit.all()
-        q.order('-__key__')
-        n = q.fetch(1)[0]
-        key = "bench(%d)" % n.num
-        page = None # memcache.get(key)
+        page = memcache.get('bench')
         if not page:
-            page = self.compute()
-            memcache.set(key, page, 3600)
+            num = memcache.get('hw')
+            if num is None:
+                q = Commit.all()
+                q.order('-__key__')
+                n = q.fetch(1)[0]
+                memcache.set('hw', num)
+            page, full = self.compute(n.num)
+            if full:
+                memcache.set('bench', page, 3600)
         self.response.out.write(page)
     
-    def compute(self):
+    def compute(self, num):
         q = Benchmark.all()
         q.filter('__key__ >', Benchmark.get_or_insert('v002.').key())
-        bs = q.fetch(10000)
-        
-        # Collect table giving all the data we need.
-        builders = {}
-        data = {}
-        for b in bs:
-            # TODO(rsc): Will want to limit benchmarks to a certain
-            # number of commits eventually, but there aren't enough
-            # commits yet to worry.
-            q = BenchmarkResult.all()
-            q.ancestor(b)
-            q.order('-__key__')
-            results = q.fetch(10000)
-            m = {}
-            revs = {}
-            for r in results:
-                if r.builder not in m:
-                    m[r.builder] = {}
-                m[r.builder][r.num] = r.nsperop
-                revs[r.num] = 0
-                builders[r.builder] = 0
-            data[b.name] = m
-
-        builders = list(builders.keys())
-        builders.sort()
-
-        revs = list(revs.keys())
-        revs.sort()
-        first = revs[0]
-        last = revs[-1]
-        if len(revs) > 80:   # At most 80 commits back
-            last = revs[-80]
-
-        names = list(data.keys())
-        names.sort()
+        benchmarks = q.fetch(10000)
 
+        # Which builders have sent benchmarks recently?
+        builders = set()
+        q = BenchmarkResult.all()
+        q.ancestor(benchmarks[0])
+        q.order('-__key__')
+        for r in q.fetch(50):
+            builders.add(r.builder)
+        builders = list(builders)
+        builders.sort()
+        
+        NB = 80
+        last = num
+        first = num+1 - NB
+        
         # Build list of rows, one per benchmark
-        benchmarks = []
-        for name in names:
-            # Build list of cells, one per builder.
-            m = data[name]
-            builds = []
-            for builder in builders:
-                # Build cell: a URL for the chart server or an empty string.
-                if builder not in m:
-                    builds.append({"url":""})
-                    continue
-                d = m[builder]
-                max = 0
-                tot = 0
-                ntot = 0
-                for i in range(first, last+1):
-                    if i not in d:
+        rows = [{"name": bm.name, "builds": [{"url": ""} for b in builders]} for bm in benchmarks]
+
+        full = True
+        try:
+            for i in range(len(rows)):
+                data = None
+                bm = benchmarks[i]
+                builds = rows[i]["builds"]
+                all = None
+                for j in range(len(builders)):
+                    cell = builds[j]
+                    b = builders[j]
+                    # Build cell: a URL for the chart server or an empty string.
+                    # Cache individual graphs because they're so damn expensive.
+                    key = "bench(%s,%s,%d)" % (bm.name, b, num)
+                    url = memcache.get(key)
+                    if url is not None:
+                        cell["url"] = url
                         continue
-                    val = d[i]
-                    if max < val:
-                        max = val
-                    tot += val
-                    ntot += 1
-                if max == 0:
-                    builds.append({"url":""})
-                    continue
-                avg = tot / ntot
-                if 2*avg > max:
-                    max = 2*avg
-                # Encoding is 0-61, which is fine enough granularity for our tiny graphs.  _ means missing.
-                encoding = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
-                s = ""
-                for i in range(first, last+1):
-                    if i not in d:
-                        s += "_"
+                    
+                    # Page in all data for benchmark for all builders,
+                    # on demand.  It might be faster to ask for just the
+                    # builder that we need, but q.filter('builder = ', b) is
+                    # broken right now (the index is corrupt).
+                    if all is None:
+                        q = BenchmarkResult.all()
+                        q.ancestor(bm)
+                        q.order('-__key__')
+                        all = q.fetch(1000)
+
+                    data = [-1 for x in range(first, last+1)]
+                    for r in all:
+                        if r.builder == b and first <= r.num and r.num <= last:
+                            data[r.num - first] = r.nsperop
+                    present = [x for x in data if x >= 0]
+                    if len(present) == 0:
+                        memcache.set(key, "", 3600)
                         continue
-                    val = d[i]
-                    s += encoding[int((len(encoding)-1)*val/max)]
-                builds.append({"url": "http://chart.apis.google.com/chart?cht=ls&chd=s:"+s})
-            benchmarks.append({"name": name, "builds": builds})
+                    avg = sum(present) / len(present)
+                    maxval = max(2*avg, max(present))
+                    # Encoding is 0-61, which is fine enough granularity for our tiny graphs.  _ means missing.
+                    encoding = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
+                    s = ''.join([x < 0 and "_" or encoding[int((len(encoding)-1)*x/maxval)] for x in data])
+                    url = "http://chart.apis.google.com/chart?cht=ls&chd=s:"+s
+                    memcache.set(key, url, 3600)
+                    cell["url"] = url
+        except DeadlineExceededError:
+            # forge ahead with partial benchmark results
+            # the url caches above should make the next page quicker to compute
+            full = False
+
+        names = [bm.name for bm in benchmarks]
 
         bs = []
         for b in builders:
@@ -395,10 +407,10 @@ class Benchmarks(webapp.RequestHandler):
                 note = f[2]
             bs.append({'goos': goos, 'goarch': goarch, 'note': note})
 
-        values = {"benchmarks": benchmarks, "builders": bs}
+        values = {"benchmarks": rows, "builders": bs}
 
         path = os.path.join(os.path.dirname(__file__), 'benchmarks.html')
-        return template.render(path, values)
+        return template.render(path, values), full
 
     def post(self):
         if not auth(self.request):
@@ -444,8 +456,10 @@ class Benchmarks(webapp.RequestHandler):
             b = Benchmark.get_or_insert('v002.' + benchmark.encode('base64'), name = benchmark, version = 2)
             r = BenchmarkResult(key_name = '%08x/%s' % (n.num, builder), parent = b, num = n.num, iterations = iterations, nsperop = time, builder = builder)
             r.put()
-        key = "bench(%d)" % n.num
-        memcache.delete(key)
+            key = "bench(%s,%s,%d)" % (benchmark, builder, n.num)
+            memcache.delete(key)
+
+        memcache.delete('bench')
         self.response.set_status(200)
 
 def node(num):
@@ -481,6 +495,9 @@ class GetBenchmarks(webapp.RequestHandler):
                 minv = r.num
             builders.add(r.builder)
 
+        builders = list(builders)
+        builders.sort()
+
         res = {}
         for b in builders:
             res[b] = [[-1] * ((maxv - minv) + 1), [-1] * ((maxv - minv) + 1)]
diff --git a/misc/dashboard/godashboard/index.yaml b/misc/dashboard/godashboard/index.yaml
index 19d2f3778..d0b08992d 100644
--- a/misc/dashboard/godashboard/index.yaml
+++ b/misc/dashboard/godashboard/index.yaml
@@ -1,14 +1,11 @@
 indexes:
 
-# AUTOGENERATED
-
-# This index.yaml is automatically updated whenever the dev_appserver
-# detects that a new type of query is run.  If you want to manage the
-# index.yaml file manually, remove the above marker line (the line
-# saying "# AUTOGENERATED").  If you want to manage some indexes
-# manually, move them above the marker line.  The index.yaml file is
-# automatically uploaded to the admin console when you next deploy
-# your application using appcfg.py.
+- kind: BenchmarkResult
+  ancestor: yes
+  properties:
+  - name: builder
+  - name: __key__
+    direction: desc
 
 - kind: BenchmarkResult
   ancestor: yes
@@ -20,3 +17,14 @@ indexes:
   properties:
   - name: __key__
     direction: desc
+
+# AUTOGENERATED
+
+# This index.yaml is automatically updated whenever the dev_appserver
+# detects that a new type of query is run.  If you want to manage the
+# index.yaml file manually, remove the above marker line (the line
+# saying "# AUTOGENERATED").  If you want to manage some indexes
+# manually, move them above the marker line.  The index.yaml file is
+# automatically uploaded to the admin console when you next deploy
+# your application using appcfg.py.
+