dashboard: add benchmarking support.

This has actually been running for a while and gathering benchmark
data. I haven't had a chance to add a UI for it yet however.

R=rsc
CC=golang-dev
http://codereview.appspot.com/194082

Committer: Russ Cox <rsc@golang.org>

6 files changed, 212 insertions, 12 deletions

diff --git a/misc/dashboard/README b/misc/dashboard/README
index 7b07a21a2..2878daef0 100644
--- a/misc/dashboard/README
+++ b/misc/dashboard/README
@@ -1,6 +1,10 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
 The files in this directory constitute the continuous builder:
 
-godashboard/: An AppEngine which acts as a server
+godashboard/: An AppEngine that acts as a server
 builder.sh, buildcontrol.sh: used by the build slaves
 
 If you wish to run a Go builder, please email golang-dev@googlegroups.com
@@ -19,7 +23,8 @@ export PATH=$PATH:/gobuild/bin
 export BUILDER=XXX
 export BUILDHOST=godashboard.appspot.com
 
-* Write ~gobuild/.gobuildkey (you need to get it from someone who knows it)
+* Write the key ~gobuild/.gobuildkey (you need to get it from someone who knows
+                                      the key)
 
 * sudo apt-get install bison gcc libc6-dev ed make
 * cd ~gobuild
diff --git a/misc/dashboard/buildcontrol.py b/misc/dashboard/buildcontrol.py
index caa1a2f47..7851de731 100644
--- a/misc/dashboard/buildcontrol.py
+++ b/misc/dashboard/buildcontrol.py
@@ -6,8 +6,10 @@
 
 # This is a utility script for implementing a Go build slave.
 
+import binascii
 import httplib
 import os
+import struct
 import subprocess
 import sys
 import time
@@ -42,10 +44,14 @@ def main(args):
         return doInit(args)
     elif args[1] == 'hwget':
         return doHWGet(args)
+    elif args[1] == 'hwset':
+        return doHWSet(args)
     elif args[1] == 'next':
         return doNext(args)
     elif args[1] == 'record':
         return doRecord(args)
+    elif args[1] == 'benchmarks':
+        return doBenchmarks(args)
     else:
         return usage(args[0])
 
@@ -55,8 +61,10 @@ def usage(name):
 Commands:
   init <rev>: init the build bot with the given commit as the first in history
   hwget <builder>: get the most recent revision built by the given builder
+  hwset <builder> <rev>: get the most recent revision built by the given builder
   next <builder>: get the next revision number to by built by the given builder
   record <builder> <rev> <ok|log file>: record a build result
+  benchmarks <builder> <rev> <log file>: record benchmark numbers
 ''' % name)
     return 1
 
@@ -78,11 +86,21 @@ def doHWGet(args, retries = 0):
     if reply.status == 200:
         print reply.read()
     elif reply.status == 500 and retries < 3:
+        time.sleep(3)
         return doHWGet(args, retries = retries + 1)
     else:
         raise Failed('get-hw returned %d' % reply.status)
     return 0
 
+def doHWSet(args):
+    if len(args) != 4:
+        return usage(args[0])
+    c = getCommit(args[3])
+    if c is None:
+        fatal('Cannot get commit %s' % args[3])
+
+    return command('hw-set', {'builder': args[2], 'hw': c.node})
+
 def doNext(args):
     if len(args) != 3:
         return usage(args[0])
@@ -96,7 +114,7 @@ def doNext(args):
 
     c = getCommit(rev)
     next = getCommit(str(c.num + 1))
-    if next is not None:
+    if next is not None and next.parent == c.node:
         print c.num + 1
     else:
         print "<none>"
@@ -117,8 +135,32 @@ def doRecord(args):
         log = file(logfile, 'r').read()
     return command('build', {'node': c.node, 'parent': c.parent, 'date': c.date, 'user': c.user, 'desc': c.desc, 'log': log, 'builder': builder})
 
-if __name__ == '__main__':
-    sys.exit(main(sys.argv))
+def doBenchmarks(args):
+    if len(args) != 5:
+        return usage(args[0])
+    builder = args[2]
+    rev = args[3]
+    c = getCommit(rev)
+    if c is None:
+        print >>sys.stderr, "Bad revision:", rev
+        return 1
+
+    benchmarks = {}
+    for line in file(args[4], 'r').readlines():
+        if 'Benchmark' in line and 'ns/op' in line:
+            parts = line.split()
+            if parts[3] == 'ns/op':
+                benchmarks[parts[0]] = (parts[1], parts[2])
+
+    e = []
+    for (name, (a, b)) in benchmarks.items():
+        e.append(struct.pack('>H', len(name)))
+        e.append(name)
+        e.append(struct.pack('>H', len(a)))
+        e.append(a)
+        e.append(struct.pack('>H', len(b)))
+        e.append(b)
+    return command('benchmarks', {'node': c.node, 'builder': builder, 'benchmarkdata': binascii.b2a_base64(''.join(e))})
 
 def encodeMultipartFormdata(fields, files):
     """fields is a sequence of (name, value) elements for regular form fields.
@@ -191,3 +233,6 @@ def command(cmd, args, retries = 0):
         return command(cmd, args, retries = retries + 1)
     if reply.status != 200:
         raise Failed('Command "%s" returned %d' % (cmd, reply.status))
+
+if __name__ == '__main__':
+    sys.exit(main(sys.argv))
diff --git a/misc/dashboard/builder.sh b/misc/dashboard/builder.sh
index 4a87ed2d5..d66ba08c3 100644
--- a/misc/dashboard/builder.sh
+++ b/misc/dashboard/builder.sh
@@ -5,7 +5,7 @@
 # license that can be found in the LICENSE file.
 
 fatal() {
-    echo $1
+    echo $0: $1 1>&2
     exit 1
 }
 
@@ -14,19 +14,19 @@ if [ ! -d go ] ; then
 fi
 
 if [ ! -f buildcontrol.py ] ; then
-    fatal "Please include buildcontrol.py in this directory"
+    fatal 'Please include buildcontrol.py in this directory'
 fi
 
 if [ "x$BUILDER" == "x" ] ; then
-    fatal "Please set \$BUILDER to the name of this builder"
+    fatal 'Please set $BUILDER to the name of this builder'
 fi
 
 if [ "x$BUILDHOST" == "x" ] ; then
-    fatal "Please set \$BUILDHOST to the hostname of the gobuild server"
+    fatal 'Please set $BUILDHOST to the hostname of the gobuild server'
 fi
 
 if [ "x$GOARCH" == "x" -o "x$GOOS" == "x" ] ; then
-    fatal "Please set $GOARCH and $GOOS"
+    fatal 'Please set $GOARCH and $GOOS'
 fi
 
 export PATH=$PATH:`pwd`/candidate/bin
@@ -59,6 +59,13 @@ while true ; do
     else
         echo "Recording success for $rev"
         python ../../buildcontrol.py record $BUILDER $rev ok || fatal "Cannot record result"
+        echo "Running benchmarks"
+        cd pkg || fatal "failed to cd to pkg"
+        make bench > ../../benchmarks 2>&1
+        if [ $? -eq 0 ] ; then
+            python ../../../buildcontrol.py benchmarks $BUILDER $rev ../../benchmarks || fatal "Cannot record benchmarks"
+        fi
+        cd .. || fatal "failed to cd out of pkg"
     fi
     cd ../.. || fatal "Cannot cd up"
 done
diff --git a/misc/dashboard/godashboard/gobuild.py b/misc/dashboard/godashboard/gobuild.py
index f984d920f..de08490e8 100644
--- a/misc/dashboard/godashboard/gobuild.py
+++ b/misc/dashboard/godashboard/gobuild.py
@@ -9,15 +9,17 @@ from google.appengine.ext import db
 from google.appengine.ext import webapp
 from google.appengine.ext.webapp import template
 from google.appengine.ext.webapp.util import run_wsgi_app
+import binascii
 import datetime
 import hashlib
 import logging
 import os
 import re
+import struct
 
 import key
 
-# The main class of state are commit objects. One of these exists for each of
+# The majority of our state are commit objects. One of these exists for each of
 # the commits known to the build system. Their key names are of the form
 # <commit number (%08x)> "-" <hg hash>. This means that a sorting by the key
 # name is sufficient to order the commits.
@@ -39,6 +41,15 @@ class Commit(db.Model):
     # successful.
     builds = db.StringListProperty()
 
+class Benchmark(db.Model):
+    name = db.StringProperty()
+
+class BenchmarkResult(db.Model):
+    num = db.IntegerProperty()
+    builder = db.StringProperty()
+    iterations = db.IntegerProperty()
+    nsperop = db.IntegerProperty()
+
 # A Log contains the textual build log of a failed build. The key name is the
 # hex digest of the SHA256 hash of the contents.
 class Log(db.Model):
@@ -96,6 +107,25 @@ class GetHighwater(webapp.RequestHandler):
         self.response.set_status(200)
         self.response.out.write(hw.commit)
 
+class SetHighwater(webapp.RequestHandler):
+    def post(self):
+        if self.request.get('key') != key.accessKey:
+            self.response.set_status(403)
+            return
+
+        builder = self.request.get('builder')
+        newhw = self.request.get('hw')
+        q = Commit.all()
+        q.filter('node =', newhw)
+        c = q.get()
+        if c is None:
+            self.response.set_status(404)
+            return
+
+        hw = Highwater(key_name = 'hw-%s' % builder)
+        hw.commit = c.node
+        hw.put()
+
 class LogHandler(webapp.RequestHandler):
     def get(self):
         self.response.headers['Content-Type'] = 'text/plain; charset=utf-8'
@@ -163,6 +193,7 @@ class Build(webapp.RequestHandler):
         q.filter('node =', parent)
         p = q.get()
         if p is None:
+            logging.error('Cannot find parent %s of node %s' % (parent, node))
             self.response.set_status(404)
             return
         parentnum, _ = p.key().name().split('-', 1)
@@ -198,6 +229,110 @@ class Build(webapp.RequestHandler):
 
         self.response.set_status(200)
 
+class Benchmarks(webapp.RequestHandler):
+    def get(self):
+        q = Benchmark.all()
+        bs = q.fetch(10000)
+
+        self.response.set_status(200)
+        self.response.headers['Content-Type'] = 'application/json; charset=utf-8'
+        self.response.out.write('{"benchmarks": [\n')
+
+        first = True
+        for b in bs:
+            if not first:
+                self.response.out.write(',"' + b.name + '"\n')
+            else:
+                self.response.out.write('"' + b.name + '"\n')
+                first = False
+        self.response.out.write(']}\n')
+
+    def post(self):
+        if self.request.get('key') != key.accessKey:
+            self.response.set_status(403)
+            return
+
+        builder = self.request.get('builder')
+        node = self.request.get('node')
+        if not validNode(node):
+            logging.error("Not valid node ('%s')", node)
+            self.response.set_status(500)
+            return
+
+        benchmarkdata = self.request.get('benchmarkdata')
+        benchmarkdata = binascii.a2b_base64(benchmarkdata)
+
+        def get_string(i):
+            l, = struct.unpack('>H', i[:2])
+            s = i[2:2+l]
+            if len(s) != l:
+                return None, None
+            return s, i[2+l:]
+
+        benchmarks = {}
+        while len(benchmarkdata) > 0:
+            name, benchmarkdata = get_string(benchmarkdata)
+            iterations_str, benchmarkdata = get_string(benchmarkdata)
+            time_str, benchmarkdata = get_string(benchmarkdata)
+            iterations = int(iterations_str)
+            time = int(time_str)
+
+            benchmarks[name] = (iterations, time)
+
+        q = Commit.all()
+        q.filter('node =', node)
+        n = q.get()
+        if n is None:
+            logging.error('Client asked for unknown commit while uploading benchmarks')
+            self.response.set_status(404)
+            return
+
+        for (benchmark, (iterations, time)) in benchmarks.items():
+            b = Benchmark.get_or_insert(benchmark.encode('base64'), name = benchmark)
+            r = BenchmarkResult(key_name = '%08x/builder' % n.num, parent = b, num = n.num, iterations = iterations, nsperop = time, builder = builder)
+            r.put()
+
+        self.response.set_status(200)
+
+class GetBenchmarks(webapp.RequestHandler):
+    def get(self):
+        self.response.headers['Content-Type'] = 'application/json; charset=utf-8'
+        benchmark = self.request.path[12:].decode('hex').encode('base64')
+
+        b = Benchmark.get_by_key_name(benchmark)
+        if b is None:
+            self.response.set_status(404)
+            return
+
+        q = BenchmarkResult.all()
+        q.ancestor(b)
+        q.order('-__key__')
+        results = q.fetch(10000)
+
+        if len(results) == 0:
+            self.response.set_status(404)
+            return
+
+        max = -1
+        min = 2000000000
+        builders = set()
+        for r in results:
+            if max < r.num:
+                max = r.num
+            if min > r.num:
+                min = r.num
+            builders.add(r.builder)
+
+        res = {}
+        for b in builders:
+            res[b] = [[-1] * ((max - min) + 1), [-1] * ((max - min) + 1)]
+
+        for r in results:
+            res[r.builder][0][r.num - min] = r.iterations
+            res[r.builder][1][r.num - min] = r.nsperop
+
+        self.response.out.write(str(res))
+
 class FixedOffset(datetime.tzinfo):
     """Fixed offset in minutes east from UTC."""
 
@@ -273,9 +408,12 @@ application = webapp.WSGIApplication(
                                      [('/', MainPage),
                                       ('/log/.*', LogHandler),
                                       ('/hw-get', GetHighwater),
+                                      ('/hw-set', SetHighwater),
 
                                       ('/init', Init),
                                       ('/build', Build),
+                                      ('/benchmarks', Benchmarks),
+                                      ('/benchmarks/.*', GetBenchmarks),
                                      ])
 
 def main():
diff --git a/misc/dashboard/godashboard/index.yaml b/misc/dashboard/godashboard/index.yaml
index 784d23d01..19d2f3778 100644
--- a/misc/dashboard/godashboard/index.yaml
+++ b/misc/dashboard/godashboard/index.yaml
@@ -10,6 +10,12 @@ indexes:
 # automatically uploaded to the admin console when you next deploy
 # your application using appcfg.py.
 
+- kind: BenchmarkResult
+  ancestor: yes
+  properties:
+  - name: __key__
+    direction: desc
+
 - kind: Commit
   properties:
   - name: __key__
diff --git a/misc/dashboard/godashboard/key.py b/misc/dashboard/godashboard/key.py
index 7495709ec..3abe410dd 100644
--- a/misc/dashboard/godashboard/key.py
+++ b/misc/dashboard/godashboard/key.py
@@ -6,4 +6,3 @@
 # builds). It's tranmitted in the clear but, given the low value of the target,
 # this should be sufficient.
 accessKey = "this is not the real key"
-