diff options
author | Ondřej Surý <ondrej@sury.org> | 2011-04-28 10:35:15 +0200 |
---|---|---|
committer | Ondřej Surý <ondrej@sury.org> | 2011-04-28 10:35:15 +0200 |
commit | c1ba1a0fec4aed430709030f98a3bdb90bfeea16 (patch) | |
tree | 3df18657e50a0313ed6defcda30e4474cb28a467 /misc | |
parent | 7b15ed9ef455b6b66c6b376898a88aef5d6a9970 (diff) | |
download | golang-c1ba1a0fec4aed430709030f98a3bdb90bfeea16.tar.gz |
Imported Upstream version 2011.04.27upstream/2011.04.27
Diffstat (limited to 'misc')
-rw-r--r-- | misc/dashboard/builder/main.go | 20 | ||||
-rw-r--r-- | misc/dashboard/godashboard/benchmark1.html | 62 | ||||
-rw-r--r-- | misc/dashboard/godashboard/benchmarks.html | 46 | ||||
-rw-r--r-- | misc/dashboard/godashboard/gobuild.py | 385 | ||||
-rw-r--r-- | misc/dashboard/godashboard/main.html | 1 | ||||
-rw-r--r-- | misc/dashboard/godashboard/package.html | 1 | ||||
-rw-r--r-- | misc/dashboard/godashboard/project.html | 1 | ||||
-rw-r--r-- | misc/goplay/goplay.go | 6 |
8 files changed, 58 insertions, 464 deletions
diff --git a/misc/dashboard/builder/main.go b/misc/dashboard/builder/main.go index 735717e28..d11cbb133 100644 --- a/misc/dashboard/builder/main.go +++ b/misc/dashboard/builder/main.go @@ -34,6 +34,7 @@ var extraEnv = []string{ "GOHOSTARCH", "PATH", "DISABLE_NET_TESTS", + "MAKEFLAGS", "GOARM", } @@ -59,6 +60,7 @@ var ( buildRevision = flag.String("rev", "", "Build specified revision and exit") buildCmd = flag.String("cmd", "./all.bash", "Build command (specify absolute or relative to go/src/)") external = flag.Bool("external", false, "Build external packages") + parallel = flag.Bool("parallel", false, "Build multiple targets in parallel") verbose = flag.Bool("v", false, "verbose") ) @@ -132,9 +134,19 @@ func main() { continue } built := false - for _, b := range builders { - if b.build() { - built = true + if *parallel { + done := make(chan bool) + for _, b := range builders { + go func(b *Builder) { + done <- b.build() + }(b) + } + for _ = range builders { + built = <-done || built + } + } else { + for _, b := range builders { + built = b.build() || built } } // only run benchmarks if we didn't build anything @@ -185,7 +197,7 @@ func NewBuilder(builder string) (*Builder, os.Error) { // get goos/goarch from builder string s := strings.Split(builder, "-", 3) - if len(s) == 2 { + if len(s) >= 2 { b.goos, b.goarch = s[0], s[1] } else { return nil, fmt.Errorf("unsupported builder form: %s", builder) diff --git a/misc/dashboard/godashboard/benchmark1.html b/misc/dashboard/godashboard/benchmark1.html deleted file mode 100644 index 2d49e7204..000000000 --- a/misc/dashboard/godashboard/benchmark1.html +++ /dev/null @@ -1,62 +0,0 @@ -<!DOCTYPE HTML> -<html> - <head> - <title>{{benchmark}} - Benchmarks - Go Dashboard</title> - <link rel="stylesheet" type="text/css" href="/static/style.css"> - </head> - - <body> - <ul class="menu"> - <li><a href="/">Build Status</a></li> - <li><a href="/package">Packages</a></li> - <li><a href="/project">Projects</a></li> - <li><a href="/benchmarks">Benchmarks</a></li> - <li><a href="http://golang.org/">golang.org</a></li> - </ul> - - <h1>Go Dashboard</h1> - - <h2>{{benchmark}}</h2> - - <a href="{{benchmark}}?fmt=json">json</a> - - {% for g in graphs %} - <h3>{{g.builder}}</h3> - {% if g.url %} - <img src="{{g.url}}&chs=600x150&chf=bg,s,00000000&chco=000000ff&chls=1,1,0"> - {% else %} - (no data available) - {% endif %} - {% endfor %} - - <br><br> - - <table class="alternate" cellpadding="0" cellspacing="0"> - <tr> - <th></th> - {% for b in builders %} - <th class="builder">{{b.goos}}<br>{{b.goarch}}<br>{{b.note}}</th> - {% endfor %} - <th></th> - <th></th> - <th></th> - </tr> - - {% for r in revs %} - <tr> - <td class="revision"><span class="hash"><a href="https://code.google.com/p/go/source/detail?r={{r.node}}">{{r.node|slice:":12"}}</a></span></td> - - {% for ns in r.ns_by_builder %} - <td class="result"> - {% if ns %} - {{ns}} - {% endif %} - </td> - {% endfor %} - <td class="user">{{r.user|escape}}</td> - <td class="date">{{r.date|escape}}</td> - <td class="desc">{{r.shortdesc|escape}}</td> - </tr> - {% endfor %} - </body> -</html> diff --git a/misc/dashboard/godashboard/benchmarks.html b/misc/dashboard/godashboard/benchmarks.html deleted file mode 100644 index d42fcfe48..000000000 --- a/misc/dashboard/godashboard/benchmarks.html +++ /dev/null @@ -1,46 +0,0 @@ -<!DOCTYPE HTML> -<html> - <head> - <title>Benchmarks - Go Dashboard</title> - <link rel="stylesheet" type="text/css" href="/static/style.css"> - </head> - - <body> - <ul class="menu"> - <li><a href="/">Build Status</a></li> - <li><a href="/package">Packages</a></li> - <li><a href="/project">Projects</a></li> - <li>Benchmarks</li> - <li><a href="http://golang.org/">golang.org</a></li> - </ul> - - <h1>Go Dashboard</h1> - - <h2>Benchmarks</h2> - - <table class="alternate" cellpadding="0" cellspacing="0"> - <tr> - <th></th> - {% for b in builders %} - <th class="builder">{{b.goos}}<br>{{b.goarch}}<br>{{b.note}}</th> - {% endfor %} - </tr> - - {% for bm in rows %} - <tr> - <td class="name"><a href="/benchmarks/{{bm.name}}">{{bm.name}}</a></td> - - {% for bl in bm.builders %} - <td class="result"> - {% if bl.url %} - <img src="{{bl.url}}" /> - {% else %} - <img src="/benchmarks/single?benchmark={{bm.name}}&builder={{bl.name}}" /> - {% endif %} - </td> - {% endfor %} - </tr> - {% endfor %} - </table> - </body> -</html> diff --git a/misc/dashboard/godashboard/gobuild.py b/misc/dashboard/godashboard/gobuild.py index 1eacdb38e..035bf842f 100644 --- a/misc/dashboard/godashboard/gobuild.py +++ b/misc/dashboard/godashboard/gobuild.py @@ -44,21 +44,12 @@ class Commit(db.Model): desc = db.BlobProperty() # This is the list of builds. Each element is a string of the form <builder - # name> "`" <log hash>. If the log hash is empty, then the build was + # name> '`' <log hash>. If the log hash is empty, then the build was # successful. builds = db.StringListProperty() fail_notification_sent = db.BooleanProperty() -class Benchmark(db.Model): - name = db.StringProperty() - version = db.IntegerProperty() - -class BenchmarkResults(db.Model): - builder = db.StringProperty() - benchmark = db.StringProperty() - data = db.ListProperty(long) # encoded as [-1, num, iterations, nsperop]* - class Cache(db.Model): data = db.BlobProperty() expire = db.IntegerProperty() @@ -69,12 +60,6 @@ class Cache(db.Model): class CompressedLog(db.Model): log = db.BlobProperty() -# For each builder, we store the last revision that it built. So, if it -# crashes, it knows where to start up from. The key names for these objects are -# "hw-" <builder name> -class Highwater(db.Model): - commit = db.StringProperty() - N = 30 def cache_get(key): @@ -165,34 +150,40 @@ class MainPage(webapp.RequestHandler): class GetHighwater(webapp.RequestHandler): def get(self): builder = self.request.get('builder') - - key = 'hw-%s' % builder - node = memcache.get(key) - if node is None: - hw = Highwater.get_by_key_name('hw-%s' % builder) - if hw is None: - # If no highwater has been recorded for this builder, - # we go back N+1 commits and return that. - q = Commit.all() - q.order('-__key__') - c = q.fetch(N+1)[-1] - node = c.node - else: - # if the proposed hw is too old, bump it forward - node = hw.commit - found = False - q = Commit.all() - q.order('-__key__') - recent = q.fetch(N+1) - for c in recent: - if c.node == node: - found = True - break - if not found: - node = recent[-1].node - memcache.set(key, node, 3600) + key = 'todo-%s' % builder + response = memcache.get(key) + if response is None: + # Fell out of memcache. Rebuild from datastore results. + # We walk the commit list looking for nodes that have not + # been built by this builder and record the *parents* of those + # nodes, because each builder builds the revision *after* the + # one return (because we might not know about the latest + # revision). + q = Commit.all() + q.order('-__key__') + todo = [] + need = False + first = None + for c in q.fetch(N+1): + if first is None: + first = c + if need: + todo.append(c.node) + need = not built(c, builder) + if not todo: + todo.append(first.node) + response = ' '.join(todo) + memcache.set(key, response, 3600) self.response.set_status(200) - self.response.out.write(node) + if self.request.get('all') != 'yes': + response = response.split()[0] + self.response.out.write(response) + +def built(c, builder): + for b in c.builds: + if b.startswith(builder+'`'): + return True + return False def auth(req): k = req.get('key') @@ -204,32 +195,10 @@ class SetHighwater(webapp.RequestHandler): self.response.set_status(403) return - builder = self.request.get('builder') - newhw = self.request.get('hw') - q = Commit.all() - q.filter('node =', newhw) - c = q.get() - if c is None: - self.response.set_status(404) - return - - # if the proposed hw is too old, bump it forward - found = False - q = Commit.all() - q.order('-__key__') - recent = q.fetch(N+1) - for c in recent: - if c.node == newhw: - found = True - break - if not found: - c = recent[-1] - - key = 'hw-%s' % builder - memcache.delete(key) - hw = Highwater(key_name = key) - hw.commit = c.node - hw.put() + # Allow for old builders. + # This is a no-op now: we figure out what to build based + # on the current dashboard status. + return class LogHandler(webapp.RequestHandler): def get(self): @@ -330,14 +299,8 @@ class Build(webapp.RequestHandler): db.run_in_transaction(add_build) - key = 'hw-%s' % builder - hw = Highwater.get_by_key_name(key) - if hw is None: - hw = Highwater(key_name = key) - hw.commit = node - hw.put() + key = 'todo-%s' % builder memcache.delete(key) - memcache.delete('hw') def mark_sent(): n = Commit.get_by_key_name(key_name) @@ -373,279 +336,12 @@ def failed(c, builder): return len(p[1]) > 0 return False -class Benchmarks(webapp.RequestHandler): - def json(self): - q = Benchmark.all() - q.filter('__key__ >', Benchmark.get_or_insert('v002.').key()) - bs = q.fetch(10000) - - self.response.set_status(200) - self.response.headers['Content-Type'] = 'text/plain; charset=utf-8' - self.response.out.write('{"benchmarks": [') - - sep = "\n\t" - for b in bs: - self.response.out.write('%s"%s"' % (sep, b.name)) - sep = ",\n\t" - self.response.out.write('\n]}\n') - - def get(self): - if self.request.get('fmt') == 'json': - return self.json() - - self.response.set_status(200) - self.response.headers['Content-Type'] = 'text/html; charset=utf-8' - page = memcache.get('bench') - if not page: - # use datastore as cache to avoid computation even - # if memcache starts dropping things on the floor - logging.error("memcache dropped bench") - page = cache_get('bench') - if not page: - logging.error("cache dropped bench") - num = memcache.get('hw') - if num is None: - q = Commit.all() - q.order('-__key__') - n = q.fetch(1)[0] - num = n.num - memcache.set('hw', num) - page = self.compute(num) - cache_set('bench', page, 600) - memcache.set('bench', page, 600) - self.response.out.write(page) - - def compute(self, num): - benchmarks, builders = benchmark_list() - - rows = [] - for bm in benchmarks: - row = {'name':bm, 'builders': []} - for bl in builders: - key = "single-%s-%s" % (bm, bl) - url = memcache.get(key) - row['builders'].append({'name': bl, 'url': url}) - rows.append(row) - - path = os.path.join(os.path.dirname(__file__), 'benchmarks.html') - data = { - "builders": [builderInfo(b) for b in builders], - "rows": rows, - } - return template.render(path, data) - - def post(self): - if not auth(self.request): - self.response.set_status(403) - return - - builder = self.request.get('builder') - node = self.request.get('node') - if not validNode(node): - logging.error("Not valid node ('%s')", node) - self.response.set_status(500) - return - - benchmarkdata = self.request.get('benchmarkdata') - benchmarkdata = binascii.a2b_base64(benchmarkdata) - - def get_string(i): - l, = struct.unpack('>H', i[:2]) - s = i[2:2+l] - if len(s) != l: - return None, None - return s, i[2+l:] - - benchmarks = {} - while len(benchmarkdata) > 0: - name, benchmarkdata = get_string(benchmarkdata) - iterations_str, benchmarkdata = get_string(benchmarkdata) - time_str, benchmarkdata = get_string(benchmarkdata) - iterations = int(iterations_str) - time = int(time_str) - - benchmarks[name] = (iterations, time) - - q = Commit.all() - q.filter('node =', node) - n = q.get() - if n is None: - logging.error('Client asked for unknown commit while uploading benchmarks') - self.response.set_status(404) - return - - for (benchmark, (iterations, time)) in benchmarks.items(): - b = Benchmark.get_or_insert('v002.' + benchmark.encode('base64'), name = benchmark, version = 2) - key = '%s;%s' % (builder, benchmark) - r1 = BenchmarkResults.get_by_key_name(key) - if r1 is not None and (len(r1.data) < 4 or r1.data[-4] != -1 or r1.data[-3] != n.num): - r1.data += [-1L, long(n.num), long(iterations), long(time)] - r1.put() - key = "bench(%s,%s,%d)" % (benchmark, builder, n.num) - memcache.delete(key) - - self.response.set_status(200) - -class SingleBenchmark(webapp.RequestHandler): - """ - Fetch data for single benchmark/builder combination - and return sparkline url as HTTP redirect, also set memcache entry. - """ - def get(self): - benchmark = self.request.get('benchmark') - builder = self.request.get('builder') - key = "single-%s-%s" % (benchmark, builder) - - url = memcache.get(key) - - if url is None: - minr, maxr, bybuilder = benchmark_data(benchmark) - for bb in bybuilder: - if bb[0] != builder: - continue - url = benchmark_sparkline(bb[2]) - - if url is None: - self.response.set_status(500, "No data found") - return - - memcache.set(key, url, 700) # slightly longer than bench timeout - - self.response.set_status(302) - self.response.headers.add_header("Location", url) - def node(num): q = Commit.all() q.filter('num =', num) n = q.get() return n -def benchmark_data(benchmark): - q = BenchmarkResults.all() - q.order('__key__') - q.filter('benchmark =', benchmark) - results = q.fetch(100) - - minr = 100000000 - maxr = 0 - for r in results: - if r.benchmark != benchmark: - continue - # data is [-1, num, iters, nsperop, -1, num, iters, nsperop, ...] - d = r.data - if not d: - continue - if [x for x in d[::4] if x != -1]: - # unexpected data framing - logging.error("bad framing for data in %s;%s" % (r.builder, r.benchmark)) - continue - revs = d[1::4] - minr = min(minr, min(revs)) - maxr = max(maxr, max(revs)) - if minr > maxr: - return 0, 0, [] - - bybuilder = [] - for r in results: - if r.benchmark != benchmark: - continue - d = r.data - if not d: - continue - nsbyrev = [-1 for x in range(minr, maxr+1)] - iterbyrev = [-1 for x in range(minr, maxr+1)] - for num, iter, ns in zip(d[1::4], d[2::4], d[3::4]): - iterbyrev[num - minr] = iter - nsbyrev[num - minr] = ns - bybuilder.append((r.builder, iterbyrev, nsbyrev)) - - return minr, maxr, bybuilder - -def benchmark_graph(builder, minhash, maxhash, ns): - valid = [x for x in ns if x >= 0] - if not valid: - return "" - m = max(max(valid), 2*sum(valid)/len(valid)) - s = "" - encoding = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789.-" - for val in ns: - if val < 0: - s += "__" - continue - val = int(val*4095.0/m) - s += encoding[val/64] + encoding[val%64] - return ("http://chart.apis.google.com/chart?cht=lc&chxt=x,y&chxl=0:|%s|%s|1:|0|%g ns|%g ns&chd=e:%s" % - (minhash[0:12], maxhash[0:12], m/2, m, s)) - -def benchmark_sparkline(ns): - valid = [x for x in ns if x >= 0] - if not valid: - return "" - m = max(max(valid), 2*sum(valid)/len(valid)) - # Encoding is 0-61, which is fine enough granularity for our tiny graphs. _ means missing. - encoding = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789" - s = ''.join([x < 0 and "_" or encoding[int((len(encoding)-1)*x/m)] for x in ns]) - url = "http://chart.apis.google.com/chart?cht=ls&chd=s:"+s+"&chs=80x20&chf=bg,s,00000000&chco=000000ff&chls=1,1,0" - return url - -def benchmark_list(): - q = BenchmarkResults.all() - q.order('__key__') - q.filter('builder = ', u'darwin-amd64') - benchmarks = [r.benchmark for r in q] - - q = BenchmarkResults.all() - q.order('__key__') - q.filter('benchmark =', u'math_test.BenchmarkSqrt') - builders = [r.builder for r in q.fetch(20)] - - return benchmarks, builders - -class GetBenchmarks(webapp.RequestHandler): - def get(self): - benchmark = self.request.path[12:] - minr, maxr, bybuilder = benchmark_data(benchmark) - minhash = node(minr).node - maxhash = node(maxr).node - - if self.request.get('fmt') == 'json': - self.response.headers['Content-Type'] = 'text/plain; charset=utf-8' - self.response.out.write('{ "min": "%s", "max": "%s", "data": {' % (minhash, maxhash)) - sep = "\n\t" - for builder, iter, ns in bybuilder: - self.response.out.write('%s{ "builder": "%s", "iterations": %s, "nsperop": %s }' % - (sep, builder, str(iter).replace("L", ""), str(ns).replace("L", ""))) - sep = ",\n\t" - self.response.out.write('\n}\n') - return - - graphs = [] - for builder, iter, ns in bybuilder: - graphs.append({"builder": builder, "url": benchmark_graph(builder, minhash, maxhash, ns)}) - - revs = [] - for i in range(minr, maxr+1): - r = nodeInfo(node(i)) - x = [] - for _, _, ns in bybuilder: - t = ns[i - minr] - if t < 0: - t = None - x.append(t) - r["ns_by_builder"] = x - revs.append(r) - revs.reverse() # same order as front page - - path = os.path.join(os.path.dirname(__file__), 'benchmark1.html') - data = { - "benchmark": benchmark, - "builders": [builderInfo(b) for b,_,_ in bybuilder], - "graphs": graphs, - "revs": revs, - } - self.response.out.write(template.render(path, data)) - - class FixedOffset(datetime.tzinfo): """Fixed offset in minutes east from UTC.""" @@ -731,9 +427,6 @@ application = webapp.WSGIApplication( ('/init', Init), ('/build', Build), - ('/benchmarks', Benchmarks), - ('/benchmarks/single', SingleBenchmark), - ('/benchmarks/.*', GetBenchmarks), ], debug=True) def main(): diff --git a/misc/dashboard/godashboard/main.html b/misc/dashboard/godashboard/main.html index 9572f181e..5390afce6 100644 --- a/misc/dashboard/godashboard/main.html +++ b/misc/dashboard/godashboard/main.html @@ -12,7 +12,6 @@ <li>Build Status</li> <li><a href="/package">Packages</a></li> <li><a href="/project">Projects</a></li> -<!-- <li><a href="/benchmarks">Benchmarks</a></li> --> <li><a href="http://golang.org/">golang.org</a></li> </ul> diff --git a/misc/dashboard/godashboard/package.html b/misc/dashboard/godashboard/package.html index 13640c8e7..9332b5a79 100644 --- a/misc/dashboard/godashboard/package.html +++ b/misc/dashboard/godashboard/package.html @@ -10,7 +10,6 @@ <li><a href="/">Build Status</a></li> <li>Packages</li> <li><a href="/project">Projects</a></li> -<!-- <li><a href="/benchmarks">Benchmarks</a></li> --> <li><a href="http://golang.org/">golang.org</a></li> </ul> diff --git a/misc/dashboard/godashboard/project.html b/misc/dashboard/godashboard/project.html index f1cf7c023..4fe1741c6 100644 --- a/misc/dashboard/godashboard/project.html +++ b/misc/dashboard/godashboard/project.html @@ -14,7 +14,6 @@ <li><a href="/">Build Status</a></li> <li><a href="/package">Packages</a></li> <li>Projects</li> -<!-- <li><a href="/benchmarks">Benchmarks</a></li> --> <li><a href="http://golang.org/">golang.org</a></li> </ul> diff --git a/misc/goplay/goplay.go b/misc/goplay/goplay.go index f887fbbda..f3e2ff565 100644 --- a/misc/goplay/goplay.go +++ b/misc/goplay/goplay.go @@ -235,8 +235,8 @@ function autoindent(el) { }, 1); } -function keyHandler() { - var e = window.event; +function keyHandler(event) { + var e = window.event || event; if (e.keyCode == 9) { // tab insertTabs(1); e.preventDefault(); @@ -290,7 +290,7 @@ function compileUpdate() { </head> <body> <table width="100%"><tr><td width="60%" valign="top"> -<textarea autofocus="true" id="edit" spellcheck="false" onkeydown="keyHandler();" onkeyup="autocompile();">«@|html»</textarea> +<textarea autofocus="true" id="edit" spellcheck="false" onkeydown="keyHandler(event);" onkeyup="autocompile();">«@|html»</textarea> <div class="hints"> (Shift-Enter to compile and run.) <input type="checkbox" id="autocompile" value="checked" /> Compile and run after each keystroke |