summaryrefslogtreecommitdiff
path: root/misc
diff options
context:
space:
mode:
authorOndřej Surý <ondrej@sury.org>2011-04-28 10:35:15 +0200
committerOndřej Surý <ondrej@sury.org>2011-04-28 10:35:15 +0200
commitc1ba1a0fec4aed430709030f98a3bdb90bfeea16 (patch)
tree3df18657e50a0313ed6defcda30e4474cb28a467 /misc
parent7b15ed9ef455b6b66c6b376898a88aef5d6a9970 (diff)
downloadgolang-c1ba1a0fec4aed430709030f98a3bdb90bfeea16.tar.gz
Imported Upstream version 2011.04.27upstream/2011.04.27
Diffstat (limited to 'misc')
-rw-r--r--misc/dashboard/builder/main.go20
-rw-r--r--misc/dashboard/godashboard/benchmark1.html62
-rw-r--r--misc/dashboard/godashboard/benchmarks.html46
-rw-r--r--misc/dashboard/godashboard/gobuild.py385
-rw-r--r--misc/dashboard/godashboard/main.html1
-rw-r--r--misc/dashboard/godashboard/package.html1
-rw-r--r--misc/dashboard/godashboard/project.html1
-rw-r--r--misc/goplay/goplay.go6
8 files changed, 58 insertions, 464 deletions
diff --git a/misc/dashboard/builder/main.go b/misc/dashboard/builder/main.go
index 735717e28..d11cbb133 100644
--- a/misc/dashboard/builder/main.go
+++ b/misc/dashboard/builder/main.go
@@ -34,6 +34,7 @@ var extraEnv = []string{
"GOHOSTARCH",
"PATH",
"DISABLE_NET_TESTS",
+ "MAKEFLAGS",
"GOARM",
}
@@ -59,6 +60,7 @@ var (
buildRevision = flag.String("rev", "", "Build specified revision and exit")
buildCmd = flag.String("cmd", "./all.bash", "Build command (specify absolute or relative to go/src/)")
external = flag.Bool("external", false, "Build external packages")
+ parallel = flag.Bool("parallel", false, "Build multiple targets in parallel")
verbose = flag.Bool("v", false, "verbose")
)
@@ -132,9 +134,19 @@ func main() {
continue
}
built := false
- for _, b := range builders {
- if b.build() {
- built = true
+ if *parallel {
+ done := make(chan bool)
+ for _, b := range builders {
+ go func(b *Builder) {
+ done <- b.build()
+ }(b)
+ }
+ for _ = range builders {
+ built = <-done || built
+ }
+ } else {
+ for _, b := range builders {
+ built = b.build() || built
}
}
// only run benchmarks if we didn't build anything
@@ -185,7 +197,7 @@ func NewBuilder(builder string) (*Builder, os.Error) {
// get goos/goarch from builder string
s := strings.Split(builder, "-", 3)
- if len(s) == 2 {
+ if len(s) >= 2 {
b.goos, b.goarch = s[0], s[1]
} else {
return nil, fmt.Errorf("unsupported builder form: %s", builder)
diff --git a/misc/dashboard/godashboard/benchmark1.html b/misc/dashboard/godashboard/benchmark1.html
deleted file mode 100644
index 2d49e7204..000000000
--- a/misc/dashboard/godashboard/benchmark1.html
+++ /dev/null
@@ -1,62 +0,0 @@
-<!DOCTYPE HTML>
-<html>
- <head>
- <title>{{benchmark}} - Benchmarks - Go Dashboard</title>
- <link rel="stylesheet" type="text/css" href="/static/style.css">
- </head>
-
- <body>
- <ul class="menu">
- <li><a href="/">Build Status</a></li>
- <li><a href="/package">Packages</a></li>
- <li><a href="/project">Projects</a></li>
- <li><a href="/benchmarks">Benchmarks</a></li>
- <li><a href="http://golang.org/">golang.org</a></li>
- </ul>
-
- <h1>Go Dashboard</h1>
-
- <h2>{{benchmark}}</h2>
-
- <a href="{{benchmark}}?fmt=json">json</a>
-
- {% for g in graphs %}
- <h3>{{g.builder}}</h3>
- {% if g.url %}
- <img src="{{g.url}}&chs=600x150&chf=bg,s,00000000&chco=000000ff&chls=1,1,0">
- {% else %}
- (no data available)
- {% endif %}
- {% endfor %}
-
- <br><br>
-
- <table class="alternate" cellpadding="0" cellspacing="0">
- <tr>
- <th></th>
- {% for b in builders %}
- <th class="builder">{{b.goos}}<br>{{b.goarch}}<br>{{b.note}}</th>
- {% endfor %}
- <th></th>
- <th></th>
- <th></th>
- </tr>
-
- {% for r in revs %}
- <tr>
- <td class="revision"><span class="hash"><a href="https://code.google.com/p/go/source/detail?r={{r.node}}">{{r.node|slice:":12"}}</a></span></td>
-
- {% for ns in r.ns_by_builder %}
- <td class="result">
- {% if ns %}
- {{ns}}
- {% endif %}
- </td>
- {% endfor %}
- <td class="user">{{r.user|escape}}</td>
- <td class="date">{{r.date|escape}}</td>
- <td class="desc">{{r.shortdesc|escape}}</td>
- </tr>
- {% endfor %}
- </body>
-</html>
diff --git a/misc/dashboard/godashboard/benchmarks.html b/misc/dashboard/godashboard/benchmarks.html
deleted file mode 100644
index d42fcfe48..000000000
--- a/misc/dashboard/godashboard/benchmarks.html
+++ /dev/null
@@ -1,46 +0,0 @@
-<!DOCTYPE HTML>
-<html>
- <head>
- <title>Benchmarks - Go Dashboard</title>
- <link rel="stylesheet" type="text/css" href="/static/style.css">
- </head>
-
- <body>
- <ul class="menu">
- <li><a href="/">Build Status</a></li>
- <li><a href="/package">Packages</a></li>
- <li><a href="/project">Projects</a></li>
- <li>Benchmarks</li>
- <li><a href="http://golang.org/">golang.org</a></li>
- </ul>
-
- <h1>Go Dashboard</h1>
-
- <h2>Benchmarks</h2>
-
- <table class="alternate" cellpadding="0" cellspacing="0">
- <tr>
- <th></th>
- {% for b in builders %}
- <th class="builder">{{b.goos}}<br>{{b.goarch}}<br>{{b.note}}</th>
- {% endfor %}
- </tr>
-
- {% for bm in rows %}
- <tr>
- <td class="name"><a href="/benchmarks/{{bm.name}}">{{bm.name}}</a></td>
-
- {% for bl in bm.builders %}
- <td class="result">
- {% if bl.url %}
- <img src="{{bl.url}}" />
- {% else %}
- <img src="/benchmarks/single?benchmark={{bm.name}}&builder={{bl.name}}" />
- {% endif %}
- </td>
- {% endfor %}
- </tr>
- {% endfor %}
- </table>
- </body>
-</html>
diff --git a/misc/dashboard/godashboard/gobuild.py b/misc/dashboard/godashboard/gobuild.py
index 1eacdb38e..035bf842f 100644
--- a/misc/dashboard/godashboard/gobuild.py
+++ b/misc/dashboard/godashboard/gobuild.py
@@ -44,21 +44,12 @@ class Commit(db.Model):
desc = db.BlobProperty()
# This is the list of builds. Each element is a string of the form <builder
- # name> "`" <log hash>. If the log hash is empty, then the build was
+ # name> '`' <log hash>. If the log hash is empty, then the build was
# successful.
builds = db.StringListProperty()
fail_notification_sent = db.BooleanProperty()
-class Benchmark(db.Model):
- name = db.StringProperty()
- version = db.IntegerProperty()
-
-class BenchmarkResults(db.Model):
- builder = db.StringProperty()
- benchmark = db.StringProperty()
- data = db.ListProperty(long) # encoded as [-1, num, iterations, nsperop]*
-
class Cache(db.Model):
data = db.BlobProperty()
expire = db.IntegerProperty()
@@ -69,12 +60,6 @@ class Cache(db.Model):
class CompressedLog(db.Model):
log = db.BlobProperty()
-# For each builder, we store the last revision that it built. So, if it
-# crashes, it knows where to start up from. The key names for these objects are
-# "hw-" <builder name>
-class Highwater(db.Model):
- commit = db.StringProperty()
-
N = 30
def cache_get(key):
@@ -165,34 +150,40 @@ class MainPage(webapp.RequestHandler):
class GetHighwater(webapp.RequestHandler):
def get(self):
builder = self.request.get('builder')
-
- key = 'hw-%s' % builder
- node = memcache.get(key)
- if node is None:
- hw = Highwater.get_by_key_name('hw-%s' % builder)
- if hw is None:
- # If no highwater has been recorded for this builder,
- # we go back N+1 commits and return that.
- q = Commit.all()
- q.order('-__key__')
- c = q.fetch(N+1)[-1]
- node = c.node
- else:
- # if the proposed hw is too old, bump it forward
- node = hw.commit
- found = False
- q = Commit.all()
- q.order('-__key__')
- recent = q.fetch(N+1)
- for c in recent:
- if c.node == node:
- found = True
- break
- if not found:
- node = recent[-1].node
- memcache.set(key, node, 3600)
+ key = 'todo-%s' % builder
+ response = memcache.get(key)
+ if response is None:
+ # Fell out of memcache. Rebuild from datastore results.
+ # We walk the commit list looking for nodes that have not
+ # been built by this builder and record the *parents* of those
+ # nodes, because each builder builds the revision *after* the
+ # one return (because we might not know about the latest
+ # revision).
+ q = Commit.all()
+ q.order('-__key__')
+ todo = []
+ need = False
+ first = None
+ for c in q.fetch(N+1):
+ if first is None:
+ first = c
+ if need:
+ todo.append(c.node)
+ need = not built(c, builder)
+ if not todo:
+ todo.append(first.node)
+ response = ' '.join(todo)
+ memcache.set(key, response, 3600)
self.response.set_status(200)
- self.response.out.write(node)
+ if self.request.get('all') != 'yes':
+ response = response.split()[0]
+ self.response.out.write(response)
+
+def built(c, builder):
+ for b in c.builds:
+ if b.startswith(builder+'`'):
+ return True
+ return False
def auth(req):
k = req.get('key')
@@ -204,32 +195,10 @@ class SetHighwater(webapp.RequestHandler):
self.response.set_status(403)
return
- builder = self.request.get('builder')
- newhw = self.request.get('hw')
- q = Commit.all()
- q.filter('node =', newhw)
- c = q.get()
- if c is None:
- self.response.set_status(404)
- return
-
- # if the proposed hw is too old, bump it forward
- found = False
- q = Commit.all()
- q.order('-__key__')
- recent = q.fetch(N+1)
- for c in recent:
- if c.node == newhw:
- found = True
- break
- if not found:
- c = recent[-1]
-
- key = 'hw-%s' % builder
- memcache.delete(key)
- hw = Highwater(key_name = key)
- hw.commit = c.node
- hw.put()
+ # Allow for old builders.
+ # This is a no-op now: we figure out what to build based
+ # on the current dashboard status.
+ return
class LogHandler(webapp.RequestHandler):
def get(self):
@@ -330,14 +299,8 @@ class Build(webapp.RequestHandler):
db.run_in_transaction(add_build)
- key = 'hw-%s' % builder
- hw = Highwater.get_by_key_name(key)
- if hw is None:
- hw = Highwater(key_name = key)
- hw.commit = node
- hw.put()
+ key = 'todo-%s' % builder
memcache.delete(key)
- memcache.delete('hw')
def mark_sent():
n = Commit.get_by_key_name(key_name)
@@ -373,279 +336,12 @@ def failed(c, builder):
return len(p[1]) > 0
return False
-class Benchmarks(webapp.RequestHandler):
- def json(self):
- q = Benchmark.all()
- q.filter('__key__ >', Benchmark.get_or_insert('v002.').key())
- bs = q.fetch(10000)
-
- self.response.set_status(200)
- self.response.headers['Content-Type'] = 'text/plain; charset=utf-8'
- self.response.out.write('{"benchmarks": [')
-
- sep = "\n\t"
- for b in bs:
- self.response.out.write('%s"%s"' % (sep, b.name))
- sep = ",\n\t"
- self.response.out.write('\n]}\n')
-
- def get(self):
- if self.request.get('fmt') == 'json':
- return self.json()
-
- self.response.set_status(200)
- self.response.headers['Content-Type'] = 'text/html; charset=utf-8'
- page = memcache.get('bench')
- if not page:
- # use datastore as cache to avoid computation even
- # if memcache starts dropping things on the floor
- logging.error("memcache dropped bench")
- page = cache_get('bench')
- if not page:
- logging.error("cache dropped bench")
- num = memcache.get('hw')
- if num is None:
- q = Commit.all()
- q.order('-__key__')
- n = q.fetch(1)[0]
- num = n.num
- memcache.set('hw', num)
- page = self.compute(num)
- cache_set('bench', page, 600)
- memcache.set('bench', page, 600)
- self.response.out.write(page)
-
- def compute(self, num):
- benchmarks, builders = benchmark_list()
-
- rows = []
- for bm in benchmarks:
- row = {'name':bm, 'builders': []}
- for bl in builders:
- key = "single-%s-%s" % (bm, bl)
- url = memcache.get(key)
- row['builders'].append({'name': bl, 'url': url})
- rows.append(row)
-
- path = os.path.join(os.path.dirname(__file__), 'benchmarks.html')
- data = {
- "builders": [builderInfo(b) for b in builders],
- "rows": rows,
- }
- return template.render(path, data)
-
- def post(self):
- if not auth(self.request):
- self.response.set_status(403)
- return
-
- builder = self.request.get('builder')
- node = self.request.get('node')
- if not validNode(node):
- logging.error("Not valid node ('%s')", node)
- self.response.set_status(500)
- return
-
- benchmarkdata = self.request.get('benchmarkdata')
- benchmarkdata = binascii.a2b_base64(benchmarkdata)
-
- def get_string(i):
- l, = struct.unpack('>H', i[:2])
- s = i[2:2+l]
- if len(s) != l:
- return None, None
- return s, i[2+l:]
-
- benchmarks = {}
- while len(benchmarkdata) > 0:
- name, benchmarkdata = get_string(benchmarkdata)
- iterations_str, benchmarkdata = get_string(benchmarkdata)
- time_str, benchmarkdata = get_string(benchmarkdata)
- iterations = int(iterations_str)
- time = int(time_str)
-
- benchmarks[name] = (iterations, time)
-
- q = Commit.all()
- q.filter('node =', node)
- n = q.get()
- if n is None:
- logging.error('Client asked for unknown commit while uploading benchmarks')
- self.response.set_status(404)
- return
-
- for (benchmark, (iterations, time)) in benchmarks.items():
- b = Benchmark.get_or_insert('v002.' + benchmark.encode('base64'), name = benchmark, version = 2)
- key = '%s;%s' % (builder, benchmark)
- r1 = BenchmarkResults.get_by_key_name(key)
- if r1 is not None and (len(r1.data) < 4 or r1.data[-4] != -1 or r1.data[-3] != n.num):
- r1.data += [-1L, long(n.num), long(iterations), long(time)]
- r1.put()
- key = "bench(%s,%s,%d)" % (benchmark, builder, n.num)
- memcache.delete(key)
-
- self.response.set_status(200)
-
-class SingleBenchmark(webapp.RequestHandler):
- """
- Fetch data for single benchmark/builder combination
- and return sparkline url as HTTP redirect, also set memcache entry.
- """
- def get(self):
- benchmark = self.request.get('benchmark')
- builder = self.request.get('builder')
- key = "single-%s-%s" % (benchmark, builder)
-
- url = memcache.get(key)
-
- if url is None:
- minr, maxr, bybuilder = benchmark_data(benchmark)
- for bb in bybuilder:
- if bb[0] != builder:
- continue
- url = benchmark_sparkline(bb[2])
-
- if url is None:
- self.response.set_status(500, "No data found")
- return
-
- memcache.set(key, url, 700) # slightly longer than bench timeout
-
- self.response.set_status(302)
- self.response.headers.add_header("Location", url)
-
def node(num):
q = Commit.all()
q.filter('num =', num)
n = q.get()
return n
-def benchmark_data(benchmark):
- q = BenchmarkResults.all()
- q.order('__key__')
- q.filter('benchmark =', benchmark)
- results = q.fetch(100)
-
- minr = 100000000
- maxr = 0
- for r in results:
- if r.benchmark != benchmark:
- continue
- # data is [-1, num, iters, nsperop, -1, num, iters, nsperop, ...]
- d = r.data
- if not d:
- continue
- if [x for x in d[::4] if x != -1]:
- # unexpected data framing
- logging.error("bad framing for data in %s;%s" % (r.builder, r.benchmark))
- continue
- revs = d[1::4]
- minr = min(minr, min(revs))
- maxr = max(maxr, max(revs))
- if minr > maxr:
- return 0, 0, []
-
- bybuilder = []
- for r in results:
- if r.benchmark != benchmark:
- continue
- d = r.data
- if not d:
- continue
- nsbyrev = [-1 for x in range(minr, maxr+1)]
- iterbyrev = [-1 for x in range(minr, maxr+1)]
- for num, iter, ns in zip(d[1::4], d[2::4], d[3::4]):
- iterbyrev[num - minr] = iter
- nsbyrev[num - minr] = ns
- bybuilder.append((r.builder, iterbyrev, nsbyrev))
-
- return minr, maxr, bybuilder
-
-def benchmark_graph(builder, minhash, maxhash, ns):
- valid = [x for x in ns if x >= 0]
- if not valid:
- return ""
- m = max(max(valid), 2*sum(valid)/len(valid))
- s = ""
- encoding = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789.-"
- for val in ns:
- if val < 0:
- s += "__"
- continue
- val = int(val*4095.0/m)
- s += encoding[val/64] + encoding[val%64]
- return ("http://chart.apis.google.com/chart?cht=lc&chxt=x,y&chxl=0:|%s|%s|1:|0|%g ns|%g ns&chd=e:%s" %
- (minhash[0:12], maxhash[0:12], m/2, m, s))
-
-def benchmark_sparkline(ns):
- valid = [x for x in ns if x >= 0]
- if not valid:
- return ""
- m = max(max(valid), 2*sum(valid)/len(valid))
- # Encoding is 0-61, which is fine enough granularity for our tiny graphs. _ means missing.
- encoding = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
- s = ''.join([x < 0 and "_" or encoding[int((len(encoding)-1)*x/m)] for x in ns])
- url = "http://chart.apis.google.com/chart?cht=ls&chd=s:"+s+"&chs=80x20&chf=bg,s,00000000&chco=000000ff&chls=1,1,0"
- return url
-
-def benchmark_list():
- q = BenchmarkResults.all()
- q.order('__key__')
- q.filter('builder = ', u'darwin-amd64')
- benchmarks = [r.benchmark for r in q]
-
- q = BenchmarkResults.all()
- q.order('__key__')
- q.filter('benchmark =', u'math_test.BenchmarkSqrt')
- builders = [r.builder for r in q.fetch(20)]
-
- return benchmarks, builders
-
-class GetBenchmarks(webapp.RequestHandler):
- def get(self):
- benchmark = self.request.path[12:]
- minr, maxr, bybuilder = benchmark_data(benchmark)
- minhash = node(minr).node
- maxhash = node(maxr).node
-
- if self.request.get('fmt') == 'json':
- self.response.headers['Content-Type'] = 'text/plain; charset=utf-8'
- self.response.out.write('{ "min": "%s", "max": "%s", "data": {' % (minhash, maxhash))
- sep = "\n\t"
- for builder, iter, ns in bybuilder:
- self.response.out.write('%s{ "builder": "%s", "iterations": %s, "nsperop": %s }' %
- (sep, builder, str(iter).replace("L", ""), str(ns).replace("L", "")))
- sep = ",\n\t"
- self.response.out.write('\n}\n')
- return
-
- graphs = []
- for builder, iter, ns in bybuilder:
- graphs.append({"builder": builder, "url": benchmark_graph(builder, minhash, maxhash, ns)})
-
- revs = []
- for i in range(minr, maxr+1):
- r = nodeInfo(node(i))
- x = []
- for _, _, ns in bybuilder:
- t = ns[i - minr]
- if t < 0:
- t = None
- x.append(t)
- r["ns_by_builder"] = x
- revs.append(r)
- revs.reverse() # same order as front page
-
- path = os.path.join(os.path.dirname(__file__), 'benchmark1.html')
- data = {
- "benchmark": benchmark,
- "builders": [builderInfo(b) for b,_,_ in bybuilder],
- "graphs": graphs,
- "revs": revs,
- }
- self.response.out.write(template.render(path, data))
-
-
class FixedOffset(datetime.tzinfo):
"""Fixed offset in minutes east from UTC."""
@@ -731,9 +427,6 @@ application = webapp.WSGIApplication(
('/init', Init),
('/build', Build),
- ('/benchmarks', Benchmarks),
- ('/benchmarks/single', SingleBenchmark),
- ('/benchmarks/.*', GetBenchmarks),
], debug=True)
def main():
diff --git a/misc/dashboard/godashboard/main.html b/misc/dashboard/godashboard/main.html
index 9572f181e..5390afce6 100644
--- a/misc/dashboard/godashboard/main.html
+++ b/misc/dashboard/godashboard/main.html
@@ -12,7 +12,6 @@
<li>Build Status</li>
<li><a href="/package">Packages</a></li>
<li><a href="/project">Projects</a></li>
-<!-- <li><a href="/benchmarks">Benchmarks</a></li> -->
<li><a href="http://golang.org/">golang.org</a></li>
</ul>
diff --git a/misc/dashboard/godashboard/package.html b/misc/dashboard/godashboard/package.html
index 13640c8e7..9332b5a79 100644
--- a/misc/dashboard/godashboard/package.html
+++ b/misc/dashboard/godashboard/package.html
@@ -10,7 +10,6 @@
<li><a href="/">Build Status</a></li>
<li>Packages</li>
<li><a href="/project">Projects</a></li>
-<!-- <li><a href="/benchmarks">Benchmarks</a></li> -->
<li><a href="http://golang.org/">golang.org</a></li>
</ul>
diff --git a/misc/dashboard/godashboard/project.html b/misc/dashboard/godashboard/project.html
index f1cf7c023..4fe1741c6 100644
--- a/misc/dashboard/godashboard/project.html
+++ b/misc/dashboard/godashboard/project.html
@@ -14,7 +14,6 @@
<li><a href="/">Build Status</a></li>
<li><a href="/package">Packages</a></li>
<li>Projects</li>
-<!-- <li><a href="/benchmarks">Benchmarks</a></li> -->
<li><a href="http://golang.org/">golang.org</a></li>
</ul>
diff --git a/misc/goplay/goplay.go b/misc/goplay/goplay.go
index f887fbbda..f3e2ff565 100644
--- a/misc/goplay/goplay.go
+++ b/misc/goplay/goplay.go
@@ -235,8 +235,8 @@ function autoindent(el) {
}, 1);
}
-function keyHandler() {
- var e = window.event;
+function keyHandler(event) {
+ var e = window.event || event;
if (e.keyCode == 9) { // tab
insertTabs(1);
e.preventDefault();
@@ -290,7 +290,7 @@ function compileUpdate() {
</head>
<body>
<table width="100%"><tr><td width="60%" valign="top">
-<textarea autofocus="true" id="edit" spellcheck="false" onkeydown="keyHandler();" onkeyup="autocompile();">«@|html»</textarea>
+<textarea autofocus="true" id="edit" spellcheck="false" onkeydown="keyHandler(event);" onkeyup="autocompile();">«@|html»</textarea>
<div class="hints">
(Shift-Enter to compile and run.)&nbsp;&nbsp;&nbsp;&nbsp;
<input type="checkbox" id="autocompile" value="checked" /> Compile and run after each keystroke