diff options
Diffstat (limited to 'misc/dashboard/godashboard')
-rw-r--r-- | misc/dashboard/godashboard/_multiprocessing.py | 5 | ||||
-rw-r--r-- | misc/dashboard/godashboard/app.yaml | 25 | ||||
-rw-r--r-- | misc/dashboard/godashboard/auth.py | 13 | ||||
-rw-r--r-- | misc/dashboard/godashboard/const.py | 13 | ||||
-rw-r--r-- | misc/dashboard/godashboard/cron.yaml | 4 | ||||
-rw-r--r-- | misc/dashboard/godashboard/fail-notify.txt | 6 | ||||
-rw-r--r-- | misc/dashboard/godashboard/gobuild.py | 558 | ||||
-rw-r--r-- | misc/dashboard/godashboard/index.yaml | 51 | ||||
-rw-r--r-- | misc/dashboard/godashboard/key.py.dummy | 10 | ||||
-rw-r--r-- | misc/dashboard/godashboard/main.html | 62 | ||||
-rw-r--r-- | misc/dashboard/godashboard/package.html | 77 | ||||
-rw-r--r-- | misc/dashboard/godashboard/package.py | 429 | ||||
-rw-r--r-- | misc/dashboard/godashboard/project-edit.html | 47 | ||||
-rw-r--r-- | misc/dashboard/godashboard/project-notify.txt | 9 | ||||
-rw-r--r-- | misc/dashboard/godashboard/project.html | 85 | ||||
-rw-r--r-- | misc/dashboard/godashboard/static/favicon.ico | bin | 0 -> 785 bytes | |||
-rw-r--r-- | misc/dashboard/godashboard/static/style.css | 118 | ||||
-rw-r--r-- | misc/dashboard/godashboard/toutf8.py | 14 |
18 files changed, 1526 insertions, 0 deletions
diff --git a/misc/dashboard/godashboard/_multiprocessing.py b/misc/dashboard/godashboard/_multiprocessing.py new file mode 100644 index 000000000..8c66c0659 --- /dev/null +++ b/misc/dashboard/godashboard/_multiprocessing.py @@ -0,0 +1,5 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +import multiprocessing diff --git a/misc/dashboard/godashboard/app.yaml b/misc/dashboard/godashboard/app.yaml new file mode 100644 index 000000000..7b77a85cc --- /dev/null +++ b/misc/dashboard/godashboard/app.yaml @@ -0,0 +1,25 @@ +application: godashboard +version: 7 +runtime: python +api_version: 1 + +handlers: +- url: /favicon\.ico + static_files: static/favicon.ico + upload: static/favicon\.ico + +- url: /static + static_dir: static + +- url: /package + script: package.py + +- url: /package/daily + script: package.py + login: admin + +- url: /project.* + script: package.py + +- url: /.* + script: gobuild.py diff --git a/misc/dashboard/godashboard/auth.py b/misc/dashboard/godashboard/auth.py new file mode 100644 index 000000000..73a54c0d4 --- /dev/null +++ b/misc/dashboard/godashboard/auth.py @@ -0,0 +1,13 @@ +# Copyright 2011 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +import hmac + +# local imports +import key + +def auth(req): + k = req.get('key') + return k == hmac.new(key.accessKey, req.get('builder')).hexdigest() or k == key.accessKey + diff --git a/misc/dashboard/godashboard/const.py b/misc/dashboard/godashboard/const.py new file mode 100644 index 000000000..b0110c635 --- /dev/null +++ b/misc/dashboard/godashboard/const.py @@ -0,0 +1,13 @@ +# Copyright 2011 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +mail_from = "Go Dashboard <builder@golang.org>" + +mail_submit_to = "adg@golang.org" +mail_submit_subject = "New Project Submitted" + +mail_fail_to = "golang-dev@googlegroups.com" +mail_fail_reply_to = "golang-dev@googlegroups.com" +mail_fail_subject = "%s broken by %s" + diff --git a/misc/dashboard/godashboard/cron.yaml b/misc/dashboard/godashboard/cron.yaml new file mode 100644 index 000000000..953b6a1cd --- /dev/null +++ b/misc/dashboard/godashboard/cron.yaml @@ -0,0 +1,4 @@ +cron: +- description: daily package maintenance + url: /package/daily + schedule: every 24 hours diff --git a/misc/dashboard/godashboard/fail-notify.txt b/misc/dashboard/godashboard/fail-notify.txt new file mode 100644 index 000000000..a699005ea --- /dev/null +++ b/misc/dashboard/godashboard/fail-notify.txt @@ -0,0 +1,6 @@ +Change {{node}} broke the {{builder}} build: +http://godashboard.appspot.com/log/{{loghash}} + +{{desc}} + +http://code.google.com/p/go/source/detail?r={{node}} diff --git a/misc/dashboard/godashboard/gobuild.py b/misc/dashboard/godashboard/gobuild.py new file mode 100644 index 000000000..685dc83a9 --- /dev/null +++ b/misc/dashboard/godashboard/gobuild.py @@ -0,0 +1,558 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +# This is the server part of the continuous build system for Go. It must be run +# by AppEngine. + +from django.utils import simplejson +from google.appengine.api import mail +from google.appengine.api import memcache +from google.appengine.ext import db +from google.appengine.ext import webapp +from google.appengine.ext.webapp import template +from google.appengine.ext.webapp.util import run_wsgi_app +import datetime +import hashlib +import logging +import os +import re +import bz2 + +# local imports +from auth import auth +import const + +# The majority of our state are commit objects. One of these exists for each of +# the commits known to the build system. Their key names are of the form +# <commit number (%08x)> "-" <hg hash>. This means that a sorting by the key +# name is sufficient to order the commits. +# +# The commit numbers are purely local. They need not match up to the commit +# numbers in an hg repo. When inserting a new commit, the parent commit must be +# given and this is used to generate the new commit number. In order to create +# the first Commit object, a special command (/init) is used. +class Commit(db.Model): + num = db.IntegerProperty() # internal, monotonic counter. + node = db.StringProperty() # Hg hash + parentnode = db.StringProperty() # Hg hash + user = db.StringProperty() + date = db.DateTimeProperty() + desc = db.BlobProperty() + + # This is the list of builds. Each element is a string of the form <builder + # name> '`' <log hash>. If the log hash is empty, then the build was + # successful. + builds = db.StringListProperty() + + fail_notification_sent = db.BooleanProperty() + +# A CompressedLog contains the textual build log of a failed build. +# The key name is the hex digest of the SHA256 hash of the contents. +# The contents is bz2 compressed. +class CompressedLog(db.Model): + log = db.BlobProperty() + +N = 30 + +def builderInfo(b): + f = b.split('-', 3) + goos = f[0] + goarch = f[1] + note = "" + if len(f) > 2: + note = f[2] + return {'name': b, 'goos': goos, 'goarch': goarch, 'note': note} + +def builderset(): + q = Commit.all() + q.order('-__key__') + results = q.fetch(N) + builders = set() + for c in results: + builders.update(set(parseBuild(build)['builder'] for build in c.builds)) + return builders + +class MainPage(webapp.RequestHandler): + def get(self): + self.response.headers['Content-Type'] = 'text/html; charset=utf-8' + + try: + page = int(self.request.get('p', 1)) + if not page > 0: + raise + except: + page = 1 + + try: + num = int(self.request.get('n', N)) + if num <= 0 or num > 200: + raise + except: + num = N + + offset = (page-1) * num + + q = Commit.all() + q.order('-__key__') + results = q.fetch(num, offset) + + revs = [toRev(r) for r in results] + builders = {} + + for r in revs: + for b in r['builds']: + builders[b['builder']] = builderInfo(b['builder']) + + for r in revs: + have = set(x['builder'] for x in r['builds']) + need = set(builders.keys()).difference(have) + for n in need: + r['builds'].append({'builder': n, 'log':'', 'ok': False}) + r['builds'].sort(cmp = byBuilder) + + builders = list(builders.items()) + builders.sort() + values = {"revs": revs, "builders": [v for k,v in builders]} + + values['num'] = num + values['prev'] = page - 1 + if len(results) == num: + values['next'] = page + 1 + + path = os.path.join(os.path.dirname(__file__), 'main.html') + self.response.out.write(template.render(path, values)) + +# A DashboardHandler is a webapp.RequestHandler but provides +# authenticated_post - called by post after authenticating +# json - writes object in json format to response output +class DashboardHandler(webapp.RequestHandler): + def post(self): + if not auth(self.request): + self.response.set_status(403) + return + self.authenticated_post() + + def authenticated_post(self): + return + + def json(self, obj): + self.response.set_status(200) + simplejson.dump(obj, self.response.out) + return + +# Todo serves /todo. It tells the builder which commits need to be built. +class Todo(DashboardHandler): + def get(self): + builder = self.request.get('builder') + key = 'todo-%s' % builder + response = memcache.get(key) + if response is None: + # Fell out of memcache. Rebuild from datastore results. + # We walk the commit list looking for nodes that have not + # been built by this builder. + q = Commit.all() + q.order('-__key__') + todo = [] + first = None + for c in q.fetch(N+1): + if first is None: + first = c + if not built(c, builder): + todo.append({'Hash': c.node}) + response = simplejson.dumps(todo) + memcache.set(key, response, 3600) + self.response.set_status(200) + self.response.out.write(response) + +def built(c, builder): + for b in c.builds: + if b.startswith(builder+'`'): + return True + return False + +# Log serves /log/. It retrieves log data by content hash. +class LogHandler(DashboardHandler): + def get(self): + self.response.headers['Content-Type'] = 'text/plain; charset=utf-8' + hash = self.request.path[5:] + l = CompressedLog.get_by_key_name(hash) + if l is None: + self.response.set_status(404) + return + log = bz2.decompress(l.log) + self.response.set_status(200) + self.response.out.write(log) + +# Init creates the commit with id 0. Since this commit doesn't have a parent, +# it cannot be created by Build. +class Init(DashboardHandler): + def authenticated_post(self): + date = parseDate(self.request.get('date')) + node = self.request.get('node') + if not validNode(node) or date is None: + logging.error("Not valid node ('%s') or bad date (%s %s)", node, date, self.request.get('date')) + self.response.set_status(500) + return + + commit = Commit(key_name = '00000000-%s' % node) + commit.num = 0 + commit.node = node + commit.parentnode = '' + commit.user = self.request.get('user').encode('utf8') + commit.date = date + commit.desc = self.request.get('desc').encode('utf8') + + commit.put() + + self.response.set_status(200) + +# The last commit when we switched to using entity groups. +# This is the root of the new commit entity group. +RootCommitKeyName = '00000f26-f32c6f1038207c55d5780231f7484f311020747e' + +# CommitHandler serves /commit. +# A GET of /commit retrieves information about the specified commit. +# A POST of /commit creates a node for the given commit. +# If the commit already exists, the POST silently succeeds (like mkdir -p). +class CommitHandler(DashboardHandler): + def get(self): + node = self.request.get('node') + if not validNode(node): + return self.json({'Status': 'FAIL', 'Error': 'malformed node hash'}) + n = nodeByHash(node) + if n is None: + return self.json({'Status': 'FAIL', 'Error': 'unknown revision'}) + return self.json({'Status': 'OK', 'Node': nodeObj(n)}) + + def authenticated_post(self): + # Require auth with the master key, not a per-builder key. + if self.request.get('builder'): + self.response.set_status(403) + return + + node = self.request.get('node') + date = parseDate(self.request.get('date')) + user = self.request.get('user').encode('utf8') + desc = self.request.get('desc').encode('utf8') + parenthash = self.request.get('parent') + + if not validNode(node) or not validNode(parenthash) or date is None: + return self.json({'Status': 'FAIL', 'Error': 'malformed node, parent, or date'}) + + n = nodeByHash(node) + if n is None: + p = nodeByHash(parenthash) + if p is None: + return self.json({'Status': 'FAIL', 'Error': 'unknown parent'}) + + # Want to create new node in a transaction so that multiple + # requests creating it do not collide and so that multiple requests + # creating different nodes get different sequence numbers. + # All queries within a transaction must include an ancestor, + # but the original datastore objects we used for the dashboard + # have no common ancestor. Instead, we use a well-known + # root node - the last one before we switched to entity groups - + # as the as the common ancestor. + root = Commit.get_by_key_name(RootCommitKeyName) + + def add_commit(): + if nodeByHash(node, ancestor=root) is not None: + return + + # Determine number for this commit. + # Once we have created one new entry it will be lastRooted.num+1, + # but the very first commit created in this scheme will have to use + # last.num's number instead (last is likely not rooted). + q = Commit.all() + q.order('-__key__') + q.ancestor(root) + last = q.fetch(1)[0] + num = last.num+1 + + n = Commit(key_name = '%08x-%s' % (num, node), parent = root) + n.num = num + n.node = node + n.parentnode = parenthash + n.user = user + n.date = date + n.desc = desc + n.put() + db.run_in_transaction(add_commit) + n = nodeByHash(node) + if n is None: + return self.json({'Status': 'FAIL', 'Error': 'failed to create commit node'}) + + return self.json({'Status': 'OK', 'Node': nodeObj(n)}) + +# Build serves /build. +# A POST to /build records a new build result. +class Build(webapp.RequestHandler): + def post(self): + if not auth(self.request): + self.response.set_status(403) + return + + builder = self.request.get('builder') + log = self.request.get('log').encode('utf-8') + + loghash = '' + if len(log) > 0: + loghash = hashlib.sha256(log).hexdigest() + l = CompressedLog(key_name=loghash) + l.log = bz2.compress(log) + l.put() + + node = self.request.get('node') + if not validNode(node): + logging.error('Invalid node %s' % (node)) + self.response.set_status(500) + return + + n = nodeByHash(node) + if n is None: + logging.error('Cannot find node %s' % (node)) + self.response.set_status(404) + return + nn = n + + def add_build(): + n = nodeByHash(node, ancestor=nn) + if n is None: + logging.error('Cannot find hash in add_build: %s %s' % (builder, node)) + return + + s = '%s`%s' % (builder, loghash) + for i, b in enumerate(n.builds): + if b.split('`', 1)[0] == builder: + # logging.error('Found result for %s %s already' % (builder, node)) + n.builds[i] = s + break + else: + # logging.error('Added result for %s %s' % (builder, node)) + n.builds.append(s) + n.put() + + db.run_in_transaction(add_build) + + key = 'todo-%s' % builder + memcache.delete(key) + + c = getBrokenCommit(node, builder) + if c is not None and not c.fail_notification_sent: + notifyBroken(c, builder) + + self.response.set_status(200) + + +def getBrokenCommit(node, builder): + """ + getBrokenCommit returns a Commit that breaks the build. + The Commit will be either the one specified by node or the one after. + """ + + # Squelch mail if already fixed. + head = firstResult(builder) + if broken(head, builder) == False: + return + + # Get current node and node before, after. + cur = nodeByHash(node) + if cur is None: + return + before = nodeBefore(cur) + after = nodeAfter(cur) + + if broken(before, builder) == False and broken(cur, builder): + return cur + if broken(cur, builder) == False and broken(after, builder): + return after + + return + +def firstResult(builder): + q = Commit.all().order('-__key__') + for c in q.fetch(20): + for i, b in enumerate(c.builds): + p = b.split('`', 1) + if p[0] == builder: + return c + return None + +def nodeBefore(c): + return nodeByHash(c.parentnode) + +def nodeAfter(c): + return Commit.all().filter('parenthash', c.node).get() + +def notifyBroken(c, builder): + def send(): + n = Commit.get(c.key()) + if n is None: + logging.error("couldn't retrieve Commit '%s'" % c.key()) + return False + if n.fail_notification_sent: + return False + n.fail_notification_sent = True + return n.put() + if not db.run_in_transaction(send): + return + + subject = const.mail_fail_subject % (builder, c.desc.split('\n')[0]) + path = os.path.join(os.path.dirname(__file__), 'fail-notify.txt') + body = template.render(path, { + "builder": builder, + "node": c.node, + "user": c.user, + "desc": c.desc, + "loghash": logHash(c, builder) + }) + mail.send_mail( + sender=const.mail_from, + to=const.mail_fail_to, + subject=subject, + body=body + ) + +def logHash(c, builder): + for i, b in enumerate(c.builds): + p = b.split('`', 1) + if p[0] == builder: + return p[1] + return "" + +def broken(c, builder): + """ + broken returns True if commit c breaks the build for the specified builder, + False if it is a good build, and None if no results exist for this builder. + """ + if c is None: + return None + for i, b in enumerate(c.builds): + p = b.split('`', 1) + if p[0] == builder: + return len(p[1]) > 0 + return None + +def node(num): + q = Commit.all() + q.filter('num =', num) + n = q.get() + return n + +def nodeByHash(hash, ancestor=None): + q = Commit.all() + q.filter('node =', hash) + if ancestor is not None: + q.ancestor(ancestor) + n = q.get() + return n + +# nodeObj returns a JSON object (ready to be passed to simplejson.dump) describing node. +def nodeObj(n): + return { + 'Hash': n.node, + 'ParentHash': n.parentnode, + 'User': n.user, + 'Date': n.date.strftime('%Y-%m-%d %H:%M %z'), + 'Desc': n.desc, + } + +class FixedOffset(datetime.tzinfo): + """Fixed offset in minutes east from UTC.""" + + def __init__(self, offset): + self.__offset = datetime.timedelta(seconds = offset) + + def utcoffset(self, dt): + return self.__offset + + def tzname(self, dt): + return None + + def dst(self, dt): + return datetime.timedelta(0) + +def validNode(node): + if len(node) != 40: + return False + for x in node: + o = ord(x) + if (o < ord('0') or o > ord('9')) and (o < ord('a') or o > ord('f')): + return False + return True + +def parseDate(date): + if '-' in date: + (a, offset) = date.split('-', 1) + try: + return datetime.datetime.fromtimestamp(float(a), FixedOffset(0-int(offset))) + except ValueError: + return None + if '+' in date: + (a, offset) = date.split('+', 1) + try: + return datetime.datetime.fromtimestamp(float(a), FixedOffset(int(offset))) + except ValueError: + return None + try: + return datetime.datetime.utcfromtimestamp(float(date)) + except ValueError: + return None + +email_re = re.compile('^[^<]+<([^>]*)>$') + +def toUsername(user): + r = email_re.match(user) + if r is None: + return user + email = r.groups()[0] + return email.replace('@golang.org', '') + +def dateToShortStr(d): + return d.strftime('%a %b %d %H:%M') + +def parseBuild(build): + [builder, logblob] = build.split('`') + return {'builder': builder, 'log': logblob, 'ok': len(logblob) == 0} + +def nodeInfo(c): + return { + "node": c.node, + "user": toUsername(c.user), + "date": dateToShortStr(c.date), + "desc": c.desc, + "shortdesc": c.desc.split('\n', 2)[0] + } + +def toRev(c): + b = nodeInfo(c) + b['builds'] = [parseBuild(build) for build in c.builds] + return b + +def byBuilder(x, y): + return cmp(x['builder'], y['builder']) + +# Give old builders work; otherwise they pound on the web site. +class Hwget(DashboardHandler): + def get(self): + self.response.out.write("8000\n") + +# This is the URL map for the server. The first three entries are public, the +# rest are only used by the builders. +application = webapp.WSGIApplication( + [('/', MainPage), + ('/hw-get', Hwget), + ('/log/.*', LogHandler), + ('/commit', CommitHandler), + ('/init', Init), + ('/todo', Todo), + ('/build', Build), + ], debug=True) + +def main(): + run_wsgi_app(application) + +if __name__ == "__main__": + main() + diff --git a/misc/dashboard/godashboard/index.yaml b/misc/dashboard/godashboard/index.yaml new file mode 100644 index 000000000..f39299d5d --- /dev/null +++ b/misc/dashboard/godashboard/index.yaml @@ -0,0 +1,51 @@ +indexes: + +- kind: BenchmarkResult + ancestor: yes + properties: + - name: builder + - name: __key__ + direction: desc + +- kind: BenchmarkResult + ancestor: yes + properties: + - name: __key__ + direction: desc + +- kind: BenchmarkResults + properties: + - name: builder + - name: benchmark + +- kind: Commit + properties: + - name: __key__ + direction: desc + +- kind: Commit + ancestor: yes + properties: + - name: __key__ + direction: desc + +- kind: Project + properties: + - name: approved + - name: category + - name: name + +- kind: Project + properties: + - name: category + - name: name + +# AUTOGENERATED + +# This index.yaml is automatically updated whenever the dev_appserver +# detects that a new type of query is run. If you want to manage the +# index.yaml file manually, remove the above marker line (the line +# saying "# AUTOGENERATED"). If you want to manage some indexes +# manually, move them above the marker line. The index.yaml file is +# automatically uploaded to the admin console when you next deploy +# your application using appcfg.py. diff --git a/misc/dashboard/godashboard/key.py.dummy b/misc/dashboard/godashboard/key.py.dummy new file mode 100644 index 000000000..5b8bab186 --- /dev/null +++ b/misc/dashboard/godashboard/key.py.dummy @@ -0,0 +1,10 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +# Copy this file to key.py after substituting the real key. + +# accessKey controls private access to the build server (i.e. to record new +# builds). It's tranmitted in the clear but, given the low value of the target, +# this should be sufficient. +accessKey = "this is not the real key" diff --git a/misc/dashboard/godashboard/main.html b/misc/dashboard/godashboard/main.html new file mode 100644 index 000000000..5390afce6 --- /dev/null +++ b/misc/dashboard/godashboard/main.html @@ -0,0 +1,62 @@ +<!DOCTYPE HTML> +<html> + <head> + <title>Build Status - Go Dashboard</title> + <link rel="stylesheet" type="text/css" href="static/style.css"> + </head> + + <body> + <a id="top"></a> + + <ul class="menu"> + <li>Build Status</li> + <li><a href="/package">Packages</a></li> + <li><a href="/project">Projects</a></li> + <li><a href="http://golang.org/">golang.org</a></li> + </ul> + + <h1>Go Dashboard</h1> + + <h2>Build Status</h2> + <table class="alternate" cellpadding="0" cellspacing="0"> + <tr> + <th></th> + {% for b in builders %} + <th class="builder">{{b.goos}}<br>{{b.goarch}}<br>{{b.note}}</th> + {% endfor %} + <th></th> + <th></th> + <th></th> + </tr> + + {% for r in revs %} + <tr> + <td class="revision"><span class="hash"><a href="https://code.google.com/p/go/source/detail?r={{r.node}}">{{r.node|slice:":12"}}</a></span></td> + + {% for b in r.builds %} + <td class="result"> + {% if b.ok %} + <span class="ok">ok</span> + {% else %} + {% if b.log %} + <a class="fail" href="/log/{{b.log}}">fail</a> + {% else %} + + {% endif %} + {% endif %} + </td> + {% endfor %} + + <td class="user">{{r.user|escape}}</td> + <td class="date">{{r.date|escape}}</td> + <td class="desc">{{r.shortdesc|escape}}</td> + </tr> + {% endfor %} + </table> + <div class="paginate"> + <a{% if prev %} href="?n={{num}}&p={{prev}}"{% else %} class="inactive"{% endif %}>prev</a> + <a{% if next %} href="?n={{num}}&p={{next}}"{% else %} class="inactive"{% endif %}>next</a> + <a{% if prev %} href="?n={{num}}&p=1"{% else %} class="inactive"{% endif %}>top</a> + </div> + </body> +</html> diff --git a/misc/dashboard/godashboard/package.html b/misc/dashboard/godashboard/package.html new file mode 100644 index 000000000..8a9d0a3a0 --- /dev/null +++ b/misc/dashboard/godashboard/package.html @@ -0,0 +1,77 @@ +<!DOCTYPE HTML> +<html> + <head> + <title>Packages - Go Dashboard</title> + <link rel="stylesheet" type="text/css" href="static/style.css"> + </head> + + <body> + <ul class="menu"> + <li><a href="/">Build Status</a></li> + <li>Packages</li> + <li><a href="/project">Projects</a></li> + <li><a href="http://golang.org/">golang.org</a></li> + </ul> + + <h1>Go Dashboard</h1> + + <p> + Packages listed on this page are written by third parties and + may or may not build or be safe to use. + </p> + + <p> + An "ok" in the <b>build</b> column indicates that the package is + <a href="http://golang.org/cmd/goinstall/">goinstallable</a> + with the latest + <a href="http://golang.org/doc/devel/release.html">release</a> of Go. + </p> + + <p> + The <b>info</b> column shows the first paragraph from the + <a href="http://blog.golang.org/2011/03/godoc-documenting-go-code.html">package doc comment</a>. + </p> + + <h2>Most Installed Packages (this week)</h2> + <table class="alternate" cellpadding="0" cellspacing="0"> + <tr><th>last install</th><th>count</th><th>build</th><th>path</th><th>info</th></tr> + {% for r in by_week_count %} + <tr> + <td class="time">{{r.last_install|date:"Y-M-d H:i"}}</td> + <td class="count">{{r.week_count}}</td> + <td class="ok">{% if r.ok %}<a title="{{r.last_ok|date:"Y-M-d H:i"}}">ok</a>{% else %} {% endif %}</td> + <td class="path"><a href="{{r.web_url}}">{{r.path}}</a></td> + <td class="info">{% if r.info %}{{r.info|escape}}{% endif %}</td> + </tr> + {% endfor %} + </table> + + <h2>Recently Installed Packages</h2> + <table class="alternate" cellpadding="0" cellspacing="0"> + <tr><th>last install</th><th>count</th><th>build</th><th>path</th><th>info</th></tr> + {% for r in by_time %} + <tr> + <td class="time">{{r.last_install|date:"Y-M-d H:i"}}</td> + <td class="count">{{r.count}}</td> + <td class="ok">{% if r.ok %}<a title="{{r.last_ok|date:"Y-M-d H:i"}}">ok</a>{% else %} {% endif %}</td> + <td class="path"><a href="{{r.web_url}}">{{r.path}}</a></td> + <td class="info">{% if r.info %}{{r.info|escape}}{% endif %}</td> + </tr> + {% endfor %} + </table> + + <h2>Most Installed Packages (all time)</h2> + <table class="alternate" cellpadding="0" cellspacing="0"> + <tr><th>last install</th><th>count</th><th>build</th><th>path</th><th>info</th></tr> + {% for r in by_count %} + <tr> + <td class="time">{{r.last_install|date:"Y-M-d H:i"}}</td> + <td class="count">{{r.count}}</td> + <td class="ok">{% if r.ok %}<a title="{{r.last_ok|date:"Y-M-d H:i"}}">ok</a>{% else %} {% endif %}</td> + <td class="path"><a href="{{r.web_url}}">{{r.path}}</a></td> + <td class="info">{% if r.info %}{{r.info|escape}}{% endif %}</td> + </tr> + {% endfor %} + </table> + </body> +</html> diff --git a/misc/dashboard/godashboard/package.py b/misc/dashboard/godashboard/package.py new file mode 100644 index 000000000..5cc2d2404 --- /dev/null +++ b/misc/dashboard/godashboard/package.py @@ -0,0 +1,429 @@ +# Copyright 2010 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +# This is the server part of the package dashboard. +# It must be run by App Engine. + +from google.appengine.api import mail +from google.appengine.api import memcache +from google.appengine.api import taskqueue +from google.appengine.api import urlfetch +from google.appengine.api import users +from google.appengine.ext import db +from google.appengine.ext import webapp +from google.appengine.ext.webapp import template +from google.appengine.ext.webapp.util import run_wsgi_app +import datetime +import logging +import os +import re +import sets +import urllib2 + +# local imports +from auth import auth +import toutf8 +import const + +template.register_template_library('toutf8') + +# Storage model for package info recorded on server. +class Package(db.Model): + path = db.StringProperty() + web_url = db.StringProperty() # derived from path + count = db.IntegerProperty() # grand total + week_count = db.IntegerProperty() # rolling weekly count + day_count = db.TextProperty(default='') # daily count + last_install = db.DateTimeProperty() + + # data contributed by gobuilder + info = db.StringProperty() + ok = db.BooleanProperty() + last_ok = db.DateTimeProperty() + + def get_day_count(self): + counts = {} + if not self.day_count: + return counts + for d in str(self.day_count).split('\n'): + date, count = d.split(' ') + counts[date] = int(count) + return counts + + def set_day_count(self, count): + days = [] + for day, count in count.items(): + days.append('%s %d' % (day, count)) + days.sort(reverse=True) + days = days[:28] + self.day_count = '\n'.join(days) + + def inc(self): + count = self.get_day_count() + today = str(datetime.date.today()) + count[today] = count.get(today, 0) + 1 + self.set_day_count(count) + self.update_week_count(count) + self.count += 1 + + def update_week_count(self, count=None): + if count is None: + count = self.get_day_count() + total = 0 + today = datetime.date.today() + for i in range(7): + day = str(today - datetime.timedelta(days=i)) + if day in count: + total += count[day] + self.week_count = total + + +# PackageDaily kicks off the daily package maintenance cron job +# and serves the associated task queue. +class PackageDaily(webapp.RequestHandler): + + def get(self): + # queue a task to update each package with a week_count > 0 + keys = Package.all(keys_only=True).filter('week_count >', 0) + for key in keys: + taskqueue.add(url='/package/daily', params={'key': key.name()}) + + def post(self): + # update a single package (in a task queue) + def update(key): + p = Package.get_by_key_name(key) + if not p: + return + p.update_week_count() + p.put() + key = self.request.get('key') + if not key: + return + db.run_in_transaction(update, key) + + +class Project(db.Model): + name = db.StringProperty(indexed=True) + descr = db.StringProperty() + web_url = db.StringProperty() + package = db.ReferenceProperty(Package) + category = db.StringProperty(indexed=True) + tags = db.ListProperty(str) + approved = db.BooleanProperty(indexed=True) + + +re_bitbucket = re.compile(r'^(bitbucket\.org/[a-z0-9A-Z_.\-]+/[a-zA-Z0-9_.\-]+)(/[a-z0-9A-Z_.\-/]+)?$') +re_googlecode = re.compile(r'^[a-z0-9\-]+\.googlecode\.com/(svn|hg|git)(/[a-z0-9A-Z_.\-/]+)?$') +re_github = re.compile(r'^github\.com/[a-z0-9A-Z_.\-]+(/[a-z0-9A-Z_.\-]+)+$') +re_launchpad = re.compile(r'^launchpad\.net/([a-z0-9A-Z_.\-]+(/[a-z0-9A-Z_.\-]+)?|~[a-z0-9A-Z_.\-]+/(\+junk|[a-z0-9A-Z_.\-]+)/[a-z0-9A-Z_.\-]+)(/[a-z0-9A-Z_.\-/]+)?$') + +def vc_to_web(path): + if re_bitbucket.match(path): + m = re_bitbucket.match(path) + check_url = 'http://' + m.group(1) + '/?cmd=heads' + web = 'http://' + m.group(1) + '/' + elif re_github.match(path): + m = re_github_web.match(path) + check_url = 'https://raw.github.com/' + m.group(1) + '/' + m.group(2) + '/master/' + web = 'http://github.com/' + m.group(1) + '/' + m.group(2) + '/' + elif re_googlecode.match(path): + m = re_googlecode.match(path) + check_url = 'http://'+path + if not m.group(2): # append / after bare '/hg' or '/git' + check_url += '/' + web = 'http://code.google.com/p/' + path[:path.index('.')] + elif re_launchpad.match(path): + check_url = web = 'https://'+path + else: + return False, False + return web, check_url + +re_bitbucket_web = re.compile(r'bitbucket\.org/([a-z0-9A-Z_.\-]+)/([a-z0-9A-Z_.\-]+)') +re_googlecode_web = re.compile(r'code.google.com/p/([a-z0-9\-]+)') +re_github_web = re.compile(r'github\.com/([a-z0-9A-Z_.\-]+)/([a-z0-9A-Z_.\-]+)') +re_launchpad_web = re.compile(r'launchpad\.net/([a-z0-9A-Z_.\-]+(/[a-z0-9A-Z_.\-]+)?|~[a-z0-9A-Z_.\-]+/(\+junk|[a-z0-9A-Z_.\-]+)/[a-z0-9A-Z_.\-]+)(/[a-z0-9A-Z_.\-/]+)?') +re_striphttp = re.compile(r'https?://(www\.)?') + +def find_googlecode_vcs(path): + # Perform http request to path/hg or path/git to check if they're + # using mercurial or git. Otherwise, assume svn. + for vcs in ['git', 'hg']: + try: + response = urlfetch.fetch('http://'+path+vcs, deadline=1) + if response.status_code == 200: + return vcs + except: pass + return 'svn' + +def web_to_vc(url): + url = re_striphttp.sub('', url) + m = re_bitbucket_web.match(url) + if m: + return 'bitbucket.org/'+m.group(1)+'/'+m.group(2) + m = re_github_web.match(url) + if m: + return 'github.com/'+m.group(1)+'/'+m.group(2) + m = re_googlecode_web.match(url) + if m: + path = m.group(1)+'.googlecode.com/' + vcs = find_googlecode_vcs(path) + return path + vcs + m = re_launchpad_web.match(url) + if m: + return m.group(0) + return False + +MaxPathLength = 100 +CacheTimeout = 3600 + +class PackagePage(webapp.RequestHandler): + def get(self): + if self.request.get('fmt') == 'json': + return self.json() + + html = memcache.get('view-package') + if not html: + tdata = {} + + q = Package.all().filter('week_count >', 0) + q.order('-week_count') + tdata['by_week_count'] = q.fetch(50) + + q = Package.all() + q.order('-last_install') + tdata['by_time'] = q.fetch(20) + + q = Package.all() + q.order('-count') + tdata['by_count'] = q.fetch(100) + + path = os.path.join(os.path.dirname(__file__), 'package.html') + html = template.render(path, tdata) + memcache.set('view-package', html, time=CacheTimeout) + + self.response.headers['Content-Type'] = 'text/html; charset=utf-8' + self.response.out.write(html) + + def json(self): + json = memcache.get('view-package-json') + if not json: + q = Package.all() + s = '{"packages": [' + sep = '' + for r in q.fetch(1000): + s += '%s\n\t{"path": "%s", "last_install": "%s", "count": "%s"}' % (sep, r.path, r.last_install, r.count) + sep = ',' + s += '\n]}\n' + json = s + memcache.set('view-package-json', json, time=CacheTimeout) + self.response.set_status(200) + self.response.headers['Content-Type'] = 'text/plain; charset=utf-8' + self.response.out.write(json) + + def can_get_url(self, url): + try: + urllib2.urlopen(urllib2.Request(url)) + return True + except: + return False + + def is_valid_package_path(self, path): + return (re_bitbucket.match(path) or + re_googlecode.match(path) or + re_github.match(path) or + re_launchpad.match(path)) + + def record_pkg(self, path): + # sanity check string + if not path or len(path) > MaxPathLength or not self.is_valid_package_path(path): + return False + + # look in datastore + key = 'pkg-' + path + p = Package.get_by_key_name(key) + if p is None: + # not in datastore - verify URL before creating + web, check_url = vc_to_web(path) + if not web: + logging.error('unrecognized path: %s', path) + return False + if not self.can_get_url(check_url): + logging.error('cannot get %s', check_url) + return False + p = Package(key_name = key, path = path, count = 0, web_url = web) + + if auth(self.request): + # builder updating package metadata + p.info = self.request.get('info') + p.ok = self.request.get('ok') == "true" + if p.ok: + p.last_ok = datetime.datetime.utcnow() + else: + # goinstall reporting an install + p.inc() + p.last_install = datetime.datetime.utcnow() + + # update package object + p.put() + return True + + def post(self): + path = self.request.get('path') + ok = db.run_in_transaction(self.record_pkg, path) + if ok: + self.response.set_status(200) + self.response.out.write('ok') + else: + logging.error('invalid path in post: %s', path) + self.response.set_status(500) + self.response.out.write('not ok') + +class ProjectPage(webapp.RequestHandler): + + def get(self): + admin = users.is_current_user_admin() + if self.request.path == "/project/login": + self.redirect(users.create_login_url("/project")) + elif self.request.path == "/project/logout": + self.redirect(users.create_logout_url("/project")) + elif self.request.path == "/project/edit" and admin: + self.edit() + elif self.request.path == "/project/assoc" and admin: + self.assoc() + else: + self.list() + + def assoc(self): + projects = Project.all() + for p in projects: + if p.package: + continue + path = web_to_vc(p.web_url) + if not path: + continue + pkg = Package.get_by_key_name("pkg-"+path) + if not pkg: + self.response.out.write('no: %s %s<br>' % (p.web_url, path)) + continue + p.package = pkg + p.put() + self.response.out.write('yes: %s %s<br>' % (p.web_url, path)) + + def post(self): + if self.request.path == "/project/edit": + self.edit(True) + else: + data = dict(map(lambda x: (x, self.request.get(x)), ["name","descr","web_url"])) + if reduce(lambda x, y: x or not y, data.values(), False): + data["submitMsg"] = "You must complete all the fields." + self.list(data) + return + p = Project.get_by_key_name("proj-"+data["name"]) + if p is not None: + data["submitMsg"] = "A project by this name already exists." + self.list(data) + return + p = Project(key_name="proj-"+data["name"], **data) + p.put() + + path = os.path.join(os.path.dirname(__file__), 'project-notify.txt') + mail.send_mail( + sender=const.mail_from, + to=const.mail_submit_to, + subject=const.mail_submit_subject, + body=template.render(path, {'project': p})) + + self.list({"submitMsg": "Your project has been submitted."}) + + def list(self, additional_data={}): + cache_key = 'view-project-data' + tag = self.request.get('tag', None) + if tag: + cache_key += '-'+tag + data = memcache.get(cache_key) + admin = users.is_current_user_admin() + if admin or not data: + projects = Project.all().order('category').order('name') + if not admin: + projects = projects.filter('approved =', True) + projects = list(projects) + + tags = sets.Set() + for p in projects: + for t in p.tags: + tags.add(t) + + if tag: + projects = filter(lambda x: tag in x.tags, projects) + + data = {} + data['tag'] = tag + data['tags'] = tags + data['projects'] = projects + data['admin']= admin + if not admin: + memcache.set(cache_key, data, time=CacheTimeout) + + for k, v in additional_data.items(): + data[k] = v + + self.response.headers['Content-Type'] = 'text/html; charset=utf-8' + path = os.path.join(os.path.dirname(__file__), 'project.html') + self.response.out.write(template.render(path, data)) + + def edit(self, save=False): + if save: + name = self.request.get("orig_name") + else: + name = self.request.get("name") + + p = Project.get_by_key_name("proj-"+name) + if not p: + self.response.out.write("Couldn't find that Project.") + return + + if save: + if self.request.get("do") == "Delete": + p.delete() + else: + pkg_name = self.request.get("package", None) + if pkg_name: + pkg = Package.get_by_key_name("pkg-"+pkg_name) + if pkg: + p.package = pkg.key() + for f in ['name', 'descr', 'web_url', 'category']: + setattr(p, f, self.request.get(f, None)) + p.approved = self.request.get("approved") == "1" + p.tags = filter(lambda x: x, self.request.get("tags", "").split(",")) + p.put() + memcache.delete('view-project-data') + self.redirect('/project') + return + + # get all project categories and tags + cats, tags = sets.Set(), sets.Set() + for r in Project.all(): + cats.add(r.category) + for t in r.tags: + tags.add(t) + + self.response.headers['Content-Type'] = 'text/html; charset=utf-8' + path = os.path.join(os.path.dirname(__file__), 'project-edit.html') + self.response.out.write(template.render(path, { + "taglist": tags, "catlist": cats, "p": p, "tags": ",".join(p.tags) })) + + def redirect(self, url): + self.response.set_status(302) + self.response.headers.add_header("Location", url) + +def main(): + app = webapp.WSGIApplication([ + ('/package', PackagePage), + ('/package/daily', PackageDaily), + ('/project.*', ProjectPage), + ], debug=True) + run_wsgi_app(app) + +if __name__ == '__main__': + main() diff --git a/misc/dashboard/godashboard/project-edit.html b/misc/dashboard/godashboard/project-edit.html new file mode 100644 index 000000000..ce18fb3fb --- /dev/null +++ b/misc/dashboard/godashboard/project-edit.html @@ -0,0 +1,47 @@ +<html> +<head> +<link href="http://ajax.googleapis.com/ajax/libs/jqueryui/1.8/themes/base/jquery-ui.css" rel="stylesheet" type="text/css"/> +<script type="text/javascript" src="http://www.google.com/jsapi"></script> +<script> +google.load("jquery", "1"); +google.load("jqueryui", "1.8.2"); +</script> +</head> +<body> +<form action="/project/edit?orig_name={{p.name}}" method="POST"> +Name:<br/> +<input type="text" name="name" value="{{p.name|escape}}"><br/> +Description:<br/> +<input type="text" name="descr" value="{{p.descr|escape}}"><br/> +Category:<br/> +<input type="text" id="cats" name="category" value="{{p.category|escape}}"><br/> +Tags: (comma-separated)<br/> +<input type="text" id="tags" name="tags" value="{{tags}}"><br/> +Web URL:<br/> +<input type="text" name="web_url" value="{{p.web_url|escape}}"><br/> +Package URL: (to link to a goinstall'd package)<br/> +<input type="text" name="package" value="{{p.package.path|escape}}"><br/> +Approved: <input type="checkbox" name="approved" value="1" {% if p.approved %}checked{% endif %}><br/> +<br/> +<input type="submit" name="do" value="Save"> +<input type="submit" name="do" value="Delete" onClick="javascript:return confirm('Delete this?');"> +</form> +<script> +var tags = [ +{% for t in taglist %} + "{{t}}"{% if not forloop.last %},{% endif %} +{% endfor %} +]; +var cats = [ +{% for c in catlist %} + "{{c}}"{% if not forloop.last %},{% endif %} +{% endfor %} +]; + +google.setOnLoadCallback(function() { + $('#tags').autocomplete({source:tags}); + $('#cats').autocomplete({source:cats}); +}); +</script> +</body> +</html> diff --git a/misc/dashboard/godashboard/project-notify.txt b/misc/dashboard/godashboard/project-notify.txt new file mode 100644 index 000000000..f55bf6421 --- /dev/null +++ b/misc/dashboard/godashboard/project-notify.txt @@ -0,0 +1,9 @@ +A new project has been submitted: + +Name: {{project.name}} +Description: {{project.descr}} +URL: {{project.web_url}} + +To edit/approve/delete: +http://godashboard.appspot.com/project/edit?name={{project.name|toutf8|urlencode}} + diff --git a/misc/dashboard/godashboard/project.html b/misc/dashboard/godashboard/project.html new file mode 100644 index 000000000..4fe1741c6 --- /dev/null +++ b/misc/dashboard/godashboard/project.html @@ -0,0 +1,85 @@ +<!DOCTYPE HTML> +<html> + <head> + <title>Projects - Go Dashboard</title> + <link rel="stylesheet" type="text/css" href="static/style.css"> + <style> + .unapproved a.name { color: red } + .tag { font-size: 0.8em; color: #666 } + </style> + </head> + + <body> + <ul class="menu"> + <li><a href="/">Build Status</a></li> + <li><a href="/package">Packages</a></li> + <li>Projects</li> + <li><a href="http://golang.org/">golang.org</a></li> + </ul> + + <h1>Go Dashboard</h1> + + <p> + These are external projects and not endorsed or supported by the Go project. + </p> + + <h2>Projects</h2> + + <div class="submit"> + <h3>Submit a Project</h3> + <p> + Using this form you can submit a project to be included in the list. + </p> + <form action="/project" method="POST"> + <table> + <tr><td>Name:<td><input type="text" name="name"> + <tr><td>Description:<td><input type="text" name="descr"> + <tr><td>URL:<td><input type="text" name="web_url"> + <tr><td> <td><input type="submit" value="Send"> + {% if submitMsg %} + <tr><td class="msg" colspan="2">{{ submitMsg }}</td></tr> + {% endif %} + </table> + </form> + </div> + + <p> + Filter by tag: + {% if tag %} + <a href="/project">all</a> + {% else %} + <b>all</b> + {% endif %} + {% for t in tags %} + {% ifequal t tag %} + <b>{{t}}</b> + {% else %} + <a href="?tag={{t}}">{{t}}</a> + {% endifequal %} + {% endfor %} + </p> + + {% for r in projects %} + {% ifchanged r.category %} + {% if not forloop.first %} + </ul> + {% endif %} + <h3>{{r.category}}</h3> + <ul> + {% endifchanged %} + <li{% if not r.approved %} class="unapproved"{% endif %}> + {% if admin %}[<a href="/project/edit?name={{r.name}}">edit</a>]{% endif %} + <a class="name" href="{{r.web_url}}">{{r.name}}</a> - {{r.descr}} + {% for tag in r.tags %} + <span class="tag">{{tag}}</span> + {% endfor %} + </li> + {% if forloop.last %} + </ul> + {% endif %} + {% endfor %} + </ul> + + + </body> +</html> diff --git a/misc/dashboard/godashboard/static/favicon.ico b/misc/dashboard/godashboard/static/favicon.ico Binary files differnew file mode 100644 index 000000000..48854ff3b --- /dev/null +++ b/misc/dashboard/godashboard/static/favicon.ico diff --git a/misc/dashboard/godashboard/static/style.css b/misc/dashboard/godashboard/static/style.css new file mode 100644 index 000000000..d6d23b536 --- /dev/null +++ b/misc/dashboard/godashboard/static/style.css @@ -0,0 +1,118 @@ +body { + font-family: sans-serif; + margin: 0; + padding: 0; +} +h1, h2, h3, ul.menu, table, p { + padding: 0 0.5em; +} +h1, h2 { + margin: 0; + background: #eee; +} +h1 { + border-bottom: 1px solid #ccc; + font-size: 1em; + padding: 0.5em; + margin-bottom: 0.5em; + text-align: right; +} +h2 { + border-top: 1px solid #ccc; + padding-left: 0.2em; +} +.submit { + float: right; + border: 1px solid #ccc; + width: 350px; + padding-bottom: 1em; + margin: 0.5em; + background: #eee; +} +.submit table { + width: 100%; +} +.submit input[type=text] { + width: 200px; +} +.submit .msg { + text-align: center; + color: red; +} +table.alternate { + white-space: nowrap; + margin: 0.5em 0; +} +table.alternate td, +table.alternate th { + padding: 0.1em 0.25em; + font-size: small; +} +table.alternate tr td:last-child { + padding-right: 0; +} +table.alternate tr:nth-child(2n) { + background-color: #f0f0f0; +} +td.result { + text-align: center; +} +span.hash { + font-family: monospace; + font-size: small; + color: #aaa; +} +td.date { + color: #aaa; +} +td.ok { + text-align: center; + color: #060; + font-weight: bold; +} +td.ok a { + cursor: help; +} +th { + text-align: left; +} +th.builder { + text-align: center; + font-weight: bold; +} +a.fail { + color: #F00; +} +a.fail:visited { + color: #900; +} +ul.menu { + margin: 0; + padding: 0; + list-style-type: none; +} +ul.menu li { + float: left; + display: block; + font-size: 1em; + padding: 0.5em; + background: #EEF; + margin-left: 0.5em; + border-left: 1px solid #999; + border-right: 1px solid #999; +} +div.paginate { + padding: 0.5em; +} +div.paginate a { + padding: 0.5em; + margin-right: 0.5em; + background: #eee; + color: blue; +} +div.paginate a.inactive { + color: #999; +} +td.time { + font-family: monospace; +} diff --git a/misc/dashboard/godashboard/toutf8.py b/misc/dashboard/godashboard/toutf8.py new file mode 100644 index 000000000..544c681b6 --- /dev/null +++ b/misc/dashboard/godashboard/toutf8.py @@ -0,0 +1,14 @@ +# Copyright 2010 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +# This is a Django custom template filter to work around the +# fact that GAE's urlencode filter doesn't handle unicode strings. + +from google.appengine.ext import webapp + +register = webapp.template.create_template_register() + +@register.filter +def toutf8(value): + return value.encode("utf-8") |