goinstall: an experiment in (external) package installation

R=adg, r
CC=cw, golang-dev
http://codereview.appspot.com/224043

2 files changed, 135 insertions, 0 deletions

diff --git a/misc/dashboard/godashboard/app.yaml b/misc/dashboard/godashboard/app.yaml
index ec4d8d9c1..1c786a6c1 100644
--- a/misc/dashboard/godashboard/app.yaml
+++ b/misc/dashboard/godashboard/app.yaml
@@ -4,5 +4,8 @@ runtime: python
 api_version: 1
 
 handlers:
+- url: /package.*
+  script: package.py
+
 - url: /.*
   script: gobuild.py
diff --git a/misc/dashboard/godashboard/package.py b/misc/dashboard/godashboard/package.py
new file mode 100644
index 000000000..351a1fadc
--- /dev/null
+++ b/misc/dashboard/godashboard/package.py
@@ -0,0 +1,132 @@
+# Copyright 2010 The Go Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+# This is the server part of the package dashboard.
+# It must be run by App Engine.
+
+from google.appengine.api import memcache
+from google.appengine.runtime import DeadlineExceededError
+from google.appengine.ext import db
+from google.appengine.ext import webapp
+from google.appengine.ext.webapp import template
+from google.appengine.ext.webapp.util import run_wsgi_app
+import binascii
+import datetime
+import hashlib
+import hmac
+import logging
+import os
+import re
+import struct
+import time
+import urllib2
+
+# Storage model for package info recorded on server.
+# Just path, count, and time of last install.
+class Package(db.Model):
+    path = db.StringProperty()
+    web_url = db.StringProperty()  # derived from path
+    count = db.IntegerProperty()
+    last_install = db.DateTimeProperty()
+
+re_bitbucket = re.compile(r'^bitbucket\.org/[a-z0-9A-Z_.\-]+/[a-z0-9A-Z_.\-]+$')
+re_googlecode = re.compile(r'^[a-z0-9\-]+\.googlecode\.com/(svn|hg)$')
+re_github = re.compile(r'^github\.com/[a-z0-9A-Z_.\-]+/[a-z0-9A-Z_.\-]+$')
+
+MaxPathLength = 100
+
+class PackagePage(webapp.RequestHandler):
+    def get(self):
+        if self.request.get('fmt') == 'json':
+            return self.json()
+
+        q = Package.all()
+        q.order('-last_install')
+        by_time = q.fetch(100)
+
+        q = Package.all()
+        q.order('-count')
+        by_count = q.fetch(100)
+
+        self.response.headers['Content-Type'] = 'text/html; charset=utf-8'
+        path = os.path.join(os.path.dirname(__file__), 'package.html')
+        self.response.out.write(template.render(path, {"by_time": by_time, "by_count": by_count}))
+
+    def json(self):
+        self.response.set_status(200)
+        self.response.headers['Content-Type'] = 'text/plain; charset=utf-8'
+        q = Package.all()
+        s = '{"packages": ['
+        sep = ''
+        for r in q.fetch(1000):
+            s += '%s\n\t{"path": "%s", "last_install": "%s", "count": "%s"}' % (sep, r.path, r.last_install, r.count)
+            sep = ','
+        s += '\n]}\n'
+        self.response.out.write(s)
+
+    def can_get_url(self, url):
+        try:
+            req = urllib2.Request(url)
+            response = urllib2.urlopen(req)
+            return True
+        except:
+            return False
+
+    def is_valid_package_path(self, path):
+        return (re_bitbucket.match(path) or
+            re_googlecode.match(path) or
+            re_github.match(path))
+
+    def record_pkg(self, path):
+        # sanity check string
+        if not path or len(path) > MaxPathLength or not self.is_valid_package_path(path):
+            return False
+
+        # look in datastore
+        key = 'pkg-' + path
+        p = Package.get_by_key_name(key)
+        if p is None:
+            # not in datastore - verify URL before creating
+            if re_bitbucket.match(path):
+                check_url = 'http://' + path + '/?cmd=heads'
+                web = 'http://' + path + '/'
+            elif re_github.match(path):
+                # github doesn't let you fetch the .git directory anymore.
+                # fetch .git/info/refs instead, like git clone would.
+                check_url = 'http://'+path+'.git/info/refs'
+                web = 'http://' + path
+            elif re_googlecode.match(path):
+                check_url = 'http://'+path
+                web = 'http://code.google.com/p/' + path[:path.index('.')]
+            else:
+                logging.error('unrecognized path: %s', path)
+                return False
+            if not self.can_get_url(check_url):
+                logging.error('cannot get %s', check_url)
+                return False
+            p = Package(key_name = key, path = path, count = 0, web_url = web)
+
+        # update package object
+        p.count += 1
+        p.last_install = datetime.datetime.utcnow()
+        p.put()
+        return True
+
+    def post(self):
+        path = self.request.get('path')
+        ok = self.record_pkg(path)
+        if ok:
+            self.response.set_status(200)
+            self.response.out.write('ok')
+        else:
+            logging.error('invalid path in post: %s', path)
+            self.response.set_status(500)
+            self.response.out.write('not ok')
+
+def main():
+    app = webapp.WSGIApplication([('/package', PackagePage)], debug=True)
+    run_wsgi_app(app)
+
+if __name__ == '__main__':
+    main()