15 files changed, 1071 insertions, 68 deletions

diff --git a/misc/dashboard/README b/misc/dashboard/README
index b2bc3c2d3..72d5546a4 100644
--- a/misc/dashboard/README
+++ b/misc/dashboard/README
@@ -24,11 +24,19 @@ export GOARCH=XXX
 export GOOS=XXX
 export GOBIN=/gobuild/bin
 export PATH=$PATH:/gobuild/bin
-export BUILDER=XXX
+export BUILDER=$GOOS-$GOARCH
 export BUILDHOST=godashboard.appspot.com
 
-* Write the key ~gobuild/.gobuildkey (you need to get it from someone who knows
-                                      the key)
+* Write the key ~gobuild/.gobuildkey 
+  You need to get it from someone who knows the key.
+  You may also use a filename of the form .gobuildkey-$BUILDER if you
+  wish to run builders for multiple targets.
+
+* Append your username and password googlecode.com credentials from
+    https://code.google.com/hosting/settings
+  to the buildkey file in the format "Username\nPassword\n".
+  (This is for uploading tarballs to the project downloads section,
+   and is an optional step.)
 
 * sudo apt-get install bison gcc libc6-dev ed make
 * cd ~gobuild
diff --git a/misc/dashboard/buildcontrol.py b/misc/dashboard/buildcontrol.py
index 91b684f79..ec503e7ff 100644
--- a/misc/dashboard/buildcontrol.py
+++ b/misc/dashboard/buildcontrol.py
@@ -18,6 +18,8 @@ buildhost = ''
 buildport = -1
 buildkey = ''
 
+upload_project = "go"
+
 def main(args):
     global buildport, buildhost, buildkey
 
@@ -35,14 +37,23 @@ def main(args):
         buildport = int(os.environ['BUILDPORT'])
 
     try:
-        buildkey = file('%s/.gobuildkey-%s' % (os.environ['HOME'], os.environ['BUILDER']), 'r').read().strip()
+        buildkeyfile = file('%s/.gobuildkey-%s' % (os.environ['HOME'], os.environ['BUILDER']), 'r')
+        buildkey = buildkeyfile.readline().strip()
     except IOError:
         try:
-            buildkey = file('%s/.gobuildkey' % os.environ['HOME'], 'r').read().strip()
+            buildkeyfile = file('%s/.gobuildkey' % os.environ['HOME'], 'r')
+            buildkey = buildkeyfile.readline().strip()
         except IOError:
             print >>sys.stderr, "Need key in ~/.gobuildkey-%s or ~/.gobuildkey" % os.environ['BUILDER']
             return
 
+    # get upload credentials
+    try:
+        username = buildkeyfile.readline().strip()
+        password = buildkeyfile.readline().strip()
+    except:
+        username, password = None, None
+
     if args[1] == 'init':
         return doInit(args)
     elif args[1] == 'hwget':
@@ -55,6 +66,8 @@ def main(args):
         return doRecord(args)
     elif args[1] == 'benchmarks':
         return doBenchmarks(args)
+    elif args[1] == 'upload':
+        return doUpload(args, username, password)
     else:
         return usage(args[0])
 
@@ -68,6 +81,7 @@ Commands:
   next <builder>: get the next revision number to by built by the given builder
   record <builder> <rev> <ok|log file>: record a build result
   benchmarks <builder> <rev> <log file>: record benchmark numbers
+  upload <builder> <summary> <tar file>: upload tarball to googlecode
 ''' % name)
     return 1
 
@@ -165,6 +179,29 @@ def doBenchmarks(args):
         e.append(b)
     return command('benchmarks', {'node': c.node, 'builder': builder, 'benchmarkdata': binascii.b2a_base64(''.join(e))})
 
+def doUpload(args, username, password):
+    # fail gracefully if no username or password set
+    if not username or not password:
+        return
+
+    if len(args) != 5:
+        return usage(args[0])
+    builder = args[2]
+    summary = args[3]
+    filename = args[4]
+
+    from googlecode_upload import upload
+    code, msg, url = upload(
+        filename, # filename
+        upload_project, # 'go'
+        username,
+        password,
+        summary,
+        builder.split('-'), # labels
+    )
+    if code != 201:
+        raise Failed('Upload returned code %s msg "%s".' % (code, msg))
+
 def encodeMultipartFormdata(fields, files):
     """fields is a sequence of (name, value) elements for regular form fields.
     files is a sequence of (name, filename, value) elements for data to be uploaded as files"""
diff --git a/misc/dashboard/buildcron.sh b/misc/dashboard/buildcron.sh
index 5f4300796..7aa70ce57 100644
--- a/misc/dashboard/buildcron.sh
+++ b/misc/dashboard/buildcron.sh
@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/usr/bin/env bash
 # Copyright 2010 The Go Authors.  All rights reserved.
 # Use of this source code is governed by a BSD-style
 # license that can be found in the LICENSE file.
@@ -48,7 +48,7 @@ fi
 mkdir -p $GOROOT/bin
 
 cd $GOROOT/..
-cp go/misc/dashboard/builder.sh go/misc/dashboard/buildcontrol.py .
+cp go/misc/dashboard/{builder.sh,buildcontrol.py,googlecode_upload.py} .
 chmod a+x builder.sh buildcontrol.py
 cd go
 ../buildcontrol.py next $BUILDER
diff --git a/misc/dashboard/builder.sh b/misc/dashboard/builder.sh
index b302acec2..4a8d117bf 100644
--- a/misc/dashboard/builder.sh
+++ b/misc/dashboard/builder.sh
@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/usr/bin/env bash
 
 # Copyright 2009 The Go Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style
@@ -31,8 +31,9 @@ fi
 
 export PATH=$PATH:`pwd`/candidate/bin
 export GOBIN=`pwd`/candidate/bin
+export GOROOT_FINAL=/usr/local/go
 
-while true ; do
+while true ; do (
     cd go || fatal "Cannot cd into 'go'"
     hg pull -u || fatal "hg sync failed"
     rev=`python ../buildcontrol.py next $BUILDER`
@@ -72,7 +73,23 @@ while true ; do
             python ../../../buildcontrol.py benchmarks $BUILDER $rev ../../benchmarks || fatal "Cannot record benchmarks"
             cd .. || fatal "failed to cd out of pkg"
         fi
+        # check if we're at a release (via the hg summary)
+        #  if so, package the tar.gz and upload to googlecode
+        SUMMARY=$(hg log -l 1 | grep summary\: | awk '{print $2}')
+        if [[ "x${SUMMARY:0:7}" == "xrelease" ]]; then
+            echo "Uploading binary to googlecode"
+            TARBALL="go.$SUMMARY.$BUILDER.tar.gz"
+            ./clean.bash --nopkg
+	    # move contents of candidate/ to candidate/go/ for archival
+            cd ../..                     || fatal "Cannot cd up"
+	    mv candidate go-candidate    || fatal "Cannot rename candidate"
+	    mkdir candidate              || fatal "Cannot mkdir candidate"
+	    mv go-candidate candidate/go || fatal "Cannot mv directory"
+	    cd candidate                 || fatal "Cannot cd candidate"
+	    # build tarball
+            tar czf ../$TARBALL go       || fatal "Cannot create tarball"
+            ../buildcontrol.py upload $BUILDER $SUMMARY ../$TARBALL || fatal "Cannot upload tarball"
+        fi
     fi
-    cd ../.. || fatal "Cannot cd up"
     sleep 10
-done
+) done
diff --git a/misc/dashboard/builder/Makefile b/misc/dashboard/builder/Makefile
new file mode 100644
index 000000000..7270a3f42
--- /dev/null
+++ b/misc/dashboard/builder/Makefile
@@ -0,0 +1,14 @@
+# Copyright 2009 The Go Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+include ../../../src/Make.inc
+
+TARG=gobuilder
+GOFILES=\
+	exec.go\
+	hg.go\
+	http.go\
+	main.go\
+
+include ../../../src/Make.cmd
diff --git a/misc/dashboard/builder/doc.go b/misc/dashboard/builder/doc.go
new file mode 100644
index 000000000..54a9adfc0
--- /dev/null
+++ b/misc/dashboard/builder/doc.go
@@ -0,0 +1,54 @@
+// Copyright 2010 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+/*
+
+Go Builder is a continuous build client for the Go project. 
+It integrates with the Go Dashboard AppEngine application.
+
+Go Builder is intended to run continuously as a background process.
+
+It periodically pulls updates from the Go Mercurial repository. 
+
+When a newer revision is found, Go Builder creates a clone of the repository,
+runs all.bash, and reports build success or failure to the Go Dashboard. 
+
+For a successful build, Go Builder will also run benchmarks 
+(cd $GOROOT/src/pkg; make bench) and send the results to the Go Dashboard.
+
+For a release revision (a change description that matches "release.YYYY-MM-DD"),
+Go Builder will create a tar.gz archive of the GOROOT and deliver it to the
+Go Google Code project's downloads section.
+
+Usage:
+
+  gobuilder goos-goarch...
+
+  Several goos-goarch combinations can be provided, and the builder will 
+  build them in serial.
+
+Optional flags:
+
+  -dashboard="godashboard.appspot.com": Go Dashboard Host
+    The location of the Go Dashboard application to which Go Builder will
+    report its results.
+
+  -bench: Run benchmarks
+
+  -release: Build and deliver binary release archive
+
+The key file should be located at $HOME/.gobuilder or, for a builder-specific
+key, $HOME/.gobuilder-$BUILDER (eg, $HOME/.gobuilder-linux-amd64).
+
+The build key file is a text file of the format:
+
+  godashboard-key
+  googlecode-username
+  googlecode-password
+
+If the Google Code credentials are not provided the archival step
+will be skipped.
+
+*/
+package documentation
diff --git a/misc/dashboard/builder/exec.go b/misc/dashboard/builder/exec.go
new file mode 100644
index 000000000..6236c915a
--- /dev/null
+++ b/misc/dashboard/builder/exec.go
@@ -0,0 +1,65 @@
+package main
+
+import (
+	"bytes"
+	"exec"
+	"io"
+	"os"
+	"strings"
+)
+
+// run is a simple wrapper for exec.Run/Close
+func run(envv []string, dir string, argv ...string) os.Error {
+	bin, err := pathLookup(argv[0])
+	if err != nil {
+		return err
+	}
+	p, err := exec.Run(bin, argv, envv, dir,
+		exec.DevNull, exec.DevNull, exec.PassThrough)
+	if err != nil {
+		return err
+	}
+	return p.Close()
+}
+
+// runLog runs a process and returns the combined stdout/stderr, 
+// as well as writing it to logfile (if specified).
+func runLog(envv []string, logfile, dir string, argv ...string) (output string, exitStatus int, err os.Error) {
+	bin, err := pathLookup(argv[0])
+	if err != nil {
+		return
+	}
+	p, err := exec.Run(bin, argv, envv, dir,
+		exec.DevNull, exec.Pipe, exec.MergeWithStdout)
+	if err != nil {
+		return
+	}
+	defer p.Close()
+	b := new(bytes.Buffer)
+	var w io.Writer = b
+	if logfile != "" {
+		f, err := os.Open(logfile, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644)
+		if err != nil {
+			return
+		}
+		defer f.Close()
+		w = io.MultiWriter(f, b)
+	}
+	_, err = io.Copy(w, p.Stdout)
+	if err != nil {
+		return
+	}
+	wait, err := p.Wait(0)
+	if err != nil {
+		return
+	}
+	return b.String(), wait.WaitStatus.ExitStatus(), nil
+}
+
+// Find bin in PATH if a relative or absolute path hasn't been specified
+func pathLookup(s string) (string, os.Error) {
+	if strings.HasPrefix(s, "/") || strings.HasPrefix(s, "./") || strings.HasPrefix(s, "../") {
+		return s, nil
+	}
+	return exec.LookPath(s)
+}
diff --git a/misc/dashboard/builder/hg.go b/misc/dashboard/builder/hg.go
new file mode 100644
index 000000000..5d2f63a17
--- /dev/null
+++ b/misc/dashboard/builder/hg.go
@@ -0,0 +1,54 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"strconv"
+	"strings"
+)
+
+type Commit struct {
+	num    int    // mercurial revision number
+	node   string // mercurial hash
+	parent string // hash of commit's parent
+	user   string // author's Name <email>
+	date   string // date of commit
+	desc   string // description
+}
+
+// getCommit returns details about the Commit specified by the revision hash
+func getCommit(rev string) (c Commit, err os.Error) {
+	defer func() {
+		if err != nil {
+			err = fmt.Errorf("getCommit: %s: %s", rev, err)
+		}
+	}()
+	parts, err := getCommitParts(rev)
+	if err != nil {
+		return
+	}
+	num, err := strconv.Atoi(parts[0])
+	if err != nil {
+		return
+	}
+	parent := ""
+	if num > 0 {
+		prev := strconv.Itoa(num - 1)
+		if pparts, err := getCommitParts(prev); err == nil {
+			parent = pparts[1]
+		}
+	}
+	user := strings.Replace(parts[2], "&lt;", "<", -1)
+	user = strings.Replace(user, "&gt;", ">", -1)
+	return Commit{num, parts[1], parent, user, parts[3], parts[4]}, nil
+}
+
+func getCommitParts(rev string) (parts []string, err os.Error) {
+	const format = "{rev}>{node}>{author|escape}>{date}>{desc}"
+	s, _, err := runLog(nil, "", goroot,
+		"hg", "log", "-r", rev, "-l", "1", "--template", format)
+	if err != nil {
+		return
+	}
+	return strings.Split(s, ">", 5), nil
+}
diff --git a/misc/dashboard/builder/http.go b/misc/dashboard/builder/http.go
new file mode 100644
index 000000000..02f281061
--- /dev/null
+++ b/misc/dashboard/builder/http.go
@@ -0,0 +1,70 @@
+package main
+
+import (
+	"bytes"
+	"encoding/base64"
+	"encoding/binary"
+	"fmt"
+	"http"
+	"os"
+	"regexp"
+)
+
+// getHighWater returns the current highwater revision hash for this builder
+func (b *Builder) getHighWater() (rev string, err os.Error) {
+	url := fmt.Sprintf("http://%s/hw-get?builder=%s", *dashboard, b.name)
+	r, _, err := http.Get(url)
+	if err != nil {
+		return
+	}
+	buf := new(bytes.Buffer)
+	_, err = buf.ReadFrom(r.Body)
+	if err != nil {
+		return
+	}
+	r.Body.Close()
+	return buf.String(), nil
+}
+
+// recordResult sends build results to the dashboard
+func (b *Builder) recordResult(buildLog string, c Commit) os.Error {
+	return httpCommand("build", map[string]string{
+		"builder": b.name,
+		"key":     b.key,
+		"node":    c.node,
+		"parent":  c.parent,
+		"user":    c.user,
+		"date":    c.date,
+		"desc":    c.desc,
+		"log":     buildLog,
+	})
+}
+
+// match lines like: "package.BechmarkFunc	100000	    999 ns/op"
+var benchmarkRegexp = regexp.MustCompile("([^\n\t ]+)[\t ]+([0-9]+)[\t ]+([0-9]+) ns/op")
+
+// recordBenchmarks sends benchmark results to the dashboard
+func (b *Builder) recordBenchmarks(benchLog string, c Commit) os.Error {
+	results := benchmarkRegexp.FindAllStringSubmatch(benchLog, -1)
+	var buf bytes.Buffer
+	b64 := base64.NewEncoder(base64.StdEncoding, &buf)
+	for _, r := range results {
+		for _, s := range r[1:] {
+			binary.Write(b64, binary.BigEndian, uint16(len(s)))
+			b64.Write([]byte(s))
+		}
+	}
+	b64.Close()
+	return httpCommand("benchmarks", map[string]string{
+		"builder":       b.name,
+		"key":           b.key,
+		"node":          c.node,
+		"benchmarkdata": buf.String(),
+	})
+}
+
+func httpCommand(cmd string, args map[string]string) os.Error {
+	url := fmt.Sprintf("http://%v/%v", *dashboard, cmd)
+	_, err := http.PostForm(url, args)
+	return err
+}
diff --git a/misc/dashboard/builder/main.go b/misc/dashboard/builder/main.go
new file mode 100644
index 000000000..32a2e10da
--- /dev/null
+++ b/misc/dashboard/builder/main.go
@@ -0,0 +1,340 @@
+package main
+
+import (
+	"container/vector"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+	"path"
+	"regexp"
+	"strconv"
+	"strings"
+	"time"
+)
+
+const (
+	codeProject  = "go"
+	codePyScript = "misc/dashboard/googlecode_upload.py"
+	hgUrl        = "https://go.googlecode.com/hg/"
+	waitInterval = 10e9 // time to wait before checking for new revs
+	mkdirPerm    = 0750
+)
+
+type Builder struct {
+	name         string
+	goos, goarch string
+	key          string
+	codeUsername string
+	codePassword string
+}
+
+type BenchRequest struct {
+	builder *Builder
+	commit  Commit
+	path    string
+}
+
+var (
+	buildroot     = flag.String("buildroot", path.Join(os.TempDir(), "gobuilder"), "Directory under which to build")
+	dashboard     = flag.String("dashboard", "godashboard.appspot.com", "Go Dashboard Host")
+	runBenchmarks = flag.Bool("bench", false, "Run benchmarks")
+	buildRelease  = flag.Bool("release", false, "Build and upload binary release archives")
+	buildRevision = flag.String("rev", "", "Build specified revision and exit")
+	buildCmd      = flag.String("cmd", "./all.bash", "Build command (specify absolute or relative to go/src/)")
+)
+
+var (
+	goroot        string
+	releaseRegexp = regexp.MustCompile(`^release\.[0-9\-.]+`)
+	benchRequests vector.Vector
+)
+
+func main() {
+	flag.Usage = func() {
+		fmt.Fprintf(os.Stderr, "usage: %s goos-goarch...\n", os.Args[0])
+		flag.PrintDefaults()
+		os.Exit(2)
+	}
+	flag.Parse()
+	if len(flag.Args()) == 0 {
+		flag.Usage()
+	}
+	goroot = path.Join(*buildroot, "goroot")
+	builders := make([]*Builder, len(flag.Args()))
+	for i, builder := range flag.Args() {
+		b, err := NewBuilder(builder)
+		if err != nil {
+			log.Exit(err)
+		}
+		builders[i] = b
+	}
+	if err := os.RemoveAll(*buildroot); err != nil {
+		log.Exitf("Error removing build root (%s): %s", *buildroot, err)
+	}
+	if err := os.Mkdir(*buildroot, mkdirPerm); err != nil {
+		log.Exitf("Error making build root (%s): %s", *buildroot, err)
+	}
+	if err := run(nil, *buildroot, "hg", "clone", hgUrl, goroot); err != nil {
+		log.Exit("Error cloning repository:", err)
+	}
+	// if specified, build revision and return
+	if *buildRevision != "" {
+		c, err := getCommit(*buildRevision)
+		if err != nil {
+			log.Exit("Error finding revision: ", err)
+		}
+		for _, b := range builders {
+			if err := b.buildCommit(c); err != nil {
+				log.Println(err)
+			}
+			runQueuedBenchmark()
+		}
+		return
+	}
+	// check for new commits and build them
+	for {
+		err := run(nil, goroot, "hg", "pull", "-u")
+		if err != nil {
+			log.Println("hg pull failed:", err)
+			time.Sleep(waitInterval)
+			continue
+		}
+		built := false
+		for _, b := range builders {
+			if b.build() {
+				built = true
+			}
+		}
+		// only run benchmarks if we didn't build anything
+		// so that they don't hold up the builder queue
+		if !built {
+			if !runQueuedBenchmark() {
+				// if we have no benchmarks to do, pause
+				time.Sleep(waitInterval)
+			}
+			// after running one benchmark, 
+			// continue to find and build new revisions.
+		}
+	}
+}
+
+func runQueuedBenchmark() bool {
+	if benchRequests.Len() == 0 {
+		return false
+	}
+	runBenchmark(benchRequests.Pop().(BenchRequest))
+	return true
+}
+
+func runBenchmark(r BenchRequest) {
+	// run benchmarks and send to dashboard
+	log.Println(r.builder.name, "benchmarking", r.commit.num)
+	defer os.RemoveAll(r.path)
+	pkg := path.Join(r.path, "go", "src", "pkg")
+	bin := path.Join(r.path, "go", "bin")
+	env := []string{
+		"GOOS=" + r.builder.goos,
+		"GOARCH=" + r.builder.goarch,
+		"PATH=" + bin + ":" + os.Getenv("PATH"),
+	}
+	logfile := path.Join(r.path, "bench.log")
+	benchLog, _, err := runLog(env, logfile, pkg, "gomake", "bench")
+	if err != nil {
+		log.Println(r.builder.name, "gomake bench:", err)
+		return
+	}
+	if err = r.builder.recordBenchmarks(benchLog, r.commit); err != nil {
+		log.Println("recordBenchmarks:", err)
+	}
+}
+
+func NewBuilder(builder string) (*Builder, os.Error) {
+	b := &Builder{name: builder}
+
+	// get goos/goarch from builder string
+	s := strings.Split(builder, "-", 3)
+	if len(s) == 2 {
+		b.goos, b.goarch = s[0], s[1]
+	} else {
+		return nil, fmt.Errorf("unsupported builder form: %s", builder)
+	}
+
+	// read keys from keyfile
+	fn := path.Join(os.Getenv("HOME"), ".gobuildkey")
+	if s := fn + "-" + b.name; isFile(s) { // builder-specific file
+		fn = s
+	}
+	c, err := ioutil.ReadFile(fn)
+	if err != nil {
+		return nil, fmt.Errorf("readKeys %s (%s): %s", b.name, fn, err)
+	}
+	v := strings.Split(string(c), "\n", -1)
+	b.key = v[0]
+	if len(v) >= 3 {
+		b.codeUsername, b.codePassword = v[1], v[2]
+	}
+
+	return b, nil
+}
+
+// build checks for a new commit for this builder
+// and builds it if one is found. 
+// It returns true if a build was attempted.
+func (b *Builder) build() bool {
+	defer func() {
+		err := recover()
+		if err != nil {
+			log.Println(b.name, "build:", err)
+		}
+	}()
+	c, err := b.nextCommit()
+	if err != nil {
+		log.Println(err)
+		return false
+	}
+	if c == nil {
+		return false
+	}
+	err = b.buildCommit(*c)
+	if err != nil {
+		log.Println(err)
+	}
+	return true
+}
+
+// nextCommit returns the next unbuilt Commit for this builder
+func (b *Builder) nextCommit() (nextC *Commit, err os.Error) {
+	defer func() {
+		if err != nil {
+			err = fmt.Errorf("%s nextCommit: %s", b.name, err)
+		}
+	}()
+	hw, err := b.getHighWater()
+	if err != nil {
+		return
+	}
+	c, err := getCommit(hw)
+	if err != nil {
+		return
+	}
+	next := c.num + 1
+	c, err = getCommit(strconv.Itoa(next))
+	if err == nil && c.num == next {
+		return &c, nil
+	}
+	return nil, nil
+}
+
+func (b *Builder) buildCommit(c Commit) (err os.Error) {
+	defer func() {
+		if err != nil {
+			err = fmt.Errorf("%s buildCommit: %d: %s", b.name, c.num, err)
+		}
+	}()
+
+	log.Println(b.name, "building", c.num)
+
+	// create place in which to do work
+	workpath := path.Join(*buildroot, b.name+"-"+strconv.Itoa(c.num))
+	err = os.Mkdir(workpath, mkdirPerm)
+	if err != nil {
+		return
+	}
+	benchRequested := false
+	defer func() {
+		if !benchRequested {
+			os.RemoveAll(workpath)
+		}
+	}()
+
+	// clone repo
+	err = run(nil, workpath, "hg", "clone", goroot, "go")
+	if err != nil {
+		return
+	}
+
+	// update to specified revision
+	err = run(nil, path.Join(workpath, "go"),
+		"hg", "update", "-r", strconv.Itoa(c.num))
+	if err != nil {
+		return
+	}
+
+	// set up environment for build/bench execution
+	env := []string{
+		"GOOS=" + b.goos,
+		"GOARCH=" + b.goarch,
+		"GOHOSTOS=" + os.Getenv("GOHOSTOS"),
+		"GOHOSTARCH=" + os.Getenv("GOHOSTARCH"),
+		"GOROOT_FINAL=/usr/local/go",
+		"PATH=" + os.Getenv("PATH"),
+	}
+	srcDir := path.Join(workpath, "go", "src")
+
+	// build
+	logfile := path.Join(workpath, "build.log")
+	buildLog, status, err := runLog(env, logfile, srcDir, *buildCmd)
+	if err != nil {
+		return fmt.Errorf("all.bash: %s", err)
+	}
+	if status != 0 {
+		// record failure
+		return b.recordResult(buildLog, c)
+	}
+
+	// record success
+	if err = b.recordResult("", c); err != nil {
+		return fmt.Errorf("recordResult: %s", err)
+	}
+
+	// send benchmark request if benchmarks are enabled
+	if *runBenchmarks {
+		benchRequests.Insert(0, BenchRequest{
+			builder: b,
+			commit:  c,
+			path:    workpath,
+		})
+		benchRequested = true
+	}
+
+	// finish here if codeUsername and codePassword aren't set
+	if b.codeUsername == "" || b.codePassword == "" || !*buildRelease {
+		return
+	}
+
+	// if this is a release, create tgz and upload to google code
+	if release := releaseRegexp.FindString(c.desc); release != "" {
+		// clean out build state
+		err = run(env, srcDir, "./clean.bash", "--nopkg")
+		if err != nil {
+			return fmt.Errorf("clean.bash: %s", err)
+		}
+		// upload binary release
+		fn := fmt.Sprintf("go.%s.%s-%s.tar.gz", release, b.goos, b.goarch)
+		err = run(nil, workpath, "tar", "czf", fn, "go")
+		if err != nil {
+			return fmt.Errorf("tar: %s", err)
+		}
+		err = run(nil, workpath, path.Join(goroot, codePyScript),
+			"-s", release,
+			"-p", codeProject,
+			"-u", b.codeUsername,
+			"-w", b.codePassword,
+			"-l", fmt.Sprintf("%s,%s", b.goos, b.goarch),
+			fn)
+	}
+
+	return
+}
+
+func isDirectory(name string) bool {
+	s, err := os.Stat(name)
+	return err == nil && s.IsDirectory()
+}
+
+func isFile(name string) bool {
+	s, err := os.Stat(name)
+	return err == nil && (s.IsRegular() || s.IsSymlink())
+}
diff --git a/misc/dashboard/godashboard/package.py b/misc/dashboard/godashboard/package.py
index 6c3bd9995..cf59bf3e8 100644
--- a/misc/dashboard/godashboard/package.py
+++ b/misc/dashboard/godashboard/package.py
@@ -17,6 +17,7 @@ from google.appengine.ext.webapp import template
 from google.appengine.ext.webapp.util import run_wsgi_app
 from google.appengine.api import users
 from google.appengine.api import mail
+from google.appengine.api import urlfetch
 import binascii
 import datetime
 import hashlib
@@ -29,6 +30,11 @@ import time
 import urllib2
 import sets
 
+# local imports
+import toutf8
+
+template.register_template_library('toutf8')
+
 # Storage model for package info recorded on server.
 # Just path, count, and time of last install.
 class Package(db.Model):
@@ -50,36 +56,91 @@ re_bitbucket = re.compile(r'^bitbucket\.org/[a-z0-9A-Z_.\-]+/[a-z0-9A-Z_.\-]+$')
 re_googlecode = re.compile(r'^[a-z0-9\-]+\.googlecode\.com/(svn|hg)$')
 re_github = re.compile(r'^github\.com/[a-z0-9A-Z_.\-]+/[a-z0-9A-Z_.\-]+$')
 
+def vc_to_web(path):
+    if re_bitbucket.match(path):
+        check_url = 'http://' + path + '/?cmd=heads'
+        web = 'http://' + path + '/'
+    elif re_github.match(path):
+        # github doesn't let you fetch the .git directory anymore.
+        # fetch .git/info/refs instead, like git clone would.
+        check_url = 'http://'+path+'.git/info/refs'
+        web = 'http://' + path
+    elif re_googlecode.match(path):
+        check_url = 'http://'+path
+        web = 'http://code.google.com/p/' + path[:path.index('.')]
+    else:
+        return False, False
+    return web, check_url
+
+re_bitbucket_web = re.compile(r'bitbucket\.org/([a-z0-9A-Z_.\-]+)/([a-z0-9A-Z_.\-]+)')
+re_googlecode_web = re.compile(r'code.google.com/p/([a-z0-9\-]+)')
+re_github_web = re.compile(r'github\.com/([a-z0-9A-Z_.\-]+)/([a-z0-9A-Z_.\-]+)')
+re_striphttp = re.compile(r'http://(www\.)?')
+
+def web_to_vc(url):
+    url = re_striphttp.sub('', url)
+    m = re_bitbucket_web.match(url)
+    if m:
+        return 'bitbucket.org/'+m.group(1)+'/'+m.group(2)
+    m = re_github_web.match(url)
+    if m:
+        return 'github.com/'+m.group(1)+'/'+m.group(2)
+    m = re_googlecode_web.match(url)
+    if m:
+        path = m.group(1)+'.googlecode.com/'
+        # perform http request to path/hg to check if they're using mercurial
+        vcs = 'svn'
+        try:
+            response = urlfetch.fetch('http://'+path+'hg', deadline=1)
+            if response.status_code == 200:
+                vcs = 'hg'
+        except: pass
+        return path + vcs
+    return False
+
 MaxPathLength = 100
+CacheTimeout = 3600
 
 class PackagePage(webapp.RequestHandler):
     def get(self):
         if self.request.get('fmt') == 'json':
             return self.json()
 
-        q = Package.all()
-        q.order('-last_install')
-        by_time = q.fetch(100)
+        html = memcache.get('view-package')
+        if not html:
+            q = Package.all()
+            q.order('-last_install')
+            by_time = q.fetch(100)
 
-        q = Package.all()
-        q.order('-count')
-        by_count = q.fetch(100)
+            q = Package.all()
+            q.order('-count')
+            by_count = q.fetch(100)
 
-        self.response.headers['Content-Type'] = 'text/html; charset=utf-8'
-        path = os.path.join(os.path.dirname(__file__), 'package.html')
-        self.response.out.write(template.render(path, {"by_time": by_time, "by_count": by_count}))
+            self.response.headers['Content-Type'] = 'text/html; charset=utf-8'
+            path = os.path.join(os.path.dirname(__file__), 'package.html')
+            html = template.render(
+                path, 
+                {"by_time": by_time, "by_count": by_count}
+            )
+            memcache.set('view-package', html, time=CacheTimeout)
+
+        self.response.out.write(html)
 
     def json(self):
-        self.response.set_status(200)
-        self.response.headers['Content-Type'] = 'text/plain; charset=utf-8'
-        q = Package.all()
-        s = '{"packages": ['
-        sep = ''
-        for r in q.fetch(1000):
-            s += '%s\n\t{"path": "%s", "last_install": "%s", "count": "%s"}' % (sep, r.path, r.last_install, r.count)
-            sep = ','
-        s += '\n]}\n'
-        self.response.out.write(s)
+        json = memcache.get('view-package-json')
+        if not json:
+            self.response.set_status(200)
+            self.response.headers['Content-Type'] = 'text/plain; charset=utf-8'
+            q = Package.all()
+            s = '{"packages": ['
+            sep = ''
+            for r in q.fetch(1000):
+                s += '%s\n\t{"path": "%s", "last_install": "%s", "count": "%s"}' % (sep, r.path, r.last_install, r.count)
+                sep = ','
+            s += '\n]}\n'
+            json = s
+            memcache.set('view-package-json', json, time=CacheTimeoout)
+        self.response.out.write(json)
 
     def can_get_url(self, url):
         try:
@@ -104,18 +165,8 @@ class PackagePage(webapp.RequestHandler):
         p = Package.get_by_key_name(key)
         if p is None:
             # not in datastore - verify URL before creating
-            if re_bitbucket.match(path):
-                check_url = 'http://' + path + '/?cmd=heads'
-                web = 'http://' + path + '/'
-            elif re_github.match(path):
-                # github doesn't let you fetch the .git directory anymore.
-                # fetch .git/info/refs instead, like git clone would.
-                check_url = 'http://'+path+'.git/info/refs'
-                web = 'http://' + path
-            elif re_googlecode.match(path):
-                check_url = 'http://'+path
-                web = 'http://code.google.com/p/' + path[:path.index('.')]
-            else:
+            web, check_url = vc_to_web(path)
+            if not web:
                 logging.error('unrecognized path: %s', path)
                 return False
             if not self.can_get_url(check_url):
@@ -150,9 +201,27 @@ class ProjectPage(webapp.RequestHandler):
             self.redirect(users.create_logout_url("/project"))
         elif self.request.path == "/project/edit" and admin:
             self.edit()
+        elif self.request.path == "/project/assoc" and admin:
+            self.assoc()
         else:
             self.list()
 
+    def assoc(self):
+        projects = Project.all()
+        for p in projects:
+            if p.package:
+                continue
+            path = web_to_vc(p.web_url)
+            if not path:
+                continue
+            pkg = Package.get_by_key_name("pkg-"+path)
+            if not pkg:
+                self.response.out.write('no: %s %s<br>' % (p.web_url, path))
+                continue
+            p.package = pkg
+            p.put()
+            self.response.out.write('yes: %s %s<br>' % (p.web_url, path))
+
     def post(self):
         if self.request.path == "/project/edit":
             self.edit(True)
@@ -177,30 +246,40 @@ class ProjectPage(webapp.RequestHandler):
 
             self.list({"submitMsg": "Your project has been submitted."})
 
-    def list(self, data={}):
-        projects = Project.all().order('category').order('name')
-
-        admin = users.is_current_user_admin()
-        if not admin:
-            projects = projects.filter('approved =', True)
-
-        projects = list(projects)
-
-        tags = sets.Set()
-        for p in projects:
-            for t in p.tags:
-                tags.add(t)
-
-        tag = self.request.get("tag", None)
+    def list(self, additional_data={}):
+        cache_key = 'view-project-data'
+        tag = self.request.get('tag', None)
         if tag:
-            projects = filter(lambda x: tag in x.tags, projects)
+            cache_key += '-'+tag
+        data = memcache.get(cache_key)
+        admin = users.is_current_user_admin()
+        if admin or not data:
+            projects = Project.all().order('category').order('name')
+            if not admin:
+                projects = projects.filter('approved =', True)
+            projects = list(projects)
+
+            tags = sets.Set()
+            for p in projects:
+                for t in p.tags:
+                    tags.add(t)
+
+            if tag:
+                projects = filter(lambda x: tag in x.tags, projects)
+
+            data = {}
+            data['tag'] = tag
+            data['tags'] = tags
+            data['projects'] = projects 
+            data['admin']= admin
+            if not admin:
+                memcache.set(cache_key, data, time=CacheTimeout)
+
+        for k, v in additional_data.items():
+            data[k] = v
 
         self.response.headers['Content-Type'] = 'text/html; charset=utf-8'
         path = os.path.join(os.path.dirname(__file__), 'project.html')
-        data["tag"] = tag
-        data["tags"] = tags
-        data["projects"] = projects 
-        data["admin"] = admin
         self.response.out.write(template.render(path, data))
 
     def edit(self, save=False):
@@ -228,7 +307,8 @@ class ProjectPage(webapp.RequestHandler):
                 p.approved = self.request.get("approved") == "1"
                 p.tags = filter(lambda x: x, self.request.get("tags", "").split(","))
                 p.put()
-            self.redirect("/project")
+            memcache.delete('view-project-data')
+            self.redirect('/project')
             return
 
         # get all project categories and tags
diff --git a/misc/dashboard/godashboard/project-edit.html b/misc/dashboard/godashboard/project-edit.html
index 5f1ca3b11..ce18fb3fb 100644
--- a/misc/dashboard/godashboard/project-edit.html
+++ b/misc/dashboard/godashboard/project-edit.html
@@ -1,11 +1,11 @@
 <html>
 <head>
+<link href="http://ajax.googleapis.com/ajax/libs/jqueryui/1.8/themes/base/jquery-ui.css" rel="stylesheet" type="text/css"/>
 <script type="text/javascript" src="http://www.google.com/jsapi"></script>
 <script>
 google.load("jquery", "1");
+google.load("jqueryui", "1.8.2");
 </script>
-<script type="text/javascript" src="/static/jquery.autocomplete.min.js"></script>
-<link rel="stylesheet" type="text/css" href="/static/jquery.autocomplete.css" />
 </head>
 <body>
 <form action="/project/edit?orig_name={{p.name}}" method="POST">
@@ -38,8 +38,10 @@ var cats = [
 {% endfor %}
 ];
 
-$('#tags').autocomplete(tags);
-$('#cats').autocomplete(cats);
+google.setOnLoadCallback(function() {
+	$('#tags').autocomplete({source:tags});
+	$('#cats').autocomplete({source:cats});
+});
 </script>
 </body>
 </html>
diff --git a/misc/dashboard/godashboard/project-notify.txt b/misc/dashboard/godashboard/project-notify.txt
index 3a165908c..f55bf6421 100644
--- a/misc/dashboard/godashboard/project-notify.txt
+++ b/misc/dashboard/godashboard/project-notify.txt
@@ -5,5 +5,5 @@ Description: {{project.descr}}
 URL: {{project.web_url}}
 
 To edit/approve/delete:
-http://godashboard.appspot.com/project/edit?name={{project.name|urlencode}}
+http://godashboard.appspot.com/project/edit?name={{project.name|toutf8|urlencode}}
 
diff --git a/misc/dashboard/godashboard/toutf8.py b/misc/dashboard/godashboard/toutf8.py
new file mode 100644
index 000000000..544c681b6
--- /dev/null
+++ b/misc/dashboard/godashboard/toutf8.py
@@ -0,0 +1,14 @@
+# Copyright 2010 The Go Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+# This is a Django custom template filter to work around the
+# fact that GAE's urlencode filter doesn't handle unicode strings.
+
+from google.appengine.ext import webapp
+
+register = webapp.template.create_template_register()
+
+@register.filter
+def toutf8(value):
+    return value.encode("utf-8")
diff --git a/misc/dashboard/googlecode_upload.py b/misc/dashboard/googlecode_upload.py
new file mode 100755
index 000000000..3b1d432ff
--- /dev/null
+++ b/misc/dashboard/googlecode_upload.py
@@ -0,0 +1,248 @@
+#!/usr/bin/env python2
+#
+# Copyright 2006, 2007 Google Inc. All Rights Reserved.
+# Author: danderson@google.com (David Anderson)
+#
+# Script for uploading files to a Google Code project.
+#
+# This is intended to be both a useful script for people who want to
+# streamline project uploads and a reference implementation for
+# uploading files to Google Code projects.
+#
+# To upload a file to Google Code, you need to provide a path to the
+# file on your local machine, a small summary of what the file is, a
+# project name, and a valid account that is a member or owner of that
+# project.  You can optionally provide a list of labels that apply to
+# the file.  The file will be uploaded under the same name that it has
+# in your local filesystem (that is, the "basename" or last path
+# component).  Run the script with '--help' to get the exact syntax
+# and available options.
+#
+# Note that the upload script requests that you enter your
+# googlecode.com password.  This is NOT your Gmail account password!
+# This is the password you use on googlecode.com for committing to
+# Subversion and uploading files.  You can find your password by going
+# to http://code.google.com/hosting/settings when logged in with your
+# Gmail account. If you have already committed to your project's
+# Subversion repository, the script will automatically retrieve your
+# credentials from there (unless disabled, see the output of '--help'
+# for details).
+#
+# If you are looking at this script as a reference for implementing
+# your own Google Code file uploader, then you should take a look at
+# the upload() function, which is the meat of the uploader.  You
+# basically need to build a multipart/form-data POST request with the
+# right fields and send it to https://PROJECT.googlecode.com/files .
+# Authenticate the request using HTTP Basic authentication, as is
+# shown below.
+#
+# Licensed under the terms of the Apache Software License 2.0:
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Questions, comments, feature requests and patches are most welcome.
+# Please direct all of these to the Google Code users group:
+#  http://groups.google.com/group/google-code-hosting
+
+"""Google Code file uploader script.
+"""
+
+__author__ = 'danderson@google.com (David Anderson)'
+
+import httplib
+import os.path
+import optparse
+import getpass
+import base64
+import sys
+
+
+def upload(file, project_name, user_name, password, summary, labels=None):
+  """Upload a file to a Google Code project's file server.
+
+  Args:
+    file: The local path to the file.
+    project_name: The name of your project on Google Code.
+    user_name: Your Google account name.
+    password: The googlecode.com password for your account.
+              Note that this is NOT your global Google Account password!
+    summary: A small description for the file.
+    labels: an optional list of label strings with which to tag the file.
+
+  Returns: a tuple:
+    http_status: 201 if the upload succeeded, something else if an
+                 error occured.
+    http_reason: The human-readable string associated with http_status
+    file_url: If the upload succeeded, the URL of the file on Google
+              Code, None otherwise.
+  """
+  # The login is the user part of user@gmail.com. If the login provided
+  # is in the full user@domain form, strip it down.
+  if user_name.endswith('@gmail.com'):
+    user_name = user_name[:user_name.index('@gmail.com')]
+
+  form_fields = [('summary', summary)]
+  if labels is not None:
+    form_fields.extend([('label', l.strip()) for l in labels])
+
+  content_type, body = encode_upload_request(form_fields, file)
+
+  upload_host = '%s.googlecode.com' % project_name
+  upload_uri = '/files'
+  auth_token = base64.b64encode('%s:%s'% (user_name, password))
+  headers = {
+    'Authorization': 'Basic %s' % auth_token,
+    'User-Agent': 'Googlecode.com uploader v0.9.4',
+    'Content-Type': content_type,
+    }
+
+  server = httplib.HTTPSConnection(upload_host)
+  server.request('POST', upload_uri, body, headers)
+  resp = server.getresponse()
+  server.close()
+
+  if resp.status == 201:
+    location = resp.getheader('Location', None)
+  else:
+    location = None
+  return resp.status, resp.reason, location
+
+
+def encode_upload_request(fields, file_path):
+  """Encode the given fields and file into a multipart form body.
+
+  fields is a sequence of (name, value) pairs. file is the path of
+  the file to upload. The file will be uploaded to Google Code with
+  the same file name.
+
+  Returns: (content_type, body) ready for httplib.HTTP instance
+  """
+  BOUNDARY = '----------Googlecode_boundary_reindeer_flotilla'
+  CRLF = '\r\n'
+
+  body = []
+
+  # Add the metadata about the upload first
+  for key, value in fields:
+    body.extend(
+      ['--' + BOUNDARY,
+       'Content-Disposition: form-data; name="%s"' % key,
+       '',
+       value,
+       ])
+
+  # Now add the file itself
+  file_name = os.path.basename(file_path)
+  f = open(file_path, 'rb')
+  file_content = f.read()
+  f.close()
+
+  body.extend(
+    ['--' + BOUNDARY,
+     'Content-Disposition: form-data; name="filename"; filename="%s"'
+     % file_name,
+     # The upload server determines the mime-type, no need to set it.
+     'Content-Type: application/octet-stream',
+     '',
+     file_content,
+     ])
+
+  # Finalize the form body
+  body.extend(['--' + BOUNDARY + '--', ''])
+
+  return 'multipart/form-data; boundary=%s' % BOUNDARY, CRLF.join(body)
+
+
+def upload_find_auth(file_path, project_name, summary, labels=None,
+                     user_name=None, password=None, tries=3):
+  """Find credentials and upload a file to a Google Code project's file server.
+
+  file_path, project_name, summary, and labels are passed as-is to upload.
+
+  Args:
+    file_path: The local path to the file.
+    project_name: The name of your project on Google Code.
+    summary: A small description for the file.
+    labels: an optional list of label strings with which to tag the file.
+    config_dir: Path to Subversion configuration directory, 'none', or None.
+    user_name: Your Google account name.
+    tries: How many attempts to make.
+  """
+
+  while tries > 0:
+    if user_name is None:
+      # Read username if not specified or loaded from svn config, or on
+      # subsequent tries.
+      sys.stdout.write('Please enter your googlecode.com username: ')
+      sys.stdout.flush()
+      user_name = sys.stdin.readline().rstrip()
+    if password is None:
+      # Read password if not loaded from svn config, or on subsequent tries.
+      print 'Please enter your googlecode.com password.'
+      print '** Note that this is NOT your Gmail account password! **'
+      print 'It is the password you use to access Subversion repositories,'
+      print 'and can be found here: http://code.google.com/hosting/settings'
+      password = getpass.getpass()
+
+    status, reason, url = upload(file_path, project_name, user_name, password,
+                                 summary, labels)
+    # Returns 403 Forbidden instead of 401 Unauthorized for bad
+    # credentials as of 2007-07-17.
+    if status in [httplib.FORBIDDEN, httplib.UNAUTHORIZED]:
+      # Rest for another try.
+      user_name = password = None
+      tries = tries - 1
+    else:
+      # We're done.
+      break
+
+  return status, reason, url
+
+
+def main():
+  parser = optparse.OptionParser(usage='googlecode-upload.py -s SUMMARY '
+                                 '-p PROJECT [options] FILE')
+  parser.add_option('-s', '--summary', dest='summary',
+                    help='Short description of the file')
+  parser.add_option('-p', '--project', dest='project',
+                    help='Google Code project name')
+  parser.add_option('-u', '--user', dest='user',
+                    help='Your Google Code username')
+  parser.add_option('-w', '--password', dest='password',
+                    help='Your Google Code password')
+  parser.add_option('-l', '--labels', dest='labels',
+                    help='An optional list of comma-separated labels to attach '
+                    'to the file')
+
+  options, args = parser.parse_args()
+
+  if not options.summary:
+    parser.error('File summary is missing.')
+  elif not options.project:
+    parser.error('Project name is missing.')
+  elif len(args) < 1:
+    parser.error('File to upload not provided.')
+  elif len(args) > 1:
+    parser.error('Only one file may be specified.')
+
+  file_path = args[0]
+
+  if options.labels:
+    labels = options.labels.split(',')
+  else:
+    labels = None
+
+  status, reason, url = upload_find_auth(file_path, options.project,
+                                         options.summary, labels,
+                                         options.user, options.password)
+  if url:
+    print 'The file was uploaded successfully.'
+    print 'URL: %s' % url
+    return 0
+  else:
+    print 'An error occurred. Your file was not uploaded.'
+    print 'Google Code upload server said: %s (%s)' % (reason, status)
+    return 1
+
+
+if __name__ == '__main__':
+  sys.exit(main())