Imported Upstream version 60upstream/60

6 files changed, 1012 insertions, 0 deletions

diff --git a/misc/dashboard/builder/Makefile b/misc/dashboard/builder/Makefile
new file mode 100644
index 000000000..f1d9c5497
--- /dev/null
+++ b/misc/dashboard/builder/Makefile
@@ -0,0 +1,14 @@
+# Copyright 2009 The Go Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+include ../../../src/Make.inc
+
+TARG=gobuilder
+GOFILES=\
+	exec.go\
+	http.go\
+	main.go\
+	package.go\
+
+include ../../../src/Make.cmd
diff --git a/misc/dashboard/builder/doc.go b/misc/dashboard/builder/doc.go
new file mode 100644
index 000000000..30d8fe948
--- /dev/null
+++ b/misc/dashboard/builder/doc.go
@@ -0,0 +1,58 @@
+// Copyright 2010 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+/*
+
+Go Builder is a continuous build client for the Go project. 
+It integrates with the Go Dashboard AppEngine application.
+
+Go Builder is intended to run continuously as a background process.
+
+It periodically pulls updates from the Go Mercurial repository. 
+
+When a newer revision is found, Go Builder creates a clone of the repository,
+runs all.bash, and reports build success or failure to the Go Dashboard. 
+
+For a release revision (a change description that matches "release.YYYY-MM-DD"),
+Go Builder will create a tar.gz archive of the GOROOT and deliver it to the
+Go Google Code project's downloads section.
+
+Usage:
+
+  gobuilder goos-goarch...
+
+  Several goos-goarch combinations can be provided, and the builder will 
+  build them in serial.
+
+Optional flags:
+
+  -dashboard="godashboard.appspot.com": Go Dashboard Host
+    The location of the Go Dashboard application to which Go Builder will
+    report its results.
+
+  -release: Build and deliver binary release archive
+
+  -rev=N: Build revision N and exit
+
+  -cmd="./all.bash": Build command (specify absolute or relative to go/src)
+
+  -v: Verbose logging
+
+  -external: External package builder mode (will not report Go build
+     state to dashboard or issue releases)
+
+The key file should be located at $HOME/.gobuildkey or, for a builder-specific
+key, $HOME/.gobuildkey-$BUILDER (eg, $HOME/.gobuildkey-linux-amd64).
+
+The build key file is a text file of the format:
+
+  godashboard-key
+  googlecode-username
+  googlecode-password
+
+If the Google Code credentials are not provided the archival step
+will be skipped.
+
+*/
+package documentation
diff --git a/misc/dashboard/builder/exec.go b/misc/dashboard/builder/exec.go
new file mode 100644
index 000000000..a042c5699
--- /dev/null
+++ b/misc/dashboard/builder/exec.go
@@ -0,0 +1,74 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+	"bytes"
+	"exec"
+	"io"
+	"log"
+	"os"
+	"strings"
+)
+
+// run is a simple wrapper for exec.Run/Close
+func run(envv []string, dir string, argv ...string) os.Error {
+	if *verbose {
+		log.Println("run", argv)
+	}
+	argv = useBash(argv)
+	cmd := exec.Command(argv[0], argv[1:]...)
+	cmd.Dir = dir
+	cmd.Env = envv
+	cmd.Stderr = os.Stderr
+	return cmd.Run()
+}
+
+// runLog runs a process and returns the combined stdout/stderr, 
+// as well as writing it to logfile (if specified). It returns
+// process combined stdout and stderr output, exit status and error.
+// The error returned is nil, if process is started successfully,
+// even if exit status is not 0.
+func runLog(envv []string, logfile, dir string, argv ...string) (string, int, os.Error) {
+	if *verbose {
+		log.Println("runLog", argv)
+	}
+	argv = useBash(argv)
+
+	b := new(bytes.Buffer)
+	var w io.Writer = b
+	if logfile != "" {
+		f, err := os.OpenFile(logfile, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666)
+		if err != nil {
+			return "", 0, err
+		}
+		defer f.Close()
+		w = io.MultiWriter(f, b)
+	}
+
+	cmd := exec.Command(argv[0], argv[1:]...)
+	cmd.Dir = dir
+	cmd.Env = envv
+	cmd.Stdout = w
+	cmd.Stderr = w
+
+	err := cmd.Run()
+	if err != nil {
+		if ws, ok := err.(*os.Waitmsg); ok {
+			return b.String(), ws.ExitStatus(), nil
+		}
+	}
+	return b.String(), 0, nil
+}
+
+// useBash prefixes a list of args with 'bash' if the first argument
+// is a bash script.
+func useBash(argv []string) []string {
+	// TODO(brainman): choose a more reliable heuristic here.
+	if strings.HasSuffix(argv[0], ".bash") {
+		argv = append([]string{"bash"}, argv...)
+	}
+	return argv
+}
diff --git a/misc/dashboard/builder/http.go b/misc/dashboard/builder/http.go
new file mode 100644
index 000000000..abef8faa4
--- /dev/null
+++ b/misc/dashboard/builder/http.go
@@ -0,0 +1,148 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"http"
+	"json"
+	"log"
+	"os"
+	"strconv"
+	"url"
+)
+
+type param map[string]string
+
+// dash runs the given method and command on the dashboard.
+// If args is not nil, it is the query or post parameters.
+// If resp is not nil, dash unmarshals the body as JSON into resp.
+func dash(meth, cmd string, resp interface{}, args param) os.Error {
+	var r *http.Response
+	var err os.Error
+	if *verbose {
+		log.Println("dash", cmd, args)
+	}
+	cmd = "http://" + *dashboard + "/" + cmd
+	vals := make(url.Values)
+	for k, v := range args {
+		vals.Add(k, v)
+	}
+	switch meth {
+	case "GET":
+		if q := vals.Encode(); q != "" {
+			cmd += "?" + q
+		}
+		r, err = http.Get(cmd)
+	case "POST":
+		r, err = http.PostForm(cmd, vals)
+	default:
+		return fmt.Errorf("unknown method %q", meth)
+	}
+	if err != nil {
+		return err
+	}
+	defer r.Body.Close()
+	var buf bytes.Buffer
+	buf.ReadFrom(r.Body)
+	if resp != nil {
+		if err = json.Unmarshal(buf.Bytes(), resp); err != nil {
+			log.Printf("json unmarshal %#q: %s\n", buf.Bytes(), err)
+			return err
+		}
+	}
+	return nil
+}
+
+func dashStatus(meth, cmd string, args param) os.Error {
+	var resp struct {
+		Status string
+		Error  string
+	}
+	err := dash(meth, cmd, &resp, args)
+	if err != nil {
+		return err
+	}
+	if resp.Status != "OK" {
+		return os.NewError("/build: " + resp.Error)
+	}
+	return nil
+}
+
+// todo returns the next hash to build.
+func (b *Builder) todo() (rev string, err os.Error) {
+	var resp []struct {
+		Hash string
+	}
+	if err = dash("GET", "todo", &resp, param{"builder": b.name}); err != nil {
+		return
+	}
+	if len(resp) > 0 {
+		rev = resp[0].Hash
+	}
+	return
+}
+
+// recordResult sends build results to the dashboard
+func (b *Builder) recordResult(buildLog string, hash string) os.Error {
+	return dash("POST", "build", nil, param{
+		"builder": b.name,
+		"key":     b.key,
+		"node":    hash,
+		"log":     buildLog,
+	})
+}
+
+// packages fetches a list of package paths from the dashboard
+func packages() (pkgs []string, err os.Error) {
+	var resp struct {
+		Packages []struct {
+			Path string
+		}
+	}
+	err = dash("GET", "package", &resp, param{"fmt": "json"})
+	if err != nil {
+		return
+	}
+	for _, p := range resp.Packages {
+		pkgs = append(pkgs, p.Path)
+	}
+	return
+}
+
+// updatePackage sends package build results and info dashboard
+func (b *Builder) updatePackage(pkg string, ok bool, buildLog, info string) os.Error {
+	return dash("POST", "package", nil, param{
+		"builder": b.name,
+		"key":     b.key,
+		"path":    pkg,
+		"ok":      strconv.Btoa(ok),
+		"log":     buildLog,
+		"info":    info,
+	})
+}
+
+// postCommit informs the dashboard of a new commit
+func postCommit(key string, l *HgLog) os.Error {
+	return dashStatus("POST", "commit", param{
+		"key":    key,
+		"node":   l.Hash,
+		"date":   l.Date,
+		"user":   l.Author,
+		"parent": l.Parent,
+		"desc":   l.Desc,
+	})
+}
+
+// dashboardCommit returns true if the dashboard knows about hash.
+func dashboardCommit(hash string) bool {
+	err := dashStatus("GET", "commit", param{"node": hash})
+	if err != nil {
+		log.Printf("check %s: %s", hash, err)
+		return false
+	}
+	return true
+}
diff --git a/misc/dashboard/builder/main.go b/misc/dashboard/builder/main.go
new file mode 100644
index 000000000..989965bc4
--- /dev/null
+++ b/misc/dashboard/builder/main.go
@@ -0,0 +1,624 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+	"path"
+	"regexp"
+	"runtime"
+	"strconv"
+	"strings"
+	"time"
+	"xml"
+)
+
+const (
+	codeProject      = "go"
+	codePyScript     = "misc/dashboard/googlecode_upload.py"
+	hgUrl            = "https://go.googlecode.com/hg/"
+	waitInterval     = 30e9 // time to wait before checking for new revs
+	mkdirPerm        = 0750
+	pkgBuildInterval = 1e9 * 60 * 60 * 24 // rebuild packages every 24 hours
+)
+
+// These variables are copied from the gobuilder's environment
+// to the envv of its subprocesses.
+var extraEnv = []string{
+	"GOHOSTOS",
+	"GOHOSTARCH",
+	"PATH",
+	"DISABLE_NET_TESTS",
+	"MAKEFLAGS",
+	"GOARM",
+}
+
+type Builder struct {
+	name         string
+	goos, goarch string
+	key          string
+	codeUsername string
+	codePassword string
+}
+
+var (
+	buildroot     = flag.String("buildroot", path.Join(os.TempDir(), "gobuilder"), "Directory under which to build")
+	commitFlag    = flag.Bool("commit", false, "upload information about new commits")
+	dashboard     = flag.String("dashboard", "godashboard.appspot.com", "Go Dashboard Host")
+	buildRelease  = flag.Bool("release", false, "Build and upload binary release archives")
+	buildRevision = flag.String("rev", "", "Build specified revision and exit")
+	buildCmd      = flag.String("cmd", "./all.bash", "Build command (specify absolute or relative to go/src/)")
+	external      = flag.Bool("external", false, "Build external packages")
+	parallel      = flag.Bool("parallel", false, "Build multiple targets in parallel")
+	verbose       = flag.Bool("v", false, "verbose")
+)
+
+var (
+	goroot      string
+	binaryTagRe = regexp.MustCompile(`^(release\.r|weekly\.)[0-9\-.]+`)
+	releaseRe   = regexp.MustCompile(`^release\.r[0-9\-.]+`)
+)
+
+func main() {
+	flag.Usage = func() {
+		fmt.Fprintf(os.Stderr, "usage: %s goos-goarch...\n", os.Args[0])
+		flag.PrintDefaults()
+		os.Exit(2)
+	}
+	flag.Parse()
+	if len(flag.Args()) == 0 && !*commitFlag {
+		flag.Usage()
+	}
+	goroot = path.Join(*buildroot, "goroot")
+	builders := make([]*Builder, len(flag.Args()))
+	for i, builder := range flag.Args() {
+		b, err := NewBuilder(builder)
+		if err != nil {
+			log.Fatal(err)
+		}
+		builders[i] = b
+	}
+
+	// set up work environment
+	if err := os.RemoveAll(*buildroot); err != nil {
+		log.Fatalf("Error removing build root (%s): %s", *buildroot, err)
+	}
+	if err := os.Mkdir(*buildroot, mkdirPerm); err != nil {
+		log.Fatalf("Error making build root (%s): %s", *buildroot, err)
+	}
+	if err := run(nil, *buildroot, "hg", "clone", hgUrl, goroot); err != nil {
+		log.Fatal("Error cloning repository:", err)
+	}
+
+	if *commitFlag {
+		if len(flag.Args()) == 0 {
+			commitWatcher()
+			return
+		}
+		go commitWatcher()
+	}
+
+	// if specified, build revision and return
+	if *buildRevision != "" {
+		hash, err := fullHash(*buildRevision)
+		if err != nil {
+			log.Fatal("Error finding revision: ", err)
+		}
+		for _, b := range builders {
+			if err := b.buildHash(hash); err != nil {
+				log.Println(err)
+			}
+		}
+		return
+	}
+
+	// external package build mode
+	if *external {
+		if len(builders) != 1 {
+			log.Fatal("only one goos-goarch should be specified with -external")
+		}
+		builders[0].buildExternal()
+	}
+
+	// go continuous build mode (default)
+	// check for new commits and build them
+	for {
+		built := false
+		t := time.Nanoseconds()
+		if *parallel {
+			done := make(chan bool)
+			for _, b := range builders {
+				go func(b *Builder) {
+					done <- b.build()
+				}(b)
+			}
+			for _ = range builders {
+				built = <-done || built
+			}
+		} else {
+			for _, b := range builders {
+				built = b.build() || built
+			}
+		}
+		// sleep if there was nothing to build
+		if !built {
+			time.Sleep(waitInterval)
+		}
+		// sleep if we're looping too fast.
+		t1 := time.Nanoseconds() - t
+		if t1 < waitInterval {
+			time.Sleep(waitInterval - t1)
+		}
+	}
+}
+
+func NewBuilder(builder string) (*Builder, os.Error) {
+	b := &Builder{name: builder}
+
+	// get goos/goarch from builder string
+	s := strings.SplitN(builder, "-", 3)
+	if len(s) >= 2 {
+		b.goos, b.goarch = s[0], s[1]
+	} else {
+		return nil, fmt.Errorf("unsupported builder form: %s", builder)
+	}
+
+	// read keys from keyfile
+	fn := path.Join(os.Getenv("HOME"), ".gobuildkey")
+	if s := fn + "-" + b.name; isFile(s) { // builder-specific file
+		fn = s
+	}
+	c, err := ioutil.ReadFile(fn)
+	if err != nil {
+		return nil, fmt.Errorf("readKeys %s (%s): %s", b.name, fn, err)
+	}
+	v := strings.Split(string(c), "\n")
+	b.key = v[0]
+	if len(v) >= 3 {
+		b.codeUsername, b.codePassword = v[1], v[2]
+	}
+
+	return b, nil
+}
+
+// buildExternal downloads and builds external packages, and
+// reports their build status to the dashboard.
+// It will re-build all packages after pkgBuildInterval nanoseconds or
+// a new release tag is found.
+func (b *Builder) buildExternal() {
+	var prevTag string
+	var nextBuild int64
+	for {
+		time.Sleep(waitInterval)
+		err := run(nil, goroot, "hg", "pull", "-u")
+		if err != nil {
+			log.Println("hg pull failed:", err)
+			continue
+		}
+		hash, tag, err := firstTag(releaseRe)
+		if err != nil {
+			log.Println(err)
+			continue
+		}
+		if *verbose {
+			log.Println("latest release:", tag)
+		}
+		// don't rebuild if there's no new release
+		// and it's been less than pkgBuildInterval
+		// nanoseconds since the last build.
+		if tag == prevTag && time.Nanoseconds() < nextBuild {
+			continue
+		}
+		// build will also build the packages
+		if err := b.buildHash(hash); err != nil {
+			log.Println(err)
+			continue
+		}
+		prevTag = tag
+		nextBuild = time.Nanoseconds() + pkgBuildInterval
+	}
+}
+
+// build checks for a new commit for this builder
+// and builds it if one is found. 
+// It returns true if a build was attempted.
+func (b *Builder) build() bool {
+	defer func() {
+		err := recover()
+		if err != nil {
+			log.Println(b.name, "build:", err)
+		}
+	}()
+	hash, err := b.todo()
+	if err != nil {
+		log.Println(err)
+		return false
+	}
+	if hash == "" {
+		return false
+	}
+	// Look for hash locally before running hg pull.
+
+	if _, err := fullHash(hash[:12]); err != nil {
+		// Don't have hash, so run hg pull.
+		if err := run(nil, goroot, "hg", "pull"); err != nil {
+			log.Println("hg pull failed:", err)
+			return false
+		}
+	}
+	err = b.buildHash(hash)
+	if err != nil {
+		log.Println(err)
+	}
+	return true
+}
+
+func (b *Builder) buildHash(hash string) (err os.Error) {
+	defer func() {
+		if err != nil {
+			err = fmt.Errorf("%s build: %s: %s", b.name, hash, err)
+		}
+	}()
+
+	log.Println(b.name, "building", hash)
+
+	// create place in which to do work
+	workpath := path.Join(*buildroot, b.name+"-"+hash[:12])
+	err = os.Mkdir(workpath, mkdirPerm)
+	if err != nil {
+		return
+	}
+	defer os.RemoveAll(workpath)
+
+	// clone repo
+	err = run(nil, workpath, "hg", "clone", goroot, "go")
+	if err != nil {
+		return
+	}
+
+	// update to specified revision
+	err = run(nil, path.Join(workpath, "go"),
+		"hg", "update", hash)
+	if err != nil {
+		return
+	}
+
+	srcDir := path.Join(workpath, "go", "src")
+
+	// build
+	logfile := path.Join(workpath, "build.log")
+	buildLog, status, err := runLog(b.envv(), logfile, srcDir, *buildCmd)
+	if err != nil {
+		return fmt.Errorf("%s: %s", *buildCmd, err)
+	}
+
+	// if we're in external mode, build all packages and return
+	if *external {
+		if status != 0 {
+			return os.NewError("go build failed")
+		}
+		return b.buildPackages(workpath, hash)
+	}
+
+	if status != 0 {
+		// record failure
+		return b.recordResult(buildLog, hash)
+	}
+
+	// record success
+	if err = b.recordResult("", hash); err != nil {
+		return fmt.Errorf("recordResult: %s", err)
+	}
+
+	// finish here if codeUsername and codePassword aren't set
+	if b.codeUsername == "" || b.codePassword == "" || !*buildRelease {
+		return
+	}
+
+	// if this is a release, create tgz and upload to google code
+	releaseHash, release, err := firstTag(binaryTagRe)
+	if hash == releaseHash {
+		// clean out build state
+		err = run(b.envv(), srcDir, "./clean.bash", "--nopkg")
+		if err != nil {
+			return fmt.Errorf("clean.bash: %s", err)
+		}
+		// upload binary release
+		fn := fmt.Sprintf("go.%s.%s-%s.tar.gz", release, b.goos, b.goarch)
+		err = run(nil, workpath, "tar", "czf", fn, "go")
+		if err != nil {
+			return fmt.Errorf("tar: %s", err)
+		}
+		err = run(nil, workpath, path.Join(goroot, codePyScript),
+			"-s", release,
+			"-p", codeProject,
+			"-u", b.codeUsername,
+			"-w", b.codePassword,
+			"-l", fmt.Sprintf("%s,%s", b.goos, b.goarch),
+			fn)
+	}
+
+	return
+}
+
+// envv returns an environment for build/bench execution
+func (b *Builder) envv() []string {
+	if runtime.GOOS == "windows" {
+		return b.envvWindows()
+	}
+	e := []string{
+		"GOOS=" + b.goos,
+		"GOARCH=" + b.goarch,
+		"GOROOT_FINAL=/usr/local/go",
+	}
+	for _, k := range extraEnv {
+		s, err := os.Getenverror(k)
+		if err == nil {
+			e = append(e, k+"="+s)
+		}
+	}
+	return e
+}
+
+// windows version of envv
+func (b *Builder) envvWindows() []string {
+	start := map[string]string{
+		"GOOS":         b.goos,
+		"GOARCH":       b.goarch,
+		"GOROOT_FINAL": "/c/go",
+		// TODO(brainman): remove once we find make that does not hang.
+		"MAKEFLAGS": "-j1",
+	}
+	for _, name := range extraEnv {
+		s, err := os.Getenverror(name)
+		if err == nil {
+			start[name] = s
+		}
+	}
+	skip := map[string]bool{
+		"GOBIN":   true,
+		"GOROOT":  true,
+		"INCLUDE": true,
+		"LIB":     true,
+	}
+	var e []string
+	for name, v := range start {
+		e = append(e, name+"="+v)
+		skip[name] = true
+	}
+	for _, kv := range os.Environ() {
+		s := strings.SplitN(kv, "=", 2)
+		name := strings.ToUpper(s[0])
+		switch {
+		case name == "":
+			// variables, like "=C:=C:\", just copy them
+			e = append(e, kv)
+		case !skip[name]:
+			e = append(e, kv)
+			skip[name] = true
+		}
+	}
+	return e
+}
+
+func isDirectory(name string) bool {
+	s, err := os.Stat(name)
+	return err == nil && s.IsDirectory()
+}
+
+func isFile(name string) bool {
+	s, err := os.Stat(name)
+	return err == nil && (s.IsRegular() || s.IsSymlink())
+}
+
+// commitWatcher polls hg for new commits and tells the dashboard about them.
+func commitWatcher() {
+	// Create builder just to get master key.
+	b, err := NewBuilder("mercurial-commit")
+	if err != nil {
+		log.Fatal(err)
+	}
+	for {
+		if *verbose {
+			log.Printf("poll...")
+		}
+		commitPoll(b.key)
+		if *verbose {
+			log.Printf("sleep...")
+		}
+		time.Sleep(60e9)
+	}
+}
+
+// HgLog represents a single Mercurial revision.
+type HgLog struct {
+	Hash   string
+	Author string
+	Date   string
+	Desc   string
+	Parent string
+
+	// Internal metadata
+	added bool
+}
+
+// logByHash is a cache of all Mercurial revisions we know about,
+// indexed by full hash.
+var logByHash = map[string]*HgLog{}
+
+// xmlLogTemplate is a template to pass to Mercurial to make
+// hg log print the log in valid XML for parsing with xml.Unmarshal.
+const xmlLogTemplate = `
+	<log>
+	<hash>{node|escape}</hash>
+	<parent>{parent|escape}</parent>
+	<author>{author|escape}</author>
+	<date>{date}</date>
+	<desc>{desc|escape}</desc>
+	</log>
+`
+
+// commitPoll pulls any new revisions from the hg server
+// and tells the server about them.
+func commitPoll(key string) {
+	// Catch unexpected panics.
+	defer func() {
+		if err := recover(); err != nil {
+			log.Printf("commitPoll panic: %s", err)
+		}
+	}()
+
+	if err := run(nil, goroot, "hg", "pull"); err != nil {
+		log.Printf("hg pull: %v", err)
+		return
+	}
+
+	const N = 20 // how many revisions to grab
+
+	data, _, err := runLog(nil, "", goroot, "hg", "log",
+		"--encoding=utf-8",
+		"--limit="+strconv.Itoa(N),
+		"--template="+xmlLogTemplate,
+	)
+	if err != nil {
+		log.Printf("hg log: %v", err)
+		return
+	}
+
+	var logStruct struct {
+		Log []HgLog
+	}
+	err = xml.Unmarshal(strings.NewReader("<top>"+data+"</top>"), &logStruct)
+	if err != nil {
+		log.Printf("unmarshal hg log: %v", err)
+		return
+	}
+
+	logs := logStruct.Log
+
+	// Pass 1.  Fill in parents and add new log entries to logsByHash.
+	// Empty parent means take parent from next log entry.
+	// Non-empty parent has form 1234:hashhashhash; we want full hash.
+	for i := range logs {
+		l := &logs[i]
+		log.Printf("hg log: %s < %s\n", l.Hash, l.Parent)
+		if l.Parent == "" && i+1 < len(logs) {
+			l.Parent = logs[i+1].Hash
+		} else if l.Parent != "" {
+			l.Parent, _ = fullHash(l.Parent)
+		}
+		if l.Parent == "" {
+			// Can't create node without parent.
+			continue
+		}
+
+		if logByHash[l.Hash] == nil {
+			// Make copy to avoid pinning entire slice when only one entry is new.
+			t := *l
+			logByHash[t.Hash] = &t
+		}
+	}
+
+	for i := range logs {
+		l := &logs[i]
+		if l.Parent == "" {
+			continue
+		}
+		addCommit(l.Hash, key)
+	}
+}
+
+// addCommit adds the commit with the named hash to the dashboard.
+// key is the secret key for authentication to the dashboard.
+// It avoids duplicate effort.
+func addCommit(hash, key string) bool {
+	l := logByHash[hash]
+	if l == nil {
+		return false
+	}
+	if l.added {
+		return true
+	}
+
+	// Check for already added, perhaps in an earlier run.
+	if dashboardCommit(hash) {
+		log.Printf("%s already on dashboard\n", hash)
+		// Record that this hash is on the dashboard,
+		// as must be all its parents.
+		for l != nil {
+			l.added = true
+			l = logByHash[l.Parent]
+		}
+		return true
+	}
+
+	// Create parent first, to maintain some semblance of order.
+	if !addCommit(l.Parent, key) {
+		return false
+	}
+
+	// Create commit.
+	if err := postCommit(key, l); err != nil {
+		log.Printf("failed to add %s to dashboard: %v", key, err)
+		return false
+	}
+	return true
+}
+
+// fullHash returns the full hash for the given Mercurial revision.
+func fullHash(rev string) (hash string, err os.Error) {
+	defer func() {
+		if err != nil {
+			err = fmt.Errorf("fullHash: %s: %s", rev, err)
+		}
+	}()
+	s, _, err := runLog(nil, "", goroot,
+		"hg", "log",
+		"--encoding=utf-8",
+		"--rev="+rev,
+		"--limit=1",
+		"--template={node}",
+	)
+	if err != nil {
+		return
+	}
+	s = strings.TrimSpace(s)
+	if s == "" {
+		return "", fmt.Errorf("cannot find revision")
+	}
+	if len(s) != 40 {
+		return "", fmt.Errorf("hg returned invalid hash " + s)
+	}
+	return s, nil
+}
+
+var revisionRe = regexp.MustCompile(`^([^ ]+) +[0-9]+:([0-9a-f]+)$`)
+
+// firstTag returns the hash and tag of the most recent tag matching re.
+func firstTag(re *regexp.Regexp) (hash string, tag string, err os.Error) {
+	o, _, err := runLog(nil, "", goroot, "hg", "tags")
+	for _, l := range strings.Split(o, "\n") {
+		if l == "" {
+			continue
+		}
+		s := revisionRe.FindStringSubmatch(l)
+		if s == nil {
+			err = os.NewError("couldn't find revision number")
+			return
+		}
+		if !re.MatchString(s[1]) {
+			continue
+		}
+		tag = s[1]
+		hash, err = fullHash(s[2])
+		return
+	}
+	err = os.NewError("no matching tag found")
+	return
+}
diff --git a/misc/dashboard/builder/package.go b/misc/dashboard/builder/package.go
new file mode 100644
index 000000000..b2a83fa13
--- /dev/null
+++ b/misc/dashboard/builder/package.go
@@ -0,0 +1,94 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+	"go/doc"
+	"go/parser"
+	"go/token"
+	"log"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+const MaxCommentLength = 500 // App Engine won't store more in a StringProperty.
+
+func (b *Builder) buildPackages(workpath string, hash string) os.Error {
+	pkgs, err := packages()
+	if err != nil {
+		return err
+	}
+	for _, p := range pkgs {
+		goroot := filepath.Join(workpath, "go")
+		gobin := filepath.Join(goroot, "bin")
+		goinstall := filepath.Join(gobin, "goinstall")
+		envv := append(b.envv(), "GOROOT="+goroot)
+
+		// add GOBIN to path
+		for i, v := range envv {
+			if strings.HasPrefix(v, "PATH=") {
+				p := filepath.SplitList(v[5:])
+				p = append([]string{gobin}, p...)
+				s := strings.Join(p, string(filepath.ListSeparator))
+				envv[i] = "PATH=" + s
+			}
+		}
+
+		// goinstall
+		buildLog, code, err := runLog(envv, "", goroot, goinstall, "-dashboard=false", p)
+		if err != nil {
+			log.Printf("goinstall %v: %v", p, err)
+		}
+
+		// get doc comment from package source
+		info, err := packageComment(p, filepath.Join(goroot, "src", "pkg", p))
+		if err != nil {
+			log.Printf("packageComment %v: %v", p, err)
+		}
+
+		// update dashboard with build state + info
+		err = b.updatePackage(p, code == 0, buildLog, info)
+		if err != nil {
+			log.Printf("updatePackage %v: %v", p, err)
+		}
+	}
+	return nil
+}
+
+func isGoFile(fi *os.FileInfo) bool {
+	return fi.IsRegular() && // exclude directories
+		!strings.HasPrefix(fi.Name, ".") && // ignore .files
+		filepath.Ext(fi.Name) == ".go"
+}
+
+func packageComment(pkg, pkgpath string) (info string, err os.Error) {
+	fset := token.NewFileSet()
+	pkgs, err := parser.ParseDir(fset, pkgpath, isGoFile, parser.PackageClauseOnly|parser.ParseComments)
+	if err != nil {
+		return
+	}
+	for name := range pkgs {
+		if name == "main" {
+			continue
+		}
+		if info != "" {
+			return "", os.NewError("multiple non-main package docs")
+		}
+		pdoc := doc.NewPackageDoc(pkgs[name], pkg)
+		info = pdoc.Doc
+	}
+	// grab only first paragraph
+	if parts := strings.SplitN(info, "\n\n", 2); len(parts) > 1 {
+		info = parts[0]
+	}
+	// replace newlines with spaces
+	info = strings.Replace(info, "\n", " ", -1)
+	// truncate
+	if len(info) > MaxCommentLength {
+		info = info[:MaxCommentLength]
+	}
+	return
+}