Imported Upstream version 2012.01.27upstream-weekly/2012.01.27

4 files changed, 300 insertions, 147 deletions

diff --git a/misc/dashboard/builder/exec.go b/misc/dashboard/builder/exec.go
index a042c5699..7f21abaa2 100644
--- a/misc/dashboard/builder/exec.go
+++ b/misc/dashboard/builder/exec.go
@@ -6,15 +6,15 @@ package main
 
 import (
 	"bytes"
-	"exec"
 	"io"
 	"log"
 	"os"
+	"os/exec"
 	"strings"
 )
 
 // run is a simple wrapper for exec.Run/Close
-func run(envv []string, dir string, argv ...string) os.Error {
+func run(envv []string, dir string, argv ...string) error {
 	if *verbose {
 		log.Println("run", argv)
 	}
@@ -31,7 +31,7 @@ func run(envv []string, dir string, argv ...string) os.Error {
 // process combined stdout and stderr output, exit status and error.
 // The error returned is nil, if process is started successfully,
 // even if exit status is not 0.
-func runLog(envv []string, logfile, dir string, argv ...string) (string, int, os.Error) {
+func runLog(envv []string, logfile, dir string, argv ...string) (string, int, error) {
 	if *verbose {
 		log.Println("runLog", argv)
 	}
@@ -56,11 +56,11 @@ func runLog(envv []string, logfile, dir string, argv ...string) (string, int, os
 
 	err := cmd.Run()
 	if err != nil {
-		if ws, ok := err.(*os.Waitmsg); ok {
+		if ws, ok := err.(*exec.ExitError); ok {
 			return b.String(), ws.ExitStatus(), nil
 		}
 	}
-	return b.String(), 0, nil
+	return b.String(), 0, err
 }
 
 // useBash prefixes a list of args with 'bash' if the first argument
diff --git a/misc/dashboard/builder/http.go b/misc/dashboard/builder/http.go
index abef8faa4..b25b417e1 100644
--- a/misc/dashboard/builder/http.go
+++ b/misc/dashboard/builder/http.go
@@ -6,98 +6,129 @@ package main
 
 import (
 	"bytes"
+	"encoding/json"
+	"errors"
 	"fmt"
-	"http"
-	"json"
+	"io"
 	"log"
-	"os"
-	"strconv"
-	"url"
+	"net/http"
+	"net/url"
+	"time"
 )
 
-type param map[string]string
+type obj map[string]interface{}
 
 // dash runs the given method and command on the dashboard.
-// If args is not nil, it is the query or post parameters.
-// If resp is not nil, dash unmarshals the body as JSON into resp.
-func dash(meth, cmd string, resp interface{}, args param) os.Error {
+// If args is non-nil it is encoded as the URL query string.
+// If req is non-nil it is JSON-encoded and passed as the body of the HTTP POST.
+// If resp is non-nil the server's response is decoded into the value pointed
+// to by resp (resp must be a pointer).
+func dash(meth, cmd string, args url.Values, req, resp interface{}) error {
 	var r *http.Response
-	var err os.Error
+	var err error
 	if *verbose {
-		log.Println("dash", cmd, args)
+		log.Println("dash", meth, cmd, args, req)
 	}
 	cmd = "http://" + *dashboard + "/" + cmd
-	vals := make(url.Values)
-	for k, v := range args {
-		vals.Add(k, v)
+	if len(args) > 0 {
+		cmd += "?" + args.Encode()
 	}
 	switch meth {
 	case "GET":
-		if q := vals.Encode(); q != "" {
-			cmd += "?" + q
+		if req != nil {
+			log.Panicf("%s to %s with req", meth, cmd)
 		}
 		r, err = http.Get(cmd)
 	case "POST":
-		r, err = http.PostForm(cmd, vals)
+		var body io.Reader
+		if req != nil {
+			b, err := json.Marshal(req)
+			if err != nil {
+				return err
+			}
+			body = bytes.NewBuffer(b)
+		}
+		r, err = http.Post(cmd, "text/json", body)
 	default:
-		return fmt.Errorf("unknown method %q", meth)
+		log.Panicf("%s: invalid method %q", cmd, meth)
+		panic("invalid method: " + meth)
 	}
 	if err != nil {
 		return err
 	}
+
 	defer r.Body.Close()
-	var buf bytes.Buffer
-	buf.ReadFrom(r.Body)
-	if resp != nil {
-		if err = json.Unmarshal(buf.Bytes(), resp); err != nil {
-			log.Printf("json unmarshal %#q: %s\n", buf.Bytes(), err)
-			return err
-		}
+	body := new(bytes.Buffer)
+	if _, err := body.ReadFrom(r.Body); err != nil {
+		return err
 	}
-	return nil
-}
 
-func dashStatus(meth, cmd string, args param) os.Error {
-	var resp struct {
-		Status string
-		Error  string
+	// Read JSON-encoded Response into provided resp
+	// and return an error if present.
+	var result = struct {
+		Response interface{}
+		Error    string
+	}{
+		// Put the provided resp in here as it can be a pointer to
+		// some value we should unmarshal into.
+		Response: resp,
 	}
-	err := dash(meth, cmd, &resp, args)
-	if err != nil {
+	if err = json.Unmarshal(body.Bytes(), &result); err != nil {
+		log.Printf("json unmarshal %#q: %s\n", body.Bytes(), err)
 		return err
 	}
-	if resp.Status != "OK" {
-		return os.NewError("/build: " + resp.Error)
+	if result.Error != "" {
+		return errors.New(result.Error)
 	}
+
 	return nil
 }
 
 // todo returns the next hash to build.
-func (b *Builder) todo() (rev string, err os.Error) {
-	var resp []struct {
-		Hash string
+func (b *Builder) todo(kind, pkg, goHash string) (rev string, err error) {
+	args := url.Values{
+		"kind":        {kind},
+		"builder":     {b.name},
+		"packagePath": {pkg},
+		"goHash":      {goHash},
 	}
-	if err = dash("GET", "todo", &resp, param{"builder": b.name}); err != nil {
-		return
+	var resp *struct {
+		Kind string
+		Data struct {
+			Hash string
+		}
 	}
-	if len(resp) > 0 {
-		rev = resp[0].Hash
+	if err = dash("GET", "todo", args, nil, &resp); err != nil {
+		return "", err
 	}
-	return
+	if resp == nil {
+		return "", nil
+	}
+	if kind != resp.Kind {
+		return "", fmt.Errorf("expecting Kind %q, got %q", kind, resp.Kind)
+	}
+	return resp.Data.Hash, nil
 }
 
 // recordResult sends build results to the dashboard
-func (b *Builder) recordResult(buildLog string, hash string) os.Error {
-	return dash("POST", "build", nil, param{
-		"builder": b.name,
-		"key":     b.key,
-		"node":    hash,
-		"log":     buildLog,
-	})
+func (b *Builder) recordResult(ok bool, pkg, hash, goHash, buildLog string, runTime time.Duration) error {
+	req := obj{
+		"Builder":     b.name,
+		"PackagePath": pkg,
+		"Hash":        hash,
+		"GoHash":      goHash,
+		"OK":          ok,
+		"Log":         buildLog,
+		"RunTime":     runTime,
+	}
+	args := url.Values{"key": {b.key}, "builder": {b.name}}
+	return dash("POST", "result", args, req, nil)
 }
 
 // packages fetches a list of package paths from the dashboard
-func packages() (pkgs []string, err os.Error) {
+func packages() (pkgs []string, err error) {
+	return nil, nil
+	/* TODO(adg): un-stub this once the new package builder design is done
 	var resp struct {
 		Packages []struct {
 			Path string
@@ -111,38 +142,58 @@ func packages() (pkgs []string, err os.Error) {
 		pkgs = append(pkgs, p.Path)
 	}
 	return
+	*/
 }
 
-// updatePackage sends package build results and info dashboard
-func (b *Builder) updatePackage(pkg string, ok bool, buildLog, info string) os.Error {
+// updatePackage sends package build results and info to the dashboard
+func (b *Builder) updatePackage(pkg string, ok bool, buildLog, info string) error {
+	return nil
+	/* TODO(adg): un-stub this once the new package builder design is done
 	return dash("POST", "package", nil, param{
 		"builder": b.name,
 		"key":     b.key,
 		"path":    pkg,
-		"ok":      strconv.Btoa(ok),
+		"ok":      strconv.FormatBool(ok),
 		"log":     buildLog,
 		"info":    info,
 	})
+	*/
 }
 
-// postCommit informs the dashboard of a new commit
-func postCommit(key string, l *HgLog) os.Error {
-	return dashStatus("POST", "commit", param{
-		"key":    key,
-		"node":   l.Hash,
-		"date":   l.Date,
-		"user":   l.Author,
-		"parent": l.Parent,
-		"desc":   l.Desc,
-	})
+func postCommit(key, pkg string, l *HgLog) error {
+	t, err := time.Parse(time.RFC3339, l.Date)
+	if err != nil {
+		return fmt.Errorf("parsing %q: %v", l.Date, t)
+	}
+	return dash("POST", "commit", url.Values{"key": {key}}, obj{
+		"PackagePath": pkg,
+		"Hash":        l.Hash,
+		"ParentHash":  l.Parent,
+		"Time":        t.Unix() * 1e6, // in microseconds, yuck!
+		"User":        l.Author,
+		"Desc":        l.Desc,
+	}, nil)
 }
 
-// dashboardCommit returns true if the dashboard knows about hash.
-func dashboardCommit(hash string) bool {
-	err := dashStatus("GET", "commit", param{"node": hash})
-	if err != nil {
-		log.Printf("check %s: %s", hash, err)
-		return false
+func dashboardCommit(pkg, hash string) bool {
+	err := dash("GET", "commit", url.Values{
+		"packagePath": {pkg},
+		"hash":        {hash},
+	}, nil, nil)
+	return err == nil
+}
+
+func dashboardPackages() []string {
+	var resp []struct {
+		Path string
+	}
+	if err := dash("GET", "packages", nil, nil, &resp); err != nil {
+		log.Println("dashboardPackages:", err)
+		return nil
+	}
+	var pkgs []string
+	for _, r := range resp {
+		pkgs = append(pkgs, r.Path)
 	}
-	return true
+	return pkgs
 }
diff --git a/misc/dashboard/builder/main.go b/misc/dashboard/builder/main.go
index a5479846d..3556a457d 100644
--- a/misc/dashboard/builder/main.go
+++ b/misc/dashboard/builder/main.go
@@ -5,38 +5,39 @@
 package main
 
 import (
+	"encoding/xml"
+	"errors"
 	"flag"
 	"fmt"
 	"io/ioutil"
 	"log"
 	"os"
 	"path"
+	"path/filepath"
 	"regexp"
 	"runtime"
 	"strconv"
 	"strings"
 	"time"
-	"xml"
 )
 
 const (
 	codeProject      = "go"
 	codePyScript     = "misc/dashboard/googlecode_upload.py"
 	hgUrl            = "https://go.googlecode.com/hg/"
-	waitInterval     = 30e9 // time to wait before checking for new revs
 	mkdirPerm        = 0750
-	pkgBuildInterval = 1e9 * 60 * 60 * 24 // rebuild packages every 24 hours
+	waitInterval     = 30 * time.Second // time to wait before checking for new revs
+	pkgBuildInterval = 24 * time.Hour   // rebuild packages every 24 hours
 )
 
 // These variables are copied from the gobuilder's environment
 // to the envv of its subprocesses.
 var extraEnv = []string{
-	"GOHOSTOS",
+	"GOARM",
 	"GOHOSTARCH",
+	"GOHOSTOS",
 	"PATH",
-	"DISABLE_NET_TESTS",
-	"MAKEFLAGS",
-	"GOARM",
+	"TMPDIR",
 }
 
 type Builder struct {
@@ -50,7 +51,7 @@ type Builder struct {
 var (
 	buildroot     = flag.String("buildroot", path.Join(os.TempDir(), "gobuilder"), "Directory under which to build")
 	commitFlag    = flag.Bool("commit", false, "upload information about new commits")
-	dashboard     = flag.String("dashboard", "godashboard.appspot.com", "Go Dashboard Host")
+	dashboard     = flag.String("dashboard", "build.golang.org", "Go Dashboard Host")
 	buildRelease  = flag.Bool("release", false, "Build and upload binary release archives")
 	buildRevision = flag.String("rev", "", "Build specified revision and exit")
 	buildCmd      = flag.String("cmd", "./all.bash", "Build command (specify absolute or relative to go/src/)")
@@ -92,7 +93,7 @@ func main() {
 	if err := os.Mkdir(*buildroot, mkdirPerm); err != nil {
 		log.Fatalf("Error making build root (%s): %s", *buildroot, err)
 	}
-	if err := run(nil, *buildroot, "hg", "clone", hgUrl, goroot); err != nil {
+	if err := hgClone(hgUrl, goroot); err != nil {
 		log.Fatal("Error cloning repository:", err)
 	}
 
@@ -106,7 +107,7 @@ func main() {
 
 	// if specified, build revision and return
 	if *buildRevision != "" {
-		hash, err := fullHash(*buildRevision)
+		hash, err := fullHash(goroot, *buildRevision)
 		if err != nil {
 			log.Fatal("Error finding revision: ", err)
 		}
@@ -130,7 +131,7 @@ func main() {
 	// check for new commits and build them
 	for {
 		built := false
-		t := time.Nanoseconds()
+		t := time.Now()
 		if *parallel {
 			done := make(chan bool)
 			for _, b := range builders {
@@ -151,14 +152,14 @@ func main() {
 			time.Sleep(waitInterval)
 		}
 		// sleep if we're looping too fast.
-		t1 := time.Nanoseconds() - t
-		if t1 < waitInterval {
-			time.Sleep(waitInterval - t1)
+		dt := time.Now().Sub(t)
+		if dt < waitInterval {
+			time.Sleep(waitInterval - dt)
 		}
 	}
 }
 
-func NewBuilder(builder string) (*Builder, os.Error) {
+func NewBuilder(builder string) (*Builder, error) {
 	b := &Builder{name: builder}
 
 	// get goos/goarch from builder string
@@ -193,7 +194,7 @@ func NewBuilder(builder string) (*Builder, os.Error) {
 // a new release tag is found.
 func (b *Builder) buildExternal() {
 	var prevTag string
-	var nextBuild int64
+	var nextBuild time.Time
 	for {
 		time.Sleep(waitInterval)
 		err := run(nil, goroot, "hg", "pull", "-u")
@@ -212,7 +213,7 @@ func (b *Builder) buildExternal() {
 		// don't rebuild if there's no new release
 		// and it's been less than pkgBuildInterval
 		// nanoseconds since the last build.
-		if tag == prevTag && time.Nanoseconds() < nextBuild {
+		if tag == prevTag && time.Now().Before(nextBuild) {
 			continue
 		}
 		// build will also build the packages
@@ -221,7 +222,7 @@ func (b *Builder) buildExternal() {
 			continue
 		}
 		prevTag = tag
-		nextBuild = time.Nanoseconds() + pkgBuildInterval
+		nextBuild = time.Now().Add(pkgBuildInterval)
 	}
 }
 
@@ -235,7 +236,7 @@ func (b *Builder) build() bool {
 			log.Println(b.name, "build:", err)
 		}
 	}()
-	hash, err := b.todo()
+	hash, err := b.todo("build-go-commit", "", "")
 	if err != nil {
 		log.Println(err)
 		return false
@@ -245,7 +246,7 @@ func (b *Builder) build() bool {
 	}
 	// Look for hash locally before running hg pull.
 
-	if _, err := fullHash(hash[:12]); err != nil {
+	if _, err := fullHash(goroot, hash[:12]); err != nil {
 		// Don't have hash, so run hg pull.
 		if err := run(nil, goroot, "hg", "pull"); err != nil {
 			log.Println("hg pull failed:", err)
@@ -259,7 +260,7 @@ func (b *Builder) build() bool {
 	return true
 }
 
-func (b *Builder) buildHash(hash string) (err os.Error) {
+func (b *Builder) buildHash(hash string) (err error) {
 	defer func() {
 		if err != nil {
 			err = fmt.Errorf("%s build: %s: %s", b.name, hash, err)
@@ -283,8 +284,7 @@ func (b *Builder) buildHash(hash string) (err os.Error) {
 	}
 
 	// update to specified revision
-	err = run(nil, path.Join(workpath, "go"),
-		"hg", "update", hash)
+	err = run(nil, path.Join(workpath, "go"), "hg", "update", hash)
 	if err != nil {
 		return
 	}
@@ -293,7 +293,9 @@ func (b *Builder) buildHash(hash string) (err os.Error) {
 
 	// build
 	logfile := path.Join(workpath, "build.log")
+	startTime := time.Now()
 	buildLog, status, err := runLog(b.envv(), logfile, srcDir, *buildCmd)
+	runTime := time.Now().Sub(startTime)
 	if err != nil {
 		return fmt.Errorf("%s: %s", *buildCmd, err)
 	}
@@ -301,21 +303,24 @@ func (b *Builder) buildHash(hash string) (err os.Error) {
 	// if we're in external mode, build all packages and return
 	if *external {
 		if status != 0 {
-			return os.NewError("go build failed")
+			return errors.New("go build failed")
 		}
-		return b.buildPackages(workpath, hash)
+		return b.buildExternalPackages(workpath, hash)
 	}
 
 	if status != 0 {
 		// record failure
-		return b.recordResult(buildLog, hash)
+		return b.recordResult(false, "", hash, "", buildLog, runTime)
 	}
 
 	// record success
-	if err = b.recordResult("", hash); err != nil {
+	if err = b.recordResult(true, "", hash, "", "", runTime); err != nil {
 		return fmt.Errorf("recordResult: %s", err)
 	}
 
+	// build goinstallable packages
+	b.buildPackages(filepath.Join(workpath, "go"), hash)
+
 	// finish here if codeUsername and codePassword aren't set
 	if b.codeUsername == "" || b.codePassword == "" || !*buildRelease {
 		return
@@ -342,11 +347,67 @@ func (b *Builder) buildHash(hash string) (err os.Error) {
 			"-w", b.codePassword,
 			"-l", fmt.Sprintf("%s,%s", b.goos, b.goarch),
 			fn)
+		if err != nil {
+			return fmt.Errorf("%s: %s", codePyScript, err)
+		}
 	}
 
 	return
 }
 
+func (b *Builder) buildPackages(goRoot, goHash string) {
+	for _, pkg := range dashboardPackages() {
+		// get the latest todo for this package
+		hash, err := b.todo("build-package", pkg, goHash)
+		if err != nil {
+			log.Printf("buildPackages %s: %v", pkg, err)
+			continue
+		}
+		if hash == "" {
+			continue
+		}
+
+		// goinstall the package
+		if *verbose {
+			log.Printf("buildPackages %s: installing %q", pkg, hash)
+		}
+		buildLog, err := b.goinstall(goRoot, pkg, hash)
+		ok := buildLog == ""
+		if err != nil {
+			ok = false
+			log.Printf("buildPackages %s: %v", pkg, err)
+		}
+
+		// record the result
+		err = b.recordResult(ok, pkg, hash, goHash, buildLog, 0)
+		if err != nil {
+			log.Printf("buildPackages %s: %v", pkg, err)
+		}
+	}
+}
+
+func (b *Builder) goinstall(goRoot, pkg, hash string) (string, error) {
+	bin := filepath.Join(goRoot, "bin/goinstall")
+	env := append(b.envv(), "GOROOT="+goRoot)
+
+	// fetch package and dependencies
+	log, status, err := runLog(env, "", goRoot, bin,
+		"-dashboard=false", "-install=false", pkg)
+	if err != nil || status != 0 {
+		return log, err
+	}
+
+	// hg update to the specified hash
+	pkgPath := filepath.Join(goRoot, "src/pkg", pkg)
+	if err := run(nil, pkgPath, "hg", "update", hash); err != nil {
+		return "", err
+	}
+
+	// build the package
+	log, _, err = runLog(env, "", goRoot, bin, "-dashboard=false", pkg)
+	return log, err
+}
+
 // envv returns an environment for build/bench execution
 func (b *Builder) envv() []string {
 	if runtime.GOOS == "windows" {
@@ -409,12 +470,12 @@ func (b *Builder) envvWindows() []string {
 
 func isDirectory(name string) bool {
 	s, err := os.Stat(name)
-	return err == nil && s.IsDirectory()
+	return err == nil && s.IsDir()
 }
 
 func isFile(name string) bool {
 	s, err := os.Stat(name)
-	return err == nil && (s.IsRegular() || s.IsSymlink())
+	return err == nil && !s.IsDir()
 }
 
 // commitWatcher polls hg for new commits and tells the dashboard about them.
@@ -424,11 +485,16 @@ func commitWatcher() {
 	if err != nil {
 		log.Fatal(err)
 	}
+	key := b.key
+
 	for {
 		if *verbose {
 			log.Printf("poll...")
 		}
-		commitPoll(b.key)
+		commitPoll(key, "")
+		for _, pkg := range dashboardPackages() {
+			commitPoll(key, pkg)
+		}
 		if *verbose {
 			log.Printf("sleep...")
 		}
@@ -436,6 +502,18 @@ func commitWatcher() {
 	}
 }
 
+func hgClone(url, path string) error {
+	return run(nil, *buildroot, "hg", "clone", url, path)
+}
+
+func hgRepoExists(path string) bool {
+	fi, err := os.Stat(filepath.Join(path, ".hg"))
+	if err != nil {
+		return false
+	}
+	return fi.IsDir()
+}
+
 // HgLog represents a single Mercurial revision.
 type HgLog struct {
 	Hash   string
@@ -455,18 +533,18 @@ var logByHash = map[string]*HgLog{}
 // xmlLogTemplate is a template to pass to Mercurial to make
 // hg log print the log in valid XML for parsing with xml.Unmarshal.
 const xmlLogTemplate = `
-	<log>
-	<hash>{node|escape}</hash>
-	<parent>{parent|escape}</parent>
-	<author>{author|escape}</author>
-	<date>{date}</date>
-	<desc>{desc|escape}</desc>
-	</log>
+	<Log>
+	<Hash>{node|escape}</Hash>
+	<Parent>{parent|escape}</Parent>
+	<Author>{author|escape}</Author>
+	<Date>{date|rfc3339date}</Date>
+	<Desc>{desc|escape}</Desc>
+	</Log>
 `
 
 // commitPoll pulls any new revisions from the hg server
 // and tells the server about them.
-func commitPoll(key string) {
+func commitPoll(key, pkg string) {
 	// Catch unexpected panics.
 	defer func() {
 		if err := recover(); err != nil {
@@ -474,14 +552,29 @@ func commitPoll(key string) {
 		}
 	}()
 
-	if err := run(nil, goroot, "hg", "pull"); err != nil {
+	pkgRoot := goroot
+
+	if pkg != "" {
+		pkgRoot = path.Join(*buildroot, pkg)
+		if !hgRepoExists(pkgRoot) {
+			if err := hgClone(repoURL(pkg), pkgRoot); err != nil {
+				log.Printf("%s: hg clone failed: %v", pkg, err)
+				if err := os.RemoveAll(pkgRoot); err != nil {
+					log.Printf("%s: %v", pkg, err)
+				}
+				return
+			}
+		}
+	}
+
+	if err := run(nil, pkgRoot, "hg", "pull"); err != nil {
 		log.Printf("hg pull: %v", err)
 		return
 	}
 
 	const N = 50 // how many revisions to grab
 
-	data, _, err := runLog(nil, "", goroot, "hg", "log",
+	data, _, err := runLog(nil, "", pkgRoot, "hg", "log",
 		"--encoding=utf-8",
 		"--limit="+strconv.Itoa(N),
 		"--template="+xmlLogTemplate,
@@ -494,7 +587,7 @@ func commitPoll(key string) {
 	var logStruct struct {
 		Log []HgLog
 	}
-	err = xml.Unmarshal(strings.NewReader("<top>"+data+"</top>"), &logStruct)
+	err = xml.Unmarshal([]byte("<Top>"+data+"</Top>"), &logStruct)
 	if err != nil {
 		log.Printf("unmarshal hg log: %v", err)
 		return
@@ -510,14 +603,11 @@ func commitPoll(key string) {
 		if l.Parent == "" && i+1 < len(logs) {
 			l.Parent = logs[i+1].Hash
 		} else if l.Parent != "" {
-			l.Parent, _ = fullHash(l.Parent)
+			l.Parent, _ = fullHash(pkgRoot, l.Parent)
 		}
-		log.Printf("hg log: %s < %s\n", l.Hash, l.Parent)
-		if l.Parent == "" {
-			// Can't create node without parent.
-			continue
+		if *verbose {
+			log.Printf("hg log %s: %s < %s\n", pkg, l.Hash, l.Parent)
 		}
-
 		if logByHash[l.Hash] == nil {
 			// Make copy to avoid pinning entire slice when only one entry is new.
 			t := *l
@@ -527,17 +617,14 @@ func commitPoll(key string) {
 
 	for i := range logs {
 		l := &logs[i]
-		if l.Parent == "" {
-			continue
-		}
-		addCommit(l.Hash, key)
+		addCommit(pkg, l.Hash, key)
 	}
 }
 
 // addCommit adds the commit with the named hash to the dashboard.
 // key is the secret key for authentication to the dashboard.
 // It avoids duplicate effort.
-func addCommit(hash, key string) bool {
+func addCommit(pkg, hash, key string) bool {
 	l := logByHash[hash]
 	if l == nil {
 		return false
@@ -547,7 +634,7 @@ func addCommit(hash, key string) bool {
 	}
 
 	// Check for already added, perhaps in an earlier run.
-	if dashboardCommit(hash) {
+	if dashboardCommit(pkg, hash) {
 		log.Printf("%s already on dashboard\n", hash)
 		// Record that this hash is on the dashboard,
 		// as must be all its parents.
@@ -559,12 +646,14 @@ func addCommit(hash, key string) bool {
 	}
 
 	// Create parent first, to maintain some semblance of order.
-	if !addCommit(l.Parent, key) {
-		return false
+	if l.Parent != "" {
+		if !addCommit(pkg, l.Parent, key) {
+			return false
+		}
 	}
 
 	// Create commit.
-	if err := postCommit(key, l); err != nil {
+	if err := postCommit(key, pkg, l); err != nil {
 		log.Printf("failed to add %s to dashboard: %v", key, err)
 		return false
 	}
@@ -572,13 +661,13 @@ func addCommit(hash, key string) bool {
 }
 
 // fullHash returns the full hash for the given Mercurial revision.
-func fullHash(rev string) (hash string, err os.Error) {
+func fullHash(root, rev string) (hash string, err error) {
 	defer func() {
 		if err != nil {
 			err = fmt.Errorf("fullHash: %s: %s", rev, err)
 		}
 	}()
-	s, _, err := runLog(nil, "", goroot,
+	s, _, err := runLog(nil, "", root,
 		"hg", "log",
 		"--encoding=utf-8",
 		"--rev="+rev,
@@ -601,7 +690,7 @@ func fullHash(rev string) (hash string, err os.Error) {
 var revisionRe = regexp.MustCompile(`^([^ ]+) +[0-9]+:([0-9a-f]+)$`)
 
 // firstTag returns the hash and tag of the most recent tag matching re.
-func firstTag(re *regexp.Regexp) (hash string, tag string, err os.Error) {
+func firstTag(re *regexp.Regexp) (hash string, tag string, err error) {
 	o, _, err := runLog(nil, "", goroot, "hg", "tags")
 	for _, l := range strings.Split(o, "\n") {
 		if l == "" {
@@ -609,16 +698,28 @@ func firstTag(re *regexp.Regexp) (hash string, tag string, err os.Error) {
 		}
 		s := revisionRe.FindStringSubmatch(l)
 		if s == nil {
-			err = os.NewError("couldn't find revision number")
+			err = errors.New("couldn't find revision number")
 			return
 		}
 		if !re.MatchString(s[1]) {
 			continue
 		}
 		tag = s[1]
-		hash, err = fullHash(s[2])
+		hash, err = fullHash(goroot, s[2])
 		return
 	}
-	err = os.NewError("no matching tag found")
+	err = errors.New("no matching tag found")
 	return
 }
+
+var repoRe = regexp.MustCompile(`^code\.google\.com/p/([a-z0-9\-]+(\.[a-z0-9\-]+)?)(/[a-z0-9A-Z_.\-/]+)?$`)
+
+// repoURL returns the repository URL for the supplied import path.
+func repoURL(importPath string) string {
+	m := repoRe.FindStringSubmatch(importPath)
+	if len(m) < 2 {
+		log.Printf("repoURL: couldn't decipher %q", importPath)
+		return ""
+	}
+	return "https://code.google.com/p/" + m[1]
+}
diff --git a/misc/dashboard/builder/package.go b/misc/dashboard/builder/package.go
index ebf4dd3c9..dcd449ab8 100644
--- a/misc/dashboard/builder/package.go
+++ b/misc/dashboard/builder/package.go
@@ -5,6 +5,7 @@
 package main
 
 import (
+	"errors"
 	"fmt"
 	"go/doc"
 	"go/parser"
@@ -17,7 +18,7 @@ import (
 
 const MaxCommentLength = 500 // App Engine won't store more in a StringProperty.
 
-func (b *Builder) buildPackages(workpath string, hash string) os.Error {
+func (b *Builder) buildExternalPackages(workpath string, hash string) error {
 	logdir := filepath.Join(*buildroot, "log")
 	if err := os.Mkdir(logdir, 0755); err != nil {
 		return err
@@ -80,14 +81,14 @@ func (b *Builder) buildPackages(workpath string, hash string) os.Error {
 	return nil
 }
 
-func isGoFile(fi *os.FileInfo) bool {
-	return fi.IsRegular() && // exclude directories
-		!strings.HasPrefix(fi.Name, ".") && // ignore .files
-		!strings.HasSuffix(fi.Name, "_test.go") && // ignore tests
-		filepath.Ext(fi.Name) == ".go"
+func isGoFile(fi os.FileInfo) bool {
+	return !fi.IsDir() && // exclude directories
+		!strings.HasPrefix(fi.Name(), ".") && // ignore .files
+		!strings.HasSuffix(fi.Name(), "_test.go") && // ignore tests
+		filepath.Ext(fi.Name()) == ".go"
 }
 
-func packageComment(pkg, pkgpath string) (info string, err os.Error) {
+func packageComment(pkg, pkgpath string) (info string, err error) {
 	fset := token.NewFileSet()
 	pkgs, err := parser.ParseDir(fset, pkgpath, isGoFile, parser.PackageClauseOnly|parser.ParseComments)
 	if err != nil {
@@ -97,12 +98,12 @@ func packageComment(pkg, pkgpath string) (info string, err os.Error) {
 		if name == "main" {
 			continue
 		}
-		pdoc := doc.NewPackageDoc(pkgs[name], pkg)
+		pdoc := doc.New(pkgs[name], pkg, doc.AllDecls)
 		if pdoc.Doc == "" {
 			continue
 		}
 		if info != "" {
-			return "", os.NewError("multiple packages with docs")
+			return "", errors.New("multiple packages with docs")
 		}
 		info = pdoc.Doc
 	}