Imported Upstream version 2011.03.07.1upstream/2011.03.07.1

11 files changed, 337 insertions, 34 deletions

diff --git a/misc/dashboard/builder/Makefile b/misc/dashboard/builder/Makefile
index 7270a3f42..cff47942a 100644
--- a/misc/dashboard/builder/Makefile
+++ b/misc/dashboard/builder/Makefile
@@ -10,5 +10,6 @@ GOFILES=\
 	hg.go\
 	http.go\
 	main.go\
+	package.go\
 
 include ../../../src/Make.cmd
diff --git a/misc/dashboard/builder/doc.go b/misc/dashboard/builder/doc.go
index 54a9adfc0..a28658a95 100644
--- a/misc/dashboard/builder/doc.go
+++ b/misc/dashboard/builder/doc.go
@@ -38,6 +38,15 @@ Optional flags:
 
   -release: Build and deliver binary release archive
 
+  -rev=N: Build revision N and exit
+  
+  -cmd="./all.bash": Build command (specify absolute or relative to go/src)
+
+  -v: Verbose logging
+
+  -external: External package builder mode (will not report Go build
+     state to dashboard, issue releases, or run benchmarks)
+
 The key file should be located at $HOME/.gobuilder or, for a builder-specific
 key, $HOME/.gobuilder-$BUILDER (eg, $HOME/.gobuilder-linux-amd64).
 
diff --git a/misc/dashboard/builder/exec.go b/misc/dashboard/builder/exec.go
index 6236c915a..53ea93ac5 100644
--- a/misc/dashboard/builder/exec.go
+++ b/misc/dashboard/builder/exec.go
@@ -1,15 +1,23 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
 package main
 
 import (
 	"bytes"
 	"exec"
 	"io"
+	"log"
 	"os"
 	"strings"
 )
 
 // run is a simple wrapper for exec.Run/Close
 func run(envv []string, dir string, argv ...string) os.Error {
+	if *verbose {
+		log.Println("run", argv)
+	}
 	bin, err := pathLookup(argv[0])
 	if err != nil {
 		return err
@@ -25,6 +33,9 @@ func run(envv []string, dir string, argv ...string) os.Error {
 // runLog runs a process and returns the combined stdout/stderr, 
 // as well as writing it to logfile (if specified).
 func runLog(envv []string, logfile, dir string, argv ...string) (output string, exitStatus int, err os.Error) {
+	if *verbose {
+		log.Println("runLog", argv)
+	}
 	bin, err := pathLookup(argv[0])
 	if err != nil {
 		return
diff --git a/misc/dashboard/builder/hg.go b/misc/dashboard/builder/hg.go
index 5d2f63a17..d4310845d 100644
--- a/misc/dashboard/builder/hg.go
+++ b/misc/dashboard/builder/hg.go
@@ -1,8 +1,13 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
 package main
 
 import (
 	"fmt"
 	"os"
+	"regexp"
 	"strconv"
 	"strings"
 )
@@ -46,9 +51,36 @@ func getCommit(rev string) (c Commit, err os.Error) {
 func getCommitParts(rev string) (parts []string, err os.Error) {
 	const format = "{rev}>{node}>{author|escape}>{date}>{desc}"
 	s, _, err := runLog(nil, "", goroot,
-		"hg", "log", "-r", rev, "-l", "1", "--template", format)
+		"hg", "log",
+		"--encoding", "utf-8",
+		"--rev", rev,
+		"--limit", "1",
+		"--template", format,
+	)
 	if err != nil {
 		return
 	}
 	return strings.Split(s, ">", 5), nil
 }
+
+var revisionRe = regexp.MustCompile(`([0-9]+):[0-9a-f]+$`)
+
+// getTag fetches a Commit by finding the first hg tag that matches re.
+func getTag(re *regexp.Regexp) (c Commit, tag string, err os.Error) {
+	o, _, err := runLog(nil, "", goroot, "hg", "tags")
+	for _, l := range strings.Split(o, "\n", -1) {
+		tag = re.FindString(l)
+		if tag == "" {
+			continue
+		}
+		s := revisionRe.FindStringSubmatch(l)
+		if s == nil {
+			err = os.NewError("couldn't find revision number")
+			return
+		}
+		c, err = getCommit(s[1])
+		return
+	}
+	err = os.NewError("no matching tag found")
+	return
+}
diff --git a/misc/dashboard/builder/http.go b/misc/dashboard/builder/http.go
index 02f281061..dba19ba8f 100644
--- a/misc/dashboard/builder/http.go
+++ b/misc/dashboard/builder/http.go
@@ -1,3 +1,7 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
 package main
 
 import (
@@ -6,8 +10,11 @@ import (
 	"encoding/binary"
 	"fmt"
 	"http"
+	"json"
+	"log"
 	"os"
 	"regexp"
+	"strconv"
 )
 
 // getHighWater returns the current highwater revision hash for this builder
@@ -63,7 +70,46 @@ func (b *Builder) recordBenchmarks(benchLog string, c Commit) os.Error {
 	})
 }
 
+// getPackages fetches a list of package paths from the dashboard
+func getPackages() (pkgs []string, err os.Error) {
+	r, _, err := http.Get(fmt.Sprintf("http://%v/package?fmt=json", *dashboard))
+	if err != nil {
+		return
+	}
+	defer r.Body.Close()
+	d := json.NewDecoder(r.Body)
+	var resp struct {
+		Packages []struct {
+			Path string
+		}
+	}
+	if err = d.Decode(&resp); err != nil {
+		return
+	}
+	for _, p := range resp.Packages {
+		pkgs = append(pkgs, p.Path)
+	}
+	return
+}
+
+// updatePackage sends package build results and info to the dashboard
+func (b *Builder) updatePackage(pkg string, state bool, buildLog, info string, c Commit) os.Error {
+	args := map[string]string{
+		"builder": b.name,
+		"key":     b.key,
+		"path":    pkg,
+		"state":   strconv.Btoa(state),
+		"log":     buildLog,
+		"info":    info,
+		"go_rev":  strconv.Itoa(c.num),
+	}
+	return httpCommand("package", args)
+}
+
 func httpCommand(cmd string, args map[string]string) os.Error {
+	if *verbose {
+		log.Println("httpCommand", cmd, args)
+	}
 	url := fmt.Sprintf("http://%v/%v", *dashboard, cmd)
 	_, err := http.PostForm(url, args)
 	return err
diff --git a/misc/dashboard/builder/main.go b/misc/dashboard/builder/main.go
index 7e80934e1..fc11d365e 100644
--- a/misc/dashboard/builder/main.go
+++ b/misc/dashboard/builder/main.go
@@ -1,3 +1,7 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
 package main
 
 import (
@@ -15,13 +19,24 @@ import (
 )
 
 const (
-	codeProject  = "go"
-	codePyScript = "misc/dashboard/googlecode_upload.py"
-	hgUrl        = "https://go.googlecode.com/hg/"
-	waitInterval = 10e9 // time to wait before checking for new revs
-	mkdirPerm    = 0750
+	codeProject      = "go"
+	codePyScript     = "misc/dashboard/googlecode_upload.py"
+	hgUrl            = "https://go.googlecode.com/hg/"
+	waitInterval     = 10e9 // time to wait before checking for new revs
+	mkdirPerm        = 0750
+	pkgBuildInterval = 1e9 * 60 * 60 * 24 // rebuild packages every 24 hours
 )
 
+// These variables are copied from the gobuilder's environment
+// to the envv of its subprocesses.
+var extraEnv = []string{
+	"GOHOSTOS",
+	"GOHOSTARCH",
+	"PATH",
+	"DISABLE_NET_TESTS",
+	"GOARM",
+}
+
 type Builder struct {
 	name         string
 	goos, goarch string
@@ -43,6 +58,8 @@ var (
 	buildRelease  = flag.Bool("release", false, "Build and upload binary release archives")
 	buildRevision = flag.String("rev", "", "Build specified revision and exit")
 	buildCmd      = flag.String("cmd", "./all.bash", "Build command (specify absolute or relative to go/src/)")
+	external      = flag.Bool("external", false, "Build external packages")
+	verbose       = flag.Bool("v", false, "verbose")
 )
 
 var (
@@ -70,6 +87,8 @@ func main() {
 		}
 		builders[i] = b
 	}
+
+	// set up work environment
 	if err := os.RemoveAll(*buildroot); err != nil {
 		log.Fatalf("Error removing build root (%s): %s", *buildroot, err)
 	}
@@ -79,6 +98,7 @@ func main() {
 	if err := run(nil, *buildroot, "hg", "clone", hgUrl, goroot); err != nil {
 		log.Fatal("Error cloning repository:", err)
 	}
+
 	// if specified, build revision and return
 	if *buildRevision != "" {
 		c, err := getCommit(*buildRevision)
@@ -93,6 +113,16 @@ func main() {
 		}
 		return
 	}
+
+	// external package build mode
+	if *external {
+		if len(builders) != 1 {
+			log.Fatal("only one goos-goarch should be specified with -external")
+		}
+		builders[0].buildExternal()
+	}
+
+	// go continuous build mode (default)
 	// check for new commits and build them
 	for {
 		err := run(nil, goroot, "hg", "pull", "-u")
@@ -179,6 +209,44 @@ func NewBuilder(builder string) (*Builder, os.Error) {
 	return b, nil
 }
 
+// buildExternal downloads and builds external packages, and
+// reports their build status to the dashboard.
+// It will re-build all packages after pkgBuildInterval nanoseconds or
+// a new release tag is found.
+func (b *Builder) buildExternal() {
+	var prevTag string
+	var nextBuild int64
+	for {
+		time.Sleep(waitInterval)
+		err := run(nil, goroot, "hg", "pull", "-u")
+		if err != nil {
+			log.Println("hg pull failed:", err)
+			continue
+		}
+		c, tag, err := getTag(releaseRegexp)
+		if err != nil {
+			log.Println(err)
+			continue
+		}
+		if *verbose {
+			log.Println("latest release:", tag)
+		}
+		// don't rebuild if there's no new release
+		// and it's been less than pkgBuildInterval
+		// nanoseconds since the last build.
+		if tag == prevTag && time.Nanoseconds() < nextBuild {
+			continue
+		}
+		// buildCommit will also build the packages
+		if err := b.buildCommit(c); err != nil {
+			log.Println(err)
+			continue
+		}
+		prevTag = tag
+		nextBuild = time.Nanoseconds() + pkgBuildInterval
+	}
+}
+
 // build checks for a new commit for this builder
 // and builds it if one is found. 
 // It returns true if a build was attempted.
@@ -262,23 +330,23 @@ func (b *Builder) buildCommit(c Commit) (err os.Error) {
 		return
 	}
 
-	// set up environment for build/bench execution
-	env := []string{
-		"GOOS=" + b.goos,
-		"GOARCH=" + b.goarch,
-		"GOHOSTOS=" + os.Getenv("GOHOSTOS"),
-		"GOHOSTARCH=" + os.Getenv("GOHOSTARCH"),
-		"GOROOT_FINAL=/usr/local/go",
-		"PATH=" + os.Getenv("PATH"),
-	}
 	srcDir := path.Join(workpath, "go", "src")
 
 	// build
 	logfile := path.Join(workpath, "build.log")
-	buildLog, status, err := runLog(env, logfile, srcDir, *buildCmd)
+	buildLog, status, err := runLog(b.envv(), logfile, srcDir, *buildCmd)
 	if err != nil {
 		return fmt.Errorf("all.bash: %s", err)
 	}
+
+	// if we're in external mode, build all packages and return
+	if *external {
+		if status != 0 {
+			return os.NewError("go build failed")
+		}
+		return b.buildPackages(workpath, c)
+	}
+
 	if status != 0 {
 		// record failure
 		return b.recordResult(buildLog, c)
@@ -307,7 +375,7 @@ func (b *Builder) buildCommit(c Commit) (err os.Error) {
 	// if this is a release, create tgz and upload to google code
 	if release := releaseRegexp.FindString(c.desc); release != "" {
 		// clean out build state
-		err = run(env, srcDir, "./clean.bash", "--nopkg")
+		err = run(b.envv(), srcDir, "./clean.bash", "--nopkg")
 		if err != nil {
 			return fmt.Errorf("clean.bash: %s", err)
 		}
@@ -329,6 +397,19 @@ func (b *Builder) buildCommit(c Commit) (err os.Error) {
 	return
 }
 
+// envv returns an environment for build/bench execution
+func (b *Builder) envv() []string {
+	e := []string{
+		"GOOS=" + b.goos,
+		"GOARCH=" + b.goarch,
+		"GOROOT_FINAL=/usr/local/go",
+	}
+	for _, k := range extraEnv {
+		e = append(e, k+"="+os.Getenv(k))
+	}
+	return e
+}
+
 func isDirectory(name string) bool {
 	s, err := os.Stat(name)
 	return err == nil && s.IsDirectory()
diff --git a/misc/dashboard/builder/package.go b/misc/dashboard/builder/package.go
new file mode 100644
index 000000000..6e9f9ff39
--- /dev/null
+++ b/misc/dashboard/builder/package.go
@@ -0,0 +1,66 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+	"go/doc"
+	"go/parser"
+	"go/token"
+	"log"
+	"os"
+	"path"
+)
+
+func (b *Builder) buildPackages(workpath string, c Commit) os.Error {
+	pkgs, err := getPackages()
+	if err != nil {
+		return err
+	}
+	for _, p := range pkgs {
+		goroot := path.Join(workpath, "go")
+		goinstall := path.Join(goroot, "bin", "goinstall")
+		envv := append(b.envv(), "GOROOT="+goroot)
+
+		// goinstall
+		buildLog, code, err := runLog(envv, "", goroot, goinstall, p)
+		if err != nil {
+			log.Printf("goinstall %v: %v", p, err)
+			continue
+		}
+		built := code != 0
+
+		// get doc comment from package source
+		info, err := getPackageComment(p, path.Join(goroot, "pkg", p))
+		if err != nil {
+			log.Printf("goinstall %v: %v", p, err)
+		}
+
+		// update dashboard with build state + info
+		err = b.updatePackage(p, built, buildLog, info, c)
+		if err != nil {
+			log.Printf("updatePackage %v: %v", p, err)
+		}
+	}
+	return nil
+}
+
+func getPackageComment(pkg, pkgpath string) (info string, err os.Error) {
+	fset := token.NewFileSet()
+	pkgs, err := parser.ParseDir(fset, pkgpath, nil, parser.PackageClauseOnly|parser.ParseComments)
+	if err != nil {
+		return
+	}
+	for name := range pkgs {
+		if name == "main" {
+			continue
+		}
+		if info != "" {
+			return "", os.NewError("multiple non-main package docs")
+		}
+		pdoc := doc.NewPackageDoc(pkgs[name], pkg)
+		info = pdoc.Doc
+	}
+	return
+}
diff --git a/misc/dashboard/godashboard/const.py b/misc/dashboard/godashboard/const.py
new file mode 100644
index 000000000..b0110c635
--- /dev/null
+++ b/misc/dashboard/godashboard/const.py
@@ -0,0 +1,13 @@
+# Copyright 2011 The Go Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+mail_from           = "Go Dashboard <builder@golang.org>"
+
+mail_submit_to      = "adg@golang.org"
+mail_submit_subject = "New Project Submitted"
+
+mail_fail_to       = "golang-dev@googlegroups.com"
+mail_fail_reply_to = "golang-dev@googlegroups.com"
+mail_fail_subject  = "%s broken by %s"
+
diff --git a/misc/dashboard/godashboard/fail-notify.txt b/misc/dashboard/godashboard/fail-notify.txt
new file mode 100644
index 000000000..a699005ea
--- /dev/null
+++ b/misc/dashboard/godashboard/fail-notify.txt
@@ -0,0 +1,6 @@
+Change {{node}} broke the {{builder}} build:
+http://godashboard.appspot.com/log/{{loghash}}
+
+{{desc}}
+
+http://code.google.com/p/go/source/detail?r={{node}}
diff --git a/misc/dashboard/godashboard/gobuild.py b/misc/dashboard/godashboard/gobuild.py
index 46aeef9f9..08d70ec64 100644
--- a/misc/dashboard/godashboard/gobuild.py
+++ b/misc/dashboard/godashboard/gobuild.py
@@ -5,6 +5,7 @@
 # This is the server part of the continuous build system for Go. It must be run
 # by AppEngine.
 
+from google.appengine.api import mail
 from google.appengine.api import memcache
 from google.appengine.runtime import DeadlineExceededError
 from google.appengine.ext import db
@@ -24,6 +25,7 @@ import bz2
 
 # local imports
 import key
+import const
 
 # The majority of our state are commit objects. One of these exists for each of
 # the commits known to the build system. Their key names are of the form
@@ -47,6 +49,8 @@ class Commit(db.Model):
     # successful.
     builds = db.StringListProperty()
 
+    fail_notification_sent = db.BooleanProperty()
+
 class Benchmark(db.Model):		
     name = db.StringProperty()		
     version = db.IntegerProperty()	
@@ -259,7 +263,7 @@ class Init(webapp.RequestHandler):
         commit.num = 0
         commit.node = node
         commit.parentnode = ''
-        commit.user = self.request.get('user')
+        commit.user = self.request.get('user').encode('utf8')
         commit.date = date
         commit.desc = self.request.get('desc').encode('utf8')
 
@@ -285,34 +289,37 @@ class Build(webapp.RequestHandler):
             l.put()
 
         date = parseDate(self.request.get('date'))
+        user = self.request.get('user').encode('utf8')
+        desc = self.request.get('desc').encode('utf8')
         node = self.request.get('node')
-        parent = self.request.get('parent')
-        if not validNode(node) or not validNode(parent) or date is None:
+        parenthash = self.request.get('parent')
+        if not validNode(node) or not validNode(parenthash) or date is None:
             logging.error("Not valid node ('%s') or bad date (%s %s)", node, date, self.request.get('date'))
             self.response.set_status(500)
             return
 
         q = Commit.all()
-        q.filter('node =', parent)
-        p = q.get()
-        if p is None:
-            logging.error('Cannot find parent %s of node %s' % (parent, node))
+        q.filter('node =', parenthash)
+        parent = q.get()
+        if parent is None:
+            logging.error('Cannot find parent %s of node %s' % (parenthash, node))
             self.response.set_status(404)
             return
-        parentnum, _ = p.key().name().split('-', 1)
+        parentnum, _ = parent.key().name().split('-', 1)
         nodenum = int(parentnum, 16) + 1
 
+        key_name = '%08x-%s' % (nodenum, node)
+
         def add_build():
-            key_name = '%08x-%s' % (nodenum, node)
             n = Commit.get_by_key_name(key_name)
             if n is None:
                 n = Commit(key_name = key_name)
                 n.num = nodenum
                 n.node = node
-                n.parentnode = parent
-                n.user = self.request.get('user')
+                n.parentnode = parenthash
+                n.user = user
                 n.date = date
-                n.desc = self.request.get('desc').encode('utf8')
+                n.desc = desc
             s = '%s`%s' % (builder, loghash)
             for i, b in enumerate(n.builds):
                 if b.split('`', 1)[0] == builder:
@@ -333,8 +340,40 @@ class Build(webapp.RequestHandler):
         memcache.delete(key)
         memcache.delete('hw')
 
+        def mark_sent():
+            n = Commit.get_by_key_name(key_name)
+            n.fail_notification_sent = True
+            n.put()
+
+        n = Commit.get_by_key_name(key_name)
+        if loghash and not failed(parent, builder) and not n.fail_notification_sent:
+            subject = const.mail_fail_subject % (builder, desc.split("\n")[0])
+            path = os.path.join(os.path.dirname(__file__), 'fail-notify.txt')
+            body = template.render(path, {
+                "builder": builder,
+                "node": node[:12],
+                "user": user,
+                "desc": desc, 
+                "loghash": loghash
+            })
+            mail.send_mail(
+                sender=const.mail_from,
+                reply_to=const.mail_fail_reply_to,
+                to=const.mail_fail_to,
+                subject=subject,
+                body=body
+            )
+            db.run_in_transaction(mark_sent)
+
         self.response.set_status(200)
 
+def failed(c, builder):
+    for i, b in enumerate(c.builds):
+        p = b.split('`', 1)
+        if p[0] == builder:
+            return len(p[1]) > 0
+    return False
+
 class Benchmarks(webapp.RequestHandler):
     def json(self):
         q = Benchmark.all()
diff --git a/misc/dashboard/godashboard/package.py b/misc/dashboard/godashboard/package.py
index cf59bf3e8..7570d2218 100644
--- a/misc/dashboard/godashboard/package.py
+++ b/misc/dashboard/godashboard/package.py
@@ -5,10 +5,6 @@
 # This is the server part of the package dashboard.
 # It must be run by App Engine.
 
-mail_to      = "adg@golang.org"
-mail_from    = "Go Dashboard <adg@golang.org>"
-mail_subject = "New Project Submitted"
-
 from google.appengine.api import memcache
 from google.appengine.runtime import DeadlineExceededError
 from google.appengine.ext import db
@@ -32,6 +28,7 @@ import sets
 
 # local imports
 import toutf8
+import const
 
 template.register_template_library('toutf8')
 
@@ -241,7 +238,9 @@ class ProjectPage(webapp.RequestHandler):
 		
             path = os.path.join(os.path.dirname(__file__), 'project-notify.txt')
             mail.send_mail(
-                sender=mail_from, to=mail_to, subject=mail_subject,
+                sender=const.mail_from,
+                to=const.mail_submit_to,
+                subject=const.mail_submit_subject,
                 body=template.render(path, {'project': p}))
 
             self.list({"submitMsg": "Your project has been submitted."})