diff options
author | Michael Stapelberg <stapelberg@debian.org> | 2013-03-04 21:27:36 +0100 |
---|---|---|
committer | Michael Stapelberg <michael@stapelberg.de> | 2013-03-04 21:27:36 +0100 |
commit | 04b08da9af0c450d645ab7389d1467308cfc2db8 (patch) | |
tree | db247935fa4f2f94408edc3acd5d0d4f997aa0d8 /misc/dashboard | |
parent | 917c5fb8ec48e22459d77e3849e6d388f93d3260 (diff) | |
download | golang-upstream/1.1_hg20130304.tar.gz |
Imported Upstream version 1.1~hg20130304upstream/1.1_hg20130304
Diffstat (limited to 'misc/dashboard')
24 files changed, 1410 insertions, 259 deletions
diff --git a/misc/dashboard/app/app.yaml b/misc/dashboard/app/app.yaml index 6e19db09c..c5a1f6cb8 100644 --- a/misc/dashboard/app/app.yaml +++ b/misc/dashboard/app/app.yaml @@ -6,7 +6,7 @@ application: golang-org version: build runtime: go -api_version: go1beta +api_version: go1 handlers: - url: /static diff --git a/misc/dashboard/app/build/build.go b/misc/dashboard/app/build/build.go index c49fa8bb2..e0c0f0048 100644 --- a/misc/dashboard/app/build/build.go +++ b/misc/dashboard/app/build/build.go @@ -49,6 +49,10 @@ func (p *Package) LastCommit(c appengine.Context) (*Commit, error) { Order("-Time"). Limit(1). GetAll(c, &commits) + if _, ok := err.(*datastore.ErrFieldMismatch); ok { + // Some fields have been removed, so it's okay to ignore this error. + err = nil + } if err != nil { return nil, err } @@ -65,6 +69,10 @@ func GetPackage(c appengine.Context, path string) (*Package, error) { if err == datastore.ErrNoSuchEntity { return nil, fmt.Errorf("package %q not found", path) } + if _, ok := err.(*datastore.ErrFieldMismatch); ok { + // Some fields have been removed, so it's okay to ignore this error. + err = nil + } return p, err } @@ -111,19 +119,35 @@ func (c *Commit) Valid() error { return nil } +// each result line is approx 105 bytes. This constant is a tradeoff between +// build history and the AppEngine datastore limit of 1mb. +const maxResults = 1000 + // AddResult adds the denormalized Reuslt data to the Commit's Result field. // It must be called from inside a datastore transaction. func (com *Commit) AddResult(c appengine.Context, r *Result) error { if err := datastore.Get(c, com.Key(c), com); err != nil { return fmt.Errorf("getting Commit: %v", err) } - com.ResultData = append(com.ResultData, r.Data()) + com.ResultData = trim(append(com.ResultData, r.Data()), maxResults) if _, err := datastore.Put(c, com.Key(c), com); err != nil { return fmt.Errorf("putting Commit: %v", err) } return nil } +func trim(s []string, n int) []string { + l := min(len(s), n) + return s[len(s)-l:] +} + +func min(a, b int) int { + if a < b { + return a + } + return b +} + // Result returns the build Result for this Commit for the given builder/goHash. func (c *Commit) Result(builder, goHash string) *Result { for _, r := range c.ResultData { @@ -160,20 +184,11 @@ func partsToHash(c *Commit, p []string) *Result { } } -// OK returns the Commit's build state for a specific builder and goHash. -func (c *Commit) OK(builder, goHash string) (ok, present bool) { - r := c.Result(builder, goHash) - if r == nil { - return false, false - } - return r.OK, true -} - // A Result describes a build result for a Commit on an OS/architecture. // // Each Result entity is a descendant of its associated Commit entity. type Result struct { - Builder string // "arch-os[-note]" + Builder string // "os-arch[-note]" Hash string PackagePath string // (empty for Go commits) @@ -184,7 +199,7 @@ type Result struct { Log string `datastore:"-"` // for JSON unmarshaling only LogHash string `datastore:",noindex"` // Key to the Log record. - RunTime int64 // time to build+test in nanoseconds + RunTime int64 // time to build+test in nanoseconds } func (r *Result) Key(c appengine.Context) *datastore.Key { @@ -297,7 +312,12 @@ func Packages(c appengine.Context, kind string) ([]*Package, error) { q := datastore.NewQuery("Package").Filter("Kind=", kind) for t := q.Run(c); ; { pkg := new(Package) - if _, err := t.Next(pkg); err == datastore.Done { + _, err := t.Next(pkg) + if _, ok := err.(*datastore.ErrFieldMismatch); ok { + // Some fields have been removed, so it's okay to ignore this error. + err = nil + } + if err == datastore.Done { break } else if err != nil { return nil, err diff --git a/misc/dashboard/app/build/handler.go b/misc/dashboard/app/build/handler.go index 5d1e3094c..1a1118641 100644 --- a/misc/dashboard/app/build/handler.go +++ b/misc/dashboard/app/build/handler.go @@ -322,7 +322,7 @@ func resultHandler(r *http.Request) (interface{}, error) { // logHandler displays log text for a given hash. // It handles paths like "/log/hash". func logHandler(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-type", "text/plain") + w.Header().Set("Content-type", "text/plain; charset=utf-8") c := appengine.NewContext(r) hash := r.URL.Path[len("/log/"):] key := datastore.NewKey(c, "Log", hash, 0, nil) diff --git a/misc/dashboard/app/build/init.go b/misc/dashboard/app/build/init.go index 5311688b7..85a766b9d 100644 --- a/misc/dashboard/app/build/init.go +++ b/misc/dashboard/app/build/init.go @@ -24,6 +24,8 @@ var subRepos = []string{ "crypto", "image", "net", + "talks", + "exp", } // Put subRepos into defaultPackages. @@ -42,7 +44,12 @@ func initHandler(w http.ResponseWriter, r *http.Request) { c := appengine.NewContext(r) defer cache.Tick(c) for _, p := range defaultPackages { - if err := datastore.Get(c, p.Key(c), new(Package)); err == nil { + err := datastore.Get(c, p.Key(c), new(Package)) + if _, ok := err.(*datastore.ErrFieldMismatch); ok { + // Some fields have been removed, so it's okay to ignore this error. + err = nil + } + if err == nil { continue } else if err != datastore.ErrNoSuchEntity { logErr(w, r, err) diff --git a/misc/dashboard/app/build/notify.go b/misc/dashboard/app/build/notify.go index e02344ca8..52b91f6c1 100644 --- a/misc/dashboard/app/build/notify.go +++ b/misc/dashboard/app/build/notify.go @@ -21,6 +21,13 @@ const ( domain = "build.golang.org" ) +// failIgnore is a set of builders that we don't email about because +// they're too flaky. +var failIgnore = map[string]bool{ + "netbsd-386-bsiegert": true, + "netbsd-amd64-bsiegert": true, +} + // notifyOnFailure checks whether the supplied Commit or the subsequent // Commit (if present) breaks the build for this builder. // If either of those commits break the build an email notification is sent @@ -30,6 +37,10 @@ const ( // This must be run in a datastore transaction, and the provided *Commit must // have been retrieved from the datastore within that transaction. func notifyOnFailure(c appengine.Context, com *Commit, builder string) error { + if failIgnore[builder] { + return nil + } + // TODO(adg): implement notifications for packages if com.PackagePath != "" { return nil @@ -37,15 +48,15 @@ func notifyOnFailure(c appengine.Context, com *Commit, builder string) error { p := &Package{Path: com.PackagePath} var broken *Commit - ok, present := com.OK(builder, "") - if !present { + cr := com.Result(builder, "") + if cr == nil { return fmt.Errorf("no result for %s/%s", com.Hash, builder) } q := datastore.NewQuery("Commit").Ancestor(p.Key(c)) - if ok { + if cr.OK { // This commit is OK. Notify if next Commit is broken. next := new(Commit) - q.Filter("ParentHash=", com.Hash) + q = q.Filter("ParentHash=", com.Hash) if err := firstMatch(c, q, next); err != nil { if err == datastore.ErrNoSuchEntity { // OK at tip, no notification necessary. @@ -53,13 +64,15 @@ func notifyOnFailure(c appengine.Context, com *Commit, builder string) error { } return err } - if ok, present := next.OK(builder, ""); present && !ok { + if nr := next.Result(builder, ""); nr != nil && !nr.OK { + c.Debugf("commit ok: %#v\nresult: %#v", com, cr) + c.Debugf("next commit broken: %#v\nnext result:%#v", next, nr) broken = next } } else { // This commit is broken. Notify if the previous Commit is OK. prev := new(Commit) - q.Filter("Hash=", com.ParentHash) + q = q.Filter("Hash=", com.ParentHash) if err := firstMatch(c, q, prev); err != nil { if err == datastore.ErrNoSuchEntity { // No previous result, let the backfill of @@ -68,7 +81,9 @@ func notifyOnFailure(c appengine.Context, com *Commit, builder string) error { } return err } - if ok, present := prev.OK(builder, ""); present && ok { + if pr := prev.Result(builder, ""); pr != nil && pr.OK { + c.Debugf("commit broken: %#v\nresult: %#v", com, cr) + c.Debugf("previous commit ok: %#v\nprevious result:%#v", prev, pr) broken = com } } diff --git a/misc/dashboard/app/build/test.go b/misc/dashboard/app/build/test.go index d8470fec1..7e5539236 100644 --- a/misc/dashboard/app/build/test.go +++ b/misc/dashboard/app/build/test.go @@ -43,14 +43,15 @@ var testPackages = []*Package{ var tCommitTime = time.Now().Add(-time.Hour * 24 * 7) -func tCommit(hash, parentHash string) *Commit { +func tCommit(hash, parentHash, path string) *Commit { tCommitTime.Add(time.Hour) // each commit should have a different time return &Commit{ - Hash: hash, - ParentHash: parentHash, - Time: tCommitTime, - User: "adg", - Desc: "change description", + PackagePath: path, + Hash: hash, + ParentHash: parentHash, + Time: tCommitTime, + User: "adg", + Desc: "change description " + hash, } } @@ -64,9 +65,9 @@ var testRequests = []struct { {"/packages?kind=subrepo", nil, nil, []*Package{testPackage}}, // Go repo - {"/commit", nil, tCommit("0001", "0000"), nil}, - {"/commit", nil, tCommit("0002", "0001"), nil}, - {"/commit", nil, tCommit("0003", "0002"), nil}, + {"/commit", nil, tCommit("0001", "0000", ""), nil}, + {"/commit", nil, tCommit("0002", "0001", ""), nil}, + {"/commit", nil, tCommit("0003", "0002", ""), nil}, {"/todo", url.Values{"kind": {"build-go-commit"}, "builder": {"linux-386"}}, nil, &Todo{Kind: "build-go-commit", Data: &Commit{Hash: "0003"}}}, {"/todo", url.Values{"kind": {"build-go-commit"}, "builder": {"linux-amd64"}}, nil, &Todo{Kind: "build-go-commit", Data: &Commit{Hash: "0003"}}}, {"/result", nil, &Result{Builder: "linux-386", Hash: "0001", OK: true}, nil}, @@ -81,12 +82,12 @@ var testRequests = []struct { {"/todo", url.Values{"kind": {"build-go-commit"}, "builder": {"linux-amd64"}}, nil, &Todo{Kind: "build-go-commit", Data: &Commit{Hash: "0002"}}}, // branches - {"/commit", nil, tCommit("0004", "0003"), nil}, - {"/commit", nil, tCommit("0005", "0002"), nil}, + {"/commit", nil, tCommit("0004", "0003", ""), nil}, + {"/commit", nil, tCommit("0005", "0002", ""), nil}, {"/todo", url.Values{"kind": {"build-go-commit"}, "builder": {"linux-386"}}, nil, &Todo{Kind: "build-go-commit", Data: &Commit{Hash: "0005"}}}, {"/result", nil, &Result{Builder: "linux-386", Hash: "0005", OK: true}, nil}, {"/todo", url.Values{"kind": {"build-go-commit"}, "builder": {"linux-386"}}, nil, &Todo{Kind: "build-go-commit", Data: &Commit{Hash: "0004"}}}, - {"/result", nil, &Result{Builder: "linux-386", Hash: "0004", OK: true}, nil}, + {"/result", nil, &Result{Builder: "linux-386", Hash: "0004", OK: false}, nil}, {"/todo", url.Values{"kind": {"build-go-commit"}, "builder": {"linux-386"}}, nil, &Todo{Kind: "build-go-commit", Data: &Commit{Hash: "0003"}}}, // logs @@ -98,9 +99,9 @@ var testRequests = []struct { {"/result", nil, &Result{Builder: "linux-386", Hash: "0003", OK: false, Log: "test"}, nil}, // non-Go repos - {"/commit", nil, &Commit{PackagePath: testPkg, Hash: "1001", ParentHash: "1000"}, nil}, - {"/commit", nil, &Commit{PackagePath: testPkg, Hash: "1002", ParentHash: "1001"}, nil}, - {"/commit", nil, &Commit{PackagePath: testPkg, Hash: "1003", ParentHash: "1002"}, nil}, + {"/commit", nil, tCommit("1001", "1000", testPkg), nil}, + {"/commit", nil, tCommit("1002", "1001", testPkg), nil}, + {"/commit", nil, tCommit("1003", "1002", testPkg), nil}, {"/todo", url.Values{"kind": {"build-package"}, "builder": {"linux-386"}, "packagePath": {testPkg}, "goHash": {"0001"}}, nil, &Todo{Kind: "build-package", Data: &Commit{Hash: "1003"}}}, {"/result", nil, &Result{PackagePath: testPkg, Builder: "linux-386", Hash: "1003", GoHash: "0001", OK: true}, nil}, {"/todo", url.Values{"kind": {"build-package"}, "builder": {"linux-386"}, "packagePath": {testPkg}, "goHash": {"0001"}}, nil, &Todo{Kind: "build-package", Data: &Commit{Hash: "1002"}}}, @@ -230,7 +231,7 @@ func testHandler(w http.ResponseWriter, r *http.Request) { return } } - fmt.Fprint(w, "PASS") + fmt.Fprint(w, "PASS\nYou should see only one mail notification (for 0003/linux-386) in the dev_appserver logs.") } func nukeEntities(c appengine.Context, kinds []string) error { diff --git a/misc/dashboard/app/build/ui.go b/misc/dashboard/app/build/ui.go index 0337aa306..cc3629a5a 100644 --- a/misc/dashboard/app/build/ui.go +++ b/misc/dashboard/app/build/ui.go @@ -97,7 +97,7 @@ type Pagination struct { func goCommits(c appengine.Context, page int) ([]*Commit, error) { q := datastore.NewQuery("Commit"). Ancestor((&Package{}).Key(c)). - Order("-Time"). + Order("-Num"). Limit(commitsPerPage). Offset(page * commitsPerPage) var commits []*Commit @@ -211,6 +211,9 @@ func builderArch(s string) string { // builderArchShort returns a short arch tag for a builder string func builderArchShort(s string) string { + if s == "linux-amd64-race" { + return "race" + } arch := builderArch(s) switch arch { case "amd64": diff --git a/misc/dashboard/builder/Makefile b/misc/dashboard/builder/Makefile index abf3755ab..4e4d408bf 100644 --- a/misc/dashboard/builder/Makefile +++ b/misc/dashboard/builder/Makefile @@ -2,8 +2,8 @@ # Use of this source code is governed by a BSD-style # license that can be found in the LICENSE file. -gobuilder: $(shell ls *.go) +builder: $(shell ls *.go) go build -o $@ $^ clean: - rm -f gobuilder + rm -f builder diff --git a/misc/dashboard/builder/doc.go b/misc/dashboard/builder/doc.go index 30d8fe948..519286170 100644 --- a/misc/dashboard/builder/doc.go +++ b/misc/dashboard/builder/doc.go @@ -4,15 +4,15 @@ /* -Go Builder is a continuous build client for the Go project. +Go Builder is a continuous build client for the Go project. It integrates with the Go Dashboard AppEngine application. Go Builder is intended to run continuously as a background process. -It periodically pulls updates from the Go Mercurial repository. +It periodically pulls updates from the Go Mercurial repository. When a newer revision is found, Go Builder creates a clone of the repository, -runs all.bash, and reports build success or failure to the Go Dashboard. +runs all.bash, and reports build success or failure to the Go Dashboard. For a release revision (a change description that matches "release.YYYY-MM-DD"), Go Builder will create a tar.gz archive of the GOROOT and deliver it to the @@ -22,7 +22,7 @@ Usage: gobuilder goos-goarch... - Several goos-goarch combinations can be provided, and the builder will + Several goos-goarch combinations can be provided, and the builder will build them in serial. Optional flags: @@ -55,4 +55,4 @@ If the Google Code credentials are not provided the archival step will be skipped. */ -package documentation +package main diff --git a/misc/dashboard/builder/exec.go b/misc/dashboard/builder/exec.go index 802d5f079..a4aabd284 100644 --- a/misc/dashboard/builder/exec.go +++ b/misc/dashboard/builder/exec.go @@ -6,14 +6,16 @@ package main import ( "bytes" + "fmt" "io" "log" "os" "os/exec" + "time" ) // run is a simple wrapper for exec.Run/Close -func run(envv []string, dir string, argv ...string) error { +func run(timeout time.Duration, envv []string, dir string, argv ...string) error { if *verbose { log.Println("run", argv) } @@ -21,43 +23,57 @@ func run(envv []string, dir string, argv ...string) error { cmd.Dir = dir cmd.Env = envv cmd.Stderr = os.Stderr - return cmd.Run() + if err := cmd.Start(); err != nil { + return err + } + return waitWithTimeout(timeout, cmd) } -// runLog runs a process and returns the combined stdout/stderr, -// as well as writing it to logfile (if specified). It returns -// process combined stdout and stderr output, exit status and error. -// The error returned is nil, if process is started successfully, -// even if exit status is not successful. -func runLog(envv []string, logfile, dir string, argv ...string) (string, int, error) { - if *verbose { - log.Println("runLog", argv) - } +// runLog runs a process and returns the combined stdout/stderr. It returns +// process combined stdout and stderr output, exit status and error. The +// error returned is nil, if process is started successfully, even if exit +// status is not successful. +func runLog(timeout time.Duration, envv []string, dir string, argv ...string) (string, bool, error) { + var b bytes.Buffer + ok, err := runOutput(timeout, envv, &b, dir, argv...) + return b.String(), ok, err +} - b := new(bytes.Buffer) - var w io.Writer = b - if logfile != "" { - f, err := os.OpenFile(logfile, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666) - if err != nil { - return "", 0, err - } - defer f.Close() - w = io.MultiWriter(f, b) +// runOutput runs a process and directs any output to the supplied writer. +// It returns exit status and error. The error returned is nil, if process +// is started successfully, even if exit status is not successful. +func runOutput(timeout time.Duration, envv []string, out io.Writer, dir string, argv ...string) (bool, error) { + if *verbose { + log.Println("runOutput", argv) } cmd := exec.Command(argv[0], argv[1:]...) cmd.Dir = dir cmd.Env = envv - cmd.Stdout = w - cmd.Stderr = w + cmd.Stdout = out + cmd.Stderr = out startErr := cmd.Start() if startErr != nil { - return "", 1, startErr + return false, startErr + } + if err := waitWithTimeout(timeout, cmd); err != nil { + return false, err } - exitStatus := 0 - if err := cmd.Wait(); err != nil { - exitStatus = 1 // TODO(bradfitz): this is fake. no callers care, so just return a bool instead. + return true, nil +} + +func waitWithTimeout(timeout time.Duration, cmd *exec.Cmd) error { + errc := make(chan error, 1) + go func() { + errc <- cmd.Wait() + }() + var err error + select { + case <-time.After(timeout): + cmd.Process.Kill() + err = fmt.Errorf("timed out after %v", timeout) + case err = <-errc: } - return b.String(), exitStatus, nil + return err } diff --git a/misc/dashboard/builder/http.go b/misc/dashboard/builder/http.go index e50ae5724..b50e84551 100644 --- a/misc/dashboard/builder/http.go +++ b/misc/dashboard/builder/http.go @@ -56,8 +56,10 @@ func dash(meth, cmd string, args url.Values, req, resp interface{}) error { if err != nil { return err } - defer r.Body.Close() + if r.StatusCode != http.StatusOK { + return fmt.Errorf("bad http response: %v", r.Status) + } body := new(bytes.Buffer) if _, err := body.ReadFrom(r.Body); err != nil { return err diff --git a/misc/dashboard/builder/main.go b/misc/dashboard/builder/main.go index 85bb7ad4b..b2b8f43a6 100644 --- a/misc/dashboard/builder/main.go +++ b/misc/dashboard/builder/main.go @@ -6,16 +6,15 @@ package main import ( "bytes" - "encoding/xml" "flag" "fmt" + "io" "io/ioutil" "log" "os" "path/filepath" "regexp" "runtime" - "strconv" "strings" "time" ) @@ -32,6 +31,7 @@ const ( // These variables are copied from the gobuilder's environment // to the envv of its subprocesses. var extraEnv = []string{ + "CC", "GOARM", "GOHOSTARCH", "GOHOSTOS", @@ -40,25 +40,27 @@ var extraEnv = []string{ } type Builder struct { + goroot *Repo name string goos, goarch string key string } var ( - buildroot = flag.String("buildroot", filepath.Join(os.TempDir(), "gobuilder"), "Directory under which to build") - commitFlag = flag.Bool("commit", false, "upload information about new commits") - dashboard = flag.String("dashboard", "build.golang.org", "Go Dashboard Host") - buildRelease = flag.Bool("release", false, "Build and upload binary release archives") - buildRevision = flag.String("rev", "", "Build specified revision and exit") - buildCmd = flag.String("cmd", filepath.Join(".", allCmd), "Build command (specify relative to go/src/)") - failAll = flag.Bool("fail", false, "fail all builds") - parallel = flag.Bool("parallel", false, "Build multiple targets in parallel") - verbose = flag.Bool("v", false, "verbose") + buildroot = flag.String("buildroot", defaultBuildRoot(), "Directory under which to build") + dashboard = flag.String("dashboard", "build.golang.org", "Go Dashboard Host") + buildRelease = flag.Bool("release", false, "Build and upload binary release archives") + buildRevision = flag.String("rev", "", "Build specified revision and exit") + buildCmd = flag.String("cmd", filepath.Join(".", allCmd), "Build command (specify relative to go/src/)") + failAll = flag.Bool("fail", false, "fail all builds") + parallel = flag.Bool("parallel", false, "Build multiple targets in parallel") + buildTimeout = flag.Duration("buildTimeout", 60*time.Minute, "Maximum time to wait for builds and tests") + cmdTimeout = flag.Duration("cmdTimeout", 5*time.Minute, "Maximum time to wait for an external command") + commitInterval = flag.Duration("commitInterval", 1*time.Minute, "Time to wait between polling for new commits (0 disables commit poller)") + verbose = flag.Bool("v", false, "verbose") ) var ( - goroot string binaryTagRe = regexp.MustCompile(`^(release\.r|weekly\.)[0-9\-.]+`) releaseRe = regexp.MustCompile(`^release\.r[0-9\-.]+`) allCmd = "all" + suffix @@ -73,26 +75,15 @@ func main() { os.Exit(2) } flag.Parse() - if len(flag.Args()) == 0 && !*commitFlag { + if len(flag.Args()) == 0 { flag.Usage() } - goroot = filepath.Join(*buildroot, "goroot") - builders := make([]*Builder, len(flag.Args())) - for i, builder := range flag.Args() { - b, err := NewBuilder(builder) - if err != nil { - log.Fatal(err) - } - builders[i] = b - } - - if *failAll { - failMode(builders) - return + goroot := &Repo{ + Path: filepath.Join(*buildroot, "goroot"), } // set up work environment, use existing enviroment if possible - if hgRepoExists(goroot) { + if goroot.Exists() || *failAll { log.Print("Found old workspace, will use it") } else { if err := os.RemoveAll(*buildroot); err != nil { @@ -101,22 +92,31 @@ func main() { if err := os.Mkdir(*buildroot, mkdirPerm); err != nil { log.Fatalf("Error making build root (%s): %s", *buildroot, err) } - if err := hgClone(hgUrl, goroot); err != nil { + var err error + goroot, err = RemoteRepo(hgUrl).Clone(goroot.Path, "tip") + if err != nil { log.Fatal("Error cloning repository:", err) } } - if *commitFlag { - if len(flag.Args()) == 0 { - commitWatcher() - return + // set up builders + builders := make([]*Builder, len(flag.Args())) + for i, name := range flag.Args() { + b, err := NewBuilder(goroot, name) + if err != nil { + log.Fatal(err) } - go commitWatcher() + builders[i] = b + } + + if *failAll { + failMode(builders) + return } // if specified, build revision and return if *buildRevision != "" { - hash, err := fullHash(goroot, *buildRevision) + hash, err := goroot.FullHash(*buildRevision) if err != nil { log.Fatal("Error finding revision: ", err) } @@ -128,7 +128,10 @@ func main() { return } - // go continuous build mode (default) + // Start commit watcher + go commitWatcher(goroot) + + // go continuous build mode // check for new commits and build them for { built := false @@ -175,15 +178,18 @@ func failMode(builders []*Builder) { } } -func NewBuilder(builder string) (*Builder, error) { - b := &Builder{name: builder} +func NewBuilder(goroot *Repo, name string) (*Builder, error) { + b := &Builder{ + goroot: goroot, + name: name, + } // get goos/goarch from builder string - s := strings.SplitN(builder, "-", 3) + s := strings.SplitN(b.name, "-", 3) if len(s) >= 2 { b.goos, b.goarch = s[0], s[1] } else { - return nil, fmt.Errorf("unsupported builder form: %s", builder) + return nil, fmt.Errorf("unsupported builder form: %s", name) } // read keys from keyfile @@ -206,7 +212,7 @@ func NewBuilder(builder string) (*Builder, error) { } // build checks for a new commit for this builder -// and builds it if one is found. +// and builds it if one is found. // It returns true if a build was attempted. func (b *Builder) build() bool { hash, err := b.todo("build-go-commit", "", "") @@ -217,16 +223,8 @@ func (b *Builder) build() bool { if hash == "" { return false } - // Look for hash locally before running hg pull. - if _, err := fullHash(goroot, hash[:12]); err != nil { - // Don't have hash, so run hg pull. - if err := run(nil, goroot, "hg", "pull"); err != nil { - log.Println("hg pull failed:", err) - return false - } - } - err = b.buildHash(hash) - if err != nil { + + if err := b.buildHash(hash); err != nil { log.Println(err) } return true @@ -242,34 +240,49 @@ func (b *Builder) buildHash(hash string) error { } defer os.RemoveAll(workpath) - // clone repo - if err := run(nil, workpath, "hg", "clone", goroot, "go"); err != nil { + // pull before cloning to ensure we have the revision + if err := b.goroot.Pull(); err != nil { return err } - // update to specified revision - if err := run(nil, filepath.Join(workpath, "go"), "hg", "update", hash); err != nil { + // clone repo at specified revision + if _, err := b.goroot.Clone(filepath.Join(workpath, "go"), hash); err != nil { return err } srcDir := filepath.Join(workpath, "go", "src") // build + var buildlog bytes.Buffer logfile := filepath.Join(workpath, "build.log") + f, err := os.Create(logfile) + if err != nil { + return err + } + defer f.Close() + w := io.MultiWriter(f, &buildlog) + cmd := *buildCmd if !filepath.IsAbs(cmd) { cmd = filepath.Join(srcDir, cmd) } startTime := time.Now() - buildLog, status, err := runLog(b.envv(), logfile, srcDir, cmd) + ok, err := runOutput(*buildTimeout, b.envv(), w, srcDir, cmd) runTime := time.Now().Sub(startTime) - if err != nil { - return fmt.Errorf("%s: %s", *buildCmd, err) + errf := func() string { + if err != nil { + return fmt.Sprintf("error: %v", err) + } + if !ok { + return "failed" + } + return "success" } + fmt.Fprintf(w, "Build complete, duration %v. Result: %v\n", runTime, errf()) - if status != 0 { + if err != nil || !ok { // record failure - return b.recordResult(false, "", hash, "", buildLog, runTime) + return b.recordResult(false, "", hash, "", buildlog.String(), runTime) } // record success @@ -278,13 +291,15 @@ func (b *Builder) buildHash(hash string) error { } // build Go sub-repositories - b.buildSubrepos(filepath.Join(workpath, "go"), hash) + goRoot := filepath.Join(workpath, "go") + goPath := workpath + b.buildSubrepos(goRoot, goPath, hash) return nil } // failBuild checks for a new commit for this builder -// and fails it if one is found. +// and fails it if one is found. // It returns true if a build was "attempted". func (b *Builder) failBuild() bool { hash, err := b.todo("build-go-commit", "", "") @@ -304,7 +319,7 @@ func (b *Builder) failBuild() bool { return true } -func (b *Builder) buildSubrepos(goRoot, goHash string) { +func (b *Builder) buildSubrepos(goRoot, goPath, goHash string) { for _, pkg := range dashboardPackages("subrepo") { // get the latest todo for this package hash, err := b.todo("build-package", pkg, goHash) @@ -320,7 +335,7 @@ func (b *Builder) buildSubrepos(goRoot, goHash string) { if *verbose { log.Printf("buildSubrepos %s: building %q", pkg, hash) } - buildLog, err := b.buildSubrepo(goRoot, pkg, hash) + buildLog, err := b.buildSubrepo(goRoot, goPath, pkg, hash) if err != nil { if buildLog == "" { buildLog = err.Error() @@ -338,45 +353,39 @@ func (b *Builder) buildSubrepos(goRoot, goHash string) { // buildSubrepo fetches the given package, updates it to the specified hash, // and runs 'go test -short pkg/...'. It returns the build log and any error. -func (b *Builder) buildSubrepo(goRoot, pkg, hash string) (string, error) { - goBin := filepath.Join(goRoot, "bin") - goTool := filepath.Join(goBin, "go") - env := append(b.envv(), "GOROOT="+goRoot) +func (b *Builder) buildSubrepo(goRoot, goPath, pkg, hash string) (string, error) { + goTool := filepath.Join(goRoot, "bin", "go") + env := append(b.envv(), "GOROOT="+goRoot, "GOPATH="+goPath) - // add goBin to PATH + // add $GOROOT/bin and $GOPATH/bin to PATH for i, e := range env { const p = "PATH=" if !strings.HasPrefix(e, p) { continue } - env[i] = p + goBin + string(os.PathListSeparator) + e[len(p):] + sep := string(os.PathListSeparator) + env[i] = p + filepath.Join(goRoot, "bin") + sep + filepath.Join(goPath, "bin") + sep + e[len(p):] } // fetch package and dependencies - log, status, err := runLog(env, "", goRoot, goTool, "get", "-d", pkg) - if err == nil && status != 0 { - err = fmt.Errorf("go exited with status %d", status) + log, ok, err := runLog(*cmdTimeout, env, goPath, goTool, "get", "-d", pkg+"/...") + if err == nil && !ok { + err = fmt.Errorf("go exited with status 1") } if err != nil { - // 'go get -d' will fail for a subrepo because its top-level - // directory does not contain a go package. No matter, just - // check whether an hg directory exists and proceed. - hgDir := filepath.Join(goRoot, "src/pkg", pkg, ".hg") - if fi, e := os.Stat(hgDir); e != nil || !fi.IsDir() { - return log, err - } + return log, err } // hg update to the specified hash - pkgPath := filepath.Join(goRoot, "src/pkg", pkg) - if err := run(nil, pkgPath, "hg", "update", hash); err != nil { + repo := Repo{Path: filepath.Join(goPath, "src", pkg)} + if err := repo.UpdateTo(hash); err != nil { return "", err } // test the package - log, status, err = runLog(env, "", goRoot, goTool, "test", "-short", pkg+"/...") - if err == nil && status != 0 { - err = fmt.Errorf("go exited with status %d", status) + log, ok, err = runLog(*buildTimeout, env, goPath, goTool, "test", "-short", pkg+"/...") + if err == nil && !ok { + err = fmt.Errorf("go exited with status 1") } return log, err } @@ -449,9 +458,13 @@ func isFile(name string) bool { } // commitWatcher polls hg for new commits and tells the dashboard about them. -func commitWatcher() { +func commitWatcher(goroot *Repo) { + if *commitInterval == 0 { + log.Printf("commitInterval is %s, disabling commitWatcher", *commitInterval) + return + } // Create builder just to get master key. - b, err := NewBuilder("mercurial-commit") + b, err := NewBuilder(goroot, "mercurial-commit") if err != nil { log.Fatal(err) } @@ -462,104 +475,46 @@ func commitWatcher() { log.Printf("poll...") } // Main Go repository. - commitPoll(key, "") + commitPoll(goroot, "", key) // Go sub-repositories. for _, pkg := range dashboardPackages("subrepo") { - commitPoll(key, pkg) + pkgroot := &Repo{ + Path: filepath.Join(*buildroot, pkg), + } + commitPoll(pkgroot, pkg, key) } if *verbose { log.Printf("sleep...") } - time.Sleep(60e9) - } -} - -func hgClone(url, path string) error { - return run(nil, *buildroot, "hg", "clone", url, path) -} - -func hgRepoExists(path string) bool { - fi, err := os.Stat(filepath.Join(path, ".hg")) - if err != nil { - return false + time.Sleep(*commitInterval) } - return fi.IsDir() -} - -// HgLog represents a single Mercurial revision. -type HgLog struct { - Hash string - Author string - Date string - Desc string - Parent string - - // Internal metadata - added bool } // logByHash is a cache of all Mercurial revisions we know about, // indexed by full hash. var logByHash = map[string]*HgLog{} -// xmlLogTemplate is a template to pass to Mercurial to make -// hg log print the log in valid XML for parsing with xml.Unmarshal. -const xmlLogTemplate = ` - <Log> - <Hash>{node|escape}</Hash> - <Parent>{parent|escape}</Parent> - <Author>{author|escape}</Author> - <Date>{date|rfc3339date}</Date> - <Desc>{desc|escape}</Desc> - </Log> -` - // commitPoll pulls any new revisions from the hg server // and tells the server about them. -func commitPoll(key, pkg string) { - pkgRoot := goroot - - if pkg != "" { - pkgRoot = filepath.Join(*buildroot, pkg) - if !hgRepoExists(pkgRoot) { - if err := hgClone(repoURL(pkg), pkgRoot); err != nil { - log.Printf("%s: hg clone failed: %v", pkg, err) - if err := os.RemoveAll(pkgRoot); err != nil { - log.Printf("%s: %v", pkg, err) - } - return +func commitPoll(repo *Repo, pkg, key string) { + if !repo.Exists() { + var err error + repo, err = RemoteRepo(repoURL(pkg)).Clone(repo.Path, "tip") + if err != nil { + log.Printf("%s: hg clone failed: %v", pkg, err) + if err := os.RemoveAll(repo.Path); err != nil { + log.Printf("%s: %v", pkg, err) } } - } - - if err := run(nil, pkgRoot, "hg", "pull"); err != nil { - log.Printf("hg pull: %v", err) return } - const N = 50 // how many revisions to grab - - data, _, err := runLog(nil, "", pkgRoot, "hg", "log", - "--encoding=utf-8", - "--limit="+strconv.Itoa(N), - "--template="+xmlLogTemplate, - ) + logs, err := repo.Log() // repo.Log calls repo.Pull internally if err != nil { log.Printf("hg log: %v", err) return } - var logStruct struct { - Log []HgLog - } - err = xml.Unmarshal([]byte("<Top>"+data+"</Top>"), &logStruct) - if err != nil { - log.Printf("unmarshal hg log: %v", err) - return - } - - logs := logStruct.Log - // Pass 1. Fill in parents and add new log entries to logsByHash. // Empty parent means take parent from next log entry. // Non-empty parent has form 1234:hashhashhash; we want full hash. @@ -568,7 +523,7 @@ func commitPoll(key, pkg string) { if l.Parent == "" && i+1 < len(logs) { l.Parent = logs[i+1].Hash } else if l.Parent != "" { - l.Parent, _ = fullHash(pkgRoot, l.Parent) + l.Parent, _ = repo.FullHash(l.Parent) } if *verbose { log.Printf("hg log %s: %s < %s\n", pkg, l.Hash, l.Parent) @@ -580,8 +535,7 @@ func commitPoll(key, pkg string) { } } - for i := range logs { - l := &logs[i] + for _, l := range logs { addCommit(pkg, l.Hash, key) } } @@ -625,28 +579,6 @@ func addCommit(pkg, hash, key string) bool { return true } -// fullHash returns the full hash for the given Mercurial revision. -func fullHash(root, rev string) (string, error) { - s, _, err := runLog(nil, "", root, - "hg", "log", - "--encoding=utf-8", - "--rev="+rev, - "--limit=1", - "--template={node}", - ) - if err != nil { - return "", nil - } - s = strings.TrimSpace(s) - if s == "" { - return "", fmt.Errorf("cannot find revision") - } - if len(s) != 40 { - return "", fmt.Errorf("hg returned invalid hash " + s) - } - return s, nil -} - var repoRe = regexp.MustCompile(`^code\.google\.com/p/([a-z0-9\-]+(\.[a-z0-9\-]+)?)(/[a-z0-9A-Z_.\-/]+)?$`) // repoURL returns the repository URL for the supplied import path. @@ -668,6 +600,19 @@ func defaultSuffix() string { return ".bash" } +// defaultBuildRoot returns default buildroot directory. +func defaultBuildRoot() string { + var d string + if runtime.GOOS == "windows" { + // will use c:\, otherwise absolute paths become too long + // during builder run, see http://golang.org/issue/3358. + d = `c:\` + } else { + d = os.TempDir() + } + return filepath.Join(d, "gobuilder") +} + func getenvOk(k string) (v string, ok bool) { v = os.Getenv(k) if v != "" { diff --git a/misc/dashboard/builder/vcs.go b/misc/dashboard/builder/vcs.go new file mode 100644 index 000000000..63198a34b --- /dev/null +++ b/misc/dashboard/builder/vcs.go @@ -0,0 +1,148 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "encoding/xml" + "fmt" + "log" + "os" + "path/filepath" + "strconv" + "strings" + "sync" +) + +// Repo represents a mercurial repository. +type Repo struct { + Path string + sync.Mutex +} + +// RemoteRepo constructs a *Repo representing a remote repository. +func RemoteRepo(url string) *Repo { + return &Repo{ + Path: url, + } +} + +// Clone clones the current Repo to a new destination +// returning a new *Repo if successful. +func (r *Repo) Clone(path, rev string) (*Repo, error) { + r.Lock() + defer r.Unlock() + if err := run(*cmdTimeout, nil, *buildroot, r.hgCmd("clone", "-r", rev, r.Path, path)...); err != nil { + return nil, err + } + return &Repo{ + Path: path, + }, nil +} + +// UpdateTo updates the working copy of this Repo to the +// supplied revision. +func (r *Repo) UpdateTo(hash string) error { + r.Lock() + defer r.Unlock() + return run(*cmdTimeout, nil, r.Path, r.hgCmd("update", hash)...) +} + +// Exists reports whether this Repo represents a valid Mecurial repository. +func (r *Repo) Exists() bool { + fi, err := os.Stat(filepath.Join(r.Path, ".hg")) + if err != nil { + return false + } + return fi.IsDir() +} + +// Pull pulls changes from the default path, that is, the path +// this Repo was cloned from. +func (r *Repo) Pull() error { + r.Lock() + defer r.Unlock() + return run(*cmdTimeout, nil, r.Path, r.hgCmd("pull")...) +} + +// Log returns the changelog for this repository. +func (r *Repo) Log() ([]HgLog, error) { + if err := r.Pull(); err != nil { + return nil, err + } + const N = 50 // how many revisions to grab + + r.Lock() + defer r.Unlock() + data, _, err := runLog(*cmdTimeout, nil, r.Path, r.hgCmd("log", + "--encoding=utf-8", + "--limit="+strconv.Itoa(N), + "--template="+xmlLogTemplate)..., + ) + if err != nil { + return nil, err + } + + var logStruct struct { + Log []HgLog + } + err = xml.Unmarshal([]byte("<Top>"+data+"</Top>"), &logStruct) + if err != nil { + log.Printf("unmarshal hg log: %v", err) + return nil, err + } + return logStruct.Log, nil +} + +// FullHash returns the full hash for the given Mercurial revision. +func (r *Repo) FullHash(rev string) (string, error) { + r.Lock() + defer r.Unlock() + s, _, err := runLog(*cmdTimeout, nil, r.Path, + r.hgCmd("log", + "--encoding=utf-8", + "--rev="+rev, + "--limit=1", + "--template={node}")..., + ) + if err != nil { + return "", nil + } + s = strings.TrimSpace(s) + if s == "" { + return "", fmt.Errorf("cannot find revision") + } + if len(s) != 40 { + return "", fmt.Errorf("hg returned invalid hash " + s) + } + return s, nil +} + +func (r *Repo) hgCmd(args ...string) []string { + return append([]string{"hg", "--config", "extensions.codereview=!"}, args...) +} + +// HgLog represents a single Mercurial revision. +type HgLog struct { + Hash string + Author string + Date string + Desc string + Parent string + + // Internal metadata + added bool +} + +// xmlLogTemplate is a template to pass to Mercurial to make +// hg log print the log in valid XML for parsing with xml.Unmarshal. +const xmlLogTemplate = ` + <Log> + <Hash>{node|escape}</Hash> + <Parent>{parent|escape}</Parent> + <Author>{author|escape}</Author> + <Date>{date|rfc3339date}</Date> + <Desc>{desc|escape}</Desc> + </Log> +` diff --git a/misc/dashboard/codereview/app.yaml b/misc/dashboard/codereview/app.yaml new file mode 100644 index 000000000..372eca5a1 --- /dev/null +++ b/misc/dashboard/codereview/app.yaml @@ -0,0 +1,24 @@ +application: gocodereview +version: 1 +runtime: go +api_version: go1 + +inbound_services: +- mail + +handlers: +- url: /static/(.*) + static_files: static/\1 + upload: static/.* +- url: /_ah/mail/.* + script: _go_app + login: admin +- url: /_ah/queue/go/delay + script: _go_app + login: admin +- url: /(gc|update-cl) + script: _go_app + login: admin +- url: /.* + script: _go_app + login: required diff --git a/misc/dashboard/codereview/cron.yaml b/misc/dashboard/codereview/cron.yaml new file mode 100644 index 000000000..3d33d32b5 --- /dev/null +++ b/misc/dashboard/codereview/cron.yaml @@ -0,0 +1,4 @@ +cron: +- description: GC + url: /gc + schedule: every 6 hours diff --git a/misc/dashboard/codereview/dashboard/cl.go b/misc/dashboard/codereview/dashboard/cl.go new file mode 100644 index 000000000..e150ea123 --- /dev/null +++ b/misc/dashboard/codereview/dashboard/cl.go @@ -0,0 +1,481 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package dashboard + +// This file handles operations on the CL entity kind. + +import ( + "bytes" + "encoding/json" + "fmt" + "html/template" + "io" + "io/ioutil" + "net/http" + "net/url" + "regexp" + "sort" + "strings" + "time" + + "appengine" + "appengine/datastore" + "appengine/taskqueue" + "appengine/urlfetch" + "appengine/user" +) + +func init() { + http.HandleFunc("/assign", handleAssign) + http.HandleFunc("/update-cl", handleUpdateCL) +} + +const codereviewBase = "http://codereview.appspot.com" +const gobotBase = "http://research.swtch.com/gobot_codereview" + +var clRegexp = regexp.MustCompile(`\d+`) + +// CL represents a code review. +type CL struct { + Number string // e.g. "5903061" + Closed bool + Owner string // email address + + Created, Modified time.Time + + Description []byte `datastore:",noindex"` + FirstLine string `datastore:",noindex"` + LGTMs []string + NotLGTMs []string + LastUpdateBy string // author of most recent review message + LastUpdate string `datastore:",noindex"` // first line of most recent review message + + // Mail information. + Subject string `datastore:",noindex"` + Recipients []string `datastore:",noindex"` + LastMessageID string `datastore:",noindex"` + + // These are person IDs (e.g. "rsc"); they may be empty + Author string + Reviewer string +} + +// Reviewed reports whether the reviewer has replied to the CL. +// The heuristic is that the CL has been replied to if it is LGTMed +// or if the last CL message was from the reviewer. +func (cl *CL) Reviewed() bool { + if cl.LastUpdateBy == cl.Reviewer { + return true + } + if person := emailToPerson[cl.LastUpdateBy]; person != "" && person == cl.Reviewer { + return true + } + for _, who := range cl.LGTMs { + if who == cl.Reviewer { + return true + } + } + return false +} + +// DisplayOwner returns the CL's owner, either as their email address +// or the person ID if it's a reviewer. It is for display only. +func (cl *CL) DisplayOwner() string { + if p, ok := emailToPerson[cl.Owner]; ok { + return p + } + return cl.Owner +} + +func (cl *CL) FirstLineHTML() template.HTML { + s := template.HTMLEscapeString(cl.FirstLine) + // Embolden the package name. + if i := strings.Index(s, ":"); i >= 0 { + s = "<b>" + s[:i] + "</b>" + s[i:] + } + return template.HTML(s) +} + +func formatEmails(e []string) template.HTML { + x := make([]string, len(e)) + for i, s := range e { + s = template.HTMLEscapeString(s) + if !strings.Contains(s, "@") { + s = "<b>" + s + "</b>" + } + s = `<span class="email">` + s + "</span>" + x[i] = s + } + return template.HTML(strings.Join(x, ", ")) +} + +func (cl *CL) LGTMHTML() template.HTML { + return formatEmails(cl.LGTMs) +} + +func (cl *CL) NotLGTMHTML() template.HTML { + return formatEmails(cl.NotLGTMs) +} + +func (cl *CL) ModifiedAgo() string { + // Just the first non-zero unit. + units := [...]struct { + suffix string + unit time.Duration + }{ + {"d", 24 * time.Hour}, + {"h", time.Hour}, + {"m", time.Minute}, + {"s", time.Second}, + } + d := time.Now().Sub(cl.Modified) + for _, u := range units { + if d > u.unit { + return fmt.Sprintf("%d%s", d/u.unit, u.suffix) + } + } + return "just now" +} + +func handleAssign(w http.ResponseWriter, r *http.Request) { + c := appengine.NewContext(r) + + if r.Method != "POST" { + http.Error(w, "Bad method "+r.Method, 400) + return + } + + u := user.Current(c) + person, ok := emailToPerson[u.Email] + if !ok { + http.Error(w, "Not allowed", http.StatusUnauthorized) + return + } + + n, rev := r.FormValue("cl"), r.FormValue("r") + if !clRegexp.MatchString(n) { + c.Errorf("Bad CL %q", n) + http.Error(w, "Bad CL", 400) + return + } + if _, ok := preferredEmail[rev]; !ok && rev != "" { + c.Errorf("Unknown reviewer %q", rev) + http.Error(w, "Unknown reviewer", 400) + return + } + + key := datastore.NewKey(c, "CL", n, 0, nil) + + if rev != "" { + // Make sure the reviewer is listed in Rietveld as a reviewer. + url := codereviewBase + "/" + n + "/fields" + resp, err := urlfetch.Client(c).Get(url + "?field=reviewers") + if err != nil { + c.Errorf("Retrieving CL reviewer list failed: %v", err) + http.Error(w, err.Error(), 500) + return + } + defer resp.Body.Close() + if resp.StatusCode != 200 { + c.Errorf("Retrieving CL reviewer list failed: got HTTP response %d", resp.StatusCode) + http.Error(w, "Failed contacting Rietveld", 500) + return + } + + var apiResp struct { + Reviewers []string `json:"reviewers"` + } + if err := json.NewDecoder(resp.Body).Decode(&apiResp); err != nil { + // probably can't be retried + msg := fmt.Sprintf("Malformed JSON from %v: %v", url, err) + c.Errorf("%s", msg) + http.Error(w, msg, 500) + return + } + found := false + for _, r := range apiResp.Reviewers { + if emailToPerson[r] == rev { + found = true + break + } + } + if !found { + c.Infof("Adding %v as a reviewer of CL %v", rev, n) + + url := fmt.Sprintf("%s?cl=%s&r=%s&obo=%s", gobotBase, n, rev, person) + resp, err := urlfetch.Client(c).Get(url) + if err != nil { + c.Errorf("Gobot GET failed: %v", err) + http.Error(w, err.Error(), 500) + return + } + defer resp.Body.Close() + if resp.StatusCode != 200 { + c.Errorf("Gobot GET failed: got HTTP response %d", resp.StatusCode) + http.Error(w, "Failed contacting Gobot", 500) + return + } + + c.Infof("Gobot said %q", resp.Status) + } + } + + // Update our own record. + err := datastore.RunInTransaction(c, func(c appengine.Context) error { + cl := new(CL) + err := datastore.Get(c, key, cl) + if err != nil { + return err + } + cl.Reviewer = rev + _, err = datastore.Put(c, key, cl) + return err + }, nil) + if err != nil { + msg := fmt.Sprintf("Assignment failed: %v", err) + c.Errorf("%s", msg) + http.Error(w, msg, 500) + return + } + c.Infof("Assigned CL %v to %v", n, rev) +} + +func UpdateCLLater(c appengine.Context, n string, delay time.Duration) { + t := taskqueue.NewPOSTTask("/update-cl", url.Values{ + "cl": []string{n}, + }) + t.Delay = delay + if _, err := taskqueue.Add(c, t, "update-cl"); err != nil { + c.Errorf("Failed adding task: %v", err) + } +} + +func handleUpdateCL(w http.ResponseWriter, r *http.Request) { + c := appengine.NewContext(r) + + n := r.FormValue("cl") + if !clRegexp.MatchString(n) { + c.Errorf("Bad CL %q", n) + http.Error(w, "Bad CL", 400) + return + } + + if err := updateCL(c, n); err != nil { + c.Errorf("Failed updating CL %v: %v", n, err) + http.Error(w, "Failed update", 500) + return + } + + io.WriteString(w, "OK") +} + +// apiMessage describes the JSON sent back by Rietveld in the CL messages list. +type apiMessage struct { + Date string `json:"date"` + Text string `json:"text"` + Sender string `json:"sender"` + Recipients []string `json:"recipients"` + Approval bool `json:"approval"` +} + +// byDate implements sort.Interface to order the messages by date, earliest first. +// The dates are sent in RFC 3339 format, so string comparison matches time value comparison. +type byDate []*apiMessage + +func (x byDate) Len() int { return len(x) } +func (x byDate) Swap(i, j int) { x[i], x[j] = x[j], x[i] } +func (x byDate) Less(i, j int) bool { return x[i].Date < x[j].Date } + +// updateCL updates a single CL. If a retryable failure occurs, an error is returned. +func updateCL(c appengine.Context, n string) error { + c.Debugf("Updating CL %v", n) + key := datastore.NewKey(c, "CL", n, 0, nil) + + url := codereviewBase + "/api/" + n + "?messages=true" + resp, err := urlfetch.Client(c).Get(url) + if err != nil { + return err + } + defer resp.Body.Close() + + raw, err := ioutil.ReadAll(resp.Body) + if err != nil { + return fmt.Errorf("Failed reading HTTP body: %v", err) + } + + // Special case for abandoned CLs. + if resp.StatusCode == 404 && bytes.Contains(raw, []byte("No issue exists with that id")) { + // Don't bother checking for errors. The CL might never have been saved, for instance. + datastore.Delete(c, key) + c.Infof("Deleted abandoned CL %v", n) + return nil + } + + if resp.StatusCode != 200 { + return fmt.Errorf("Update: got HTTP response %d", resp.StatusCode) + } + + var apiResp struct { + Description string `json:"description"` + Reviewers []string `json:"reviewers"` + Created string `json:"created"` + OwnerEmail string `json:"owner_email"` + Modified string `json:"modified"` + Closed bool `json:"closed"` + Subject string `json:"subject"` + Messages []*apiMessage `json:"messages"` + } + if err := json.Unmarshal(raw, &apiResp); err != nil { + // probably can't be retried + c.Errorf("Malformed JSON from %v: %v", url, err) + return nil + } + //c.Infof("RAW: %+v", apiResp) + sort.Sort(byDate(apiResp.Messages)) + + cl := &CL{ + Number: n, + Closed: apiResp.Closed, + Owner: apiResp.OwnerEmail, + Description: []byte(apiResp.Description), + FirstLine: apiResp.Description, + Subject: apiResp.Subject, + Author: emailToPerson[apiResp.OwnerEmail], + } + cl.Created, err = time.Parse("2006-01-02 15:04:05.000000", apiResp.Created) + if err != nil { + c.Errorf("Bad creation time %q: %v", apiResp.Created, err) + } + cl.Modified, err = time.Parse("2006-01-02 15:04:05.000000", apiResp.Modified) + if err != nil { + c.Errorf("Bad modification time %q: %v", apiResp.Modified, err) + } + if i := strings.Index(cl.FirstLine, "\n"); i >= 0 { + cl.FirstLine = cl.FirstLine[:i] + } + // Treat zero reviewers as a signal that the CL is completed. + // This could be after the CL has been submitted, but before the CL author has synced, + // but it could also be a CL manually edited to remove reviewers. + if len(apiResp.Reviewers) == 0 { + cl.Closed = true + } + + lgtm := make(map[string]bool) + notLGTM := make(map[string]bool) + rcpt := make(map[string]bool) + for _, msg := range apiResp.Messages { + s, rev := msg.Sender, false + if p, ok := emailToPerson[s]; ok { + s, rev = p, true + } + + line := firstLine(msg.Text) + if line != "" { + cl.LastUpdateBy = msg.Sender + cl.LastUpdate = line + } + + // CLs submitted by someone other than the CL owner do not immediately + // transition to "closed". Let's simulate the intention by treating + // messages starting with "*** Submitted as " from a reviewer as a + // signal that the CL is now closed. + if rev && strings.HasPrefix(msg.Text, "*** Submitted as ") { + cl.Closed = true + } + + if msg.Approval { + lgtm[s] = true + delete(notLGTM, s) // "LGTM" overrules previous "NOT LGTM" + } + if strings.Contains(line, "NOT LGTM") { + notLGTM[s] = true + delete(lgtm, s) // "NOT LGTM" overrules previous "LGTM" + } + + for _, r := range msg.Recipients { + rcpt[r] = true + } + } + for l := range lgtm { + cl.LGTMs = append(cl.LGTMs, l) + } + for l := range notLGTM { + cl.NotLGTMs = append(cl.NotLGTMs, l) + } + for r := range rcpt { + cl.Recipients = append(cl.Recipients, r) + } + sort.Strings(cl.LGTMs) + sort.Strings(cl.NotLGTMs) + sort.Strings(cl.Recipients) + + err = datastore.RunInTransaction(c, func(c appengine.Context) error { + ocl := new(CL) + err := datastore.Get(c, key, ocl) + if err != nil && err != datastore.ErrNoSuchEntity { + return err + } else if err == nil { + // LastMessageID and Reviewer need preserving. + cl.LastMessageID = ocl.LastMessageID + cl.Reviewer = ocl.Reviewer + } + _, err = datastore.Put(c, key, cl) + return err + }, nil) + if err != nil { + return err + } + c.Infof("Updated CL %v", n) + return nil +} + +// trailingSpaceRE matches trailing spaces. +var trailingSpaceRE = regexp.MustCompile(`(?m)[ \t\r]+$`) + +// removeRE is the list of patterns to skip over at the beginning of a +// message when looking for message text. +var removeRE = regexp.MustCompile(`(?m-s)\A(` + + // Skip leading "Hello so-and-so," generated by codereview plugin. + `(Hello(.|\n)*?\n\n)` + + + // Skip quoted text. + `|((On.*|.* writes|.* wrote):\n)` + + `|((>.*\n)+)` + + + // Skip lines with no letters. + `|(([^A-Za-z]*\n)+)` + + + // Skip links to comments and file info. + `|(http://codereview.*\n([^ ]+:[0-9]+:.*\n)?)` + + `|(File .*:\n)` + + + `)`, +) + +// firstLine returns the first interesting line of the message text. +func firstLine(text string) string { + // Cut trailing spaces. + text = trailingSpaceRE.ReplaceAllString(text, "") + + // Skip uninteresting lines. + for { + text = strings.TrimSpace(text) + m := removeRE.FindStringIndex(text) + if m == nil || m[0] != 0 { + break + } + text = text[m[1]:] + } + + // Chop line at newline or else at 74 bytes. + i := strings.Index(text, "\n") + if i >= 0 { + text = text[:i] + } + if len(text) > 74 { + text = text[:70] + "..." + } + return text +} diff --git a/misc/dashboard/codereview/dashboard/front.go b/misc/dashboard/codereview/dashboard/front.go new file mode 100644 index 000000000..c7b0f0fbf --- /dev/null +++ b/misc/dashboard/codereview/dashboard/front.go @@ -0,0 +1,299 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package dashboard + +// This file handles the front page. + +import ( + "bytes" + "html/template" + "io" + "net/http" + "strings" + "sync" + "time" + + "appengine" + "appengine/datastore" + "appengine/user" +) + +func init() { + http.HandleFunc("/", handleFront) + http.HandleFunc("/favicon.ico", http.NotFound) +} + +// maximum number of active CLs to show in person-specific tables. +const maxCLs = 100 + +func handleFront(w http.ResponseWriter, r *http.Request) { + c := appengine.NewContext(r) + + data := &frontPageData{ + Reviewers: personList, + User: user.Current(c).Email, + IsAdmin: user.IsAdmin(c), + } + var currentPerson string + u := data.User + you := "you" + if e := r.FormValue("user"); e != "" { + u = e + you = e + } + currentPerson, data.UserIsReviewer = emailToPerson[u] + if !data.UserIsReviewer { + currentPerson = u + } + + var wg sync.WaitGroup + errc := make(chan error, 10) + activeCLs := datastore.NewQuery("CL"). + Filter("Closed =", false). + Order("-Modified") + + tableFetch := func(index int, f func(tbl *clTable) error) { + wg.Add(1) + go func() { + defer wg.Done() + start := time.Now() + if err := f(&data.Tables[index]); err != nil { + errc <- err + } + data.Timing[index] = time.Now().Sub(start) + }() + } + + data.Tables[0].Title = "CLs assigned to " + you + " for review" + if data.UserIsReviewer { + tableFetch(0, func(tbl *clTable) error { + q := activeCLs.Filter("Reviewer =", currentPerson).Limit(maxCLs) + tbl.Assignable = true + _, err := q.GetAll(c, &tbl.CLs) + return err + }) + } + + tableFetch(1, func(tbl *clTable) error { + q := activeCLs + if data.UserIsReviewer { + q = q.Filter("Author =", currentPerson) + } else { + q = q.Filter("Owner =", currentPerson) + } + q = q.Limit(maxCLs) + tbl.Title = "CLs sent by " + you + tbl.Assignable = true + _, err := q.GetAll(c, &tbl.CLs) + return err + }) + + tableFetch(2, func(tbl *clTable) error { + q := activeCLs.Limit(50) + tbl.Title = "Other active CLs" + tbl.Assignable = true + if _, err := q.GetAll(c, &tbl.CLs); err != nil { + return err + } + // filter + for i := len(tbl.CLs) - 1; i >= 0; i-- { + cl := tbl.CLs[i] + if cl.Owner == currentPerson || cl.Author == currentPerson || cl.Reviewer == currentPerson { + // Preserve order. + copy(tbl.CLs[i:], tbl.CLs[i+1:]) + tbl.CLs = tbl.CLs[:len(tbl.CLs)-1] + } + } + return nil + }) + + tableFetch(3, func(tbl *clTable) error { + q := datastore.NewQuery("CL"). + Filter("Closed =", true). + Order("-Modified"). + Limit(10) + tbl.Title = "Recently closed CLs" + tbl.Assignable = false + _, err := q.GetAll(c, &tbl.CLs) + return err + }) + + // Not really a table fetch. + tableFetch(0, func(_ *clTable) error { + var err error + data.LogoutURL, err = user.LogoutURL(c, "/") + return err + }) + + wg.Wait() + + select { + case err := <-errc: + c.Errorf("%v", err) + http.Error(w, err.Error(), http.StatusInternalServerError) + return + default: + } + + var b bytes.Buffer + if err := frontPage.ExecuteTemplate(&b, "front", &data); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + io.Copy(w, &b) +} + +type frontPageData struct { + Tables [4]clTable + Timing [4]time.Duration + + Reviewers []string + UserIsReviewer bool + + User, LogoutURL string // actual logged in user + IsAdmin bool +} + +type clTable struct { + Title string + Assignable bool + CLs []*CL +} + +var frontPage = template.Must(template.New("front").Funcs(template.FuncMap{ + "selected": func(a, b string) string { + if a == b { + return "selected" + } + return "" + }, + "shortemail": func(s string) string { + if i := strings.Index(s, "@"); i >= 0 { + s = s[:i] + } + return s + }, +}).Parse(` +<!doctype html> +<html> + <head> + <title>Go code reviews</title> + <link rel="icon" type="image/png" href="/static/icon.png" /> + <style type="text/css"> + body { + font-family: Helvetica, sans-serif; + } + img#gopherstamp { + float: right; + height: auto; + width: 250px; + } + h1, h2, h3 { + color: #777; + margin-bottom: 0; + } + table { + border-spacing: 0; + } + td { + vertical-align: top; + padding: 2px 5px; + } + tr.unreplied td.email { + border-left: 2px solid blue; + } + tr.pending td { + background: #fc8; + } + tr.failed td { + background: #f88; + } + tr.saved td { + background: #8f8; + } + .cls { + margin-top: 0; + } + a { + color: blue; + text-decoration: none; /* no link underline */ + } + address { + font-size: 10px; + text-align: right; + } + .email { + font-family: monospace; + } + </style> + <script src="https://ajax.googleapis.com/ajax/libs/jquery/1.7.2/jquery.min.js"></script> + <head> + <body> + +<img id="gopherstamp" src="/static/gopherstamp.jpg" /> +<h1>Go code reviews</h1> + +<table class="cls"> +{{range $i, $tbl := .Tables}} +<tr><td colspan="5"><h3>{{$tbl.Title}}</h3></td></tr> +{{if .CLs}} +{{range $cl := .CLs}} + <tr id="cl-{{$cl.Number}}" class="{{if not $i}}{{if not .Reviewed}}unreplied{{end}}{{end}}"> + <td class="email">{{$cl.DisplayOwner}}</td> + <td> + {{if $tbl.Assignable}} + <select id="cl-rev-{{$cl.Number}}" {{if not $.UserIsReviewer}}disabled{{end}}> + <option></option> + {{range $.Reviewers}} + <option {{selected . $cl.Reviewer}}>{{.}}</option> + {{end}} + </select> + <script type="text/javascript"> + $(function() { + $('#cl-rev-{{$cl.Number}}').change(function() { + var r = $(this).val(); + var row = $('tr#cl-{{$cl.Number}}'); + row.addClass('pending'); + $.post('/assign', { + 'cl': '{{$cl.Number}}', + 'r': r + }).success(function() { + row.removeClass('pending'); + row.addClass('saved'); + }).error(function() { + row.removeClass('pending'); + row.addClass('failed'); + }); + }); + }); + </script> + {{end}} + </td> + <td> + <a href="http://codereview.appspot.com/{{.Number}}/" title="{{ printf "%s" .Description}}">{{.Number}}: {{.FirstLineHTML}}</a> + {{if and .LGTMs $tbl.Assignable}}<br /><span style="font-size: smaller;">LGTMs: {{.LGTMHTML}}</span>{{end}} + {{if and .NotLGTMs $tbl.Assignable}}<br /><span style="font-size: smaller; color: #f74545;">NOT LGTMs: {{.NotLGTMHTML}}</span>{{end}} + {{if .LastUpdateBy}}<br /><span style="font-size: smaller; color: #777777;">(<span title="{{.LastUpdateBy}}">{{.LastUpdateBy | shortemail}}</span>) {{.LastUpdate}}</span>{{end}} + </td> + <td title="Last modified">{{.ModifiedAgo}}</td> + <td>{{if $.IsAdmin}}<a href="/update-cl?cl={{.Number}}" title="Update this CL">⟳</a>{{end}}</td> + </tr> +{{end}} +{{else}} +<tr><td colspan="5"><em>none</em></td></tr> +{{end}} +{{end}} +</table> + +<hr /> +<address> +You are <span class="email">{{.User}}</span> · <a href="{{.LogoutURL}}">logout</a><br /> +datastore timing: {{range .Timing}} {{.}}{{end}} +</address> + + </body> +</html> +`)) diff --git a/misc/dashboard/codereview/dashboard/gc.go b/misc/dashboard/codereview/dashboard/gc.go new file mode 100644 index 000000000..a80b375f6 --- /dev/null +++ b/misc/dashboard/codereview/dashboard/gc.go @@ -0,0 +1,47 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package dashboard + +// This file handles garbage collection of old CLs. + +import ( + "net/http" + "time" + + "appengine" + "appengine/datastore" +) + +func init() { + http.HandleFunc("/gc", handleGC) +} + +func handleGC(w http.ResponseWriter, r *http.Request) { + c := appengine.NewContext(r) + + // Delete closed CLs that haven't been modified in 168 hours (7 days). + cutoff := time.Now().Add(-168 * time.Hour) + q := datastore.NewQuery("CL"). + Filter("Closed =", true). + Filter("Modified <", cutoff). + Limit(100). + KeysOnly() + keys, err := q.GetAll(c, nil) + if err != nil { + c.Errorf("GetAll failed for old CLs: %v", err) + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + if len(keys) == 0 { + return + } + + if err := datastore.DeleteMulti(c, keys); err != nil { + c.Errorf("DeleteMulti failed for old CLs: %v", err) + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + c.Infof("Deleted %d old CLs", len(keys)) +} diff --git a/misc/dashboard/codereview/dashboard/mail.go b/misc/dashboard/codereview/dashboard/mail.go new file mode 100644 index 000000000..838d08222 --- /dev/null +++ b/misc/dashboard/codereview/dashboard/mail.go @@ -0,0 +1,68 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package dashboard + +// This file handles receiving mail. + +import ( + "net/http" + "net/mail" + "regexp" + "time" + + "appengine" + "appengine/datastore" +) + +func init() { + http.HandleFunc("/_ah/mail/", handleMail) +} + +var subjectRegexp = regexp.MustCompile(`.*code review (\d+):.*`) + +func handleMail(w http.ResponseWriter, r *http.Request) { + c := appengine.NewContext(r) + + defer r.Body.Close() + msg, err := mail.ReadMessage(r.Body) + if err != nil { + c.Errorf("mail.ReadMessage: %v", err) + return + } + + subj := msg.Header.Get("Subject") + m := subjectRegexp.FindStringSubmatch(subj) + if len(m) != 2 { + c.Debugf("Subject %q did not match /%v/", subj, subjectRegexp) + return + } + + c.Infof("Found issue %q", m[1]) + + // Track the MessageID. + key := datastore.NewKey(c, "CL", m[1], 0, nil) + err = datastore.RunInTransaction(c, func(c appengine.Context) error { + cl := new(CL) + err := datastore.Get(c, key, cl) + if err != nil && err != datastore.ErrNoSuchEntity { + return err + } + if err == datastore.ErrNoSuchEntity { + // Must set sentinel values for time.Time fields + // if this is a new entity. + cl.Created = time.Unix(0, 0) + cl.Modified = time.Unix(0, 0) + } + cl.LastMessageID = msg.Header.Get("Message-ID") + _, err = datastore.Put(c, key, cl) + return err + }, nil) + if err != nil { + c.Errorf("datastore transaction failed: %v", err) + } + + // Update the CL after a delay to give Rietveld a chance to catch up. + UpdateCLLater(c, m[1], 10*time.Second) +} diff --git a/misc/dashboard/codereview/dashboard/people.go b/misc/dashboard/codereview/dashboard/people.go new file mode 100644 index 000000000..facda7baf --- /dev/null +++ b/misc/dashboard/codereview/dashboard/people.go @@ -0,0 +1,42 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package dashboard + +// This file handles identities of people. + +import ( + "sort" +) + +var ( + emailToPerson = make(map[string]string) // email => person + preferredEmail = make(map[string]string) // person => email + personList []string +) + +func init() { + // People we assume have golang.org and google.com accounts, + // and prefer to use their golang.org address for code review. + gophers := [...]string{ + "adg", + "bradfitz", + "campoy", + "dsymonds", + "gri", + "iant", + "nigeltao", + "r", + "rsc", + "sameer", + } + for _, p := range gophers { + personList = append(personList, p) + emailToPerson[p+"@golang.org"] = p + emailToPerson[p+"@google.com"] = p + preferredEmail[p] = p + "@golang.org" + } + + sort.Strings(personList) +} diff --git a/misc/dashboard/codereview/index.yaml b/misc/dashboard/codereview/index.yaml new file mode 100644 index 000000000..a87073cc4 --- /dev/null +++ b/misc/dashboard/codereview/index.yaml @@ -0,0 +1,25 @@ +indexes: + +- kind: CL + properties: + - name: Author + - name: Modified + direction: desc + +- kind: CL + properties: + - name: Owner + - name: Modified + direction: desc + +- kind: CL + properties: + - name: Closed + - name: Modified + direction: desc + +- kind: CL + properties: + - name: Reviewer + - name: Modified + direction: desc diff --git a/misc/dashboard/codereview/queue.yaml b/misc/dashboard/codereview/queue.yaml new file mode 100644 index 000000000..1a35facaf --- /dev/null +++ b/misc/dashboard/codereview/queue.yaml @@ -0,0 +1,4 @@ +queue: +- name: update-cl + rate: 12/m + bucket_size: 1 diff --git a/misc/dashboard/codereview/static/gopherstamp.jpg b/misc/dashboard/codereview/static/gopherstamp.jpg Binary files differnew file mode 100644 index 000000000..b17f3c82a --- /dev/null +++ b/misc/dashboard/codereview/static/gopherstamp.jpg diff --git a/misc/dashboard/codereview/static/icon.png b/misc/dashboard/codereview/static/icon.png Binary files differnew file mode 100644 index 000000000..c929ac8e9 --- /dev/null +++ b/misc/dashboard/codereview/static/icon.png |