diff options
Diffstat (limited to 'misc/dashboard')
-rw-r--r-- | misc/dashboard/builder/Makefile | 1 | ||||
-rw-r--r-- | misc/dashboard/builder/doc.go | 7 | ||||
-rw-r--r-- | misc/dashboard/builder/exec.go | 68 | ||||
-rw-r--r-- | misc/dashboard/builder/hg.go | 86 | ||||
-rw-r--r-- | misc/dashboard/builder/http.go | 156 | ||||
-rw-r--r-- | misc/dashboard/builder/main.go | 406 | ||||
-rw-r--r-- | misc/dashboard/builder/package.go | 10 | ||||
-rw-r--r-- | misc/dashboard/godashboard/app.yaml | 6 | ||||
-rw-r--r-- | misc/dashboard/godashboard/gobuild.py | 354 | ||||
-rw-r--r-- | misc/dashboard/godashboard/index.yaml | 6 | ||||
-rw-r--r-- | misc/dashboard/godashboard/static/favicon.ico | bin | 0 -> 785 bytes | |||
-rwxr-xr-x | misc/dashboard/googlecode_upload.py | 2 |
12 files changed, 679 insertions, 423 deletions
diff --git a/misc/dashboard/builder/Makefile b/misc/dashboard/builder/Makefile index cff47942a..f1d9c5497 100644 --- a/misc/dashboard/builder/Makefile +++ b/misc/dashboard/builder/Makefile @@ -7,7 +7,6 @@ include ../../../src/Make.inc TARG=gobuilder GOFILES=\ exec.go\ - hg.go\ http.go\ main.go\ package.go\ diff --git a/misc/dashboard/builder/doc.go b/misc/dashboard/builder/doc.go index 7bb7ccbe3..30d8fe948 100644 --- a/misc/dashboard/builder/doc.go +++ b/misc/dashboard/builder/doc.go @@ -14,9 +14,6 @@ It periodically pulls updates from the Go Mercurial repository. When a newer revision is found, Go Builder creates a clone of the repository, runs all.bash, and reports build success or failure to the Go Dashboard. -For a successful build, Go Builder will also run benchmarks -(cd $GOROOT/src/pkg; make bench) and send the results to the Go Dashboard. - For a release revision (a change description that matches "release.YYYY-MM-DD"), Go Builder will create a tar.gz archive of the GOROOT and deliver it to the Go Google Code project's downloads section. @@ -34,8 +31,6 @@ Optional flags: The location of the Go Dashboard application to which Go Builder will report its results. - -bench: Run benchmarks - -release: Build and deliver binary release archive -rev=N: Build revision N and exit @@ -45,7 +40,7 @@ Optional flags: -v: Verbose logging -external: External package builder mode (will not report Go build - state to dashboard, issue releases, or run benchmarks) + state to dashboard or issue releases) The key file should be located at $HOME/.gobuildkey or, for a builder-specific key, $HOME/.gobuildkey-$BUILDER (eg, $HOME/.gobuildkey-linux-amd64). diff --git a/misc/dashboard/builder/exec.go b/misc/dashboard/builder/exec.go index 3c6fbdced..a042c5699 100644 --- a/misc/dashboard/builder/exec.go +++ b/misc/dashboard/builder/exec.go @@ -18,59 +18,57 @@ func run(envv []string, dir string, argv ...string) os.Error { if *verbose { log.Println("run", argv) } - bin, err := pathLookup(argv[0]) - if err != nil { - return err - } - p, err := exec.Run(bin, argv, envv, dir, - exec.DevNull, exec.DevNull, exec.PassThrough) - if err != nil { - return err - } - return p.Close() + argv = useBash(argv) + cmd := exec.Command(argv[0], argv[1:]...) + cmd.Dir = dir + cmd.Env = envv + cmd.Stderr = os.Stderr + return cmd.Run() } // runLog runs a process and returns the combined stdout/stderr, -// as well as writing it to logfile (if specified). -func runLog(envv []string, logfile, dir string, argv ...string) (output string, exitStatus int, err os.Error) { +// as well as writing it to logfile (if specified). It returns +// process combined stdout and stderr output, exit status and error. +// The error returned is nil, if process is started successfully, +// even if exit status is not 0. +func runLog(envv []string, logfile, dir string, argv ...string) (string, int, os.Error) { if *verbose { log.Println("runLog", argv) } - bin, err := pathLookup(argv[0]) - if err != nil { - return - } - p, err := exec.Run(bin, argv, envv, dir, - exec.DevNull, exec.Pipe, exec.MergeWithStdout) - if err != nil { - return - } - defer p.Close() + argv = useBash(argv) + b := new(bytes.Buffer) var w io.Writer = b if logfile != "" { f, err := os.OpenFile(logfile, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666) if err != nil { - return + return "", 0, err } defer f.Close() w = io.MultiWriter(f, b) } - _, err = io.Copy(w, p.Stdout) - if err != nil { - return - } - wait, err := p.Wait(0) + + cmd := exec.Command(argv[0], argv[1:]...) + cmd.Dir = dir + cmd.Env = envv + cmd.Stdout = w + cmd.Stderr = w + + err := cmd.Run() if err != nil { - return + if ws, ok := err.(*os.Waitmsg); ok { + return b.String(), ws.ExitStatus(), nil + } } - return b.String(), wait.WaitStatus.ExitStatus(), nil + return b.String(), 0, nil } -// Find bin in PATH if a relative or absolute path hasn't been specified -func pathLookup(s string) (string, os.Error) { - if strings.HasPrefix(s, "/") || strings.HasPrefix(s, "./") || strings.HasPrefix(s, "../") { - return s, nil +// useBash prefixes a list of args with 'bash' if the first argument +// is a bash script. +func useBash(argv []string) []string { + // TODO(brainman): choose a more reliable heuristic here. + if strings.HasSuffix(argv[0], ".bash") { + argv = append([]string{"bash"}, argv...) } - return exec.LookPath(s) + return argv } diff --git a/misc/dashboard/builder/hg.go b/misc/dashboard/builder/hg.go deleted file mode 100644 index d4310845d..000000000 --- a/misc/dashboard/builder/hg.go +++ /dev/null @@ -1,86 +0,0 @@ -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package main - -import ( - "fmt" - "os" - "regexp" - "strconv" - "strings" -) - -type Commit struct { - num int // mercurial revision number - node string // mercurial hash - parent string // hash of commit's parent - user string // author's Name <email> - date string // date of commit - desc string // description -} - -// getCommit returns details about the Commit specified by the revision hash -func getCommit(rev string) (c Commit, err os.Error) { - defer func() { - if err != nil { - err = fmt.Errorf("getCommit: %s: %s", rev, err) - } - }() - parts, err := getCommitParts(rev) - if err != nil { - return - } - num, err := strconv.Atoi(parts[0]) - if err != nil { - return - } - parent := "" - if num > 0 { - prev := strconv.Itoa(num - 1) - if pparts, err := getCommitParts(prev); err == nil { - parent = pparts[1] - } - } - user := strings.Replace(parts[2], "<", "<", -1) - user = strings.Replace(user, ">", ">", -1) - return Commit{num, parts[1], parent, user, parts[3], parts[4]}, nil -} - -func getCommitParts(rev string) (parts []string, err os.Error) { - const format = "{rev}>{node}>{author|escape}>{date}>{desc}" - s, _, err := runLog(nil, "", goroot, - "hg", "log", - "--encoding", "utf-8", - "--rev", rev, - "--limit", "1", - "--template", format, - ) - if err != nil { - return - } - return strings.Split(s, ">", 5), nil -} - -var revisionRe = regexp.MustCompile(`([0-9]+):[0-9a-f]+$`) - -// getTag fetches a Commit by finding the first hg tag that matches re. -func getTag(re *regexp.Regexp) (c Commit, tag string, err os.Error) { - o, _, err := runLog(nil, "", goroot, "hg", "tags") - for _, l := range strings.Split(o, "\n", -1) { - tag = re.FindString(l) - if tag == "" { - continue - } - s := revisionRe.FindStringSubmatch(l) - if s == nil { - err = os.NewError("couldn't find revision number") - return - } - c, err = getCommit(s[1]) - return - } - err = os.NewError("no matching tag found") - return -} diff --git a/misc/dashboard/builder/http.go b/misc/dashboard/builder/http.go index dba19ba8f..5e1da0c87 100644 --- a/misc/dashboard/builder/http.go +++ b/misc/dashboard/builder/http.go @@ -6,84 +6,104 @@ package main import ( "bytes" - "encoding/base64" - "encoding/binary" "fmt" "http" "json" "log" "os" - "regexp" "strconv" ) -// getHighWater returns the current highwater revision hash for this builder -func (b *Builder) getHighWater() (rev string, err os.Error) { - url := fmt.Sprintf("http://%s/hw-get?builder=%s", *dashboard, b.name) - r, _, err := http.Get(url) +type param map[string]string + +// dash runs the given method and command on the dashboard. +// If args is not nil, it is the query or post parameters. +// If resp is not nil, dash unmarshals the body as JSON into resp. +func dash(meth, cmd string, resp interface{}, args param) os.Error { + var r *http.Response + var err os.Error + if *verbose { + log.Println("dash", cmd, args) + } + cmd = "http://" + *dashboard + "/" + cmd + vals := make(http.Values) + for k, v := range args { + vals.Add(k, v) + } + switch meth { + case "GET": + if q := vals.Encode(); q != "" { + cmd += "?" + q + } + r, err = http.Get(cmd) + case "POST": + r, err = http.PostForm(cmd, vals) + default: + return fmt.Errorf("unknown method %q", meth) + } if err != nil { - return + return err + } + defer r.Body.Close() + var buf bytes.Buffer + buf.ReadFrom(r.Body) + if resp != nil { + if err = json.Unmarshal(buf.Bytes(), resp); err != nil { + log.Printf("json unmarshal %#q: %s\n", buf.Bytes(), err) + return err + } } - buf := new(bytes.Buffer) - _, err = buf.ReadFrom(r.Body) + return nil +} + +func dashStatus(meth, cmd string, args param) os.Error { + var resp struct { + Status string + Error string + } + err := dash(meth, cmd, &resp, args) if err != nil { + return err + } + if resp.Status != "OK" { + return os.NewError("/build: " + resp.Error) + } + return nil +} + +// todo returns the next hash to build. +func (b *Builder) todo() (rev string, err os.Error) { + var resp []struct { + Hash string + } + if err = dash("GET", "todo", &resp, param{"builder": b.name}); err != nil { return } - r.Body.Close() - return buf.String(), nil + if len(resp) > 0 { + rev = resp[0].Hash + } + return } // recordResult sends build results to the dashboard -func (b *Builder) recordResult(buildLog string, c Commit) os.Error { - return httpCommand("build", map[string]string{ +func (b *Builder) recordResult(buildLog string, hash string) os.Error { + return dash("POST", "build", nil, param{ "builder": b.name, "key": b.key, - "node": c.node, - "parent": c.parent, - "user": c.user, - "date": c.date, - "desc": c.desc, + "node": hash, "log": buildLog, }) } -// match lines like: "package.BechmarkFunc 100000 999 ns/op" -var benchmarkRegexp = regexp.MustCompile("([^\n\t ]+)[\t ]+([0-9]+)[\t ]+([0-9]+) ns/op") - -// recordBenchmarks sends benchmark results to the dashboard -func (b *Builder) recordBenchmarks(benchLog string, c Commit) os.Error { - results := benchmarkRegexp.FindAllStringSubmatch(benchLog, -1) - var buf bytes.Buffer - b64 := base64.NewEncoder(base64.StdEncoding, &buf) - for _, r := range results { - for _, s := range r[1:] { - binary.Write(b64, binary.BigEndian, uint16(len(s))) - b64.Write([]byte(s)) - } - } - b64.Close() - return httpCommand("benchmarks", map[string]string{ - "builder": b.name, - "key": b.key, - "node": c.node, - "benchmarkdata": buf.String(), - }) -} - -// getPackages fetches a list of package paths from the dashboard -func getPackages() (pkgs []string, err os.Error) { - r, _, err := http.Get(fmt.Sprintf("http://%v/package?fmt=json", *dashboard)) - if err != nil { - return - } - defer r.Body.Close() - d := json.NewDecoder(r.Body) +// packages fetches a list of package paths from the dashboard +func packages() (pkgs []string, err os.Error) { var resp struct { Packages []struct { Path string } } - if err = d.Decode(&resp); err != nil { + err = dash("GET", "package", &resp, param{"fmt": "json"}) + if err != nil { return } for _, p := range resp.Packages { @@ -93,24 +113,36 @@ func getPackages() (pkgs []string, err os.Error) { } // updatePackage sends package build results and info to the dashboard -func (b *Builder) updatePackage(pkg string, state bool, buildLog, info string, c Commit) os.Error { - args := map[string]string{ +func (b *Builder) updatePackage(pkg string, state bool, buildLog, info string, hash string) os.Error { + return dash("POST", "package", nil, param{ "builder": b.name, "key": b.key, "path": pkg, "state": strconv.Btoa(state), "log": buildLog, "info": info, - "go_rev": strconv.Itoa(c.num), - } - return httpCommand("package", args) + "go_rev": hash[:12], + }) } -func httpCommand(cmd string, args map[string]string) os.Error { - if *verbose { - log.Println("httpCommand", cmd, args) +// postCommit informs the dashboard of a new commit +func postCommit(key string, l *HgLog) os.Error { + return dashStatus("POST", "commit", param{ + "key": key, + "node": l.Hash, + "date": l.Date, + "user": l.Author, + "parent": l.Parent, + "desc": l.Desc, + }) +} + +// dashboardCommit returns true if the dashboard knows about hash. +func dashboardCommit(hash string) bool { + err := dashStatus("GET", "commit", param{"node": hash}) + if err != nil { + log.Printf("check %s: %s", hash, err) + return false } - url := fmt.Sprintf("http://%v/%v", *dashboard, cmd) - _, err := http.PostForm(url, args) - return err + return true } diff --git a/misc/dashboard/builder/main.go b/misc/dashboard/builder/main.go index d11cbb133..9377fbe32 100644 --- a/misc/dashboard/builder/main.go +++ b/misc/dashboard/builder/main.go @@ -5,7 +5,6 @@ package main import ( - "container/vector" "flag" "fmt" "io/ioutil" @@ -13,16 +12,18 @@ import ( "os" "path" "regexp" + "runtime" "strconv" "strings" "time" + "xml" ) const ( codeProject = "go" codePyScript = "misc/dashboard/googlecode_upload.py" hgUrl = "https://go.googlecode.com/hg/" - waitInterval = 10e9 // time to wait before checking for new revs + waitInterval = 30e9 // time to wait before checking for new revs mkdirPerm = 0750 pkgBuildInterval = 1e9 * 60 * 60 * 24 // rebuild packages every 24 hours ) @@ -46,16 +47,10 @@ type Builder struct { codePassword string } -type BenchRequest struct { - builder *Builder - commit Commit - path string -} - var ( buildroot = flag.String("buildroot", path.Join(os.TempDir(), "gobuilder"), "Directory under which to build") + commitFlag = flag.Bool("commit", false, "upload information about new commits") dashboard = flag.String("dashboard", "godashboard.appspot.com", "Go Dashboard Host") - runBenchmarks = flag.Bool("bench", false, "Run benchmarks") buildRelease = flag.Bool("release", false, "Build and upload binary release archives") buildRevision = flag.String("rev", "", "Build specified revision and exit") buildCmd = flag.String("cmd", "./all.bash", "Build command (specify absolute or relative to go/src/)") @@ -67,7 +62,6 @@ var ( var ( goroot string releaseRegexp = regexp.MustCompile(`^(release|weekly)\.[0-9\-.]+`) - benchRequests vector.Vector ) func main() { @@ -77,7 +71,7 @@ func main() { os.Exit(2) } flag.Parse() - if len(flag.Args()) == 0 { + if len(flag.Args()) == 0 && !*commitFlag { flag.Usage() } goroot = path.Join(*buildroot, "goroot") @@ -101,17 +95,24 @@ func main() { log.Fatal("Error cloning repository:", err) } + if *commitFlag { + if len(flag.Args()) == 0 { + commitWatcher() + return + } + go commitWatcher() + } + // if specified, build revision and return if *buildRevision != "" { - c, err := getCommit(*buildRevision) + hash, err := fullHash(*buildRevision) if err != nil { log.Fatal("Error finding revision: ", err) } for _, b := range builders { - if err := b.buildCommit(c); err != nil { + if err := b.buildHash(hash); err != nil { log.Println(err) } - runQueuedBenchmark() } return } @@ -127,13 +128,8 @@ func main() { // go continuous build mode (default) // check for new commits and build them for { - err := run(nil, goroot, "hg", "pull", "-u") - if err != nil { - log.Println("hg pull failed:", err) - time.Sleep(waitInterval) - continue - } built := false + t := time.Nanoseconds() if *parallel { done := make(chan bool) for _, b := range builders { @@ -149,46 +145,15 @@ func main() { built = b.build() || built } } - // only run benchmarks if we didn't build anything - // so that they don't hold up the builder queue + // sleep if there was nothing to build if !built { - if !runQueuedBenchmark() { - // if we have no benchmarks to do, pause - time.Sleep(waitInterval) - } - // after running one benchmark, - // continue to find and build new revisions. + time.Sleep(waitInterval) + } + // sleep if we're looping too fast. + t1 := time.Nanoseconds() - t + if t1 < waitInterval { + time.Sleep(waitInterval - t1) } - } -} - -func runQueuedBenchmark() bool { - if benchRequests.Len() == 0 { - return false - } - runBenchmark(benchRequests.Pop().(BenchRequest)) - return true -} - -func runBenchmark(r BenchRequest) { - // run benchmarks and send to dashboard - log.Println(r.builder.name, "benchmarking", r.commit.num) - defer os.RemoveAll(r.path) - pkg := path.Join(r.path, "go", "src", "pkg") - bin := path.Join(r.path, "go", "bin") - env := []string{ - "GOOS=" + r.builder.goos, - "GOARCH=" + r.builder.goarch, - "PATH=" + bin + ":" + os.Getenv("PATH"), - } - logfile := path.Join(r.path, "bench.log") - benchLog, _, err := runLog(env, logfile, pkg, "gomake", "bench") - if err != nil { - log.Println(r.builder.name, "gomake bench:", err) - return - } - if err = r.builder.recordBenchmarks(benchLog, r.commit); err != nil { - log.Println("recordBenchmarks:", err) } } @@ -235,7 +200,7 @@ func (b *Builder) buildExternal() { log.Println("hg pull failed:", err) continue } - c, tag, err := getTag(releaseRegexp) + hash, tag, err := firstTag(releaseRegexp) if err != nil { log.Println(err) continue @@ -249,8 +214,8 @@ func (b *Builder) buildExternal() { if tag == prevTag && time.Nanoseconds() < nextBuild { continue } - // buildCommit will also build the packages - if err := b.buildCommit(c); err != nil { + // build will also build the packages + if err := b.buildHash(hash); err != nil { log.Println(err) continue } @@ -269,65 +234,46 @@ func (b *Builder) build() bool { log.Println(b.name, "build:", err) } }() - c, err := b.nextCommit() + hash, err := b.todo() if err != nil { log.Println(err) return false } - if c == nil { + if hash == "" { return false } - err = b.buildCommit(*c) - if err != nil { - log.Println(err) - } - return true -} + // Look for hash locally before running hg pull. -// nextCommit returns the next unbuilt Commit for this builder -func (b *Builder) nextCommit() (nextC *Commit, err os.Error) { - defer func() { - if err != nil { - err = fmt.Errorf("%s nextCommit: %s", b.name, err) + if _, err := fullHash(hash[:12]); err != nil { + // Don't have hash, so run hg pull. + if err := run(nil, goroot, "hg", "pull"); err != nil { + log.Println("hg pull failed:", err) + return false } - }() - hw, err := b.getHighWater() - if err != nil { - return } - c, err := getCommit(hw) + err = b.buildHash(hash) if err != nil { - return - } - next := c.num + 1 - c, err = getCommit(strconv.Itoa(next)) - if err == nil && c.num == next { - return &c, nil + log.Println(err) } - return nil, nil + return true } -func (b *Builder) buildCommit(c Commit) (err os.Error) { +func (b *Builder) buildHash(hash string) (err os.Error) { defer func() { if err != nil { - err = fmt.Errorf("%s buildCommit: %d: %s", b.name, c.num, err) + err = fmt.Errorf("%s build: %s: %s", b.name, hash, err) } }() - log.Println(b.name, "building", c.num) + log.Println(b.name, "building", hash) // create place in which to do work - workpath := path.Join(*buildroot, b.name+"-"+strconv.Itoa(c.num)) + workpath := path.Join(*buildroot, b.name+"-"+hash[:12]) err = os.Mkdir(workpath, mkdirPerm) if err != nil { return } - benchRequested := false - defer func() { - if !benchRequested { - os.RemoveAll(workpath) - } - }() + defer os.RemoveAll(workpath) // clone repo err = run(nil, workpath, "hg", "clone", goroot, "go") @@ -337,7 +283,7 @@ func (b *Builder) buildCommit(c Commit) (err os.Error) { // update to specified revision err = run(nil, path.Join(workpath, "go"), - "hg", "update", "-r", strconv.Itoa(c.num)) + "hg", "update", hash) if err != nil { return } @@ -348,7 +294,7 @@ func (b *Builder) buildCommit(c Commit) (err os.Error) { logfile := path.Join(workpath, "build.log") buildLog, status, err := runLog(b.envv(), logfile, srcDir, *buildCmd) if err != nil { - return fmt.Errorf("all.bash: %s", err) + return fmt.Errorf("%s: %s", *buildCmd, err) } // if we're in external mode, build all packages and return @@ -356,36 +302,27 @@ func (b *Builder) buildCommit(c Commit) (err os.Error) { if status != 0 { return os.NewError("go build failed") } - return b.buildPackages(workpath, c) + return b.buildPackages(workpath, hash) } if status != 0 { // record failure - return b.recordResult(buildLog, c) + return b.recordResult(buildLog, hash) } // record success - if err = b.recordResult("", c); err != nil { + if err = b.recordResult("", hash); err != nil { return fmt.Errorf("recordResult: %s", err) } - // send benchmark request if benchmarks are enabled - if *runBenchmarks { - benchRequests.Insert(0, BenchRequest{ - builder: b, - commit: c, - path: workpath, - }) - benchRequested = true - } - // finish here if codeUsername and codePassword aren't set if b.codeUsername == "" || b.codePassword == "" || !*buildRelease { return } // if this is a release, create tgz and upload to google code - if release := releaseRegexp.FindString(c.desc); release != "" { + releaseHash, release, err := firstTag(releaseRegexp) + if hash == releaseHash { // clean out build state err = run(b.envv(), srcDir, "./clean.bash", "--nopkg") if err != nil { @@ -411,6 +348,9 @@ func (b *Builder) buildCommit(c Commit) (err os.Error) { // envv returns an environment for build/bench execution func (b *Builder) envv() []string { + if runtime.GOOS == "windows" { + return b.envvWindows() + } e := []string{ "GOOS=" + b.goos, "GOARCH=" + b.goarch, @@ -422,6 +362,42 @@ func (b *Builder) envv() []string { return e } +// windows version of envv +func (b *Builder) envvWindows() []string { + start := map[string]string{ + "GOOS": b.goos, + "GOARCH": b.goarch, + "GOROOT_FINAL": "/c/go", + } + for _, name := range extraEnv { + start[name] = os.Getenv(name) + } + skip := map[string]bool{ + "GOBIN": true, + "GOROOT": true, + "INCLUDE": true, + "LIB": true, + } + var e []string + for name, v := range start { + e = append(e, name+"="+v) + skip[name] = true + } + for _, kv := range os.Environ() { + s := strings.Split(kv, "=", 2) + name := strings.ToUpper(s[0]) + switch { + case name == "": + // variables, like "=C:=C:\", just copy them + e = append(e, kv) + case !skip[name]: + e = append(e, kv) + skip[name] = true + } + } + return e +} + func isDirectory(name string) bool { s, err := os.Stat(name) return err == nil && s.IsDirectory() @@ -431,3 +407,209 @@ func isFile(name string) bool { s, err := os.Stat(name) return err == nil && (s.IsRegular() || s.IsSymlink()) } + +// commitWatcher polls hg for new commits and tells the dashboard about them. +func commitWatcher() { + // Create builder just to get master key. + b, err := NewBuilder("mercurial-commit") + if err != nil { + log.Fatal(err) + } + for { + if *verbose { + log.Printf("poll...") + } + commitPoll(b.key) + if *verbose { + log.Printf("sleep...") + } + time.Sleep(60e9) + } +} + +// HgLog represents a single Mercurial revision. +type HgLog struct { + Hash string + Author string + Date string + Desc string + Parent string + + // Internal metadata + added bool +} + +// logByHash is a cache of all Mercurial revisions we know about, +// indexed by full hash. +var logByHash = map[string]*HgLog{} + +// xmlLogTemplate is a template to pass to Mercurial to make +// hg log print the log in valid XML for parsing with xml.Unmarshal. +const xmlLogTemplate = ` + <log> + <hash>{node|escape}</hash> + <parent>{parent|escape}</parent> + <author>{author|escape}</author> + <date>{date}</date> + <desc>{desc|escape}</desc> + </log> +` + +// commitPoll pulls any new revisions from the hg server +// and tells the server about them. +func commitPoll(key string) { + // Catch unexpected panics. + defer func() { + if err := recover(); err != nil { + log.Printf("commitPoll panic: %s", err) + } + }() + + if err := run(nil, goroot, "hg", "pull"); err != nil { + log.Printf("hg pull: %v", err) + return + } + + const N = 20 // how many revisions to grab + + data, _, err := runLog(nil, "", goroot, "hg", "log", + "--encoding=utf-8", + "--limit="+strconv.Itoa(N), + "--template="+xmlLogTemplate, + ) + if err != nil { + log.Printf("hg log: %v", err) + return + } + + var logStruct struct { + Log []HgLog + } + err = xml.Unmarshal(strings.NewReader("<top>"+data+"</top>"), &logStruct) + if err != nil { + log.Printf("unmarshal hg log: %v", err) + return + } + + logs := logStruct.Log + + // Pass 1. Fill in parents and add new log entries to logsByHash. + // Empty parent means take parent from next log entry. + // Non-empty parent has form 1234:hashhashhash; we want full hash. + for i := range logs { + l := &logs[i] + log.Printf("hg log: %s < %s\n", l.Hash, l.Parent) + if l.Parent == "" && i+1 < len(logs) { + l.Parent = logs[i+1].Hash + } else if l.Parent != "" { + l.Parent, _ = fullHash(l.Parent) + } + if l.Parent == "" { + // Can't create node without parent. + continue + } + + if logByHash[l.Hash] == nil { + // Make copy to avoid pinning entire slice when only one entry is new. + t := *l + logByHash[t.Hash] = &t + } + } + + for i := range logs { + l := &logs[i] + if l.Parent == "" { + continue + } + addCommit(l.Hash, key) + } +} + +// addCommit adds the commit with the named hash to the dashboard. +// key is the secret key for authentication to the dashboard. +// It avoids duplicate effort. +func addCommit(hash, key string) bool { + l := logByHash[hash] + if l == nil { + return false + } + if l.added { + return true + } + + // Check for already added, perhaps in an earlier run. + if dashboardCommit(hash) { + log.Printf("%s already on dashboard\n", hash) + // Record that this hash is on the dashboard, + // as must be all its parents. + for l != nil { + l.added = true + l = logByHash[l.Parent] + } + return true + } + + // Create parent first, to maintain some semblance of order. + if !addCommit(l.Parent, key) { + return false + } + + // Create commit. + if err := postCommit(key, l); err != nil { + log.Printf("failed to add %s to dashboard: %v", key, err) + return false + } + return true +} + +// fullHash returns the full hash for the given Mercurial revision. +func fullHash(rev string) (hash string, err os.Error) { + defer func() { + if err != nil { + err = fmt.Errorf("fullHash: %s: %s", rev, err) + } + }() + s, _, err := runLog(nil, "", goroot, + "hg", "log", + "--encoding=utf-8", + "--rev="+rev, + "--limit=1", + "--template={node}", + ) + if err != nil { + return + } + s = strings.TrimSpace(s) + if s == "" { + return "", fmt.Errorf("cannot find revision") + } + if len(s) != 20 { + return "", fmt.Errorf("hg returned invalid hash " + s) + } + return s, nil +} + +var revisionRe = regexp.MustCompile(`^([^ ]+) +[0-9]+:([0-9a-f]+)$`) + +// firstTag returns the hash and tag of the most recent tag matching re. +func firstTag(re *regexp.Regexp) (hash string, tag string, err os.Error) { + o, _, err := runLog(nil, "", goroot, "hg", "tags") + for _, l := range strings.Split(o, "\n", -1) { + if l == "" { + continue + } + s := revisionRe.FindStringSubmatch(l) + if s == nil { + err = os.NewError("couldn't find revision number") + return + } + if !re.MatchString(s[1]) { + continue + } + tag = s[1] + hash, err = fullHash(s[3]) + return + } + err = os.NewError("no matching tag found") + return +} diff --git a/misc/dashboard/builder/package.go b/misc/dashboard/builder/package.go index 6e9f9ff39..ee65d7669 100644 --- a/misc/dashboard/builder/package.go +++ b/misc/dashboard/builder/package.go @@ -13,8 +13,8 @@ import ( "path" ) -func (b *Builder) buildPackages(workpath string, c Commit) os.Error { - pkgs, err := getPackages() +func (b *Builder) buildPackages(workpath string, hash string) os.Error { + pkgs, err := packages() if err != nil { return err } @@ -32,13 +32,13 @@ func (b *Builder) buildPackages(workpath string, c Commit) os.Error { built := code != 0 // get doc comment from package source - info, err := getPackageComment(p, path.Join(goroot, "pkg", p)) + info, err := packageComment(p, path.Join(goroot, "pkg", p)) if err != nil { log.Printf("goinstall %v: %v", p, err) } // update dashboard with build state + info - err = b.updatePackage(p, built, buildLog, info, c) + err = b.updatePackage(p, built, buildLog, info, hash) if err != nil { log.Printf("updatePackage %v: %v", p, err) } @@ -46,7 +46,7 @@ func (b *Builder) buildPackages(workpath string, c Commit) os.Error { return nil } -func getPackageComment(pkg, pkgpath string) (info string, err os.Error) { +func packageComment(pkg, pkgpath string) (info string, err os.Error) { fset := token.NewFileSet() pkgs, err := parser.ParseDir(fset, pkgpath, nil, parser.PackageClauseOnly|parser.ParseComments) if err != nil { diff --git a/misc/dashboard/godashboard/app.yaml b/misc/dashboard/godashboard/app.yaml index aec559dcc..83611cf90 100644 --- a/misc/dashboard/godashboard/app.yaml +++ b/misc/dashboard/godashboard/app.yaml @@ -1,9 +1,13 @@ application: godashboard -version: 5 +version: 7 runtime: python api_version: 1 handlers: +- url: /favicon\.ico + static_files: static/favicon.ico + upload: static/favicon\.ico + - url: /static static_dir: static diff --git a/misc/dashboard/godashboard/gobuild.py b/misc/dashboard/godashboard/gobuild.py index 035bf842f..5678f2e1b 100644 --- a/misc/dashboard/godashboard/gobuild.py +++ b/misc/dashboard/godashboard/gobuild.py @@ -5,21 +5,19 @@ # This is the server part of the continuous build system for Go. It must be run # by AppEngine. +from django.utils import simplejson from google.appengine.api import mail from google.appengine.api import memcache from google.appengine.ext import db from google.appengine.ext import webapp from google.appengine.ext.webapp import template from google.appengine.ext.webapp.util import run_wsgi_app -import binascii import datetime import hashlib import hmac import logging import os import re -import struct -import time import bz2 # local imports @@ -50,10 +48,6 @@ class Commit(db.Model): fail_notification_sent = db.BooleanProperty() -class Cache(db.Model): - data = db.BlobProperty() - expire = db.IntegerProperty() - # A CompressedLog contains the textual build log of a failed build. # The key name is the hex digest of the SHA256 hash of the contents. # The contents is bz2 compressed. @@ -62,23 +56,6 @@ class CompressedLog(db.Model): N = 30 -def cache_get(key): - c = Cache.get_by_key_name(key) - if c is None or c.expire < time.time(): - return None - return c.data - -def cache_set(key, val, timeout): - c = Cache(key_name = key) - c.data = val - c.expire = int(time.time() + timeout) - c.put() - -def cache_del(key): - c = Cache.get_by_key_name(key) - if c is not None: - c.delete() - def builderInfo(b): f = b.split('-', 3) goos = f[0] @@ -96,7 +73,7 @@ def builderset(): for c in results: builders.update(set(parseBuild(build)['builder'] for build in c.builds)) return builders - + class MainPage(webapp.RequestHandler): def get(self): self.response.headers['Content-Type'] = 'text/html; charset=utf-8' @@ -147,7 +124,30 @@ class MainPage(webapp.RequestHandler): path = os.path.join(os.path.dirname(__file__), 'main.html') self.response.out.write(template.render(path, values)) -class GetHighwater(webapp.RequestHandler): +# A DashboardHandler is a webapp.RequestHandler but provides +# authenticated_post - called by post after authenticating +# json - writes object in json format to response output +class DashboardHandler(webapp.RequestHandler): + def post(self): + if not auth(self.request): + self.response.set_status(403) + return + self.authenticated_post() + + def authenticated_post(self): + return + + def json(self, obj): + self.response.set_status(200) + simplejson.dump(obj, self.response.out) + return + +def auth(req): + k = req.get('key') + return k == hmac.new(key.accessKey, req.get('builder')).hexdigest() or k == key.accessKey + +# Todo serves /todo. It tells the builder which commits need to be built. +class Todo(DashboardHandler): def get(self): builder = self.request.get('builder') key = 'todo-%s' % builder @@ -155,28 +155,19 @@ class GetHighwater(webapp.RequestHandler): if response is None: # Fell out of memcache. Rebuild from datastore results. # We walk the commit list looking for nodes that have not - # been built by this builder and record the *parents* of those - # nodes, because each builder builds the revision *after* the - # one return (because we might not know about the latest - # revision). + # been built by this builder. q = Commit.all() q.order('-__key__') todo = [] - need = False first = None for c in q.fetch(N+1): if first is None: first = c - if need: - todo.append(c.node) - need = not built(c, builder) - if not todo: - todo.append(first.node) - response = ' '.join(todo) + if not built(c, builder): + todo.append({'Hash': c.node}) + response = simplejson.dumps(todo) memcache.set(key, response, 3600) self.response.set_status(200) - if self.request.get('all') != 'yes': - response = response.split()[0] self.response.out.write(response) def built(c, builder): @@ -185,22 +176,8 @@ def built(c, builder): return True return False -def auth(req): - k = req.get('key') - return k == hmac.new(key.accessKey, req.get('builder')).hexdigest() or k == key.accessKey - -class SetHighwater(webapp.RequestHandler): - def post(self): - if not auth(self.request): - self.response.set_status(403) - return - - # Allow for old builders. - # This is a no-op now: we figure out what to build based - # on the current dashboard status. - return - -class LogHandler(webapp.RequestHandler): +# Log serves /log/. It retrieves log data by content hash. +class LogHandler(DashboardHandler): def get(self): self.response.headers['Content-Type'] = 'text/plain; charset=utf-8' hash = self.request.path[5:] @@ -214,12 +191,8 @@ class LogHandler(webapp.RequestHandler): # Init creates the commit with id 0. Since this commit doesn't have a parent, # it cannot be created by Build. -class Init(webapp.RequestHandler): - def post(self): - if not auth(self.request): - self.response.set_status(403) - return - +class Init(DashboardHandler): + def authenticated_post(self): date = parseDate(self.request.get('date')) node = self.request.get('node') if not validNode(node) or date is None: @@ -239,7 +212,86 @@ class Init(webapp.RequestHandler): self.response.set_status(200) -# Build is the main command: it records the result of a new build. +# The last commit when we switched to using entity groups. +# This is the root of the new commit entity group. +RootCommitKeyName = '00000f26-f32c6f1038207c55d5780231f7484f311020747e' + +# CommitHandler serves /commit. +# A GET of /commit retrieves information about the specified commit. +# A POST of /commit creates a node for the given commit. +# If the commit already exists, the POST silently succeeds (like mkdir -p). +class CommitHandler(DashboardHandler): + def get(self): + node = self.request.get('node') + if not validNode(node): + return self.json({'Status': 'FAIL', 'Error': 'malformed node hash'}) + n = nodeByHash(node) + if n is None: + return self.json({'Status': 'FAIL', 'Error': 'unknown revision'}) + return self.json({'Status': 'OK', 'Node': nodeObj(n)}) + + def authenticated_post(self): + # Require auth with the master key, not a per-builder key. + if self.request.get('builder'): + self.response.set_status(403) + return + + node = self.request.get('node') + date = parseDate(self.request.get('date')) + user = self.request.get('user').encode('utf8') + desc = self.request.get('desc').encode('utf8') + parenthash = self.request.get('parent') + + if not validNode(node) or not validNode(parenthash) or date is None: + return self.json({'Status': 'FAIL', 'Error': 'malformed node, parent, or date'}) + + n = nodeByHash(node) + if n is None: + p = nodeByHash(parenthash) + if p is None: + return self.json({'Status': 'FAIL', 'Error': 'unknown parent'}) + + # Want to create new node in a transaction so that multiple + # requests creating it do not collide and so that multiple requests + # creating different nodes get different sequence numbers. + # All queries within a transaction must include an ancestor, + # but the original datastore objects we used for the dashboard + # have no common ancestor. Instead, we use a well-known + # root node - the last one before we switched to entity groups - + # as the as the common ancestor. + root = Commit.get_by_key_name(RootCommitKeyName) + + def add_commit(): + if nodeByHash(node, ancestor=root) is not None: + return + + # Determine number for this commit. + # Once we have created one new entry it will be lastRooted.num+1, + # but the very first commit created in this scheme will have to use + # last.num's number instead (last is likely not rooted). + q = Commit.all() + q.order('-__key__') + q.ancestor(root) + last = q.fetch(1)[0] + num = last.num+1 + + n = Commit(key_name = '%08x-%s' % (num, node), parent = root) + n.num = num + n.node = node + n.parentnode = parenthash + n.user = user + n.date = date + n.desc = desc + n.put() + db.run_in_transaction(add_commit) + n = nodeByHash(node) + if n is None: + return self.json({'Status': 'FAIL', 'Error': 'failed to create commit node'}) + + return self.json({'Status': 'OK', 'Node': nodeObj(n)}) + +# Build serves /build. +# A POST to /build records a new build result. class Build(webapp.RequestHandler): def post(self): if not auth(self.request): @@ -256,44 +308,33 @@ class Build(webapp.RequestHandler): l.log = bz2.compress(log) l.put() - date = parseDate(self.request.get('date')) - user = self.request.get('user').encode('utf8') - desc = self.request.get('desc').encode('utf8') node = self.request.get('node') - parenthash = self.request.get('parent') - if not validNode(node) or not validNode(parenthash) or date is None: - logging.error("Not valid node ('%s') or bad date (%s %s)", node, date, self.request.get('date')) + if not validNode(node): + logging.error('Invalid node %s' % (node)) self.response.set_status(500) return - q = Commit.all() - q.filter('node =', parenthash) - parent = q.get() - if parent is None: - logging.error('Cannot find parent %s of node %s' % (parenthash, node)) + n = nodeByHash(node) + if n is None: + logging.error('Cannot find node %s' % (node)) self.response.set_status(404) return - parentnum, _ = parent.key().name().split('-', 1) - nodenum = int(parentnum, 16) + 1 - - key_name = '%08x-%s' % (nodenum, node) + nn = n def add_build(): - n = Commit.get_by_key_name(key_name) + n = nodeByHash(node, ancestor=nn) if n is None: - n = Commit(key_name = key_name) - n.num = nodenum - n.node = node - n.parentnode = parenthash - n.user = user - n.date = date - n.desc = desc + logging.error('Cannot find hash in add_build: %s %s' % (builder, node)) + return + s = '%s`%s' % (builder, loghash) for i, b in enumerate(n.builds): if b.split('`', 1)[0] == builder: + # logging.error('Found result for %s %s already' % (builder, node)) n.builds[i] = s break else: + # logging.error('Added result for %s %s' % (builder, node)) n.builds.append(s) n.put() @@ -302,39 +343,101 @@ class Build(webapp.RequestHandler): key = 'todo-%s' % builder memcache.delete(key) - def mark_sent(): - n = Commit.get_by_key_name(key_name) - n.fail_notification_sent = True - n.put() - - n = Commit.get_by_key_name(key_name) - if loghash and not failed(parent, builder) and not n.fail_notification_sent: - subject = const.mail_fail_subject % (builder, desc.split("\n")[0]) - path = os.path.join(os.path.dirname(__file__), 'fail-notify.txt') - body = template.render(path, { - "builder": builder, - "node": node[:12], - "user": user, - "desc": desc, - "loghash": loghash - }) - mail.send_mail( - sender=const.mail_from, - reply_to=const.mail_fail_reply_to, - to=const.mail_fail_to, - subject=subject, - body=body - ) - db.run_in_transaction(mark_sent) + c = getBrokenCommit(node, builder) + if c is not None and not c.fail_notification_sent: + notifyBroken(c, builder) self.response.set_status(200) -def failed(c, builder): + +def getBrokenCommit(node, builder): + """ + getBrokenCommit returns a Commit that breaks the build. + The Commit will be either the one specified by node or the one after. + """ + + # Squelch mail if already fixed. + head = firstResult(builder) + if broken(head, builder) == False: + return + + # Get current node and node before, after. + cur = nodeByHash(node) + if cur is None: + return + before = nodeBefore(cur) + after = nodeAfter(cur) + + if broken(before, builder) == False and broken(cur, builder): + return cur + if broken(cur, builder) == False and broken(after, builder): + return after + + return + +def firstResult(builder): + q = Commit.all().order('-__key__') + for c in q.fetch(20): + for i, b in enumerate(c.builds): + p = b.split('`', 1) + if p[0] == builder: + return c + return None + +def nodeBefore(c): + return nodeByHash(c.parentnode) + +def nodeAfter(c): + return Commit.all().filter('parenthash', c.node).get() + +def notifyBroken(c, builder): + def send(): + n = Commit.get(c.key()) + if n is None: + logging.error("couldn't retrieve Commit '%s'" % c.key()) + return False + if n.fail_notification_sent: + return False + n.fail_notification_sent = True + return n.put() + if not db.run_in_transaction(send): + return + + subject = const.mail_fail_subject % (builder, c.desc.split('\n')[0]) + path = os.path.join(os.path.dirname(__file__), 'fail-notify.txt') + body = template.render(path, { + "builder": builder, + "node": c.node, + "user": c.user, + "desc": c.desc, + "loghash": logHash(c, builder) + }) + mail.send_mail( + sender=const.mail_from, + to=const.mail_fail_to, + subject=subject, + body=body + ) + +def logHash(c, builder): + for i, b in enumerate(c.builds): + p = b.split('`', 1) + if p[0] == builder: + return p[1] + return "" + +def broken(c, builder): + """ + broken returns True if commit c breaks the build for the specified builder, + False if it is a good build, and None if no results exist for this builder. + """ + if c is None: + return None for i, b in enumerate(c.builds): p = b.split('`', 1) if p[0] == builder: return len(p[1]) > 0 - return False + return None def node(num): q = Commit.all() @@ -342,6 +445,24 @@ def node(num): n = q.get() return n +def nodeByHash(hash, ancestor=None): + q = Commit.all() + q.filter('node =', hash) + if ancestor is not None: + q.ancestor(ancestor) + n = q.get() + return n + +# nodeObj returns a JSON object (ready to be passed to simplejson.dump) describing node. +def nodeObj(n): + return { + 'Hash': n.node, + 'ParentHash': n.parentnode, + 'User': n.user, + 'Date': n.date.strftime('%Y-%m-%d %H:%M %z'), + 'Desc': n.desc, + } + class FixedOffset(datetime.tzinfo): """Fixed offset in minutes east from UTC.""" @@ -417,15 +538,20 @@ def toRev(c): def byBuilder(x, y): return cmp(x['builder'], y['builder']) +# Give old builders work; otherwise they pound on the web site. +class Hwget(DashboardHandler): + def get(self): + self.response.out.write("8000\n") + # This is the URL map for the server. The first three entries are public, the # rest are only used by the builders. application = webapp.WSGIApplication( [('/', MainPage), + ('/hw-get', Hwget), ('/log/.*', LogHandler), - ('/hw-get', GetHighwater), - ('/hw-set', SetHighwater), - + ('/commit', CommitHandler), ('/init', Init), + ('/todo', Todo), ('/build', Build), ], debug=True) diff --git a/misc/dashboard/godashboard/index.yaml b/misc/dashboard/godashboard/index.yaml index 148824bb6..4a00c4a6f 100644 --- a/misc/dashboard/godashboard/index.yaml +++ b/misc/dashboard/godashboard/index.yaml @@ -23,6 +23,12 @@ indexes: - name: __key__ direction: desc +- kind: Commit + ancestor: yes + properties: + - name: __key__ + direction: desc + - kind: Project properties: - name: approved diff --git a/misc/dashboard/godashboard/static/favicon.ico b/misc/dashboard/godashboard/static/favicon.ico Binary files differnew file mode 100644 index 000000000..48854ff3b --- /dev/null +++ b/misc/dashboard/godashboard/static/favicon.ico diff --git a/misc/dashboard/googlecode_upload.py b/misc/dashboard/googlecode_upload.py index 3b1d432ff..e87db884a 100755 --- a/misc/dashboard/googlecode_upload.py +++ b/misc/dashboard/googlecode_upload.py @@ -70,7 +70,7 @@ def upload(file, project_name, user_name, password, summary, labels=None): Returns: a tuple: http_status: 201 if the upload succeeded, something else if an - error occured. + error occurred. http_reason: The human-readable string associated with http_status file_url: If the upload succeeded, the URL of the file on Google Code, None otherwise. |