diff options
Diffstat (limited to 'misc')
36 files changed, 1119 insertions, 493 deletions
diff --git a/misc/IntelliJIDEA/Go.xml b/misc/IntelliJIDEA/Go.xml new file mode 100644 index 000000000..09265a2e0 --- /dev/null +++ b/misc/IntelliJIDEA/Go.xml @@ -0,0 +1,98 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Copyright 2011 The Go Authors. All rights reserved. + Use of this source code is governed by a BSD-style + license that can be found in the LICENSE file. + +Copy this custom language definition & configuration file to + * Mac : ~/Library/Preferences/IntelliJIdea10/filetypes/ + * Linux & Windows : ~/.IntelliJIdea10/config/filetypes/ +--> + +<filetype binary="false" default_extension="" description="Go" name="Go"> + <highlighting> + <options> + <option name="LINE_COMMENT" value="//"/> + <option name="COMMENT_START" value="/*"/> + <option name="COMMENT_END" value="*/"/> + <option name="HEX_PREFIX" value="0x"/> + <option name="NUM_POSTFIXES" value=""/> + <option name="HAS_BRACKETS" value="true"/> + <option name="HAS_BRACES" value="true"/> + <option name="HAS_PARENS" value="true"/> + <option name="HAS_STRING_ESCAPES" value="true"/> + </options> + <keywords ignore_case="false"> + <keyword name="break"/> + <keyword name="case"/> + <keyword name="chan"/> + <keyword name="const"/> + <keyword name="continue"/> + <keyword name="default"/> + <keyword name="defer"/> + <keyword name="else"/> + <keyword name="fallthrough"/> + <keyword name="for"/> + <keyword name="func"/> + <keyword name="go"/> + <keyword name="goto"/> + <keyword name="if"/> + <keyword name="import"/> + <keyword name="interface"/> + <keyword name="map"/> + <keyword name="package"/> + <keyword name="range"/> + <keyword name="return"/> + <keyword name="select"/> + <keyword name="struct"/> + <keyword name="switch"/> + <keyword name="type"/> + <keyword name="var"/> + </keywords> + <keywords2> + <keyword name="bool"/> + <keyword name="byte"/> + <keyword name="complex64"/> + <keyword name="complex128"/> + <keyword name="float32"/> + <keyword name="float64"/> + <keyword name="int"/> + <keyword name="int8"/> + <keyword name="int16"/> + <keyword name="int32"/> + <keyword name="int64"/> + <keyword name="string"/> + <keyword name="uint"/> + <keyword name="uint8"/> + <keyword name="uint16"/> + <keyword name="uint32"/> + <keyword name="uint64"/> + <keyword name="uintptr"/> + </keywords2> + <keywords3> + <keyword name="append"/> + <keyword name="cap"/> + <keyword name="close"/> + <keyword name="complex"/> + <keyword name="copy"/> + <keyword name="imag"/> + <keyword name="len"/> + <keyword name="make"/> + <keyword name="new"/> + <keyword name="panic"/> + <keyword name="print"/> + <keyword name="println"/> + <keyword name="real"/> + <keyword name="recover"/> + </keywords3> + <keywords4> + <keyword name="false"/> + <keyword name="iota"/> + <keyword name="nil"/> + <keyword name="true"/> + </keywords4> + </highlighting> + <extensionMap> + <mapping ext="go"/> + </extensionMap> +</filetype> diff --git a/misc/cgo/gmp/gmp.go b/misc/cgo/gmp/gmp.go index f7bbe9c51..7faa71b69 100644 --- a/misc/cgo/gmp/gmp.go +++ b/misc/cgo/gmp/gmp.go @@ -86,9 +86,8 @@ explicitly in Go to pointers to arrays, as they do (implicitly) in C. Garbage collection is the big problem. It is fine for the Go world to have pointers into the C world and to free those pointers when they -are no longer needed. To help, the garbage collector calls an -object's destroy() method prior to collecting it. C pointers can be -wrapped by Go objects with appropriate destroy methods. +are no longer needed. To help, the Go code can define Go objects +holding the C pointers and use runtime.SetFinalizer on those Go objects. It is much more difficult for the C world to have pointers into the Go world, because the Go garbage collector is unaware of the memory diff --git a/misc/cgo/test/Makefile b/misc/cgo/test/Makefile index 893540d97..43c45f416 100644 --- a/misc/cgo/test/Makefile +++ b/misc/cgo/test/Makefile @@ -10,6 +10,7 @@ CGOFILES=\ align.go\ basic.go\ callback.go\ + env.go\ issue1222.go\ issue1328.go\ issue1560.go\ diff --git a/misc/cgo/test/align.go b/misc/cgo/test/align.go index 2d2979595..07ab9ef50 100644 --- a/misc/cgo/test/align.go +++ b/misc/cgo/test/align.go @@ -58,7 +58,7 @@ import ( "testing" ) -func TestAlign(t *testing.T) { +func testAlign(t *testing.T) { var evt C.SDL_KeyboardEvent C.makeEvent(&evt) if C.same(&evt, evt.typ, evt.which, evt.state, evt.keysym.scancode, evt.keysym.sym, evt.keysym.mod, evt.keysym.unicode) == 0 { diff --git a/misc/cgo/test/basic.go b/misc/cgo/test/basic.go index a94074c52..b9d0953bd 100644 --- a/misc/cgo/test/basic.go +++ b/misc/cgo/test/basic.go @@ -90,31 +90,31 @@ func Atol(s string) int { return int(n) } -func TestConst(t *testing.T) { +func testConst(t *testing.T) { C.myConstFunc(nil, 0, nil) } -func TestEnum(t *testing.T) { +func testEnum(t *testing.T) { if C.Enum1 != 1 || C.Enum2 != 2 { t.Error("bad enum", C.Enum1, C.Enum2) } } -func TestAtol(t *testing.T) { +func testAtol(t *testing.T) { l := Atol("123") if l != 123 { t.Error("Atol 123: ", l) } } -func TestErrno(t *testing.T) { +func testErrno(t *testing.T) { n, err := Strtol("asdf", 123) if n != 0 || err != os.EINVAL { t.Error("Strtol: ", n, err) } } -func TestMultipleAssign(t *testing.T) { +func testMultipleAssign(t *testing.T) { p := C.CString("234") n, m := C.strtol(p, nil, 345), C.strtol(p, nil, 10) if n != 0 || m != 234 { diff --git a/misc/cgo/test/callback.go b/misc/cgo/test/callback.go index 450a7cbf2..3edee9758 100644 --- a/misc/cgo/test/callback.go +++ b/misc/cgo/test/callback.go @@ -27,7 +27,7 @@ func goCallback(p unsafe.Pointer) { (*(*func())(unsafe.Pointer(&p)))() } -func TestCallback(t *testing.T) { +func testCallback(t *testing.T) { var x = false nestedCall(func() { x = true }) if !x { @@ -35,13 +35,13 @@ func TestCallback(t *testing.T) { } } -func TestCallbackGC(t *testing.T) { +func testCallbackGC(t *testing.T) { nestedCall(runtime.GC) } func lockedOSThread() bool // in runtime.c -func TestCallbackPanic(t *testing.T) { +func testCallbackPanic(t *testing.T) { // Make sure panic during callback unwinds properly. if lockedOSThread() { t.Fatal("locked OS thread on entry to TestCallbackPanic") @@ -62,14 +62,14 @@ func TestCallbackPanic(t *testing.T) { panic("nestedCall returned") } -func TestCallbackPanicLoop(t *testing.T) { +func testCallbackPanicLoop(t *testing.T) { // Make sure we don't blow out m->g0 stack. for i := 0; i < 100000; i++ { TestCallbackPanic(t) } } -func TestCallbackPanicLocked(t *testing.T) { +func testCallbackPanicLocked(t *testing.T) { runtime.LockOSThread() defer runtime.UnlockOSThread() @@ -94,7 +94,7 @@ func TestCallbackPanicLocked(t *testing.T) { // Callback with zero arguments used to make the stack misaligned, // which broke the garbage collector and other things. -func TestZeroArgCallback(t *testing.T) { +func testZeroArgCallback(t *testing.T) { defer func() { s := recover() if s != nil { @@ -118,7 +118,7 @@ func goFoo() { func variadic(x ...interface{}) {} -func TestBlocking(t *testing.T) { +func testBlocking(t *testing.T) { c := make(chan int) go func() { for i := 0; i < 10; i++ { diff --git a/misc/cgo/test/cgo_test.go b/misc/cgo/test/cgo_test.go index 967dc0e92..94fba15db 100644 --- a/misc/cgo/test/cgo_test.go +++ b/misc/cgo/test/cgo_test.go @@ -1,5 +1,28 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + package cgotest -// dummy file so gotest thinks there are tests. -// the actual tests are in the main go files, next -// to the code they test. +import "testing" + +// The actual test functions are in non-_test.go files +// so that they can use cgo (import "C"). +// These wrappers are here for gotest to find. + +func TestAlign(t *testing.T) { testAlign(t) } +func TestConst(t *testing.T) { testConst(t) } +func TestEnum(t *testing.T) { testEnum(t) } +func TestAtol(t *testing.T) { testAtol(t) } +func TestErrno(t *testing.T) { testErrno(t) } +func TestMultipleAssign(t *testing.T) { testMultipleAssign(t) } +func TestCallback(t *testing.T) { testCallback(t) } +func TestCallbackGC(t *testing.T) { testCallbackGC(t) } +func TestCallbackPanic(t *testing.T) { testCallbackPanic(t) } +func TestCallbackPanicLoop(t *testing.T) { testCallbackPanicLoop(t) } +func TestCallbackPanicLocked(t *testing.T) { testCallbackPanicLocked(t) } +func TestZeroArgCallback(t *testing.T) { testZeroArgCallback(t) } +func TestBlocking(t *testing.T) { testBlocking(t) } +func Test1328(t *testing.T) { test1328(t) } +func TestParallelSleep(t *testing.T) { testParallelSleep(t) } +func TestSetEnv(t *testing.T) { testSetEnv(t) } diff --git a/misc/cgo/test/env.go b/misc/cgo/test/env.go new file mode 100644 index 000000000..1fb4e684c --- /dev/null +++ b/misc/cgo/test/env.go @@ -0,0 +1,32 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cgotest + +/* +#include <stdlib.h> +*/ +import "C" +import ( + "os" + "testing" + "unsafe" +) + +// This is really an os package test but here for convenience. +func testSetEnv(t *testing.T) { + const key = "CGO_OS_TEST_KEY" + const val = "CGO_OS_TEST_VALUE" + os.Setenv(key, val) + keyc := C.CString(key) + defer C.free(unsafe.Pointer(keyc)) + v := C.getenv(keyc) + if v == (*C.char)(unsafe.Pointer(uintptr(0))) { + t.Fatal("getenv returned NULL") + } + vs := C.GoString(v) + if vs != val { + t.Fatalf("getenv() = %q; want %q", vs, val) + } +} diff --git a/misc/cgo/test/issue1328.go b/misc/cgo/test/issue1328.go index f29d7057e..e01207dd9 100644 --- a/misc/cgo/test/issue1328.go +++ b/misc/cgo/test/issue1328.go @@ -25,6 +25,6 @@ func BackIntoGo() { func xvariadic(x ...interface{}) { } -func Test1328(t *testing.T) { +func test1328(t *testing.T) { C.IntoC() } diff --git a/misc/cgo/test/issue1560.go b/misc/cgo/test/issue1560.go index 75d31c035..e534cce47 100644 --- a/misc/cgo/test/issue1560.go +++ b/misc/cgo/test/issue1560.go @@ -35,7 +35,7 @@ func BackgroundSleep(n int) { }() } -func TestParallelSleep(t *testing.T) { +func testParallelSleep(t *testing.T) { dt := -time.Nanoseconds() parallelSleep(1) dt += time.Nanoseconds() diff --git a/misc/chrome/gophertool/README.txt b/misc/chrome/gophertool/README.txt new file mode 100644 index 000000000..a7c0b4b26 --- /dev/null +++ b/misc/chrome/gophertool/README.txt @@ -0,0 +1,8 @@ +To install: + +1) chrome://extensions/ +2) click "[+] Developer Mode" in top right +3) "Load unpacked extension..." +4) pick $GOROOT/misc/chrome/gophertool + +Done. It'll now auto-reload from source. diff --git a/misc/chrome/gophertool/background.html b/misc/chrome/gophertool/background.html new file mode 100644 index 000000000..058c18142 --- /dev/null +++ b/misc/chrome/gophertool/background.html @@ -0,0 +1,24 @@ +<html> +<!-- + Copyright 2011 The Go Authors. All rights reserved. + Use of this source code is governed by a BSD-style + license that can be found in the LICENSE file. +--> +<head> +<script src="gopher.js"></script> +<script> + +chrome.omnibox.onInputEntered.addListener(function(t) { + var url = urlForInput(t); + if (url) { + chrome.tabs.getSelected(null, function(tab) { + if (!tab) return; + chrome.tabs.update(tab.id, { "url": url, "selected": true }); + }); + } +}); + +</script> +</head> +</html> + diff --git a/misc/chrome/gophertool/gopher.js b/misc/chrome/gophertool/gopher.js new file mode 100644 index 000000000..847c1c70d --- /dev/null +++ b/misc/chrome/gophertool/gopher.js @@ -0,0 +1,34 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +var numericRE = /^\d+$/; +var commitRE = /^(?:\d+:)?([0-9a-f]{6,20})$/; // e.g "8486:ab29d2698a47" or "ab29d2698a47" +var pkgRE = /^[a-z0-9_\/]+$/; + +function urlForInput(t) { + if (!t) { + return null; + } + + if (numericRE.test(t)) { + if (t < 1000000) { + return "http://code.google.com/p/go/issues/detail?id=" + t; + } + return "http://codereview.appspot.com/" + t + "/"; + } + + var match = commitRE.exec(t); + if (match) { + return "http://code.google.com/p/go/source/detail?r=" + match[1]; + } + + if (pkgRE.test(t)) { + // TODO: make this smarter, using a list of packages + substring matches. + // Get the list from godoc itself in JSON format? + // TODO: prefer localhost:6060 to golang.org if localhost:6060 is responding. + return "http://golang.org/pkg/" + t; + } + + return null; +} diff --git a/misc/chrome/gophertool/gopher.png b/misc/chrome/gophertool/gopher.png Binary files differnew file mode 100644 index 000000000..0d1abb741 --- /dev/null +++ b/misc/chrome/gophertool/gopher.png diff --git a/misc/chrome/gophertool/manifest.json b/misc/chrome/gophertool/manifest.json new file mode 100644 index 000000000..3a2540a86 --- /dev/null +++ b/misc/chrome/gophertool/manifest.json @@ -0,0 +1,17 @@ +{ + "name": "Hacking Gopher", + "version": "1.0", + "description": "Go Hacking utility", + "background_page": "background.html", + "browser_action": { + "default_icon": "gopher.png", + "popup": "popup.html" + }, + "omnibox": { "keyword": "golang" }, + "icons": { + "16": "gopher.png" + }, + "permissions": [ + "tabs" + ] +} diff --git a/misc/chrome/gophertool/popup.html b/misc/chrome/gophertool/popup.html new file mode 100644 index 000000000..ebbc71f3a --- /dev/null +++ b/misc/chrome/gophertool/popup.html @@ -0,0 +1,54 @@ +<html> +<!-- + Copyright 2011 The Go Authors. All rights reserved. + Use of this source code is governed by a BSD-style + license that can be found in the LICENSE file. +--> +<head> +<script src="gopher.js"></script> +<script> + +function focusinput() { + document.getElementById("inputbox").focus(); +} + +function navigate() { + var box = document.getElementById("inputbox"); + box.focus(); + + var t = box.value; + if (t == "") { + return false; + } + + var success = function(url) { + console.log("matched " + t + " to: " + url) + box.value = ""; + openURL(url); + return false; // cancel form submission + }; + + var url = urlForInput(t); + if (url) { + return success(url); + } + + console.log("no match for text: " + t) + return false; +} + +function openURL(url) { + chrome.tabs.create({ "url": url }) +} + +</script> +</head> +<body onload="focusinput()" style='margin: 0.5em; font-family: sans;'> +<small><a href="#" onclick="openURL('http://code.google.com/p/go/issues/list')">issue</a>, +<a href="#" onclick="openURL('http://codereview.appspot.com/')">codereview</a>, +<a href="#" onclick="openURL('http://code.google.com/p/go/source/list')">commit</a>, or +<a href="#" onclick="openURL('http://golang.org/pkg/')">pkg</a> id/name:</small> +<form style='margin: 0' onsubmit="return navigate();"><nobr><input id="inputbox" size=10 /><input type="submit" value="go" /></nobr></form> +<small>Also: <a href="#" onclick="openURL('http://godashboard.appspot.com/')">buildbots</small> +</body> +</html> diff --git a/misc/dashboard/builder/Makefile b/misc/dashboard/builder/Makefile index cff47942a..f1d9c5497 100644 --- a/misc/dashboard/builder/Makefile +++ b/misc/dashboard/builder/Makefile @@ -7,7 +7,6 @@ include ../../../src/Make.inc TARG=gobuilder GOFILES=\ exec.go\ - hg.go\ http.go\ main.go\ package.go\ diff --git a/misc/dashboard/builder/doc.go b/misc/dashboard/builder/doc.go index 7bb7ccbe3..30d8fe948 100644 --- a/misc/dashboard/builder/doc.go +++ b/misc/dashboard/builder/doc.go @@ -14,9 +14,6 @@ It periodically pulls updates from the Go Mercurial repository. When a newer revision is found, Go Builder creates a clone of the repository, runs all.bash, and reports build success or failure to the Go Dashboard. -For a successful build, Go Builder will also run benchmarks -(cd $GOROOT/src/pkg; make bench) and send the results to the Go Dashboard. - For a release revision (a change description that matches "release.YYYY-MM-DD"), Go Builder will create a tar.gz archive of the GOROOT and deliver it to the Go Google Code project's downloads section. @@ -34,8 +31,6 @@ Optional flags: The location of the Go Dashboard application to which Go Builder will report its results. - -bench: Run benchmarks - -release: Build and deliver binary release archive -rev=N: Build revision N and exit @@ -45,7 +40,7 @@ Optional flags: -v: Verbose logging -external: External package builder mode (will not report Go build - state to dashboard, issue releases, or run benchmarks) + state to dashboard or issue releases) The key file should be located at $HOME/.gobuildkey or, for a builder-specific key, $HOME/.gobuildkey-$BUILDER (eg, $HOME/.gobuildkey-linux-amd64). diff --git a/misc/dashboard/builder/exec.go b/misc/dashboard/builder/exec.go index 3c6fbdced..a042c5699 100644 --- a/misc/dashboard/builder/exec.go +++ b/misc/dashboard/builder/exec.go @@ -18,59 +18,57 @@ func run(envv []string, dir string, argv ...string) os.Error { if *verbose { log.Println("run", argv) } - bin, err := pathLookup(argv[0]) - if err != nil { - return err - } - p, err := exec.Run(bin, argv, envv, dir, - exec.DevNull, exec.DevNull, exec.PassThrough) - if err != nil { - return err - } - return p.Close() + argv = useBash(argv) + cmd := exec.Command(argv[0], argv[1:]...) + cmd.Dir = dir + cmd.Env = envv + cmd.Stderr = os.Stderr + return cmd.Run() } // runLog runs a process and returns the combined stdout/stderr, -// as well as writing it to logfile (if specified). -func runLog(envv []string, logfile, dir string, argv ...string) (output string, exitStatus int, err os.Error) { +// as well as writing it to logfile (if specified). It returns +// process combined stdout and stderr output, exit status and error. +// The error returned is nil, if process is started successfully, +// even if exit status is not 0. +func runLog(envv []string, logfile, dir string, argv ...string) (string, int, os.Error) { if *verbose { log.Println("runLog", argv) } - bin, err := pathLookup(argv[0]) - if err != nil { - return - } - p, err := exec.Run(bin, argv, envv, dir, - exec.DevNull, exec.Pipe, exec.MergeWithStdout) - if err != nil { - return - } - defer p.Close() + argv = useBash(argv) + b := new(bytes.Buffer) var w io.Writer = b if logfile != "" { f, err := os.OpenFile(logfile, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666) if err != nil { - return + return "", 0, err } defer f.Close() w = io.MultiWriter(f, b) } - _, err = io.Copy(w, p.Stdout) - if err != nil { - return - } - wait, err := p.Wait(0) + + cmd := exec.Command(argv[0], argv[1:]...) + cmd.Dir = dir + cmd.Env = envv + cmd.Stdout = w + cmd.Stderr = w + + err := cmd.Run() if err != nil { - return + if ws, ok := err.(*os.Waitmsg); ok { + return b.String(), ws.ExitStatus(), nil + } } - return b.String(), wait.WaitStatus.ExitStatus(), nil + return b.String(), 0, nil } -// Find bin in PATH if a relative or absolute path hasn't been specified -func pathLookup(s string) (string, os.Error) { - if strings.HasPrefix(s, "/") || strings.HasPrefix(s, "./") || strings.HasPrefix(s, "../") { - return s, nil +// useBash prefixes a list of args with 'bash' if the first argument +// is a bash script. +func useBash(argv []string) []string { + // TODO(brainman): choose a more reliable heuristic here. + if strings.HasSuffix(argv[0], ".bash") { + argv = append([]string{"bash"}, argv...) } - return exec.LookPath(s) + return argv } diff --git a/misc/dashboard/builder/hg.go b/misc/dashboard/builder/hg.go deleted file mode 100644 index d4310845d..000000000 --- a/misc/dashboard/builder/hg.go +++ /dev/null @@ -1,86 +0,0 @@ -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package main - -import ( - "fmt" - "os" - "regexp" - "strconv" - "strings" -) - -type Commit struct { - num int // mercurial revision number - node string // mercurial hash - parent string // hash of commit's parent - user string // author's Name <email> - date string // date of commit - desc string // description -} - -// getCommit returns details about the Commit specified by the revision hash -func getCommit(rev string) (c Commit, err os.Error) { - defer func() { - if err != nil { - err = fmt.Errorf("getCommit: %s: %s", rev, err) - } - }() - parts, err := getCommitParts(rev) - if err != nil { - return - } - num, err := strconv.Atoi(parts[0]) - if err != nil { - return - } - parent := "" - if num > 0 { - prev := strconv.Itoa(num - 1) - if pparts, err := getCommitParts(prev); err == nil { - parent = pparts[1] - } - } - user := strings.Replace(parts[2], "<", "<", -1) - user = strings.Replace(user, ">", ">", -1) - return Commit{num, parts[1], parent, user, parts[3], parts[4]}, nil -} - -func getCommitParts(rev string) (parts []string, err os.Error) { - const format = "{rev}>{node}>{author|escape}>{date}>{desc}" - s, _, err := runLog(nil, "", goroot, - "hg", "log", - "--encoding", "utf-8", - "--rev", rev, - "--limit", "1", - "--template", format, - ) - if err != nil { - return - } - return strings.Split(s, ">", 5), nil -} - -var revisionRe = regexp.MustCompile(`([0-9]+):[0-9a-f]+$`) - -// getTag fetches a Commit by finding the first hg tag that matches re. -func getTag(re *regexp.Regexp) (c Commit, tag string, err os.Error) { - o, _, err := runLog(nil, "", goroot, "hg", "tags") - for _, l := range strings.Split(o, "\n", -1) { - tag = re.FindString(l) - if tag == "" { - continue - } - s := revisionRe.FindStringSubmatch(l) - if s == nil { - err = os.NewError("couldn't find revision number") - return - } - c, err = getCommit(s[1]) - return - } - err = os.NewError("no matching tag found") - return -} diff --git a/misc/dashboard/builder/http.go b/misc/dashboard/builder/http.go index dba19ba8f..5e1da0c87 100644 --- a/misc/dashboard/builder/http.go +++ b/misc/dashboard/builder/http.go @@ -6,84 +6,104 @@ package main import ( "bytes" - "encoding/base64" - "encoding/binary" "fmt" "http" "json" "log" "os" - "regexp" "strconv" ) -// getHighWater returns the current highwater revision hash for this builder -func (b *Builder) getHighWater() (rev string, err os.Error) { - url := fmt.Sprintf("http://%s/hw-get?builder=%s", *dashboard, b.name) - r, _, err := http.Get(url) +type param map[string]string + +// dash runs the given method and command on the dashboard. +// If args is not nil, it is the query or post parameters. +// If resp is not nil, dash unmarshals the body as JSON into resp. +func dash(meth, cmd string, resp interface{}, args param) os.Error { + var r *http.Response + var err os.Error + if *verbose { + log.Println("dash", cmd, args) + } + cmd = "http://" + *dashboard + "/" + cmd + vals := make(http.Values) + for k, v := range args { + vals.Add(k, v) + } + switch meth { + case "GET": + if q := vals.Encode(); q != "" { + cmd += "?" + q + } + r, err = http.Get(cmd) + case "POST": + r, err = http.PostForm(cmd, vals) + default: + return fmt.Errorf("unknown method %q", meth) + } if err != nil { - return + return err + } + defer r.Body.Close() + var buf bytes.Buffer + buf.ReadFrom(r.Body) + if resp != nil { + if err = json.Unmarshal(buf.Bytes(), resp); err != nil { + log.Printf("json unmarshal %#q: %s\n", buf.Bytes(), err) + return err + } } - buf := new(bytes.Buffer) - _, err = buf.ReadFrom(r.Body) + return nil +} + +func dashStatus(meth, cmd string, args param) os.Error { + var resp struct { + Status string + Error string + } + err := dash(meth, cmd, &resp, args) if err != nil { + return err + } + if resp.Status != "OK" { + return os.NewError("/build: " + resp.Error) + } + return nil +} + +// todo returns the next hash to build. +func (b *Builder) todo() (rev string, err os.Error) { + var resp []struct { + Hash string + } + if err = dash("GET", "todo", &resp, param{"builder": b.name}); err != nil { return } - r.Body.Close() - return buf.String(), nil + if len(resp) > 0 { + rev = resp[0].Hash + } + return } // recordResult sends build results to the dashboard -func (b *Builder) recordResult(buildLog string, c Commit) os.Error { - return httpCommand("build", map[string]string{ +func (b *Builder) recordResult(buildLog string, hash string) os.Error { + return dash("POST", "build", nil, param{ "builder": b.name, "key": b.key, - "node": c.node, - "parent": c.parent, - "user": c.user, - "date": c.date, - "desc": c.desc, + "node": hash, "log": buildLog, }) } -// match lines like: "package.BechmarkFunc 100000 999 ns/op" -var benchmarkRegexp = regexp.MustCompile("([^\n\t ]+)[\t ]+([0-9]+)[\t ]+([0-9]+) ns/op") - -// recordBenchmarks sends benchmark results to the dashboard -func (b *Builder) recordBenchmarks(benchLog string, c Commit) os.Error { - results := benchmarkRegexp.FindAllStringSubmatch(benchLog, -1) - var buf bytes.Buffer - b64 := base64.NewEncoder(base64.StdEncoding, &buf) - for _, r := range results { - for _, s := range r[1:] { - binary.Write(b64, binary.BigEndian, uint16(len(s))) - b64.Write([]byte(s)) - } - } - b64.Close() - return httpCommand("benchmarks", map[string]string{ - "builder": b.name, - "key": b.key, - "node": c.node, - "benchmarkdata": buf.String(), - }) -} - -// getPackages fetches a list of package paths from the dashboard -func getPackages() (pkgs []string, err os.Error) { - r, _, err := http.Get(fmt.Sprintf("http://%v/package?fmt=json", *dashboard)) - if err != nil { - return - } - defer r.Body.Close() - d := json.NewDecoder(r.Body) +// packages fetches a list of package paths from the dashboard +func packages() (pkgs []string, err os.Error) { var resp struct { Packages []struct { Path string } } - if err = d.Decode(&resp); err != nil { + err = dash("GET", "package", &resp, param{"fmt": "json"}) + if err != nil { return } for _, p := range resp.Packages { @@ -93,24 +113,36 @@ func getPackages() (pkgs []string, err os.Error) { } // updatePackage sends package build results and info to the dashboard -func (b *Builder) updatePackage(pkg string, state bool, buildLog, info string, c Commit) os.Error { - args := map[string]string{ +func (b *Builder) updatePackage(pkg string, state bool, buildLog, info string, hash string) os.Error { + return dash("POST", "package", nil, param{ "builder": b.name, "key": b.key, "path": pkg, "state": strconv.Btoa(state), "log": buildLog, "info": info, - "go_rev": strconv.Itoa(c.num), - } - return httpCommand("package", args) + "go_rev": hash[:12], + }) } -func httpCommand(cmd string, args map[string]string) os.Error { - if *verbose { - log.Println("httpCommand", cmd, args) +// postCommit informs the dashboard of a new commit +func postCommit(key string, l *HgLog) os.Error { + return dashStatus("POST", "commit", param{ + "key": key, + "node": l.Hash, + "date": l.Date, + "user": l.Author, + "parent": l.Parent, + "desc": l.Desc, + }) +} + +// dashboardCommit returns true if the dashboard knows about hash. +func dashboardCommit(hash string) bool { + err := dashStatus("GET", "commit", param{"node": hash}) + if err != nil { + log.Printf("check %s: %s", hash, err) + return false } - url := fmt.Sprintf("http://%v/%v", *dashboard, cmd) - _, err := http.PostForm(url, args) - return err + return true } diff --git a/misc/dashboard/builder/main.go b/misc/dashboard/builder/main.go index d11cbb133..9377fbe32 100644 --- a/misc/dashboard/builder/main.go +++ b/misc/dashboard/builder/main.go @@ -5,7 +5,6 @@ package main import ( - "container/vector" "flag" "fmt" "io/ioutil" @@ -13,16 +12,18 @@ import ( "os" "path" "regexp" + "runtime" "strconv" "strings" "time" + "xml" ) const ( codeProject = "go" codePyScript = "misc/dashboard/googlecode_upload.py" hgUrl = "https://go.googlecode.com/hg/" - waitInterval = 10e9 // time to wait before checking for new revs + waitInterval = 30e9 // time to wait before checking for new revs mkdirPerm = 0750 pkgBuildInterval = 1e9 * 60 * 60 * 24 // rebuild packages every 24 hours ) @@ -46,16 +47,10 @@ type Builder struct { codePassword string } -type BenchRequest struct { - builder *Builder - commit Commit - path string -} - var ( buildroot = flag.String("buildroot", path.Join(os.TempDir(), "gobuilder"), "Directory under which to build") + commitFlag = flag.Bool("commit", false, "upload information about new commits") dashboard = flag.String("dashboard", "godashboard.appspot.com", "Go Dashboard Host") - runBenchmarks = flag.Bool("bench", false, "Run benchmarks") buildRelease = flag.Bool("release", false, "Build and upload binary release archives") buildRevision = flag.String("rev", "", "Build specified revision and exit") buildCmd = flag.String("cmd", "./all.bash", "Build command (specify absolute or relative to go/src/)") @@ -67,7 +62,6 @@ var ( var ( goroot string releaseRegexp = regexp.MustCompile(`^(release|weekly)\.[0-9\-.]+`) - benchRequests vector.Vector ) func main() { @@ -77,7 +71,7 @@ func main() { os.Exit(2) } flag.Parse() - if len(flag.Args()) == 0 { + if len(flag.Args()) == 0 && !*commitFlag { flag.Usage() } goroot = path.Join(*buildroot, "goroot") @@ -101,17 +95,24 @@ func main() { log.Fatal("Error cloning repository:", err) } + if *commitFlag { + if len(flag.Args()) == 0 { + commitWatcher() + return + } + go commitWatcher() + } + // if specified, build revision and return if *buildRevision != "" { - c, err := getCommit(*buildRevision) + hash, err := fullHash(*buildRevision) if err != nil { log.Fatal("Error finding revision: ", err) } for _, b := range builders { - if err := b.buildCommit(c); err != nil { + if err := b.buildHash(hash); err != nil { log.Println(err) } - runQueuedBenchmark() } return } @@ -127,13 +128,8 @@ func main() { // go continuous build mode (default) // check for new commits and build them for { - err := run(nil, goroot, "hg", "pull", "-u") - if err != nil { - log.Println("hg pull failed:", err) - time.Sleep(waitInterval) - continue - } built := false + t := time.Nanoseconds() if *parallel { done := make(chan bool) for _, b := range builders { @@ -149,46 +145,15 @@ func main() { built = b.build() || built } } - // only run benchmarks if we didn't build anything - // so that they don't hold up the builder queue + // sleep if there was nothing to build if !built { - if !runQueuedBenchmark() { - // if we have no benchmarks to do, pause - time.Sleep(waitInterval) - } - // after running one benchmark, - // continue to find and build new revisions. + time.Sleep(waitInterval) + } + // sleep if we're looping too fast. + t1 := time.Nanoseconds() - t + if t1 < waitInterval { + time.Sleep(waitInterval - t1) } - } -} - -func runQueuedBenchmark() bool { - if benchRequests.Len() == 0 { - return false - } - runBenchmark(benchRequests.Pop().(BenchRequest)) - return true -} - -func runBenchmark(r BenchRequest) { - // run benchmarks and send to dashboard - log.Println(r.builder.name, "benchmarking", r.commit.num) - defer os.RemoveAll(r.path) - pkg := path.Join(r.path, "go", "src", "pkg") - bin := path.Join(r.path, "go", "bin") - env := []string{ - "GOOS=" + r.builder.goos, - "GOARCH=" + r.builder.goarch, - "PATH=" + bin + ":" + os.Getenv("PATH"), - } - logfile := path.Join(r.path, "bench.log") - benchLog, _, err := runLog(env, logfile, pkg, "gomake", "bench") - if err != nil { - log.Println(r.builder.name, "gomake bench:", err) - return - } - if err = r.builder.recordBenchmarks(benchLog, r.commit); err != nil { - log.Println("recordBenchmarks:", err) } } @@ -235,7 +200,7 @@ func (b *Builder) buildExternal() { log.Println("hg pull failed:", err) continue } - c, tag, err := getTag(releaseRegexp) + hash, tag, err := firstTag(releaseRegexp) if err != nil { log.Println(err) continue @@ -249,8 +214,8 @@ func (b *Builder) buildExternal() { if tag == prevTag && time.Nanoseconds() < nextBuild { continue } - // buildCommit will also build the packages - if err := b.buildCommit(c); err != nil { + // build will also build the packages + if err := b.buildHash(hash); err != nil { log.Println(err) continue } @@ -269,65 +234,46 @@ func (b *Builder) build() bool { log.Println(b.name, "build:", err) } }() - c, err := b.nextCommit() + hash, err := b.todo() if err != nil { log.Println(err) return false } - if c == nil { + if hash == "" { return false } - err = b.buildCommit(*c) - if err != nil { - log.Println(err) - } - return true -} + // Look for hash locally before running hg pull. -// nextCommit returns the next unbuilt Commit for this builder -func (b *Builder) nextCommit() (nextC *Commit, err os.Error) { - defer func() { - if err != nil { - err = fmt.Errorf("%s nextCommit: %s", b.name, err) + if _, err := fullHash(hash[:12]); err != nil { + // Don't have hash, so run hg pull. + if err := run(nil, goroot, "hg", "pull"); err != nil { + log.Println("hg pull failed:", err) + return false } - }() - hw, err := b.getHighWater() - if err != nil { - return } - c, err := getCommit(hw) + err = b.buildHash(hash) if err != nil { - return - } - next := c.num + 1 - c, err = getCommit(strconv.Itoa(next)) - if err == nil && c.num == next { - return &c, nil + log.Println(err) } - return nil, nil + return true } -func (b *Builder) buildCommit(c Commit) (err os.Error) { +func (b *Builder) buildHash(hash string) (err os.Error) { defer func() { if err != nil { - err = fmt.Errorf("%s buildCommit: %d: %s", b.name, c.num, err) + err = fmt.Errorf("%s build: %s: %s", b.name, hash, err) } }() - log.Println(b.name, "building", c.num) + log.Println(b.name, "building", hash) // create place in which to do work - workpath := path.Join(*buildroot, b.name+"-"+strconv.Itoa(c.num)) + workpath := path.Join(*buildroot, b.name+"-"+hash[:12]) err = os.Mkdir(workpath, mkdirPerm) if err != nil { return } - benchRequested := false - defer func() { - if !benchRequested { - os.RemoveAll(workpath) - } - }() + defer os.RemoveAll(workpath) // clone repo err = run(nil, workpath, "hg", "clone", goroot, "go") @@ -337,7 +283,7 @@ func (b *Builder) buildCommit(c Commit) (err os.Error) { // update to specified revision err = run(nil, path.Join(workpath, "go"), - "hg", "update", "-r", strconv.Itoa(c.num)) + "hg", "update", hash) if err != nil { return } @@ -348,7 +294,7 @@ func (b *Builder) buildCommit(c Commit) (err os.Error) { logfile := path.Join(workpath, "build.log") buildLog, status, err := runLog(b.envv(), logfile, srcDir, *buildCmd) if err != nil { - return fmt.Errorf("all.bash: %s", err) + return fmt.Errorf("%s: %s", *buildCmd, err) } // if we're in external mode, build all packages and return @@ -356,36 +302,27 @@ func (b *Builder) buildCommit(c Commit) (err os.Error) { if status != 0 { return os.NewError("go build failed") } - return b.buildPackages(workpath, c) + return b.buildPackages(workpath, hash) } if status != 0 { // record failure - return b.recordResult(buildLog, c) + return b.recordResult(buildLog, hash) } // record success - if err = b.recordResult("", c); err != nil { + if err = b.recordResult("", hash); err != nil { return fmt.Errorf("recordResult: %s", err) } - // send benchmark request if benchmarks are enabled - if *runBenchmarks { - benchRequests.Insert(0, BenchRequest{ - builder: b, - commit: c, - path: workpath, - }) - benchRequested = true - } - // finish here if codeUsername and codePassword aren't set if b.codeUsername == "" || b.codePassword == "" || !*buildRelease { return } // if this is a release, create tgz and upload to google code - if release := releaseRegexp.FindString(c.desc); release != "" { + releaseHash, release, err := firstTag(releaseRegexp) + if hash == releaseHash { // clean out build state err = run(b.envv(), srcDir, "./clean.bash", "--nopkg") if err != nil { @@ -411,6 +348,9 @@ func (b *Builder) buildCommit(c Commit) (err os.Error) { // envv returns an environment for build/bench execution func (b *Builder) envv() []string { + if runtime.GOOS == "windows" { + return b.envvWindows() + } e := []string{ "GOOS=" + b.goos, "GOARCH=" + b.goarch, @@ -422,6 +362,42 @@ func (b *Builder) envv() []string { return e } +// windows version of envv +func (b *Builder) envvWindows() []string { + start := map[string]string{ + "GOOS": b.goos, + "GOARCH": b.goarch, + "GOROOT_FINAL": "/c/go", + } + for _, name := range extraEnv { + start[name] = os.Getenv(name) + } + skip := map[string]bool{ + "GOBIN": true, + "GOROOT": true, + "INCLUDE": true, + "LIB": true, + } + var e []string + for name, v := range start { + e = append(e, name+"="+v) + skip[name] = true + } + for _, kv := range os.Environ() { + s := strings.Split(kv, "=", 2) + name := strings.ToUpper(s[0]) + switch { + case name == "": + // variables, like "=C:=C:\", just copy them + e = append(e, kv) + case !skip[name]: + e = append(e, kv) + skip[name] = true + } + } + return e +} + func isDirectory(name string) bool { s, err := os.Stat(name) return err == nil && s.IsDirectory() @@ -431,3 +407,209 @@ func isFile(name string) bool { s, err := os.Stat(name) return err == nil && (s.IsRegular() || s.IsSymlink()) } + +// commitWatcher polls hg for new commits and tells the dashboard about them. +func commitWatcher() { + // Create builder just to get master key. + b, err := NewBuilder("mercurial-commit") + if err != nil { + log.Fatal(err) + } + for { + if *verbose { + log.Printf("poll...") + } + commitPoll(b.key) + if *verbose { + log.Printf("sleep...") + } + time.Sleep(60e9) + } +} + +// HgLog represents a single Mercurial revision. +type HgLog struct { + Hash string + Author string + Date string + Desc string + Parent string + + // Internal metadata + added bool +} + +// logByHash is a cache of all Mercurial revisions we know about, +// indexed by full hash. +var logByHash = map[string]*HgLog{} + +// xmlLogTemplate is a template to pass to Mercurial to make +// hg log print the log in valid XML for parsing with xml.Unmarshal. +const xmlLogTemplate = ` + <log> + <hash>{node|escape}</hash> + <parent>{parent|escape}</parent> + <author>{author|escape}</author> + <date>{date}</date> + <desc>{desc|escape}</desc> + </log> +` + +// commitPoll pulls any new revisions from the hg server +// and tells the server about them. +func commitPoll(key string) { + // Catch unexpected panics. + defer func() { + if err := recover(); err != nil { + log.Printf("commitPoll panic: %s", err) + } + }() + + if err := run(nil, goroot, "hg", "pull"); err != nil { + log.Printf("hg pull: %v", err) + return + } + + const N = 20 // how many revisions to grab + + data, _, err := runLog(nil, "", goroot, "hg", "log", + "--encoding=utf-8", + "--limit="+strconv.Itoa(N), + "--template="+xmlLogTemplate, + ) + if err != nil { + log.Printf("hg log: %v", err) + return + } + + var logStruct struct { + Log []HgLog + } + err = xml.Unmarshal(strings.NewReader("<top>"+data+"</top>"), &logStruct) + if err != nil { + log.Printf("unmarshal hg log: %v", err) + return + } + + logs := logStruct.Log + + // Pass 1. Fill in parents and add new log entries to logsByHash. + // Empty parent means take parent from next log entry. + // Non-empty parent has form 1234:hashhashhash; we want full hash. + for i := range logs { + l := &logs[i] + log.Printf("hg log: %s < %s\n", l.Hash, l.Parent) + if l.Parent == "" && i+1 < len(logs) { + l.Parent = logs[i+1].Hash + } else if l.Parent != "" { + l.Parent, _ = fullHash(l.Parent) + } + if l.Parent == "" { + // Can't create node without parent. + continue + } + + if logByHash[l.Hash] == nil { + // Make copy to avoid pinning entire slice when only one entry is new. + t := *l + logByHash[t.Hash] = &t + } + } + + for i := range logs { + l := &logs[i] + if l.Parent == "" { + continue + } + addCommit(l.Hash, key) + } +} + +// addCommit adds the commit with the named hash to the dashboard. +// key is the secret key for authentication to the dashboard. +// It avoids duplicate effort. +func addCommit(hash, key string) bool { + l := logByHash[hash] + if l == nil { + return false + } + if l.added { + return true + } + + // Check for already added, perhaps in an earlier run. + if dashboardCommit(hash) { + log.Printf("%s already on dashboard\n", hash) + // Record that this hash is on the dashboard, + // as must be all its parents. + for l != nil { + l.added = true + l = logByHash[l.Parent] + } + return true + } + + // Create parent first, to maintain some semblance of order. + if !addCommit(l.Parent, key) { + return false + } + + // Create commit. + if err := postCommit(key, l); err != nil { + log.Printf("failed to add %s to dashboard: %v", key, err) + return false + } + return true +} + +// fullHash returns the full hash for the given Mercurial revision. +func fullHash(rev string) (hash string, err os.Error) { + defer func() { + if err != nil { + err = fmt.Errorf("fullHash: %s: %s", rev, err) + } + }() + s, _, err := runLog(nil, "", goroot, + "hg", "log", + "--encoding=utf-8", + "--rev="+rev, + "--limit=1", + "--template={node}", + ) + if err != nil { + return + } + s = strings.TrimSpace(s) + if s == "" { + return "", fmt.Errorf("cannot find revision") + } + if len(s) != 20 { + return "", fmt.Errorf("hg returned invalid hash " + s) + } + return s, nil +} + +var revisionRe = regexp.MustCompile(`^([^ ]+) +[0-9]+:([0-9a-f]+)$`) + +// firstTag returns the hash and tag of the most recent tag matching re. +func firstTag(re *regexp.Regexp) (hash string, tag string, err os.Error) { + o, _, err := runLog(nil, "", goroot, "hg", "tags") + for _, l := range strings.Split(o, "\n", -1) { + if l == "" { + continue + } + s := revisionRe.FindStringSubmatch(l) + if s == nil { + err = os.NewError("couldn't find revision number") + return + } + if !re.MatchString(s[1]) { + continue + } + tag = s[1] + hash, err = fullHash(s[3]) + return + } + err = os.NewError("no matching tag found") + return +} diff --git a/misc/dashboard/builder/package.go b/misc/dashboard/builder/package.go index 6e9f9ff39..ee65d7669 100644 --- a/misc/dashboard/builder/package.go +++ b/misc/dashboard/builder/package.go @@ -13,8 +13,8 @@ import ( "path" ) -func (b *Builder) buildPackages(workpath string, c Commit) os.Error { - pkgs, err := getPackages() +func (b *Builder) buildPackages(workpath string, hash string) os.Error { + pkgs, err := packages() if err != nil { return err } @@ -32,13 +32,13 @@ func (b *Builder) buildPackages(workpath string, c Commit) os.Error { built := code != 0 // get doc comment from package source - info, err := getPackageComment(p, path.Join(goroot, "pkg", p)) + info, err := packageComment(p, path.Join(goroot, "pkg", p)) if err != nil { log.Printf("goinstall %v: %v", p, err) } // update dashboard with build state + info - err = b.updatePackage(p, built, buildLog, info, c) + err = b.updatePackage(p, built, buildLog, info, hash) if err != nil { log.Printf("updatePackage %v: %v", p, err) } @@ -46,7 +46,7 @@ func (b *Builder) buildPackages(workpath string, c Commit) os.Error { return nil } -func getPackageComment(pkg, pkgpath string) (info string, err os.Error) { +func packageComment(pkg, pkgpath string) (info string, err os.Error) { fset := token.NewFileSet() pkgs, err := parser.ParseDir(fset, pkgpath, nil, parser.PackageClauseOnly|parser.ParseComments) if err != nil { diff --git a/misc/dashboard/godashboard/app.yaml b/misc/dashboard/godashboard/app.yaml index aec559dcc..83611cf90 100644 --- a/misc/dashboard/godashboard/app.yaml +++ b/misc/dashboard/godashboard/app.yaml @@ -1,9 +1,13 @@ application: godashboard -version: 5 +version: 7 runtime: python api_version: 1 handlers: +- url: /favicon\.ico + static_files: static/favicon.ico + upload: static/favicon\.ico + - url: /static static_dir: static diff --git a/misc/dashboard/godashboard/gobuild.py b/misc/dashboard/godashboard/gobuild.py index 035bf842f..5678f2e1b 100644 --- a/misc/dashboard/godashboard/gobuild.py +++ b/misc/dashboard/godashboard/gobuild.py @@ -5,21 +5,19 @@ # This is the server part of the continuous build system for Go. It must be run # by AppEngine. +from django.utils import simplejson from google.appengine.api import mail from google.appengine.api import memcache from google.appengine.ext import db from google.appengine.ext import webapp from google.appengine.ext.webapp import template from google.appengine.ext.webapp.util import run_wsgi_app -import binascii import datetime import hashlib import hmac import logging import os import re -import struct -import time import bz2 # local imports @@ -50,10 +48,6 @@ class Commit(db.Model): fail_notification_sent = db.BooleanProperty() -class Cache(db.Model): - data = db.BlobProperty() - expire = db.IntegerProperty() - # A CompressedLog contains the textual build log of a failed build. # The key name is the hex digest of the SHA256 hash of the contents. # The contents is bz2 compressed. @@ -62,23 +56,6 @@ class CompressedLog(db.Model): N = 30 -def cache_get(key): - c = Cache.get_by_key_name(key) - if c is None or c.expire < time.time(): - return None - return c.data - -def cache_set(key, val, timeout): - c = Cache(key_name = key) - c.data = val - c.expire = int(time.time() + timeout) - c.put() - -def cache_del(key): - c = Cache.get_by_key_name(key) - if c is not None: - c.delete() - def builderInfo(b): f = b.split('-', 3) goos = f[0] @@ -96,7 +73,7 @@ def builderset(): for c in results: builders.update(set(parseBuild(build)['builder'] for build in c.builds)) return builders - + class MainPage(webapp.RequestHandler): def get(self): self.response.headers['Content-Type'] = 'text/html; charset=utf-8' @@ -147,7 +124,30 @@ class MainPage(webapp.RequestHandler): path = os.path.join(os.path.dirname(__file__), 'main.html') self.response.out.write(template.render(path, values)) -class GetHighwater(webapp.RequestHandler): +# A DashboardHandler is a webapp.RequestHandler but provides +# authenticated_post - called by post after authenticating +# json - writes object in json format to response output +class DashboardHandler(webapp.RequestHandler): + def post(self): + if not auth(self.request): + self.response.set_status(403) + return + self.authenticated_post() + + def authenticated_post(self): + return + + def json(self, obj): + self.response.set_status(200) + simplejson.dump(obj, self.response.out) + return + +def auth(req): + k = req.get('key') + return k == hmac.new(key.accessKey, req.get('builder')).hexdigest() or k == key.accessKey + +# Todo serves /todo. It tells the builder which commits need to be built. +class Todo(DashboardHandler): def get(self): builder = self.request.get('builder') key = 'todo-%s' % builder @@ -155,28 +155,19 @@ class GetHighwater(webapp.RequestHandler): if response is None: # Fell out of memcache. Rebuild from datastore results. # We walk the commit list looking for nodes that have not - # been built by this builder and record the *parents* of those - # nodes, because each builder builds the revision *after* the - # one return (because we might not know about the latest - # revision). + # been built by this builder. q = Commit.all() q.order('-__key__') todo = [] - need = False first = None for c in q.fetch(N+1): if first is None: first = c - if need: - todo.append(c.node) - need = not built(c, builder) - if not todo: - todo.append(first.node) - response = ' '.join(todo) + if not built(c, builder): + todo.append({'Hash': c.node}) + response = simplejson.dumps(todo) memcache.set(key, response, 3600) self.response.set_status(200) - if self.request.get('all') != 'yes': - response = response.split()[0] self.response.out.write(response) def built(c, builder): @@ -185,22 +176,8 @@ def built(c, builder): return True return False -def auth(req): - k = req.get('key') - return k == hmac.new(key.accessKey, req.get('builder')).hexdigest() or k == key.accessKey - -class SetHighwater(webapp.RequestHandler): - def post(self): - if not auth(self.request): - self.response.set_status(403) - return - - # Allow for old builders. - # This is a no-op now: we figure out what to build based - # on the current dashboard status. - return - -class LogHandler(webapp.RequestHandler): +# Log serves /log/. It retrieves log data by content hash. +class LogHandler(DashboardHandler): def get(self): self.response.headers['Content-Type'] = 'text/plain; charset=utf-8' hash = self.request.path[5:] @@ -214,12 +191,8 @@ class LogHandler(webapp.RequestHandler): # Init creates the commit with id 0. Since this commit doesn't have a parent, # it cannot be created by Build. -class Init(webapp.RequestHandler): - def post(self): - if not auth(self.request): - self.response.set_status(403) - return - +class Init(DashboardHandler): + def authenticated_post(self): date = parseDate(self.request.get('date')) node = self.request.get('node') if not validNode(node) or date is None: @@ -239,7 +212,86 @@ class Init(webapp.RequestHandler): self.response.set_status(200) -# Build is the main command: it records the result of a new build. +# The last commit when we switched to using entity groups. +# This is the root of the new commit entity group. +RootCommitKeyName = '00000f26-f32c6f1038207c55d5780231f7484f311020747e' + +# CommitHandler serves /commit. +# A GET of /commit retrieves information about the specified commit. +# A POST of /commit creates a node for the given commit. +# If the commit already exists, the POST silently succeeds (like mkdir -p). +class CommitHandler(DashboardHandler): + def get(self): + node = self.request.get('node') + if not validNode(node): + return self.json({'Status': 'FAIL', 'Error': 'malformed node hash'}) + n = nodeByHash(node) + if n is None: + return self.json({'Status': 'FAIL', 'Error': 'unknown revision'}) + return self.json({'Status': 'OK', 'Node': nodeObj(n)}) + + def authenticated_post(self): + # Require auth with the master key, not a per-builder key. + if self.request.get('builder'): + self.response.set_status(403) + return + + node = self.request.get('node') + date = parseDate(self.request.get('date')) + user = self.request.get('user').encode('utf8') + desc = self.request.get('desc').encode('utf8') + parenthash = self.request.get('parent') + + if not validNode(node) or not validNode(parenthash) or date is None: + return self.json({'Status': 'FAIL', 'Error': 'malformed node, parent, or date'}) + + n = nodeByHash(node) + if n is None: + p = nodeByHash(parenthash) + if p is None: + return self.json({'Status': 'FAIL', 'Error': 'unknown parent'}) + + # Want to create new node in a transaction so that multiple + # requests creating it do not collide and so that multiple requests + # creating different nodes get different sequence numbers. + # All queries within a transaction must include an ancestor, + # but the original datastore objects we used for the dashboard + # have no common ancestor. Instead, we use a well-known + # root node - the last one before we switched to entity groups - + # as the as the common ancestor. + root = Commit.get_by_key_name(RootCommitKeyName) + + def add_commit(): + if nodeByHash(node, ancestor=root) is not None: + return + + # Determine number for this commit. + # Once we have created one new entry it will be lastRooted.num+1, + # but the very first commit created in this scheme will have to use + # last.num's number instead (last is likely not rooted). + q = Commit.all() + q.order('-__key__') + q.ancestor(root) + last = q.fetch(1)[0] + num = last.num+1 + + n = Commit(key_name = '%08x-%s' % (num, node), parent = root) + n.num = num + n.node = node + n.parentnode = parenthash + n.user = user + n.date = date + n.desc = desc + n.put() + db.run_in_transaction(add_commit) + n = nodeByHash(node) + if n is None: + return self.json({'Status': 'FAIL', 'Error': 'failed to create commit node'}) + + return self.json({'Status': 'OK', 'Node': nodeObj(n)}) + +# Build serves /build. +# A POST to /build records a new build result. class Build(webapp.RequestHandler): def post(self): if not auth(self.request): @@ -256,44 +308,33 @@ class Build(webapp.RequestHandler): l.log = bz2.compress(log) l.put() - date = parseDate(self.request.get('date')) - user = self.request.get('user').encode('utf8') - desc = self.request.get('desc').encode('utf8') node = self.request.get('node') - parenthash = self.request.get('parent') - if not validNode(node) or not validNode(parenthash) or date is None: - logging.error("Not valid node ('%s') or bad date (%s %s)", node, date, self.request.get('date')) + if not validNode(node): + logging.error('Invalid node %s' % (node)) self.response.set_status(500) return - q = Commit.all() - q.filter('node =', parenthash) - parent = q.get() - if parent is None: - logging.error('Cannot find parent %s of node %s' % (parenthash, node)) + n = nodeByHash(node) + if n is None: + logging.error('Cannot find node %s' % (node)) self.response.set_status(404) return - parentnum, _ = parent.key().name().split('-', 1) - nodenum = int(parentnum, 16) + 1 - - key_name = '%08x-%s' % (nodenum, node) + nn = n def add_build(): - n = Commit.get_by_key_name(key_name) + n = nodeByHash(node, ancestor=nn) if n is None: - n = Commit(key_name = key_name) - n.num = nodenum - n.node = node - n.parentnode = parenthash - n.user = user - n.date = date - n.desc = desc + logging.error('Cannot find hash in add_build: %s %s' % (builder, node)) + return + s = '%s`%s' % (builder, loghash) for i, b in enumerate(n.builds): if b.split('`', 1)[0] == builder: + # logging.error('Found result for %s %s already' % (builder, node)) n.builds[i] = s break else: + # logging.error('Added result for %s %s' % (builder, node)) n.builds.append(s) n.put() @@ -302,39 +343,101 @@ class Build(webapp.RequestHandler): key = 'todo-%s' % builder memcache.delete(key) - def mark_sent(): - n = Commit.get_by_key_name(key_name) - n.fail_notification_sent = True - n.put() - - n = Commit.get_by_key_name(key_name) - if loghash and not failed(parent, builder) and not n.fail_notification_sent: - subject = const.mail_fail_subject % (builder, desc.split("\n")[0]) - path = os.path.join(os.path.dirname(__file__), 'fail-notify.txt') - body = template.render(path, { - "builder": builder, - "node": node[:12], - "user": user, - "desc": desc, - "loghash": loghash - }) - mail.send_mail( - sender=const.mail_from, - reply_to=const.mail_fail_reply_to, - to=const.mail_fail_to, - subject=subject, - body=body - ) - db.run_in_transaction(mark_sent) + c = getBrokenCommit(node, builder) + if c is not None and not c.fail_notification_sent: + notifyBroken(c, builder) self.response.set_status(200) -def failed(c, builder): + +def getBrokenCommit(node, builder): + """ + getBrokenCommit returns a Commit that breaks the build. + The Commit will be either the one specified by node or the one after. + """ + + # Squelch mail if already fixed. + head = firstResult(builder) + if broken(head, builder) == False: + return + + # Get current node and node before, after. + cur = nodeByHash(node) + if cur is None: + return + before = nodeBefore(cur) + after = nodeAfter(cur) + + if broken(before, builder) == False and broken(cur, builder): + return cur + if broken(cur, builder) == False and broken(after, builder): + return after + + return + +def firstResult(builder): + q = Commit.all().order('-__key__') + for c in q.fetch(20): + for i, b in enumerate(c.builds): + p = b.split('`', 1) + if p[0] == builder: + return c + return None + +def nodeBefore(c): + return nodeByHash(c.parentnode) + +def nodeAfter(c): + return Commit.all().filter('parenthash', c.node).get() + +def notifyBroken(c, builder): + def send(): + n = Commit.get(c.key()) + if n is None: + logging.error("couldn't retrieve Commit '%s'" % c.key()) + return False + if n.fail_notification_sent: + return False + n.fail_notification_sent = True + return n.put() + if not db.run_in_transaction(send): + return + + subject = const.mail_fail_subject % (builder, c.desc.split('\n')[0]) + path = os.path.join(os.path.dirname(__file__), 'fail-notify.txt') + body = template.render(path, { + "builder": builder, + "node": c.node, + "user": c.user, + "desc": c.desc, + "loghash": logHash(c, builder) + }) + mail.send_mail( + sender=const.mail_from, + to=const.mail_fail_to, + subject=subject, + body=body + ) + +def logHash(c, builder): + for i, b in enumerate(c.builds): + p = b.split('`', 1) + if p[0] == builder: + return p[1] + return "" + +def broken(c, builder): + """ + broken returns True if commit c breaks the build for the specified builder, + False if it is a good build, and None if no results exist for this builder. + """ + if c is None: + return None for i, b in enumerate(c.builds): p = b.split('`', 1) if p[0] == builder: return len(p[1]) > 0 - return False + return None def node(num): q = Commit.all() @@ -342,6 +445,24 @@ def node(num): n = q.get() return n +def nodeByHash(hash, ancestor=None): + q = Commit.all() + q.filter('node =', hash) + if ancestor is not None: + q.ancestor(ancestor) + n = q.get() + return n + +# nodeObj returns a JSON object (ready to be passed to simplejson.dump) describing node. +def nodeObj(n): + return { + 'Hash': n.node, + 'ParentHash': n.parentnode, + 'User': n.user, + 'Date': n.date.strftime('%Y-%m-%d %H:%M %z'), + 'Desc': n.desc, + } + class FixedOffset(datetime.tzinfo): """Fixed offset in minutes east from UTC.""" @@ -417,15 +538,20 @@ def toRev(c): def byBuilder(x, y): return cmp(x['builder'], y['builder']) +# Give old builders work; otherwise they pound on the web site. +class Hwget(DashboardHandler): + def get(self): + self.response.out.write("8000\n") + # This is the URL map for the server. The first three entries are public, the # rest are only used by the builders. application = webapp.WSGIApplication( [('/', MainPage), + ('/hw-get', Hwget), ('/log/.*', LogHandler), - ('/hw-get', GetHighwater), - ('/hw-set', SetHighwater), - + ('/commit', CommitHandler), ('/init', Init), + ('/todo', Todo), ('/build', Build), ], debug=True) diff --git a/misc/dashboard/godashboard/index.yaml b/misc/dashboard/godashboard/index.yaml index 148824bb6..4a00c4a6f 100644 --- a/misc/dashboard/godashboard/index.yaml +++ b/misc/dashboard/godashboard/index.yaml @@ -23,6 +23,12 @@ indexes: - name: __key__ direction: desc +- kind: Commit + ancestor: yes + properties: + - name: __key__ + direction: desc + - kind: Project properties: - name: approved diff --git a/misc/dashboard/godashboard/static/favicon.ico b/misc/dashboard/godashboard/static/favicon.ico Binary files differnew file mode 100644 index 000000000..48854ff3b --- /dev/null +++ b/misc/dashboard/godashboard/static/favicon.ico diff --git a/misc/dashboard/googlecode_upload.py b/misc/dashboard/googlecode_upload.py index 3b1d432ff..e87db884a 100755 --- a/misc/dashboard/googlecode_upload.py +++ b/misc/dashboard/googlecode_upload.py @@ -70,7 +70,7 @@ def upload(file, project_name, user_name, password, summary, labels=None): Returns: a tuple: http_status: 201 if the upload succeeded, something else if an - error occured. + error occurred. http_reason: The human-readable string associated with http_status file_url: If the upload succeeded, the URL of the file on Google Code, None otherwise. diff --git a/misc/emacs/go-mode-load.el b/misc/emacs/go-mode-load.el index 0ace46dfa..d453166a4 100644 --- a/misc/emacs/go-mode-load.el +++ b/misc/emacs/go-mode-load.el @@ -19,7 +19,7 @@ ;;;### (autoloads (gofmt-before-save gofmt go-mode) "go-mode" "go-mode.el" -;;;;;; (19847 61431)) +;;;;;; (19917 17808)) ;;; Generated autoloads from go-mode.el (autoload 'go-mode "go-mode" "\ diff --git a/misc/emacs/go-mode.el b/misc/emacs/go-mode.el index 692cabfe5..03f0a2a8b 100644 --- a/misc/emacs/go-mode.el +++ b/misc/emacs/go-mode.el @@ -507,7 +507,9 @@ Replace the current buffer on success; display errors on failure." (let ((srcbuf (current-buffer))) (with-temp-buffer (let ((outbuf (current-buffer)) - (errbuf (get-buffer-create "*Gofmt Errors*"))) + (errbuf (get-buffer-create "*Gofmt Errors*")) + (coding-system-for-read 'utf-8) ;; use utf-8 with subprocesses + (coding-system-for-write 'utf-8)) (with-current-buffer errbuf (erase-buffer)) (with-current-buffer srcbuf (save-restriction @@ -521,7 +523,7 @@ Replace the current buffer on success; display errors on failure." (erase-buffer) (insert-buffer-substring outbuf) (goto-char (min old-point (point-max))) - (if old-mark (set-mark (min old-mark (point-max)))) + (if old-mark (push-mark (min old-mark (point-max)) t)) (kill-buffer errbuf)) ;; gofmt failed: display the errors diff --git a/misc/godoc/README b/misc/godoc/README new file mode 100644 index 000000000..3c8d830e4 --- /dev/null +++ b/misc/godoc/README @@ -0,0 +1,22 @@ +Instructions to get an initial godoc running on a local app engine emulator +--------------------------------------------------------------------------- + +To run godoc under the app engine emulator, create a ("goroot") godoc +directory that contains the app.yaml file, the doc and lib directories +from the Go distribution, as well as a godoc directory with the godoc +sources from src/cmd/godoc. In the godoc source directory, replace +main.go with init.go. The directory structure should look as follows: + +godoc // "goroot" directory + app.yaml // app engine control file + doc // goroot/doc directory + favicon.ico + godoc // contains godoc sources + godoc.go // unchanged godoc file + init.go // this file instead of godoc/main.go + ... // remaining godoc files + lib // goroot/lib directory + +Run app engine emulator locally: dev_appserver.py -a <hostname> godoc +where godoc is the top-level "goroot" directory. The godoc home page +is then served at: <hostname>:8080 . diff --git a/misc/godoc/app.yaml b/misc/godoc/app.yaml new file mode 100644 index 000000000..f8b46db31 --- /dev/null +++ b/misc/godoc/app.yaml @@ -0,0 +1,12 @@ +# Copyright 2011 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +application: godoc +version: 1 +runtime: go +api_version: 1 + +handlers: +- url: /.* + script: _go_app diff --git a/misc/godoc/init.go b/misc/godoc/init.go new file mode 100644 index 000000000..0fd0bd542 --- /dev/null +++ b/misc/godoc/init.go @@ -0,0 +1,35 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file replaces main.go when running godoc under the app engine emulator. +// See the README file for instructions. + +package main + +import ( + "http" + "log" + "os" + "path/filepath" +) + +func serveError(w http.ResponseWriter, r *http.Request, relpath string, err os.Error) { + contents := applyTemplate(errorHTML, "errorHTML", err) // err may contain an absolute path! + w.WriteHeader(http.StatusNotFound) + servePage(w, "File "+relpath, "", "", contents) +} + +func init() { + // set goroot + cwd, err := os.Getwd() + if err != nil { + log.Fatalf("cwd: %s", err) + } + log.Printf("cwd = %s", cwd) + *goroot = filepath.Clean(cwd) + + initHandlers() + readTemplates() + registerPublicHandlers(http.DefaultServeMux) +} diff --git a/misc/goplay/goplay.go b/misc/goplay/goplay.go index f3e2ff565..f1dc1bca5 100644 --- a/misc/goplay/goplay.go +++ b/misc/goplay/goplay.go @@ -5,7 +5,6 @@ package main import ( - "bytes" "exec" "flag" "http" @@ -140,32 +139,7 @@ func error(w http.ResponseWriter, out []byte, err os.Error) { // run executes the specified command and returns its output and an error. func run(cmd ...string) ([]byte, os.Error) { - // find the specified binary - bin, err := exec.LookPath(cmd[0]) - if err != nil { - // report binary as well as the error - return nil, os.NewError(cmd[0] + ": " + err.String()) - } - - // run the binary and read its combined stdout and stderr into a buffer - p, err := exec.Run(bin, cmd, os.Environ(), "", exec.DevNull, exec.Pipe, exec.MergeWithStdout) - if err != nil { - return nil, err - } - var buf bytes.Buffer - io.Copy(&buf, p.Stdout) - w, err := p.Wait(0) - p.Close() - if err != nil { - return nil, err - } - - // set the error return value if the program had a non-zero exit status - if !w.Exited() || w.ExitStatus() != 0 { - err = os.ErrorString("running " + cmd[0] + ": " + w.String()) - } - - return buf.Bytes(), err + return exec.Command(cmd[0], cmd[1:]...).CombinedOutput() } var frontPage, output *template.Template // HTML templates diff --git a/misc/vim/ftplugin/go/fmt.vim b/misc/vim/ftplugin/go/fmt.vim index 18a2156f5..a299dfcee 100644 --- a/misc/vim/ftplugin/go/fmt.vim +++ b/misc/vim/ftplugin/go/fmt.vim @@ -13,7 +13,7 @@ " replacing the buffer with stderr output. " -command! Fmt call s:GoFormat() +command! -buffer Fmt call s:GoFormat() function! s:GoFormat() let view = winsaveview() diff --git a/misc/vim/indent/go.vim b/misc/vim/indent/go.vim index 2e9f191f5..faf4d79e2 100644 --- a/misc/vim/indent/go.vim +++ b/misc/vim/indent/go.vim @@ -4,27 +4,62 @@ " " indent/go.vim: Vim indent file for Go. " +" TODO: +" - function invocations split across lines +" - general line splits (line ends in an operator) if exists("b:did_indent") finish endif let b:did_indent = 1 -" C indentation is mostly correct -setlocal cindent - -" Options set: -" +0 -- Don't indent continuation lines (because Go doesn't use semicolons -" much) -" L0 -- Don't move jump labels (NOTE: this isn't correct when working with -" gofmt, but it does keep struct literals properly indented.) -" :0 -- Align case labels with switch statement -" l1 -- Always align case body relative to case labels -" J1 -- Indent JSON-style objects (properly indents struct-literals) -" (0, Ws -- Indent lines inside of unclosed parentheses by one shiftwidth -" m1 -- Align closing parenthesis line with first non-blank of matching -" parenthesis line -" -" Known issue: Trying to do a multi-line struct literal in a short variable -" declaration will not indent properly. -setlocal cinoptions+=+0,L0,:0,l1,J1,(0,Ws,m1 +" C indentation is too far off useful, mainly due to Go's := operator. +" Let's just define our own. +setlocal nolisp +setlocal autoindent +setlocal indentexpr=GoIndent(v:lnum) +setlocal indentkeys+=<:>,0=},0=) + +if exists("*GoIndent") + finish +endif + +function! GoIndent(lnum) + let prevlnum = prevnonblank(a:lnum-1) + if prevlnum == 0 + " top of file + return 0 + endif + + " grab the previous and current line, stripping comments. + let prevl = substitute(getline(prevlnum), '//.*$', '', '') + let thisl = substitute(getline(a:lnum), '//.*$', '', '') + let previ = indent(prevlnum) + + let ind = previ + + if prevl =~ '[({]\s*$' + " previous line opened a block + let ind += &sw + endif + if prevl =~# '^\s*\(case .*\|default\):$' + " previous line is part of a switch statement + let ind += &sw + endif + " TODO: handle if the previous line is a label. + + if thisl =~ '^\s*[)}]' + " this line closed a block + let ind -= &sw + endif + + " Colons are tricky. + " We want to outdent if it's part of a switch ("case foo:" or "default:"). + " We ignore trying to deal with jump labels because (a) they're rare, and + " (b) they're hard to disambiguate from a composite literal key. + if thisl =~# '^\s*\(case .*\|default\):$' + let ind -= &sw + endif + + return ind +endfunction |
