diff options
Diffstat (limited to 'misc/dashboard')
26 files changed, 2812 insertions, 0 deletions
diff --git a/misc/dashboard/README b/misc/dashboard/README new file mode 100644 index 000000000..c00311ef7 --- /dev/null +++ b/misc/dashboard/README @@ -0,0 +1,26 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +The files in this directory constitute the continuous builder: + +godashboard/: an AppEngine server +builder/: gobuilder, a Go continuous build client + +If you wish to run a Go builder, please email golang-dev@googlegroups.com + +To run a builder: + +* Write the key ~gobuild/.gobuildkey + You need to get it from someone who knows the key. + You may also use a filename of the form .gobuildkey-$BUILDER if you + wish to run builders for multiple targets. + +* Append your username and password googlecode.com credentials from + https://code.google.com/hosting/settings + to the buildkey file in the format "Username\nPassword\n". + (This is for uploading tarballs to the project downloads section, + and is an optional step.) + +* Build and run gobuilder (see its documentation for command-line options). + diff --git a/misc/dashboard/builder/Makefile b/misc/dashboard/builder/Makefile new file mode 100644 index 000000000..f1d9c5497 --- /dev/null +++ b/misc/dashboard/builder/Makefile @@ -0,0 +1,14 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +include ../../../src/Make.inc + +TARG=gobuilder +GOFILES=\ + exec.go\ + http.go\ + main.go\ + package.go\ + +include ../../../src/Make.cmd diff --git a/misc/dashboard/builder/doc.go b/misc/dashboard/builder/doc.go new file mode 100644 index 000000000..30d8fe948 --- /dev/null +++ b/misc/dashboard/builder/doc.go @@ -0,0 +1,58 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* + +Go Builder is a continuous build client for the Go project. +It integrates with the Go Dashboard AppEngine application. + +Go Builder is intended to run continuously as a background process. + +It periodically pulls updates from the Go Mercurial repository. + +When a newer revision is found, Go Builder creates a clone of the repository, +runs all.bash, and reports build success or failure to the Go Dashboard. + +For a release revision (a change description that matches "release.YYYY-MM-DD"), +Go Builder will create a tar.gz archive of the GOROOT and deliver it to the +Go Google Code project's downloads section. + +Usage: + + gobuilder goos-goarch... + + Several goos-goarch combinations can be provided, and the builder will + build them in serial. + +Optional flags: + + -dashboard="godashboard.appspot.com": Go Dashboard Host + The location of the Go Dashboard application to which Go Builder will + report its results. + + -release: Build and deliver binary release archive + + -rev=N: Build revision N and exit + + -cmd="./all.bash": Build command (specify absolute or relative to go/src) + + -v: Verbose logging + + -external: External package builder mode (will not report Go build + state to dashboard or issue releases) + +The key file should be located at $HOME/.gobuildkey or, for a builder-specific +key, $HOME/.gobuildkey-$BUILDER (eg, $HOME/.gobuildkey-linux-amd64). + +The build key file is a text file of the format: + + godashboard-key + googlecode-username + googlecode-password + +If the Google Code credentials are not provided the archival step +will be skipped. + +*/ +package documentation diff --git a/misc/dashboard/builder/exec.go b/misc/dashboard/builder/exec.go new file mode 100644 index 000000000..a042c5699 --- /dev/null +++ b/misc/dashboard/builder/exec.go @@ -0,0 +1,74 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "bytes" + "exec" + "io" + "log" + "os" + "strings" +) + +// run is a simple wrapper for exec.Run/Close +func run(envv []string, dir string, argv ...string) os.Error { + if *verbose { + log.Println("run", argv) + } + argv = useBash(argv) + cmd := exec.Command(argv[0], argv[1:]...) + cmd.Dir = dir + cmd.Env = envv + cmd.Stderr = os.Stderr + return cmd.Run() +} + +// runLog runs a process and returns the combined stdout/stderr, +// as well as writing it to logfile (if specified). It returns +// process combined stdout and stderr output, exit status and error. +// The error returned is nil, if process is started successfully, +// even if exit status is not 0. +func runLog(envv []string, logfile, dir string, argv ...string) (string, int, os.Error) { + if *verbose { + log.Println("runLog", argv) + } + argv = useBash(argv) + + b := new(bytes.Buffer) + var w io.Writer = b + if logfile != "" { + f, err := os.OpenFile(logfile, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666) + if err != nil { + return "", 0, err + } + defer f.Close() + w = io.MultiWriter(f, b) + } + + cmd := exec.Command(argv[0], argv[1:]...) + cmd.Dir = dir + cmd.Env = envv + cmd.Stdout = w + cmd.Stderr = w + + err := cmd.Run() + if err != nil { + if ws, ok := err.(*os.Waitmsg); ok { + return b.String(), ws.ExitStatus(), nil + } + } + return b.String(), 0, nil +} + +// useBash prefixes a list of args with 'bash' if the first argument +// is a bash script. +func useBash(argv []string) []string { + // TODO(brainman): choose a more reliable heuristic here. + if strings.HasSuffix(argv[0], ".bash") { + argv = append([]string{"bash"}, argv...) + } + return argv +} diff --git a/misc/dashboard/builder/http.go b/misc/dashboard/builder/http.go new file mode 100644 index 000000000..abef8faa4 --- /dev/null +++ b/misc/dashboard/builder/http.go @@ -0,0 +1,148 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "bytes" + "fmt" + "http" + "json" + "log" + "os" + "strconv" + "url" +) + +type param map[string]string + +// dash runs the given method and command on the dashboard. +// If args is not nil, it is the query or post parameters. +// If resp is not nil, dash unmarshals the body as JSON into resp. +func dash(meth, cmd string, resp interface{}, args param) os.Error { + var r *http.Response + var err os.Error + if *verbose { + log.Println("dash", cmd, args) + } + cmd = "http://" + *dashboard + "/" + cmd + vals := make(url.Values) + for k, v := range args { + vals.Add(k, v) + } + switch meth { + case "GET": + if q := vals.Encode(); q != "" { + cmd += "?" + q + } + r, err = http.Get(cmd) + case "POST": + r, err = http.PostForm(cmd, vals) + default: + return fmt.Errorf("unknown method %q", meth) + } + if err != nil { + return err + } + defer r.Body.Close() + var buf bytes.Buffer + buf.ReadFrom(r.Body) + if resp != nil { + if err = json.Unmarshal(buf.Bytes(), resp); err != nil { + log.Printf("json unmarshal %#q: %s\n", buf.Bytes(), err) + return err + } + } + return nil +} + +func dashStatus(meth, cmd string, args param) os.Error { + var resp struct { + Status string + Error string + } + err := dash(meth, cmd, &resp, args) + if err != nil { + return err + } + if resp.Status != "OK" { + return os.NewError("/build: " + resp.Error) + } + return nil +} + +// todo returns the next hash to build. +func (b *Builder) todo() (rev string, err os.Error) { + var resp []struct { + Hash string + } + if err = dash("GET", "todo", &resp, param{"builder": b.name}); err != nil { + return + } + if len(resp) > 0 { + rev = resp[0].Hash + } + return +} + +// recordResult sends build results to the dashboard +func (b *Builder) recordResult(buildLog string, hash string) os.Error { + return dash("POST", "build", nil, param{ + "builder": b.name, + "key": b.key, + "node": hash, + "log": buildLog, + }) +} + +// packages fetches a list of package paths from the dashboard +func packages() (pkgs []string, err os.Error) { + var resp struct { + Packages []struct { + Path string + } + } + err = dash("GET", "package", &resp, param{"fmt": "json"}) + if err != nil { + return + } + for _, p := range resp.Packages { + pkgs = append(pkgs, p.Path) + } + return +} + +// updatePackage sends package build results and info dashboard +func (b *Builder) updatePackage(pkg string, ok bool, buildLog, info string) os.Error { + return dash("POST", "package", nil, param{ + "builder": b.name, + "key": b.key, + "path": pkg, + "ok": strconv.Btoa(ok), + "log": buildLog, + "info": info, + }) +} + +// postCommit informs the dashboard of a new commit +func postCommit(key string, l *HgLog) os.Error { + return dashStatus("POST", "commit", param{ + "key": key, + "node": l.Hash, + "date": l.Date, + "user": l.Author, + "parent": l.Parent, + "desc": l.Desc, + }) +} + +// dashboardCommit returns true if the dashboard knows about hash. +func dashboardCommit(hash string) bool { + err := dashStatus("GET", "commit", param{"node": hash}) + if err != nil { + log.Printf("check %s: %s", hash, err) + return false + } + return true +} diff --git a/misc/dashboard/builder/main.go b/misc/dashboard/builder/main.go new file mode 100644 index 000000000..989965bc4 --- /dev/null +++ b/misc/dashboard/builder/main.go @@ -0,0 +1,624 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "flag" + "fmt" + "io/ioutil" + "log" + "os" + "path" + "regexp" + "runtime" + "strconv" + "strings" + "time" + "xml" +) + +const ( + codeProject = "go" + codePyScript = "misc/dashboard/googlecode_upload.py" + hgUrl = "https://go.googlecode.com/hg/" + waitInterval = 30e9 // time to wait before checking for new revs + mkdirPerm = 0750 + pkgBuildInterval = 1e9 * 60 * 60 * 24 // rebuild packages every 24 hours +) + +// These variables are copied from the gobuilder's environment +// to the envv of its subprocesses. +var extraEnv = []string{ + "GOHOSTOS", + "GOHOSTARCH", + "PATH", + "DISABLE_NET_TESTS", + "MAKEFLAGS", + "GOARM", +} + +type Builder struct { + name string + goos, goarch string + key string + codeUsername string + codePassword string +} + +var ( + buildroot = flag.String("buildroot", path.Join(os.TempDir(), "gobuilder"), "Directory under which to build") + commitFlag = flag.Bool("commit", false, "upload information about new commits") + dashboard = flag.String("dashboard", "godashboard.appspot.com", "Go Dashboard Host") + buildRelease = flag.Bool("release", false, "Build and upload binary release archives") + buildRevision = flag.String("rev", "", "Build specified revision and exit") + buildCmd = flag.String("cmd", "./all.bash", "Build command (specify absolute or relative to go/src/)") + external = flag.Bool("external", false, "Build external packages") + parallel = flag.Bool("parallel", false, "Build multiple targets in parallel") + verbose = flag.Bool("v", false, "verbose") +) + +var ( + goroot string + binaryTagRe = regexp.MustCompile(`^(release\.r|weekly\.)[0-9\-.]+`) + releaseRe = regexp.MustCompile(`^release\.r[0-9\-.]+`) +) + +func main() { + flag.Usage = func() { + fmt.Fprintf(os.Stderr, "usage: %s goos-goarch...\n", os.Args[0]) + flag.PrintDefaults() + os.Exit(2) + } + flag.Parse() + if len(flag.Args()) == 0 && !*commitFlag { + flag.Usage() + } + goroot = path.Join(*buildroot, "goroot") + builders := make([]*Builder, len(flag.Args())) + for i, builder := range flag.Args() { + b, err := NewBuilder(builder) + if err != nil { + log.Fatal(err) + } + builders[i] = b + } + + // set up work environment + if err := os.RemoveAll(*buildroot); err != nil { + log.Fatalf("Error removing build root (%s): %s", *buildroot, err) + } + if err := os.Mkdir(*buildroot, mkdirPerm); err != nil { + log.Fatalf("Error making build root (%s): %s", *buildroot, err) + } + if err := run(nil, *buildroot, "hg", "clone", hgUrl, goroot); err != nil { + log.Fatal("Error cloning repository:", err) + } + + if *commitFlag { + if len(flag.Args()) == 0 { + commitWatcher() + return + } + go commitWatcher() + } + + // if specified, build revision and return + if *buildRevision != "" { + hash, err := fullHash(*buildRevision) + if err != nil { + log.Fatal("Error finding revision: ", err) + } + for _, b := range builders { + if err := b.buildHash(hash); err != nil { + log.Println(err) + } + } + return + } + + // external package build mode + if *external { + if len(builders) != 1 { + log.Fatal("only one goos-goarch should be specified with -external") + } + builders[0].buildExternal() + } + + // go continuous build mode (default) + // check for new commits and build them + for { + built := false + t := time.Nanoseconds() + if *parallel { + done := make(chan bool) + for _, b := range builders { + go func(b *Builder) { + done <- b.build() + }(b) + } + for _ = range builders { + built = <-done || built + } + } else { + for _, b := range builders { + built = b.build() || built + } + } + // sleep if there was nothing to build + if !built { + time.Sleep(waitInterval) + } + // sleep if we're looping too fast. + t1 := time.Nanoseconds() - t + if t1 < waitInterval { + time.Sleep(waitInterval - t1) + } + } +} + +func NewBuilder(builder string) (*Builder, os.Error) { + b := &Builder{name: builder} + + // get goos/goarch from builder string + s := strings.SplitN(builder, "-", 3) + if len(s) >= 2 { + b.goos, b.goarch = s[0], s[1] + } else { + return nil, fmt.Errorf("unsupported builder form: %s", builder) + } + + // read keys from keyfile + fn := path.Join(os.Getenv("HOME"), ".gobuildkey") + if s := fn + "-" + b.name; isFile(s) { // builder-specific file + fn = s + } + c, err := ioutil.ReadFile(fn) + if err != nil { + return nil, fmt.Errorf("readKeys %s (%s): %s", b.name, fn, err) + } + v := strings.Split(string(c), "\n") + b.key = v[0] + if len(v) >= 3 { + b.codeUsername, b.codePassword = v[1], v[2] + } + + return b, nil +} + +// buildExternal downloads and builds external packages, and +// reports their build status to the dashboard. +// It will re-build all packages after pkgBuildInterval nanoseconds or +// a new release tag is found. +func (b *Builder) buildExternal() { + var prevTag string + var nextBuild int64 + for { + time.Sleep(waitInterval) + err := run(nil, goroot, "hg", "pull", "-u") + if err != nil { + log.Println("hg pull failed:", err) + continue + } + hash, tag, err := firstTag(releaseRe) + if err != nil { + log.Println(err) + continue + } + if *verbose { + log.Println("latest release:", tag) + } + // don't rebuild if there's no new release + // and it's been less than pkgBuildInterval + // nanoseconds since the last build. + if tag == prevTag && time.Nanoseconds() < nextBuild { + continue + } + // build will also build the packages + if err := b.buildHash(hash); err != nil { + log.Println(err) + continue + } + prevTag = tag + nextBuild = time.Nanoseconds() + pkgBuildInterval + } +} + +// build checks for a new commit for this builder +// and builds it if one is found. +// It returns true if a build was attempted. +func (b *Builder) build() bool { + defer func() { + err := recover() + if err != nil { + log.Println(b.name, "build:", err) + } + }() + hash, err := b.todo() + if err != nil { + log.Println(err) + return false + } + if hash == "" { + return false + } + // Look for hash locally before running hg pull. + + if _, err := fullHash(hash[:12]); err != nil { + // Don't have hash, so run hg pull. + if err := run(nil, goroot, "hg", "pull"); err != nil { + log.Println("hg pull failed:", err) + return false + } + } + err = b.buildHash(hash) + if err != nil { + log.Println(err) + } + return true +} + +func (b *Builder) buildHash(hash string) (err os.Error) { + defer func() { + if err != nil { + err = fmt.Errorf("%s build: %s: %s", b.name, hash, err) + } + }() + + log.Println(b.name, "building", hash) + + // create place in which to do work + workpath := path.Join(*buildroot, b.name+"-"+hash[:12]) + err = os.Mkdir(workpath, mkdirPerm) + if err != nil { + return + } + defer os.RemoveAll(workpath) + + // clone repo + err = run(nil, workpath, "hg", "clone", goroot, "go") + if err != nil { + return + } + + // update to specified revision + err = run(nil, path.Join(workpath, "go"), + "hg", "update", hash) + if err != nil { + return + } + + srcDir := path.Join(workpath, "go", "src") + + // build + logfile := path.Join(workpath, "build.log") + buildLog, status, err := runLog(b.envv(), logfile, srcDir, *buildCmd) + if err != nil { + return fmt.Errorf("%s: %s", *buildCmd, err) + } + + // if we're in external mode, build all packages and return + if *external { + if status != 0 { + return os.NewError("go build failed") + } + return b.buildPackages(workpath, hash) + } + + if status != 0 { + // record failure + return b.recordResult(buildLog, hash) + } + + // record success + if err = b.recordResult("", hash); err != nil { + return fmt.Errorf("recordResult: %s", err) + } + + // finish here if codeUsername and codePassword aren't set + if b.codeUsername == "" || b.codePassword == "" || !*buildRelease { + return + } + + // if this is a release, create tgz and upload to google code + releaseHash, release, err := firstTag(binaryTagRe) + if hash == releaseHash { + // clean out build state + err = run(b.envv(), srcDir, "./clean.bash", "--nopkg") + if err != nil { + return fmt.Errorf("clean.bash: %s", err) + } + // upload binary release + fn := fmt.Sprintf("go.%s.%s-%s.tar.gz", release, b.goos, b.goarch) + err = run(nil, workpath, "tar", "czf", fn, "go") + if err != nil { + return fmt.Errorf("tar: %s", err) + } + err = run(nil, workpath, path.Join(goroot, codePyScript), + "-s", release, + "-p", codeProject, + "-u", b.codeUsername, + "-w", b.codePassword, + "-l", fmt.Sprintf("%s,%s", b.goos, b.goarch), + fn) + } + + return +} + +// envv returns an environment for build/bench execution +func (b *Builder) envv() []string { + if runtime.GOOS == "windows" { + return b.envvWindows() + } + e := []string{ + "GOOS=" + b.goos, + "GOARCH=" + b.goarch, + "GOROOT_FINAL=/usr/local/go", + } + for _, k := range extraEnv { + s, err := os.Getenverror(k) + if err == nil { + e = append(e, k+"="+s) + } + } + return e +} + +// windows version of envv +func (b *Builder) envvWindows() []string { + start := map[string]string{ + "GOOS": b.goos, + "GOARCH": b.goarch, + "GOROOT_FINAL": "/c/go", + // TODO(brainman): remove once we find make that does not hang. + "MAKEFLAGS": "-j1", + } + for _, name := range extraEnv { + s, err := os.Getenverror(name) + if err == nil { + start[name] = s + } + } + skip := map[string]bool{ + "GOBIN": true, + "GOROOT": true, + "INCLUDE": true, + "LIB": true, + } + var e []string + for name, v := range start { + e = append(e, name+"="+v) + skip[name] = true + } + for _, kv := range os.Environ() { + s := strings.SplitN(kv, "=", 2) + name := strings.ToUpper(s[0]) + switch { + case name == "": + // variables, like "=C:=C:\", just copy them + e = append(e, kv) + case !skip[name]: + e = append(e, kv) + skip[name] = true + } + } + return e +} + +func isDirectory(name string) bool { + s, err := os.Stat(name) + return err == nil && s.IsDirectory() +} + +func isFile(name string) bool { + s, err := os.Stat(name) + return err == nil && (s.IsRegular() || s.IsSymlink()) +} + +// commitWatcher polls hg for new commits and tells the dashboard about them. +func commitWatcher() { + // Create builder just to get master key. + b, err := NewBuilder("mercurial-commit") + if err != nil { + log.Fatal(err) + } + for { + if *verbose { + log.Printf("poll...") + } + commitPoll(b.key) + if *verbose { + log.Printf("sleep...") + } + time.Sleep(60e9) + } +} + +// HgLog represents a single Mercurial revision. +type HgLog struct { + Hash string + Author string + Date string + Desc string + Parent string + + // Internal metadata + added bool +} + +// logByHash is a cache of all Mercurial revisions we know about, +// indexed by full hash. +var logByHash = map[string]*HgLog{} + +// xmlLogTemplate is a template to pass to Mercurial to make +// hg log print the log in valid XML for parsing with xml.Unmarshal. +const xmlLogTemplate = ` + <log> + <hash>{node|escape}</hash> + <parent>{parent|escape}</parent> + <author>{author|escape}</author> + <date>{date}</date> + <desc>{desc|escape}</desc> + </log> +` + +// commitPoll pulls any new revisions from the hg server +// and tells the server about them. +func commitPoll(key string) { + // Catch unexpected panics. + defer func() { + if err := recover(); err != nil { + log.Printf("commitPoll panic: %s", err) + } + }() + + if err := run(nil, goroot, "hg", "pull"); err != nil { + log.Printf("hg pull: %v", err) + return + } + + const N = 20 // how many revisions to grab + + data, _, err := runLog(nil, "", goroot, "hg", "log", + "--encoding=utf-8", + "--limit="+strconv.Itoa(N), + "--template="+xmlLogTemplate, + ) + if err != nil { + log.Printf("hg log: %v", err) + return + } + + var logStruct struct { + Log []HgLog + } + err = xml.Unmarshal(strings.NewReader("<top>"+data+"</top>"), &logStruct) + if err != nil { + log.Printf("unmarshal hg log: %v", err) + return + } + + logs := logStruct.Log + + // Pass 1. Fill in parents and add new log entries to logsByHash. + // Empty parent means take parent from next log entry. + // Non-empty parent has form 1234:hashhashhash; we want full hash. + for i := range logs { + l := &logs[i] + log.Printf("hg log: %s < %s\n", l.Hash, l.Parent) + if l.Parent == "" && i+1 < len(logs) { + l.Parent = logs[i+1].Hash + } else if l.Parent != "" { + l.Parent, _ = fullHash(l.Parent) + } + if l.Parent == "" { + // Can't create node without parent. + continue + } + + if logByHash[l.Hash] == nil { + // Make copy to avoid pinning entire slice when only one entry is new. + t := *l + logByHash[t.Hash] = &t + } + } + + for i := range logs { + l := &logs[i] + if l.Parent == "" { + continue + } + addCommit(l.Hash, key) + } +} + +// addCommit adds the commit with the named hash to the dashboard. +// key is the secret key for authentication to the dashboard. +// It avoids duplicate effort. +func addCommit(hash, key string) bool { + l := logByHash[hash] + if l == nil { + return false + } + if l.added { + return true + } + + // Check for already added, perhaps in an earlier run. + if dashboardCommit(hash) { + log.Printf("%s already on dashboard\n", hash) + // Record that this hash is on the dashboard, + // as must be all its parents. + for l != nil { + l.added = true + l = logByHash[l.Parent] + } + return true + } + + // Create parent first, to maintain some semblance of order. + if !addCommit(l.Parent, key) { + return false + } + + // Create commit. + if err := postCommit(key, l); err != nil { + log.Printf("failed to add %s to dashboard: %v", key, err) + return false + } + return true +} + +// fullHash returns the full hash for the given Mercurial revision. +func fullHash(rev string) (hash string, err os.Error) { + defer func() { + if err != nil { + err = fmt.Errorf("fullHash: %s: %s", rev, err) + } + }() + s, _, err := runLog(nil, "", goroot, + "hg", "log", + "--encoding=utf-8", + "--rev="+rev, + "--limit=1", + "--template={node}", + ) + if err != nil { + return + } + s = strings.TrimSpace(s) + if s == "" { + return "", fmt.Errorf("cannot find revision") + } + if len(s) != 40 { + return "", fmt.Errorf("hg returned invalid hash " + s) + } + return s, nil +} + +var revisionRe = regexp.MustCompile(`^([^ ]+) +[0-9]+:([0-9a-f]+)$`) + +// firstTag returns the hash and tag of the most recent tag matching re. +func firstTag(re *regexp.Regexp) (hash string, tag string, err os.Error) { + o, _, err := runLog(nil, "", goroot, "hg", "tags") + for _, l := range strings.Split(o, "\n") { + if l == "" { + continue + } + s := revisionRe.FindStringSubmatch(l) + if s == nil { + err = os.NewError("couldn't find revision number") + return + } + if !re.MatchString(s[1]) { + continue + } + tag = s[1] + hash, err = fullHash(s[2]) + return + } + err = os.NewError("no matching tag found") + return +} diff --git a/misc/dashboard/builder/package.go b/misc/dashboard/builder/package.go new file mode 100644 index 000000000..b2a83fa13 --- /dev/null +++ b/misc/dashboard/builder/package.go @@ -0,0 +1,94 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "go/doc" + "go/parser" + "go/token" + "log" + "os" + "path/filepath" + "strings" +) + +const MaxCommentLength = 500 // App Engine won't store more in a StringProperty. + +func (b *Builder) buildPackages(workpath string, hash string) os.Error { + pkgs, err := packages() + if err != nil { + return err + } + for _, p := range pkgs { + goroot := filepath.Join(workpath, "go") + gobin := filepath.Join(goroot, "bin") + goinstall := filepath.Join(gobin, "goinstall") + envv := append(b.envv(), "GOROOT="+goroot) + + // add GOBIN to path + for i, v := range envv { + if strings.HasPrefix(v, "PATH=") { + p := filepath.SplitList(v[5:]) + p = append([]string{gobin}, p...) + s := strings.Join(p, string(filepath.ListSeparator)) + envv[i] = "PATH=" + s + } + } + + // goinstall + buildLog, code, err := runLog(envv, "", goroot, goinstall, "-dashboard=false", p) + if err != nil { + log.Printf("goinstall %v: %v", p, err) + } + + // get doc comment from package source + info, err := packageComment(p, filepath.Join(goroot, "src", "pkg", p)) + if err != nil { + log.Printf("packageComment %v: %v", p, err) + } + + // update dashboard with build state + info + err = b.updatePackage(p, code == 0, buildLog, info) + if err != nil { + log.Printf("updatePackage %v: %v", p, err) + } + } + return nil +} + +func isGoFile(fi *os.FileInfo) bool { + return fi.IsRegular() && // exclude directories + !strings.HasPrefix(fi.Name, ".") && // ignore .files + filepath.Ext(fi.Name) == ".go" +} + +func packageComment(pkg, pkgpath string) (info string, err os.Error) { + fset := token.NewFileSet() + pkgs, err := parser.ParseDir(fset, pkgpath, isGoFile, parser.PackageClauseOnly|parser.ParseComments) + if err != nil { + return + } + for name := range pkgs { + if name == "main" { + continue + } + if info != "" { + return "", os.NewError("multiple non-main package docs") + } + pdoc := doc.NewPackageDoc(pkgs[name], pkg) + info = pdoc.Doc + } + // grab only first paragraph + if parts := strings.SplitN(info, "\n\n", 2); len(parts) > 1 { + info = parts[0] + } + // replace newlines with spaces + info = strings.Replace(info, "\n", " ", -1) + // truncate + if len(info) > MaxCommentLength { + info = info[:MaxCommentLength] + } + return +} diff --git a/misc/dashboard/godashboard/_multiprocessing.py b/misc/dashboard/godashboard/_multiprocessing.py new file mode 100644 index 000000000..8c66c0659 --- /dev/null +++ b/misc/dashboard/godashboard/_multiprocessing.py @@ -0,0 +1,5 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +import multiprocessing diff --git a/misc/dashboard/godashboard/app.yaml b/misc/dashboard/godashboard/app.yaml new file mode 100644 index 000000000..7b77a85cc --- /dev/null +++ b/misc/dashboard/godashboard/app.yaml @@ -0,0 +1,25 @@ +application: godashboard +version: 7 +runtime: python +api_version: 1 + +handlers: +- url: /favicon\.ico + static_files: static/favicon.ico + upload: static/favicon\.ico + +- url: /static + static_dir: static + +- url: /package + script: package.py + +- url: /package/daily + script: package.py + login: admin + +- url: /project.* + script: package.py + +- url: /.* + script: gobuild.py diff --git a/misc/dashboard/godashboard/auth.py b/misc/dashboard/godashboard/auth.py new file mode 100644 index 000000000..73a54c0d4 --- /dev/null +++ b/misc/dashboard/godashboard/auth.py @@ -0,0 +1,13 @@ +# Copyright 2011 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +import hmac + +# local imports +import key + +def auth(req): + k = req.get('key') + return k == hmac.new(key.accessKey, req.get('builder')).hexdigest() or k == key.accessKey + diff --git a/misc/dashboard/godashboard/const.py b/misc/dashboard/godashboard/const.py new file mode 100644 index 000000000..b0110c635 --- /dev/null +++ b/misc/dashboard/godashboard/const.py @@ -0,0 +1,13 @@ +# Copyright 2011 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +mail_from = "Go Dashboard <builder@golang.org>" + +mail_submit_to = "adg@golang.org" +mail_submit_subject = "New Project Submitted" + +mail_fail_to = "golang-dev@googlegroups.com" +mail_fail_reply_to = "golang-dev@googlegroups.com" +mail_fail_subject = "%s broken by %s" + diff --git a/misc/dashboard/godashboard/cron.yaml b/misc/dashboard/godashboard/cron.yaml new file mode 100644 index 000000000..953b6a1cd --- /dev/null +++ b/misc/dashboard/godashboard/cron.yaml @@ -0,0 +1,4 @@ +cron: +- description: daily package maintenance + url: /package/daily + schedule: every 24 hours diff --git a/misc/dashboard/godashboard/fail-notify.txt b/misc/dashboard/godashboard/fail-notify.txt new file mode 100644 index 000000000..a699005ea --- /dev/null +++ b/misc/dashboard/godashboard/fail-notify.txt @@ -0,0 +1,6 @@ +Change {{node}} broke the {{builder}} build: +http://godashboard.appspot.com/log/{{loghash}} + +{{desc}} + +http://code.google.com/p/go/source/detail?r={{node}} diff --git a/misc/dashboard/godashboard/gobuild.py b/misc/dashboard/godashboard/gobuild.py new file mode 100644 index 000000000..685dc83a9 --- /dev/null +++ b/misc/dashboard/godashboard/gobuild.py @@ -0,0 +1,558 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +# This is the server part of the continuous build system for Go. It must be run +# by AppEngine. + +from django.utils import simplejson +from google.appengine.api import mail +from google.appengine.api import memcache +from google.appengine.ext import db +from google.appengine.ext import webapp +from google.appengine.ext.webapp import template +from google.appengine.ext.webapp.util import run_wsgi_app +import datetime +import hashlib +import logging +import os +import re +import bz2 + +# local imports +from auth import auth +import const + +# The majority of our state are commit objects. One of these exists for each of +# the commits known to the build system. Their key names are of the form +# <commit number (%08x)> "-" <hg hash>. This means that a sorting by the key +# name is sufficient to order the commits. +# +# The commit numbers are purely local. They need not match up to the commit +# numbers in an hg repo. When inserting a new commit, the parent commit must be +# given and this is used to generate the new commit number. In order to create +# the first Commit object, a special command (/init) is used. +class Commit(db.Model): + num = db.IntegerProperty() # internal, monotonic counter. + node = db.StringProperty() # Hg hash + parentnode = db.StringProperty() # Hg hash + user = db.StringProperty() + date = db.DateTimeProperty() + desc = db.BlobProperty() + + # This is the list of builds. Each element is a string of the form <builder + # name> '`' <log hash>. If the log hash is empty, then the build was + # successful. + builds = db.StringListProperty() + + fail_notification_sent = db.BooleanProperty() + +# A CompressedLog contains the textual build log of a failed build. +# The key name is the hex digest of the SHA256 hash of the contents. +# The contents is bz2 compressed. +class CompressedLog(db.Model): + log = db.BlobProperty() + +N = 30 + +def builderInfo(b): + f = b.split('-', 3) + goos = f[0] + goarch = f[1] + note = "" + if len(f) > 2: + note = f[2] + return {'name': b, 'goos': goos, 'goarch': goarch, 'note': note} + +def builderset(): + q = Commit.all() + q.order('-__key__') + results = q.fetch(N) + builders = set() + for c in results: + builders.update(set(parseBuild(build)['builder'] for build in c.builds)) + return builders + +class MainPage(webapp.RequestHandler): + def get(self): + self.response.headers['Content-Type'] = 'text/html; charset=utf-8' + + try: + page = int(self.request.get('p', 1)) + if not page > 0: + raise + except: + page = 1 + + try: + num = int(self.request.get('n', N)) + if num <= 0 or num > 200: + raise + except: + num = N + + offset = (page-1) * num + + q = Commit.all() + q.order('-__key__') + results = q.fetch(num, offset) + + revs = [toRev(r) for r in results] + builders = {} + + for r in revs: + for b in r['builds']: + builders[b['builder']] = builderInfo(b['builder']) + + for r in revs: + have = set(x['builder'] for x in r['builds']) + need = set(builders.keys()).difference(have) + for n in need: + r['builds'].append({'builder': n, 'log':'', 'ok': False}) + r['builds'].sort(cmp = byBuilder) + + builders = list(builders.items()) + builders.sort() + values = {"revs": revs, "builders": [v for k,v in builders]} + + values['num'] = num + values['prev'] = page - 1 + if len(results) == num: + values['next'] = page + 1 + + path = os.path.join(os.path.dirname(__file__), 'main.html') + self.response.out.write(template.render(path, values)) + +# A DashboardHandler is a webapp.RequestHandler but provides +# authenticated_post - called by post after authenticating +# json - writes object in json format to response output +class DashboardHandler(webapp.RequestHandler): + def post(self): + if not auth(self.request): + self.response.set_status(403) + return + self.authenticated_post() + + def authenticated_post(self): + return + + def json(self, obj): + self.response.set_status(200) + simplejson.dump(obj, self.response.out) + return + +# Todo serves /todo. It tells the builder which commits need to be built. +class Todo(DashboardHandler): + def get(self): + builder = self.request.get('builder') + key = 'todo-%s' % builder + response = memcache.get(key) + if response is None: + # Fell out of memcache. Rebuild from datastore results. + # We walk the commit list looking for nodes that have not + # been built by this builder. + q = Commit.all() + q.order('-__key__') + todo = [] + first = None + for c in q.fetch(N+1): + if first is None: + first = c + if not built(c, builder): + todo.append({'Hash': c.node}) + response = simplejson.dumps(todo) + memcache.set(key, response, 3600) + self.response.set_status(200) + self.response.out.write(response) + +def built(c, builder): + for b in c.builds: + if b.startswith(builder+'`'): + return True + return False + +# Log serves /log/. It retrieves log data by content hash. +class LogHandler(DashboardHandler): + def get(self): + self.response.headers['Content-Type'] = 'text/plain; charset=utf-8' + hash = self.request.path[5:] + l = CompressedLog.get_by_key_name(hash) + if l is None: + self.response.set_status(404) + return + log = bz2.decompress(l.log) + self.response.set_status(200) + self.response.out.write(log) + +# Init creates the commit with id 0. Since this commit doesn't have a parent, +# it cannot be created by Build. +class Init(DashboardHandler): + def authenticated_post(self): + date = parseDate(self.request.get('date')) + node = self.request.get('node') + if not validNode(node) or date is None: + logging.error("Not valid node ('%s') or bad date (%s %s)", node, date, self.request.get('date')) + self.response.set_status(500) + return + + commit = Commit(key_name = '00000000-%s' % node) + commit.num = 0 + commit.node = node + commit.parentnode = '' + commit.user = self.request.get('user').encode('utf8') + commit.date = date + commit.desc = self.request.get('desc').encode('utf8') + + commit.put() + + self.response.set_status(200) + +# The last commit when we switched to using entity groups. +# This is the root of the new commit entity group. +RootCommitKeyName = '00000f26-f32c6f1038207c55d5780231f7484f311020747e' + +# CommitHandler serves /commit. +# A GET of /commit retrieves information about the specified commit. +# A POST of /commit creates a node for the given commit. +# If the commit already exists, the POST silently succeeds (like mkdir -p). +class CommitHandler(DashboardHandler): + def get(self): + node = self.request.get('node') + if not validNode(node): + return self.json({'Status': 'FAIL', 'Error': 'malformed node hash'}) + n = nodeByHash(node) + if n is None: + return self.json({'Status': 'FAIL', 'Error': 'unknown revision'}) + return self.json({'Status': 'OK', 'Node': nodeObj(n)}) + + def authenticated_post(self): + # Require auth with the master key, not a per-builder key. + if self.request.get('builder'): + self.response.set_status(403) + return + + node = self.request.get('node') + date = parseDate(self.request.get('date')) + user = self.request.get('user').encode('utf8') + desc = self.request.get('desc').encode('utf8') + parenthash = self.request.get('parent') + + if not validNode(node) or not validNode(parenthash) or date is None: + return self.json({'Status': 'FAIL', 'Error': 'malformed node, parent, or date'}) + + n = nodeByHash(node) + if n is None: + p = nodeByHash(parenthash) + if p is None: + return self.json({'Status': 'FAIL', 'Error': 'unknown parent'}) + + # Want to create new node in a transaction so that multiple + # requests creating it do not collide and so that multiple requests + # creating different nodes get different sequence numbers. + # All queries within a transaction must include an ancestor, + # but the original datastore objects we used for the dashboard + # have no common ancestor. Instead, we use a well-known + # root node - the last one before we switched to entity groups - + # as the as the common ancestor. + root = Commit.get_by_key_name(RootCommitKeyName) + + def add_commit(): + if nodeByHash(node, ancestor=root) is not None: + return + + # Determine number for this commit. + # Once we have created one new entry it will be lastRooted.num+1, + # but the very first commit created in this scheme will have to use + # last.num's number instead (last is likely not rooted). + q = Commit.all() + q.order('-__key__') + q.ancestor(root) + last = q.fetch(1)[0] + num = last.num+1 + + n = Commit(key_name = '%08x-%s' % (num, node), parent = root) + n.num = num + n.node = node + n.parentnode = parenthash + n.user = user + n.date = date + n.desc = desc + n.put() + db.run_in_transaction(add_commit) + n = nodeByHash(node) + if n is None: + return self.json({'Status': 'FAIL', 'Error': 'failed to create commit node'}) + + return self.json({'Status': 'OK', 'Node': nodeObj(n)}) + +# Build serves /build. +# A POST to /build records a new build result. +class Build(webapp.RequestHandler): + def post(self): + if not auth(self.request): + self.response.set_status(403) + return + + builder = self.request.get('builder') + log = self.request.get('log').encode('utf-8') + + loghash = '' + if len(log) > 0: + loghash = hashlib.sha256(log).hexdigest() + l = CompressedLog(key_name=loghash) + l.log = bz2.compress(log) + l.put() + + node = self.request.get('node') + if not validNode(node): + logging.error('Invalid node %s' % (node)) + self.response.set_status(500) + return + + n = nodeByHash(node) + if n is None: + logging.error('Cannot find node %s' % (node)) + self.response.set_status(404) + return + nn = n + + def add_build(): + n = nodeByHash(node, ancestor=nn) + if n is None: + logging.error('Cannot find hash in add_build: %s %s' % (builder, node)) + return + + s = '%s`%s' % (builder, loghash) + for i, b in enumerate(n.builds): + if b.split('`', 1)[0] == builder: + # logging.error('Found result for %s %s already' % (builder, node)) + n.builds[i] = s + break + else: + # logging.error('Added result for %s %s' % (builder, node)) + n.builds.append(s) + n.put() + + db.run_in_transaction(add_build) + + key = 'todo-%s' % builder + memcache.delete(key) + + c = getBrokenCommit(node, builder) + if c is not None and not c.fail_notification_sent: + notifyBroken(c, builder) + + self.response.set_status(200) + + +def getBrokenCommit(node, builder): + """ + getBrokenCommit returns a Commit that breaks the build. + The Commit will be either the one specified by node or the one after. + """ + + # Squelch mail if already fixed. + head = firstResult(builder) + if broken(head, builder) == False: + return + + # Get current node and node before, after. + cur = nodeByHash(node) + if cur is None: + return + before = nodeBefore(cur) + after = nodeAfter(cur) + + if broken(before, builder) == False and broken(cur, builder): + return cur + if broken(cur, builder) == False and broken(after, builder): + return after + + return + +def firstResult(builder): + q = Commit.all().order('-__key__') + for c in q.fetch(20): + for i, b in enumerate(c.builds): + p = b.split('`', 1) + if p[0] == builder: + return c + return None + +def nodeBefore(c): + return nodeByHash(c.parentnode) + +def nodeAfter(c): + return Commit.all().filter('parenthash', c.node).get() + +def notifyBroken(c, builder): + def send(): + n = Commit.get(c.key()) + if n is None: + logging.error("couldn't retrieve Commit '%s'" % c.key()) + return False + if n.fail_notification_sent: + return False + n.fail_notification_sent = True + return n.put() + if not db.run_in_transaction(send): + return + + subject = const.mail_fail_subject % (builder, c.desc.split('\n')[0]) + path = os.path.join(os.path.dirname(__file__), 'fail-notify.txt') + body = template.render(path, { + "builder": builder, + "node": c.node, + "user": c.user, + "desc": c.desc, + "loghash": logHash(c, builder) + }) + mail.send_mail( + sender=const.mail_from, + to=const.mail_fail_to, + subject=subject, + body=body + ) + +def logHash(c, builder): + for i, b in enumerate(c.builds): + p = b.split('`', 1) + if p[0] == builder: + return p[1] + return "" + +def broken(c, builder): + """ + broken returns True if commit c breaks the build for the specified builder, + False if it is a good build, and None if no results exist for this builder. + """ + if c is None: + return None + for i, b in enumerate(c.builds): + p = b.split('`', 1) + if p[0] == builder: + return len(p[1]) > 0 + return None + +def node(num): + q = Commit.all() + q.filter('num =', num) + n = q.get() + return n + +def nodeByHash(hash, ancestor=None): + q = Commit.all() + q.filter('node =', hash) + if ancestor is not None: + q.ancestor(ancestor) + n = q.get() + return n + +# nodeObj returns a JSON object (ready to be passed to simplejson.dump) describing node. +def nodeObj(n): + return { + 'Hash': n.node, + 'ParentHash': n.parentnode, + 'User': n.user, + 'Date': n.date.strftime('%Y-%m-%d %H:%M %z'), + 'Desc': n.desc, + } + +class FixedOffset(datetime.tzinfo): + """Fixed offset in minutes east from UTC.""" + + def __init__(self, offset): + self.__offset = datetime.timedelta(seconds = offset) + + def utcoffset(self, dt): + return self.__offset + + def tzname(self, dt): + return None + + def dst(self, dt): + return datetime.timedelta(0) + +def validNode(node): + if len(node) != 40: + return False + for x in node: + o = ord(x) + if (o < ord('0') or o > ord('9')) and (o < ord('a') or o > ord('f')): + return False + return True + +def parseDate(date): + if '-' in date: + (a, offset) = date.split('-', 1) + try: + return datetime.datetime.fromtimestamp(float(a), FixedOffset(0-int(offset))) + except ValueError: + return None + if '+' in date: + (a, offset) = date.split('+', 1) + try: + return datetime.datetime.fromtimestamp(float(a), FixedOffset(int(offset))) + except ValueError: + return None + try: + return datetime.datetime.utcfromtimestamp(float(date)) + except ValueError: + return None + +email_re = re.compile('^[^<]+<([^>]*)>$') + +def toUsername(user): + r = email_re.match(user) + if r is None: + return user + email = r.groups()[0] + return email.replace('@golang.org', '') + +def dateToShortStr(d): + return d.strftime('%a %b %d %H:%M') + +def parseBuild(build): + [builder, logblob] = build.split('`') + return {'builder': builder, 'log': logblob, 'ok': len(logblob) == 0} + +def nodeInfo(c): + return { + "node": c.node, + "user": toUsername(c.user), + "date": dateToShortStr(c.date), + "desc": c.desc, + "shortdesc": c.desc.split('\n', 2)[0] + } + +def toRev(c): + b = nodeInfo(c) + b['builds'] = [parseBuild(build) for build in c.builds] + return b + +def byBuilder(x, y): + return cmp(x['builder'], y['builder']) + +# Give old builders work; otherwise they pound on the web site. +class Hwget(DashboardHandler): + def get(self): + self.response.out.write("8000\n") + +# This is the URL map for the server. The first three entries are public, the +# rest are only used by the builders. +application = webapp.WSGIApplication( + [('/', MainPage), + ('/hw-get', Hwget), + ('/log/.*', LogHandler), + ('/commit', CommitHandler), + ('/init', Init), + ('/todo', Todo), + ('/build', Build), + ], debug=True) + +def main(): + run_wsgi_app(application) + +if __name__ == "__main__": + main() + diff --git a/misc/dashboard/godashboard/index.yaml b/misc/dashboard/godashboard/index.yaml new file mode 100644 index 000000000..f39299d5d --- /dev/null +++ b/misc/dashboard/godashboard/index.yaml @@ -0,0 +1,51 @@ +indexes: + +- kind: BenchmarkResult + ancestor: yes + properties: + - name: builder + - name: __key__ + direction: desc + +- kind: BenchmarkResult + ancestor: yes + properties: + - name: __key__ + direction: desc + +- kind: BenchmarkResults + properties: + - name: builder + - name: benchmark + +- kind: Commit + properties: + - name: __key__ + direction: desc + +- kind: Commit + ancestor: yes + properties: + - name: __key__ + direction: desc + +- kind: Project + properties: + - name: approved + - name: category + - name: name + +- kind: Project + properties: + - name: category + - name: name + +# AUTOGENERATED + +# This index.yaml is automatically updated whenever the dev_appserver +# detects that a new type of query is run. If you want to manage the +# index.yaml file manually, remove the above marker line (the line +# saying "# AUTOGENERATED"). If you want to manage some indexes +# manually, move them above the marker line. The index.yaml file is +# automatically uploaded to the admin console when you next deploy +# your application using appcfg.py. diff --git a/misc/dashboard/godashboard/key.py.dummy b/misc/dashboard/godashboard/key.py.dummy new file mode 100644 index 000000000..5b8bab186 --- /dev/null +++ b/misc/dashboard/godashboard/key.py.dummy @@ -0,0 +1,10 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +# Copy this file to key.py after substituting the real key. + +# accessKey controls private access to the build server (i.e. to record new +# builds). It's tranmitted in the clear but, given the low value of the target, +# this should be sufficient. +accessKey = "this is not the real key" diff --git a/misc/dashboard/godashboard/main.html b/misc/dashboard/godashboard/main.html new file mode 100644 index 000000000..5390afce6 --- /dev/null +++ b/misc/dashboard/godashboard/main.html @@ -0,0 +1,62 @@ +<!DOCTYPE HTML> +<html> + <head> + <title>Build Status - Go Dashboard</title> + <link rel="stylesheet" type="text/css" href="static/style.css"> + </head> + + <body> + <a id="top"></a> + + <ul class="menu"> + <li>Build Status</li> + <li><a href="/package">Packages</a></li> + <li><a href="/project">Projects</a></li> + <li><a href="http://golang.org/">golang.org</a></li> + </ul> + + <h1>Go Dashboard</h1> + + <h2>Build Status</h2> + <table class="alternate" cellpadding="0" cellspacing="0"> + <tr> + <th></th> + {% for b in builders %} + <th class="builder">{{b.goos}}<br>{{b.goarch}}<br>{{b.note}}</th> + {% endfor %} + <th></th> + <th></th> + <th></th> + </tr> + + {% for r in revs %} + <tr> + <td class="revision"><span class="hash"><a href="https://code.google.com/p/go/source/detail?r={{r.node}}">{{r.node|slice:":12"}}</a></span></td> + + {% for b in r.builds %} + <td class="result"> + {% if b.ok %} + <span class="ok">ok</span> + {% else %} + {% if b.log %} + <a class="fail" href="/log/{{b.log}}">fail</a> + {% else %} + + {% endif %} + {% endif %} + </td> + {% endfor %} + + <td class="user">{{r.user|escape}}</td> + <td class="date">{{r.date|escape}}</td> + <td class="desc">{{r.shortdesc|escape}}</td> + </tr> + {% endfor %} + </table> + <div class="paginate"> + <a{% if prev %} href="?n={{num}}&p={{prev}}"{% else %} class="inactive"{% endif %}>prev</a> + <a{% if next %} href="?n={{num}}&p={{next}}"{% else %} class="inactive"{% endif %}>next</a> + <a{% if prev %} href="?n={{num}}&p=1"{% else %} class="inactive"{% endif %}>top</a> + </div> + </body> +</html> diff --git a/misc/dashboard/godashboard/package.html b/misc/dashboard/godashboard/package.html new file mode 100644 index 000000000..8a9d0a3a0 --- /dev/null +++ b/misc/dashboard/godashboard/package.html @@ -0,0 +1,77 @@ +<!DOCTYPE HTML> +<html> + <head> + <title>Packages - Go Dashboard</title> + <link rel="stylesheet" type="text/css" href="static/style.css"> + </head> + + <body> + <ul class="menu"> + <li><a href="/">Build Status</a></li> + <li>Packages</li> + <li><a href="/project">Projects</a></li> + <li><a href="http://golang.org/">golang.org</a></li> + </ul> + + <h1>Go Dashboard</h1> + + <p> + Packages listed on this page are written by third parties and + may or may not build or be safe to use. + </p> + + <p> + An "ok" in the <b>build</b> column indicates that the package is + <a href="http://golang.org/cmd/goinstall/">goinstallable</a> + with the latest + <a href="http://golang.org/doc/devel/release.html">release</a> of Go. + </p> + + <p> + The <b>info</b> column shows the first paragraph from the + <a href="http://blog.golang.org/2011/03/godoc-documenting-go-code.html">package doc comment</a>. + </p> + + <h2>Most Installed Packages (this week)</h2> + <table class="alternate" cellpadding="0" cellspacing="0"> + <tr><th>last install</th><th>count</th><th>build</th><th>path</th><th>info</th></tr> + {% for r in by_week_count %} + <tr> + <td class="time">{{r.last_install|date:"Y-M-d H:i"}}</td> + <td class="count">{{r.week_count}}</td> + <td class="ok">{% if r.ok %}<a title="{{r.last_ok|date:"Y-M-d H:i"}}">ok</a>{% else %} {% endif %}</td> + <td class="path"><a href="{{r.web_url}}">{{r.path}}</a></td> + <td class="info">{% if r.info %}{{r.info|escape}}{% endif %}</td> + </tr> + {% endfor %} + </table> + + <h2>Recently Installed Packages</h2> + <table class="alternate" cellpadding="0" cellspacing="0"> + <tr><th>last install</th><th>count</th><th>build</th><th>path</th><th>info</th></tr> + {% for r in by_time %} + <tr> + <td class="time">{{r.last_install|date:"Y-M-d H:i"}}</td> + <td class="count">{{r.count}}</td> + <td class="ok">{% if r.ok %}<a title="{{r.last_ok|date:"Y-M-d H:i"}}">ok</a>{% else %} {% endif %}</td> + <td class="path"><a href="{{r.web_url}}">{{r.path}}</a></td> + <td class="info">{% if r.info %}{{r.info|escape}}{% endif %}</td> + </tr> + {% endfor %} + </table> + + <h2>Most Installed Packages (all time)</h2> + <table class="alternate" cellpadding="0" cellspacing="0"> + <tr><th>last install</th><th>count</th><th>build</th><th>path</th><th>info</th></tr> + {% for r in by_count %} + <tr> + <td class="time">{{r.last_install|date:"Y-M-d H:i"}}</td> + <td class="count">{{r.count}}</td> + <td class="ok">{% if r.ok %}<a title="{{r.last_ok|date:"Y-M-d H:i"}}">ok</a>{% else %} {% endif %}</td> + <td class="path"><a href="{{r.web_url}}">{{r.path}}</a></td> + <td class="info">{% if r.info %}{{r.info|escape}}{% endif %}</td> + </tr> + {% endfor %} + </table> + </body> +</html> diff --git a/misc/dashboard/godashboard/package.py b/misc/dashboard/godashboard/package.py new file mode 100644 index 000000000..5cc2d2404 --- /dev/null +++ b/misc/dashboard/godashboard/package.py @@ -0,0 +1,429 @@ +# Copyright 2010 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +# This is the server part of the package dashboard. +# It must be run by App Engine. + +from google.appengine.api import mail +from google.appengine.api import memcache +from google.appengine.api import taskqueue +from google.appengine.api import urlfetch +from google.appengine.api import users +from google.appengine.ext import db +from google.appengine.ext import webapp +from google.appengine.ext.webapp import template +from google.appengine.ext.webapp.util import run_wsgi_app +import datetime +import logging +import os +import re +import sets +import urllib2 + +# local imports +from auth import auth +import toutf8 +import const + +template.register_template_library('toutf8') + +# Storage model for package info recorded on server. +class Package(db.Model): + path = db.StringProperty() + web_url = db.StringProperty() # derived from path + count = db.IntegerProperty() # grand total + week_count = db.IntegerProperty() # rolling weekly count + day_count = db.TextProperty(default='') # daily count + last_install = db.DateTimeProperty() + + # data contributed by gobuilder + info = db.StringProperty() + ok = db.BooleanProperty() + last_ok = db.DateTimeProperty() + + def get_day_count(self): + counts = {} + if not self.day_count: + return counts + for d in str(self.day_count).split('\n'): + date, count = d.split(' ') + counts[date] = int(count) + return counts + + def set_day_count(self, count): + days = [] + for day, count in count.items(): + days.append('%s %d' % (day, count)) + days.sort(reverse=True) + days = days[:28] + self.day_count = '\n'.join(days) + + def inc(self): + count = self.get_day_count() + today = str(datetime.date.today()) + count[today] = count.get(today, 0) + 1 + self.set_day_count(count) + self.update_week_count(count) + self.count += 1 + + def update_week_count(self, count=None): + if count is None: + count = self.get_day_count() + total = 0 + today = datetime.date.today() + for i in range(7): + day = str(today - datetime.timedelta(days=i)) + if day in count: + total += count[day] + self.week_count = total + + +# PackageDaily kicks off the daily package maintenance cron job +# and serves the associated task queue. +class PackageDaily(webapp.RequestHandler): + + def get(self): + # queue a task to update each package with a week_count > 0 + keys = Package.all(keys_only=True).filter('week_count >', 0) + for key in keys: + taskqueue.add(url='/package/daily', params={'key': key.name()}) + + def post(self): + # update a single package (in a task queue) + def update(key): + p = Package.get_by_key_name(key) + if not p: + return + p.update_week_count() + p.put() + key = self.request.get('key') + if not key: + return + db.run_in_transaction(update, key) + + +class Project(db.Model): + name = db.StringProperty(indexed=True) + descr = db.StringProperty() + web_url = db.StringProperty() + package = db.ReferenceProperty(Package) + category = db.StringProperty(indexed=True) + tags = db.ListProperty(str) + approved = db.BooleanProperty(indexed=True) + + +re_bitbucket = re.compile(r'^(bitbucket\.org/[a-z0-9A-Z_.\-]+/[a-zA-Z0-9_.\-]+)(/[a-z0-9A-Z_.\-/]+)?$') +re_googlecode = re.compile(r'^[a-z0-9\-]+\.googlecode\.com/(svn|hg|git)(/[a-z0-9A-Z_.\-/]+)?$') +re_github = re.compile(r'^github\.com/[a-z0-9A-Z_.\-]+(/[a-z0-9A-Z_.\-]+)+$') +re_launchpad = re.compile(r'^launchpad\.net/([a-z0-9A-Z_.\-]+(/[a-z0-9A-Z_.\-]+)?|~[a-z0-9A-Z_.\-]+/(\+junk|[a-z0-9A-Z_.\-]+)/[a-z0-9A-Z_.\-]+)(/[a-z0-9A-Z_.\-/]+)?$') + +def vc_to_web(path): + if re_bitbucket.match(path): + m = re_bitbucket.match(path) + check_url = 'http://' + m.group(1) + '/?cmd=heads' + web = 'http://' + m.group(1) + '/' + elif re_github.match(path): + m = re_github_web.match(path) + check_url = 'https://raw.github.com/' + m.group(1) + '/' + m.group(2) + '/master/' + web = 'http://github.com/' + m.group(1) + '/' + m.group(2) + '/' + elif re_googlecode.match(path): + m = re_googlecode.match(path) + check_url = 'http://'+path + if not m.group(2): # append / after bare '/hg' or '/git' + check_url += '/' + web = 'http://code.google.com/p/' + path[:path.index('.')] + elif re_launchpad.match(path): + check_url = web = 'https://'+path + else: + return False, False + return web, check_url + +re_bitbucket_web = re.compile(r'bitbucket\.org/([a-z0-9A-Z_.\-]+)/([a-z0-9A-Z_.\-]+)') +re_googlecode_web = re.compile(r'code.google.com/p/([a-z0-9\-]+)') +re_github_web = re.compile(r'github\.com/([a-z0-9A-Z_.\-]+)/([a-z0-9A-Z_.\-]+)') +re_launchpad_web = re.compile(r'launchpad\.net/([a-z0-9A-Z_.\-]+(/[a-z0-9A-Z_.\-]+)?|~[a-z0-9A-Z_.\-]+/(\+junk|[a-z0-9A-Z_.\-]+)/[a-z0-9A-Z_.\-]+)(/[a-z0-9A-Z_.\-/]+)?') +re_striphttp = re.compile(r'https?://(www\.)?') + +def find_googlecode_vcs(path): + # Perform http request to path/hg or path/git to check if they're + # using mercurial or git. Otherwise, assume svn. + for vcs in ['git', 'hg']: + try: + response = urlfetch.fetch('http://'+path+vcs, deadline=1) + if response.status_code == 200: + return vcs + except: pass + return 'svn' + +def web_to_vc(url): + url = re_striphttp.sub('', url) + m = re_bitbucket_web.match(url) + if m: + return 'bitbucket.org/'+m.group(1)+'/'+m.group(2) + m = re_github_web.match(url) + if m: + return 'github.com/'+m.group(1)+'/'+m.group(2) + m = re_googlecode_web.match(url) + if m: + path = m.group(1)+'.googlecode.com/' + vcs = find_googlecode_vcs(path) + return path + vcs + m = re_launchpad_web.match(url) + if m: + return m.group(0) + return False + +MaxPathLength = 100 +CacheTimeout = 3600 + +class PackagePage(webapp.RequestHandler): + def get(self): + if self.request.get('fmt') == 'json': + return self.json() + + html = memcache.get('view-package') + if not html: + tdata = {} + + q = Package.all().filter('week_count >', 0) + q.order('-week_count') + tdata['by_week_count'] = q.fetch(50) + + q = Package.all() + q.order('-last_install') + tdata['by_time'] = q.fetch(20) + + q = Package.all() + q.order('-count') + tdata['by_count'] = q.fetch(100) + + path = os.path.join(os.path.dirname(__file__), 'package.html') + html = template.render(path, tdata) + memcache.set('view-package', html, time=CacheTimeout) + + self.response.headers['Content-Type'] = 'text/html; charset=utf-8' + self.response.out.write(html) + + def json(self): + json = memcache.get('view-package-json') + if not json: + q = Package.all() + s = '{"packages": [' + sep = '' + for r in q.fetch(1000): + s += '%s\n\t{"path": "%s", "last_install": "%s", "count": "%s"}' % (sep, r.path, r.last_install, r.count) + sep = ',' + s += '\n]}\n' + json = s + memcache.set('view-package-json', json, time=CacheTimeout) + self.response.set_status(200) + self.response.headers['Content-Type'] = 'text/plain; charset=utf-8' + self.response.out.write(json) + + def can_get_url(self, url): + try: + urllib2.urlopen(urllib2.Request(url)) + return True + except: + return False + + def is_valid_package_path(self, path): + return (re_bitbucket.match(path) or + re_googlecode.match(path) or + re_github.match(path) or + re_launchpad.match(path)) + + def record_pkg(self, path): + # sanity check string + if not path or len(path) > MaxPathLength or not self.is_valid_package_path(path): + return False + + # look in datastore + key = 'pkg-' + path + p = Package.get_by_key_name(key) + if p is None: + # not in datastore - verify URL before creating + web, check_url = vc_to_web(path) + if not web: + logging.error('unrecognized path: %s', path) + return False + if not self.can_get_url(check_url): + logging.error('cannot get %s', check_url) + return False + p = Package(key_name = key, path = path, count = 0, web_url = web) + + if auth(self.request): + # builder updating package metadata + p.info = self.request.get('info') + p.ok = self.request.get('ok') == "true" + if p.ok: + p.last_ok = datetime.datetime.utcnow() + else: + # goinstall reporting an install + p.inc() + p.last_install = datetime.datetime.utcnow() + + # update package object + p.put() + return True + + def post(self): + path = self.request.get('path') + ok = db.run_in_transaction(self.record_pkg, path) + if ok: + self.response.set_status(200) + self.response.out.write('ok') + else: + logging.error('invalid path in post: %s', path) + self.response.set_status(500) + self.response.out.write('not ok') + +class ProjectPage(webapp.RequestHandler): + + def get(self): + admin = users.is_current_user_admin() + if self.request.path == "/project/login": + self.redirect(users.create_login_url("/project")) + elif self.request.path == "/project/logout": + self.redirect(users.create_logout_url("/project")) + elif self.request.path == "/project/edit" and admin: + self.edit() + elif self.request.path == "/project/assoc" and admin: + self.assoc() + else: + self.list() + + def assoc(self): + projects = Project.all() + for p in projects: + if p.package: + continue + path = web_to_vc(p.web_url) + if not path: + continue + pkg = Package.get_by_key_name("pkg-"+path) + if not pkg: + self.response.out.write('no: %s %s<br>' % (p.web_url, path)) + continue + p.package = pkg + p.put() + self.response.out.write('yes: %s %s<br>' % (p.web_url, path)) + + def post(self): + if self.request.path == "/project/edit": + self.edit(True) + else: + data = dict(map(lambda x: (x, self.request.get(x)), ["name","descr","web_url"])) + if reduce(lambda x, y: x or not y, data.values(), False): + data["submitMsg"] = "You must complete all the fields." + self.list(data) + return + p = Project.get_by_key_name("proj-"+data["name"]) + if p is not None: + data["submitMsg"] = "A project by this name already exists." + self.list(data) + return + p = Project(key_name="proj-"+data["name"], **data) + p.put() + + path = os.path.join(os.path.dirname(__file__), 'project-notify.txt') + mail.send_mail( + sender=const.mail_from, + to=const.mail_submit_to, + subject=const.mail_submit_subject, + body=template.render(path, {'project': p})) + + self.list({"submitMsg": "Your project has been submitted."}) + + def list(self, additional_data={}): + cache_key = 'view-project-data' + tag = self.request.get('tag', None) + if tag: + cache_key += '-'+tag + data = memcache.get(cache_key) + admin = users.is_current_user_admin() + if admin or not data: + projects = Project.all().order('category').order('name') + if not admin: + projects = projects.filter('approved =', True) + projects = list(projects) + + tags = sets.Set() + for p in projects: + for t in p.tags: + tags.add(t) + + if tag: + projects = filter(lambda x: tag in x.tags, projects) + + data = {} + data['tag'] = tag + data['tags'] = tags + data['projects'] = projects + data['admin']= admin + if not admin: + memcache.set(cache_key, data, time=CacheTimeout) + + for k, v in additional_data.items(): + data[k] = v + + self.response.headers['Content-Type'] = 'text/html; charset=utf-8' + path = os.path.join(os.path.dirname(__file__), 'project.html') + self.response.out.write(template.render(path, data)) + + def edit(self, save=False): + if save: + name = self.request.get("orig_name") + else: + name = self.request.get("name") + + p = Project.get_by_key_name("proj-"+name) + if not p: + self.response.out.write("Couldn't find that Project.") + return + + if save: + if self.request.get("do") == "Delete": + p.delete() + else: + pkg_name = self.request.get("package", None) + if pkg_name: + pkg = Package.get_by_key_name("pkg-"+pkg_name) + if pkg: + p.package = pkg.key() + for f in ['name', 'descr', 'web_url', 'category']: + setattr(p, f, self.request.get(f, None)) + p.approved = self.request.get("approved") == "1" + p.tags = filter(lambda x: x, self.request.get("tags", "").split(",")) + p.put() + memcache.delete('view-project-data') + self.redirect('/project') + return + + # get all project categories and tags + cats, tags = sets.Set(), sets.Set() + for r in Project.all(): + cats.add(r.category) + for t in r.tags: + tags.add(t) + + self.response.headers['Content-Type'] = 'text/html; charset=utf-8' + path = os.path.join(os.path.dirname(__file__), 'project-edit.html') + self.response.out.write(template.render(path, { + "taglist": tags, "catlist": cats, "p": p, "tags": ",".join(p.tags) })) + + def redirect(self, url): + self.response.set_status(302) + self.response.headers.add_header("Location", url) + +def main(): + app = webapp.WSGIApplication([ + ('/package', PackagePage), + ('/package/daily', PackageDaily), + ('/project.*', ProjectPage), + ], debug=True) + run_wsgi_app(app) + +if __name__ == '__main__': + main() diff --git a/misc/dashboard/godashboard/project-edit.html b/misc/dashboard/godashboard/project-edit.html new file mode 100644 index 000000000..ce18fb3fb --- /dev/null +++ b/misc/dashboard/godashboard/project-edit.html @@ -0,0 +1,47 @@ +<html> +<head> +<link href="http://ajax.googleapis.com/ajax/libs/jqueryui/1.8/themes/base/jquery-ui.css" rel="stylesheet" type="text/css"/> +<script type="text/javascript" src="http://www.google.com/jsapi"></script> +<script> +google.load("jquery", "1"); +google.load("jqueryui", "1.8.2"); +</script> +</head> +<body> +<form action="/project/edit?orig_name={{p.name}}" method="POST"> +Name:<br/> +<input type="text" name="name" value="{{p.name|escape}}"><br/> +Description:<br/> +<input type="text" name="descr" value="{{p.descr|escape}}"><br/> +Category:<br/> +<input type="text" id="cats" name="category" value="{{p.category|escape}}"><br/> +Tags: (comma-separated)<br/> +<input type="text" id="tags" name="tags" value="{{tags}}"><br/> +Web URL:<br/> +<input type="text" name="web_url" value="{{p.web_url|escape}}"><br/> +Package URL: (to link to a goinstall'd package)<br/> +<input type="text" name="package" value="{{p.package.path|escape}}"><br/> +Approved: <input type="checkbox" name="approved" value="1" {% if p.approved %}checked{% endif %}><br/> +<br/> +<input type="submit" name="do" value="Save"> +<input type="submit" name="do" value="Delete" onClick="javascript:return confirm('Delete this?');"> +</form> +<script> +var tags = [ +{% for t in taglist %} + "{{t}}"{% if not forloop.last %},{% endif %} +{% endfor %} +]; +var cats = [ +{% for c in catlist %} + "{{c}}"{% if not forloop.last %},{% endif %} +{% endfor %} +]; + +google.setOnLoadCallback(function() { + $('#tags').autocomplete({source:tags}); + $('#cats').autocomplete({source:cats}); +}); +</script> +</body> +</html> diff --git a/misc/dashboard/godashboard/project-notify.txt b/misc/dashboard/godashboard/project-notify.txt new file mode 100644 index 000000000..f55bf6421 --- /dev/null +++ b/misc/dashboard/godashboard/project-notify.txt @@ -0,0 +1,9 @@ +A new project has been submitted: + +Name: {{project.name}} +Description: {{project.descr}} +URL: {{project.web_url}} + +To edit/approve/delete: +http://godashboard.appspot.com/project/edit?name={{project.name|toutf8|urlencode}} + diff --git a/misc/dashboard/godashboard/project.html b/misc/dashboard/godashboard/project.html new file mode 100644 index 000000000..4fe1741c6 --- /dev/null +++ b/misc/dashboard/godashboard/project.html @@ -0,0 +1,85 @@ +<!DOCTYPE HTML> +<html> + <head> + <title>Projects - Go Dashboard</title> + <link rel="stylesheet" type="text/css" href="static/style.css"> + <style> + .unapproved a.name { color: red } + .tag { font-size: 0.8em; color: #666 } + </style> + </head> + + <body> + <ul class="menu"> + <li><a href="/">Build Status</a></li> + <li><a href="/package">Packages</a></li> + <li>Projects</li> + <li><a href="http://golang.org/">golang.org</a></li> + </ul> + + <h1>Go Dashboard</h1> + + <p> + These are external projects and not endorsed or supported by the Go project. + </p> + + <h2>Projects</h2> + + <div class="submit"> + <h3>Submit a Project</h3> + <p> + Using this form you can submit a project to be included in the list. + </p> + <form action="/project" method="POST"> + <table> + <tr><td>Name:<td><input type="text" name="name"> + <tr><td>Description:<td><input type="text" name="descr"> + <tr><td>URL:<td><input type="text" name="web_url"> + <tr><td> <td><input type="submit" value="Send"> + {% if submitMsg %} + <tr><td class="msg" colspan="2">{{ submitMsg }}</td></tr> + {% endif %} + </table> + </form> + </div> + + <p> + Filter by tag: + {% if tag %} + <a href="/project">all</a> + {% else %} + <b>all</b> + {% endif %} + {% for t in tags %} + {% ifequal t tag %} + <b>{{t}}</b> + {% else %} + <a href="?tag={{t}}">{{t}}</a> + {% endifequal %} + {% endfor %} + </p> + + {% for r in projects %} + {% ifchanged r.category %} + {% if not forloop.first %} + </ul> + {% endif %} + <h3>{{r.category}}</h3> + <ul> + {% endifchanged %} + <li{% if not r.approved %} class="unapproved"{% endif %}> + {% if admin %}[<a href="/project/edit?name={{r.name}}">edit</a>]{% endif %} + <a class="name" href="{{r.web_url}}">{{r.name}}</a> - {{r.descr}} + {% for tag in r.tags %} + <span class="tag">{{tag}}</span> + {% endfor %} + </li> + {% if forloop.last %} + </ul> + {% endif %} + {% endfor %} + </ul> + + + </body> +</html> diff --git a/misc/dashboard/godashboard/static/favicon.ico b/misc/dashboard/godashboard/static/favicon.ico Binary files differnew file mode 100644 index 000000000..48854ff3b --- /dev/null +++ b/misc/dashboard/godashboard/static/favicon.ico diff --git a/misc/dashboard/godashboard/static/style.css b/misc/dashboard/godashboard/static/style.css new file mode 100644 index 000000000..d6d23b536 --- /dev/null +++ b/misc/dashboard/godashboard/static/style.css @@ -0,0 +1,118 @@ +body { + font-family: sans-serif; + margin: 0; + padding: 0; +} +h1, h2, h3, ul.menu, table, p { + padding: 0 0.5em; +} +h1, h2 { + margin: 0; + background: #eee; +} +h1 { + border-bottom: 1px solid #ccc; + font-size: 1em; + padding: 0.5em; + margin-bottom: 0.5em; + text-align: right; +} +h2 { + border-top: 1px solid #ccc; + padding-left: 0.2em; +} +.submit { + float: right; + border: 1px solid #ccc; + width: 350px; + padding-bottom: 1em; + margin: 0.5em; + background: #eee; +} +.submit table { + width: 100%; +} +.submit input[type=text] { + width: 200px; +} +.submit .msg { + text-align: center; + color: red; +} +table.alternate { + white-space: nowrap; + margin: 0.5em 0; +} +table.alternate td, +table.alternate th { + padding: 0.1em 0.25em; + font-size: small; +} +table.alternate tr td:last-child { + padding-right: 0; +} +table.alternate tr:nth-child(2n) { + background-color: #f0f0f0; +} +td.result { + text-align: center; +} +span.hash { + font-family: monospace; + font-size: small; + color: #aaa; +} +td.date { + color: #aaa; +} +td.ok { + text-align: center; + color: #060; + font-weight: bold; +} +td.ok a { + cursor: help; +} +th { + text-align: left; +} +th.builder { + text-align: center; + font-weight: bold; +} +a.fail { + color: #F00; +} +a.fail:visited { + color: #900; +} +ul.menu { + margin: 0; + padding: 0; + list-style-type: none; +} +ul.menu li { + float: left; + display: block; + font-size: 1em; + padding: 0.5em; + background: #EEF; + margin-left: 0.5em; + border-left: 1px solid #999; + border-right: 1px solid #999; +} +div.paginate { + padding: 0.5em; +} +div.paginate a { + padding: 0.5em; + margin-right: 0.5em; + background: #eee; + color: blue; +} +div.paginate a.inactive { + color: #999; +} +td.time { + font-family: monospace; +} diff --git a/misc/dashboard/godashboard/toutf8.py b/misc/dashboard/godashboard/toutf8.py new file mode 100644 index 000000000..544c681b6 --- /dev/null +++ b/misc/dashboard/godashboard/toutf8.py @@ -0,0 +1,14 @@ +# Copyright 2010 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +# This is a Django custom template filter to work around the +# fact that GAE's urlencode filter doesn't handle unicode strings. + +from google.appengine.ext import webapp + +register = webapp.template.create_template_register() + +@register.filter +def toutf8(value): + return value.encode("utf-8") diff --git a/misc/dashboard/googlecode_upload.py b/misc/dashboard/googlecode_upload.py new file mode 100755 index 000000000..e87db884a --- /dev/null +++ b/misc/dashboard/googlecode_upload.py @@ -0,0 +1,248 @@ +#!/usr/bin/env python2 +# +# Copyright 2006, 2007 Google Inc. All Rights Reserved. +# Author: danderson@google.com (David Anderson) +# +# Script for uploading files to a Google Code project. +# +# This is intended to be both a useful script for people who want to +# streamline project uploads and a reference implementation for +# uploading files to Google Code projects. +# +# To upload a file to Google Code, you need to provide a path to the +# file on your local machine, a small summary of what the file is, a +# project name, and a valid account that is a member or owner of that +# project. You can optionally provide a list of labels that apply to +# the file. The file will be uploaded under the same name that it has +# in your local filesystem (that is, the "basename" or last path +# component). Run the script with '--help' to get the exact syntax +# and available options. +# +# Note that the upload script requests that you enter your +# googlecode.com password. This is NOT your Gmail account password! +# This is the password you use on googlecode.com for committing to +# Subversion and uploading files. You can find your password by going +# to http://code.google.com/hosting/settings when logged in with your +# Gmail account. If you have already committed to your project's +# Subversion repository, the script will automatically retrieve your +# credentials from there (unless disabled, see the output of '--help' +# for details). +# +# If you are looking at this script as a reference for implementing +# your own Google Code file uploader, then you should take a look at +# the upload() function, which is the meat of the uploader. You +# basically need to build a multipart/form-data POST request with the +# right fields and send it to https://PROJECT.googlecode.com/files . +# Authenticate the request using HTTP Basic authentication, as is +# shown below. +# +# Licensed under the terms of the Apache Software License 2.0: +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Questions, comments, feature requests and patches are most welcome. +# Please direct all of these to the Google Code users group: +# http://groups.google.com/group/google-code-hosting + +"""Google Code file uploader script. +""" + +__author__ = 'danderson@google.com (David Anderson)' + +import httplib +import os.path +import optparse +import getpass +import base64 +import sys + + +def upload(file, project_name, user_name, password, summary, labels=None): + """Upload a file to a Google Code project's file server. + + Args: + file: The local path to the file. + project_name: The name of your project on Google Code. + user_name: Your Google account name. + password: The googlecode.com password for your account. + Note that this is NOT your global Google Account password! + summary: A small description for the file. + labels: an optional list of label strings with which to tag the file. + + Returns: a tuple: + http_status: 201 if the upload succeeded, something else if an + error occurred. + http_reason: The human-readable string associated with http_status + file_url: If the upload succeeded, the URL of the file on Google + Code, None otherwise. + """ + # The login is the user part of user@gmail.com. If the login provided + # is in the full user@domain form, strip it down. + if user_name.endswith('@gmail.com'): + user_name = user_name[:user_name.index('@gmail.com')] + + form_fields = [('summary', summary)] + if labels is not None: + form_fields.extend([('label', l.strip()) for l in labels]) + + content_type, body = encode_upload_request(form_fields, file) + + upload_host = '%s.googlecode.com' % project_name + upload_uri = '/files' + auth_token = base64.b64encode('%s:%s'% (user_name, password)) + headers = { + 'Authorization': 'Basic %s' % auth_token, + 'User-Agent': 'Googlecode.com uploader v0.9.4', + 'Content-Type': content_type, + } + + server = httplib.HTTPSConnection(upload_host) + server.request('POST', upload_uri, body, headers) + resp = server.getresponse() + server.close() + + if resp.status == 201: + location = resp.getheader('Location', None) + else: + location = None + return resp.status, resp.reason, location + + +def encode_upload_request(fields, file_path): + """Encode the given fields and file into a multipart form body. + + fields is a sequence of (name, value) pairs. file is the path of + the file to upload. The file will be uploaded to Google Code with + the same file name. + + Returns: (content_type, body) ready for httplib.HTTP instance + """ + BOUNDARY = '----------Googlecode_boundary_reindeer_flotilla' + CRLF = '\r\n' + + body = [] + + # Add the metadata about the upload first + for key, value in fields: + body.extend( + ['--' + BOUNDARY, + 'Content-Disposition: form-data; name="%s"' % key, + '', + value, + ]) + + # Now add the file itself + file_name = os.path.basename(file_path) + f = open(file_path, 'rb') + file_content = f.read() + f.close() + + body.extend( + ['--' + BOUNDARY, + 'Content-Disposition: form-data; name="filename"; filename="%s"' + % file_name, + # The upload server determines the mime-type, no need to set it. + 'Content-Type: application/octet-stream', + '', + file_content, + ]) + + # Finalize the form body + body.extend(['--' + BOUNDARY + '--', '']) + + return 'multipart/form-data; boundary=%s' % BOUNDARY, CRLF.join(body) + + +def upload_find_auth(file_path, project_name, summary, labels=None, + user_name=None, password=None, tries=3): + """Find credentials and upload a file to a Google Code project's file server. + + file_path, project_name, summary, and labels are passed as-is to upload. + + Args: + file_path: The local path to the file. + project_name: The name of your project on Google Code. + summary: A small description for the file. + labels: an optional list of label strings with which to tag the file. + config_dir: Path to Subversion configuration directory, 'none', or None. + user_name: Your Google account name. + tries: How many attempts to make. + """ + + while tries > 0: + if user_name is None: + # Read username if not specified or loaded from svn config, or on + # subsequent tries. + sys.stdout.write('Please enter your googlecode.com username: ') + sys.stdout.flush() + user_name = sys.stdin.readline().rstrip() + if password is None: + # Read password if not loaded from svn config, or on subsequent tries. + print 'Please enter your googlecode.com password.' + print '** Note that this is NOT your Gmail account password! **' + print 'It is the password you use to access Subversion repositories,' + print 'and can be found here: http://code.google.com/hosting/settings' + password = getpass.getpass() + + status, reason, url = upload(file_path, project_name, user_name, password, + summary, labels) + # Returns 403 Forbidden instead of 401 Unauthorized for bad + # credentials as of 2007-07-17. + if status in [httplib.FORBIDDEN, httplib.UNAUTHORIZED]: + # Rest for another try. + user_name = password = None + tries = tries - 1 + else: + # We're done. + break + + return status, reason, url + + +def main(): + parser = optparse.OptionParser(usage='googlecode-upload.py -s SUMMARY ' + '-p PROJECT [options] FILE') + parser.add_option('-s', '--summary', dest='summary', + help='Short description of the file') + parser.add_option('-p', '--project', dest='project', + help='Google Code project name') + parser.add_option('-u', '--user', dest='user', + help='Your Google Code username') + parser.add_option('-w', '--password', dest='password', + help='Your Google Code password') + parser.add_option('-l', '--labels', dest='labels', + help='An optional list of comma-separated labels to attach ' + 'to the file') + + options, args = parser.parse_args() + + if not options.summary: + parser.error('File summary is missing.') + elif not options.project: + parser.error('Project name is missing.') + elif len(args) < 1: + parser.error('File to upload not provided.') + elif len(args) > 1: + parser.error('Only one file may be specified.') + + file_path = args[0] + + if options.labels: + labels = options.labels.split(',') + else: + labels = None + + status, reason, url = upload_find_auth(file_path, options.project, + options.summary, labels, + options.user, options.password) + if url: + print 'The file was uploaded successfully.' + print 'URL: %s' % url + return 0 + else: + print 'An error occurred. Your file was not uploaded.' + print 'Google Code upload server said: %s (%s)' % (reason, status) + return 1 + + +if __name__ == '__main__': + sys.exit(main()) |