diff options
| author | Ondřej Surý <ondrej@sury.org> | 2011-09-13 13:11:55 +0200 |
|---|---|---|
| committer | Ondřej Surý <ondrej@sury.org> | 2011-09-13 13:11:55 +0200 |
| commit | 80f18fc933cf3f3e829c5455a1023d69f7b86e52 (patch) | |
| tree | 4b825dc642cb6eb9a060e54bf8d69288fbee4904 /misc/dashboard | |
| parent | 28592ee1ea1f5cdffcf85472f9de0285d928cf12 (diff) | |
| download | golang-80f18fc933cf3f3e829c5455a1023d69f7b86e52.tar.gz | |
Imported Upstream version 60
Diffstat (limited to 'misc/dashboard')
25 files changed, 0 insertions, 2712 deletions
diff --git a/misc/dashboard/README b/misc/dashboard/README deleted file mode 100644 index c00311ef7..000000000 --- a/misc/dashboard/README +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -The files in this directory constitute the continuous builder: - -godashboard/: an AppEngine server -builder/: gobuilder, a Go continuous build client - -If you wish to run a Go builder, please email golang-dev@googlegroups.com - -To run a builder: - -* Write the key ~gobuild/.gobuildkey - You need to get it from someone who knows the key. - You may also use a filename of the form .gobuildkey-$BUILDER if you - wish to run builders for multiple targets. - -* Append your username and password googlecode.com credentials from - https://code.google.com/hosting/settings - to the buildkey file in the format "Username\nPassword\n". - (This is for uploading tarballs to the project downloads section, - and is an optional step.) - -* Build and run gobuilder (see its documentation for command-line options). - diff --git a/misc/dashboard/builder/Makefile b/misc/dashboard/builder/Makefile deleted file mode 100644 index f1d9c5497..000000000 --- a/misc/dashboard/builder/Makefile +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright 2009 The Go Authors. All rights reserved. -# Use of this source code is governed by a BSD-style -# license that can be found in the LICENSE file. - -include ../../../src/Make.inc - -TARG=gobuilder -GOFILES=\ - exec.go\ - http.go\ - main.go\ - package.go\ - -include ../../../src/Make.cmd diff --git a/misc/dashboard/builder/doc.go b/misc/dashboard/builder/doc.go deleted file mode 100644 index 30d8fe948..000000000 --- a/misc/dashboard/builder/doc.go +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -/* - -Go Builder is a continuous build client for the Go project. -It integrates with the Go Dashboard AppEngine application. - -Go Builder is intended to run continuously as a background process. - -It periodically pulls updates from the Go Mercurial repository. - -When a newer revision is found, Go Builder creates a clone of the repository, -runs all.bash, and reports build success or failure to the Go Dashboard. - -For a release revision (a change description that matches "release.YYYY-MM-DD"), -Go Builder will create a tar.gz archive of the GOROOT and deliver it to the -Go Google Code project's downloads section. - -Usage: - - gobuilder goos-goarch... - - Several goos-goarch combinations can be provided, and the builder will - build them in serial. - -Optional flags: - - -dashboard="godashboard.appspot.com": Go Dashboard Host - The location of the Go Dashboard application to which Go Builder will - report its results. - - -release: Build and deliver binary release archive - - -rev=N: Build revision N and exit - - -cmd="./all.bash": Build command (specify absolute or relative to go/src) - - -v: Verbose logging - - -external: External package builder mode (will not report Go build - state to dashboard or issue releases) - -The key file should be located at $HOME/.gobuildkey or, for a builder-specific -key, $HOME/.gobuildkey-$BUILDER (eg, $HOME/.gobuildkey-linux-amd64). - -The build key file is a text file of the format: - - godashboard-key - googlecode-username - googlecode-password - -If the Google Code credentials are not provided the archival step -will be skipped. - -*/ -package documentation diff --git a/misc/dashboard/builder/exec.go b/misc/dashboard/builder/exec.go deleted file mode 100644 index a042c5699..000000000 --- a/misc/dashboard/builder/exec.go +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package main - -import ( - "bytes" - "exec" - "io" - "log" - "os" - "strings" -) - -// run is a simple wrapper for exec.Run/Close -func run(envv []string, dir string, argv ...string) os.Error { - if *verbose { - log.Println("run", argv) - } - argv = useBash(argv) - cmd := exec.Command(argv[0], argv[1:]...) - cmd.Dir = dir - cmd.Env = envv - cmd.Stderr = os.Stderr - return cmd.Run() -} - -// runLog runs a process and returns the combined stdout/stderr, -// as well as writing it to logfile (if specified). It returns -// process combined stdout and stderr output, exit status and error. -// The error returned is nil, if process is started successfully, -// even if exit status is not 0. -func runLog(envv []string, logfile, dir string, argv ...string) (string, int, os.Error) { - if *verbose { - log.Println("runLog", argv) - } - argv = useBash(argv) - - b := new(bytes.Buffer) - var w io.Writer = b - if logfile != "" { - f, err := os.OpenFile(logfile, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666) - if err != nil { - return "", 0, err - } - defer f.Close() - w = io.MultiWriter(f, b) - } - - cmd := exec.Command(argv[0], argv[1:]...) - cmd.Dir = dir - cmd.Env = envv - cmd.Stdout = w - cmd.Stderr = w - - err := cmd.Run() - if err != nil { - if ws, ok := err.(*os.Waitmsg); ok { - return b.String(), ws.ExitStatus(), nil - } - } - return b.String(), 0, nil -} - -// useBash prefixes a list of args with 'bash' if the first argument -// is a bash script. -func useBash(argv []string) []string { - // TODO(brainman): choose a more reliable heuristic here. - if strings.HasSuffix(argv[0], ".bash") { - argv = append([]string{"bash"}, argv...) - } - return argv -} diff --git a/misc/dashboard/builder/http.go b/misc/dashboard/builder/http.go deleted file mode 100644 index 98400c51a..000000000 --- a/misc/dashboard/builder/http.go +++ /dev/null @@ -1,147 +0,0 @@ -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package main - -import ( - "bytes" - "fmt" - "http" - "json" - "log" - "os" - "strconv" -) - -type param map[string]string - -// dash runs the given method and command on the dashboard. -// If args is not nil, it is the query or post parameters. -// If resp is not nil, dash unmarshals the body as JSON into resp. -func dash(meth, cmd string, resp interface{}, args param) os.Error { - var r *http.Response - var err os.Error - if *verbose { - log.Println("dash", cmd, args) - } - cmd = "http://" + *dashboard + "/" + cmd - vals := make(http.Values) - for k, v := range args { - vals.Add(k, v) - } - switch meth { - case "GET": - if q := vals.Encode(); q != "" { - cmd += "?" + q - } - r, err = http.Get(cmd) - case "POST": - r, err = http.PostForm(cmd, vals) - default: - return fmt.Errorf("unknown method %q", meth) - } - if err != nil { - return err - } - defer r.Body.Close() - var buf bytes.Buffer - buf.ReadFrom(r.Body) - if resp != nil { - if err = json.Unmarshal(buf.Bytes(), resp); err != nil { - log.Printf("json unmarshal %#q: %s\n", buf.Bytes(), err) - return err - } - } - return nil -} - -func dashStatus(meth, cmd string, args param) os.Error { - var resp struct { - Status string - Error string - } - err := dash(meth, cmd, &resp, args) - if err != nil { - return err - } - if resp.Status != "OK" { - return os.NewError("/build: " + resp.Error) - } - return nil -} - -// todo returns the next hash to build. -func (b *Builder) todo() (rev string, err os.Error) { - var resp []struct { - Hash string - } - if err = dash("GET", "todo", &resp, param{"builder": b.name}); err != nil { - return - } - if len(resp) > 0 { - rev = resp[0].Hash - } - return -} - -// recordResult sends build results to the dashboard -func (b *Builder) recordResult(buildLog string, hash string) os.Error { - return dash("POST", "build", nil, param{ - "builder": b.name, - "key": b.key, - "node": hash, - "log": buildLog, - }) -} - -// packages fetches a list of package paths from the dashboard -func packages() (pkgs []string, err os.Error) { - var resp struct { - Packages []struct { - Path string - } - } - err = dash("GET", "package", &resp, param{"fmt": "json"}) - if err != nil { - return - } - for _, p := range resp.Packages { - pkgs = append(pkgs, p.Path) - } - return -} - -// updatePackage sends package build results and info dashboard -func (b *Builder) updatePackage(pkg string, ok bool, buildLog, info string) os.Error { - return dash("POST", "package", nil, param{ - "builder": b.name, - "key": b.key, - "path": pkg, - "ok": strconv.Btoa(ok), - "log": buildLog, - "info": info, - }) -} - -// postCommit informs the dashboard of a new commit -func postCommit(key string, l *HgLog) os.Error { - return dashStatus("POST", "commit", param{ - "key": key, - "node": l.Hash, - "date": l.Date, - "user": l.Author, - "parent": l.Parent, - "desc": l.Desc, - }) -} - -// dashboardCommit returns true if the dashboard knows about hash. -func dashboardCommit(hash string) bool { - err := dashStatus("GET", "commit", param{"node": hash}) - if err != nil { - log.Printf("check %s: %s", hash, err) - return false - } - return true -} diff --git a/misc/dashboard/builder/main.go b/misc/dashboard/builder/main.go deleted file mode 100644 index 989965bc4..000000000 --- a/misc/dashboard/builder/main.go +++ /dev/null @@ -1,624 +0,0 @@ -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package main - -import ( - "flag" - "fmt" - "io/ioutil" - "log" - "os" - "path" - "regexp" - "runtime" - "strconv" - "strings" - "time" - "xml" -) - -const ( - codeProject = "go" - codePyScript = "misc/dashboard/googlecode_upload.py" - hgUrl = "https://go.googlecode.com/hg/" - waitInterval = 30e9 // time to wait before checking for new revs - mkdirPerm = 0750 - pkgBuildInterval = 1e9 * 60 * 60 * 24 // rebuild packages every 24 hours -) - -// These variables are copied from the gobuilder's environment -// to the envv of its subprocesses. -var extraEnv = []string{ - "GOHOSTOS", - "GOHOSTARCH", - "PATH", - "DISABLE_NET_TESTS", - "MAKEFLAGS", - "GOARM", -} - -type Builder struct { - name string - goos, goarch string - key string - codeUsername string - codePassword string -} - -var ( - buildroot = flag.String("buildroot", path.Join(os.TempDir(), "gobuilder"), "Directory under which to build") - commitFlag = flag.Bool("commit", false, "upload information about new commits") - dashboard = flag.String("dashboard", "godashboard.appspot.com", "Go Dashboard Host") - buildRelease = flag.Bool("release", false, "Build and upload binary release archives") - buildRevision = flag.String("rev", "", "Build specified revision and exit") - buildCmd = flag.String("cmd", "./all.bash", "Build command (specify absolute or relative to go/src/)") - external = flag.Bool("external", false, "Build external packages") - parallel = flag.Bool("parallel", false, "Build multiple targets in parallel") - verbose = flag.Bool("v", false, "verbose") -) - -var ( - goroot string - binaryTagRe = regexp.MustCompile(`^(release\.r|weekly\.)[0-9\-.]+`) - releaseRe = regexp.MustCompile(`^release\.r[0-9\-.]+`) -) - -func main() { - flag.Usage = func() { - fmt.Fprintf(os.Stderr, "usage: %s goos-goarch...\n", os.Args[0]) - flag.PrintDefaults() - os.Exit(2) - } - flag.Parse() - if len(flag.Args()) == 0 && !*commitFlag { - flag.Usage() - } - goroot = path.Join(*buildroot, "goroot") - builders := make([]*Builder, len(flag.Args())) - for i, builder := range flag.Args() { - b, err := NewBuilder(builder) - if err != nil { - log.Fatal(err) - } - builders[i] = b - } - - // set up work environment - if err := os.RemoveAll(*buildroot); err != nil { - log.Fatalf("Error removing build root (%s): %s", *buildroot, err) - } - if err := os.Mkdir(*buildroot, mkdirPerm); err != nil { - log.Fatalf("Error making build root (%s): %s", *buildroot, err) - } - if err := run(nil, *buildroot, "hg", "clone", hgUrl, goroot); err != nil { - log.Fatal("Error cloning repository:", err) - } - - if *commitFlag { - if len(flag.Args()) == 0 { - commitWatcher() - return - } - go commitWatcher() - } - - // if specified, build revision and return - if *buildRevision != "" { - hash, err := fullHash(*buildRevision) - if err != nil { - log.Fatal("Error finding revision: ", err) - } - for _, b := range builders { - if err := b.buildHash(hash); err != nil { - log.Println(err) - } - } - return - } - - // external package build mode - if *external { - if len(builders) != 1 { - log.Fatal("only one goos-goarch should be specified with -external") - } - builders[0].buildExternal() - } - - // go continuous build mode (default) - // check for new commits and build them - for { - built := false - t := time.Nanoseconds() - if *parallel { - done := make(chan bool) - for _, b := range builders { - go func(b *Builder) { - done <- b.build() - }(b) - } - for _ = range builders { - built = <-done || built - } - } else { - for _, b := range builders { - built = b.build() || built - } - } - // sleep if there was nothing to build - if !built { - time.Sleep(waitInterval) - } - // sleep if we're looping too fast. - t1 := time.Nanoseconds() - t - if t1 < waitInterval { - time.Sleep(waitInterval - t1) - } - } -} - -func NewBuilder(builder string) (*Builder, os.Error) { - b := &Builder{name: builder} - - // get goos/goarch from builder string - s := strings.SplitN(builder, "-", 3) - if len(s) >= 2 { - b.goos, b.goarch = s[0], s[1] - } else { - return nil, fmt.Errorf("unsupported builder form: %s", builder) - } - - // read keys from keyfile - fn := path.Join(os.Getenv("HOME"), ".gobuildkey") - if s := fn + "-" + b.name; isFile(s) { // builder-specific file - fn = s - } - c, err := ioutil.ReadFile(fn) - if err != nil { - return nil, fmt.Errorf("readKeys %s (%s): %s", b.name, fn, err) - } - v := strings.Split(string(c), "\n") - b.key = v[0] - if len(v) >= 3 { - b.codeUsername, b.codePassword = v[1], v[2] - } - - return b, nil -} - -// buildExternal downloads and builds external packages, and -// reports their build status to the dashboard. -// It will re-build all packages after pkgBuildInterval nanoseconds or -// a new release tag is found. -func (b *Builder) buildExternal() { - var prevTag string - var nextBuild int64 - for { - time.Sleep(waitInterval) - err := run(nil, goroot, "hg", "pull", "-u") - if err != nil { - log.Println("hg pull failed:", err) - continue - } - hash, tag, err := firstTag(releaseRe) - if err != nil { - log.Println(err) - continue - } - if *verbose { - log.Println("latest release:", tag) - } - // don't rebuild if there's no new release - // and it's been less than pkgBuildInterval - // nanoseconds since the last build. - if tag == prevTag && time.Nanoseconds() < nextBuild { - continue - } - // build will also build the packages - if err := b.buildHash(hash); err != nil { - log.Println(err) - continue - } - prevTag = tag - nextBuild = time.Nanoseconds() + pkgBuildInterval - } -} - -// build checks for a new commit for this builder -// and builds it if one is found. -// It returns true if a build was attempted. -func (b *Builder) build() bool { - defer func() { - err := recover() - if err != nil { - log.Println(b.name, "build:", err) - } - }() - hash, err := b.todo() - if err != nil { - log.Println(err) - return false - } - if hash == "" { - return false - } - // Look for hash locally before running hg pull. - - if _, err := fullHash(hash[:12]); err != nil { - // Don't have hash, so run hg pull. - if err := run(nil, goroot, "hg", "pull"); err != nil { - log.Println("hg pull failed:", err) - return false - } - } - err = b.buildHash(hash) - if err != nil { - log.Println(err) - } - return true -} - -func (b *Builder) buildHash(hash string) (err os.Error) { - defer func() { - if err != nil { - err = fmt.Errorf("%s build: %s: %s", b.name, hash, err) - } - }() - - log.Println(b.name, "building", hash) - - // create place in which to do work - workpath := path.Join(*buildroot, b.name+"-"+hash[:12]) - err = os.Mkdir(workpath, mkdirPerm) - if err != nil { - return - } - defer os.RemoveAll(workpath) - - // clone repo - err = run(nil, workpath, "hg", "clone", goroot, "go") - if err != nil { - return - } - - // update to specified revision - err = run(nil, path.Join(workpath, "go"), - "hg", "update", hash) - if err != nil { - return - } - - srcDir := path.Join(workpath, "go", "src") - - // build - logfile := path.Join(workpath, "build.log") - buildLog, status, err := runLog(b.envv(), logfile, srcDir, *buildCmd) - if err != nil { - return fmt.Errorf("%s: %s", *buildCmd, err) - } - - // if we're in external mode, build all packages and return - if *external { - if status != 0 { - return os.NewError("go build failed") - } - return b.buildPackages(workpath, hash) - } - - if status != 0 { - // record failure - return b.recordResult(buildLog, hash) - } - - // record success - if err = b.recordResult("", hash); err != nil { - return fmt.Errorf("recordResult: %s", err) - } - - // finish here if codeUsername and codePassword aren't set - if b.codeUsername == "" || b.codePassword == "" || !*buildRelease { - return - } - - // if this is a release, create tgz and upload to google code - releaseHash, release, err := firstTag(binaryTagRe) - if hash == releaseHash { - // clean out build state - err = run(b.envv(), srcDir, "./clean.bash", "--nopkg") - if err != nil { - return fmt.Errorf("clean.bash: %s", err) - } - // upload binary release - fn := fmt.Sprintf("go.%s.%s-%s.tar.gz", release, b.goos, b.goarch) - err = run(nil, workpath, "tar", "czf", fn, "go") - if err != nil { - return fmt.Errorf("tar: %s", err) - } - err = run(nil, workpath, path.Join(goroot, codePyScript), - "-s", release, - "-p", codeProject, - "-u", b.codeUsername, - "-w", b.codePassword, - "-l", fmt.Sprintf("%s,%s", b.goos, b.goarch), - fn) - } - - return -} - -// envv returns an environment for build/bench execution -func (b *Builder) envv() []string { - if runtime.GOOS == "windows" { - return b.envvWindows() - } - e := []string{ - "GOOS=" + b.goos, - "GOARCH=" + b.goarch, - "GOROOT_FINAL=/usr/local/go", - } - for _, k := range extraEnv { - s, err := os.Getenverror(k) - if err == nil { - e = append(e, k+"="+s) - } - } - return e -} - -// windows version of envv -func (b *Builder) envvWindows() []string { - start := map[string]string{ - "GOOS": b.goos, - "GOARCH": b.goarch, - "GOROOT_FINAL": "/c/go", - // TODO(brainman): remove once we find make that does not hang. - "MAKEFLAGS": "-j1", - } - for _, name := range extraEnv { - s, err := os.Getenverror(name) - if err == nil { - start[name] = s - } - } - skip := map[string]bool{ - "GOBIN": true, - "GOROOT": true, - "INCLUDE": true, - "LIB": true, - } - var e []string - for name, v := range start { - e = append(e, name+"="+v) - skip[name] = true - } - for _, kv := range os.Environ() { - s := strings.SplitN(kv, "=", 2) - name := strings.ToUpper(s[0]) - switch { - case name == "": - // variables, like "=C:=C:\", just copy them - e = append(e, kv) - case !skip[name]: - e = append(e, kv) - skip[name] = true - } - } - return e -} - -func isDirectory(name string) bool { - s, err := os.Stat(name) - return err == nil && s.IsDirectory() -} - -func isFile(name string) bool { - s, err := os.Stat(name) - return err == nil && (s.IsRegular() || s.IsSymlink()) -} - -// commitWatcher polls hg for new commits and tells the dashboard about them. -func commitWatcher() { - // Create builder just to get master key. - b, err := NewBuilder("mercurial-commit") - if err != nil { - log.Fatal(err) - } - for { - if *verbose { - log.Printf("poll...") - } - commitPoll(b.key) - if *verbose { - log.Printf("sleep...") - } - time.Sleep(60e9) - } -} - -// HgLog represents a single Mercurial revision. -type HgLog struct { - Hash string - Author string - Date string - Desc string - Parent string - - // Internal metadata - added bool -} - -// logByHash is a cache of all Mercurial revisions we know about, -// indexed by full hash. -var logByHash = map[string]*HgLog{} - -// xmlLogTemplate is a template to pass to Mercurial to make -// hg log print the log in valid XML for parsing with xml.Unmarshal. -const xmlLogTemplate = ` - <log> - <hash>{node|escape}</hash> - <parent>{parent|escape}</parent> - <author>{author|escape}</author> - <date>{date}</date> - <desc>{desc|escape}</desc> - </log> -` - -// commitPoll pulls any new revisions from the hg server -// and tells the server about them. -func commitPoll(key string) { - // Catch unexpected panics. - defer func() { - if err := recover(); err != nil { - log.Printf("commitPoll panic: %s", err) - } - }() - - if err := run(nil, goroot, "hg", "pull"); err != nil { - log.Printf("hg pull: %v", err) - return - } - - const N = 20 // how many revisions to grab - - data, _, err := runLog(nil, "", goroot, "hg", "log", - "--encoding=utf-8", - "--limit="+strconv.Itoa(N), - "--template="+xmlLogTemplate, - ) - if err != nil { - log.Printf("hg log: %v", err) - return - } - - var logStruct struct { - Log []HgLog - } - err = xml.Unmarshal(strings.NewReader("<top>"+data+"</top>"), &logStruct) - if err != nil { - log.Printf("unmarshal hg log: %v", err) - return - } - - logs := logStruct.Log - - // Pass 1. Fill in parents and add new log entries to logsByHash. - // Empty parent means take parent from next log entry. - // Non-empty parent has form 1234:hashhashhash; we want full hash. - for i := range logs { - l := &logs[i] - log.Printf("hg log: %s < %s\n", l.Hash, l.Parent) - if l.Parent == "" && i+1 < len(logs) { - l.Parent = logs[i+1].Hash - } else if l.Parent != "" { - l.Parent, _ = fullHash(l.Parent) - } - if l.Parent == "" { - // Can't create node without parent. - continue - } - - if logByHash[l.Hash] == nil { - // Make copy to avoid pinning entire slice when only one entry is new. - t := *l - logByHash[t.Hash] = &t - } - } - - for i := range logs { - l := &logs[i] - if l.Parent == "" { - continue - } - addCommit(l.Hash, key) - } -} - -// addCommit adds the commit with the named hash to the dashboard. -// key is the secret key for authentication to the dashboard. -// It avoids duplicate effort. -func addCommit(hash, key string) bool { - l := logByHash[hash] - if l == nil { - return false - } - if l.added { - return true - } - - // Check for already added, perhaps in an earlier run. - if dashboardCommit(hash) { - log.Printf("%s already on dashboard\n", hash) - // Record that this hash is on the dashboard, - // as must be all its parents. - for l != nil { - l.added = true - l = logByHash[l.Parent] - } - return true - } - - // Create parent first, to maintain some semblance of order. - if !addCommit(l.Parent, key) { - return false - } - - // Create commit. - if err := postCommit(key, l); err != nil { - log.Printf("failed to add %s to dashboard: %v", key, err) - return false - } - return true -} - -// fullHash returns the full hash for the given Mercurial revision. -func fullHash(rev string) (hash string, err os.Error) { - defer func() { - if err != nil { - err = fmt.Errorf("fullHash: %s: %s", rev, err) - } - }() - s, _, err := runLog(nil, "", goroot, - "hg", "log", - "--encoding=utf-8", - "--rev="+rev, - "--limit=1", - "--template={node}", - ) - if err != nil { - return - } - s = strings.TrimSpace(s) - if s == "" { - return "", fmt.Errorf("cannot find revision") - } - if len(s) != 40 { - return "", fmt.Errorf("hg returned invalid hash " + s) - } - return s, nil -} - -var revisionRe = regexp.MustCompile(`^([^ ]+) +[0-9]+:([0-9a-f]+)$`) - -// firstTag returns the hash and tag of the most recent tag matching re. -func firstTag(re *regexp.Regexp) (hash string, tag string, err os.Error) { - o, _, err := runLog(nil, "", goroot, "hg", "tags") - for _, l := range strings.Split(o, "\n") { - if l == "" { - continue - } - s := revisionRe.FindStringSubmatch(l) - if s == nil { - err = os.NewError("couldn't find revision number") - return - } - if !re.MatchString(s[1]) { - continue - } - tag = s[1] - hash, err = fullHash(s[2]) - return - } - err = os.NewError("no matching tag found") - return -} diff --git a/misc/dashboard/builder/package.go b/misc/dashboard/builder/package.go deleted file mode 100644 index b6674428d..000000000 --- a/misc/dashboard/builder/package.go +++ /dev/null @@ -1,94 +0,0 @@ -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package main - -import ( - "go/doc" - "go/parser" - "go/token" - "log" - "os" - "path/filepath" - "strings" -) - -const MaxCommentLength = 500 // App Engine won't store more in a StringProperty. - -func (b *Builder) buildPackages(workpath string, hash string) os.Error { - pkgs, err := packages() - if err != nil { - return err - } - for _, p := range pkgs { - goroot := filepath.Join(workpath, "go") - gobin := filepath.Join(goroot, "bin") - goinstall := filepath.Join(gobin, "goinstall") - envv := append(b.envv(), "GOROOT="+goroot) - - // add GOBIN to path - for i, v := range envv { - if strings.HasPrefix(v, "PATH=") { - p := filepath.SplitList(v[5:]) - p = append([]string{gobin}, p...) - s := strings.Join(p, string(filepath.ListSeparator)) - envv[i] = "PATH=" + s - } - } - - // goinstall - buildLog, code, err := runLog(envv, "", goroot, goinstall, "-log=false", p) - if err != nil { - log.Printf("goinstall %v: %v", p, err) - } - - // get doc comment from package source - info, err := packageComment(p, filepath.Join(goroot, "src", "pkg", p)) - if err != nil { - log.Printf("packageComment %v: %v", p, err) - } - - // update dashboard with build state + info - err = b.updatePackage(p, code == 0, buildLog, info) - if err != nil { - log.Printf("updatePackage %v: %v", p, err) - } - } - return nil -} - -func isGoFile(fi *os.FileInfo) bool { - return fi.IsRegular() && // exclude directories - !strings.HasPrefix(fi.Name, ".") && // ignore .files - filepath.Ext(fi.Name) == ".go" -} - -func packageComment(pkg, pkgpath string) (info string, err os.Error) { - fset := token.NewFileSet() - pkgs, err := parser.ParseDir(fset, pkgpath, isGoFile, parser.PackageClauseOnly|parser.ParseComments) - if err != nil { - return - } - for name := range pkgs { - if name == "main" { - continue - } - if info != "" { - return "", os.NewError("multiple non-main package docs") - } - pdoc := doc.NewPackageDoc(pkgs[name], pkg) - info = pdoc.Doc - } - // grab only first paragraph - if parts := strings.SplitN(info, "\n\n", 2); len(parts) > 1 { - info = parts[0] - } - // replace newlines with spaces - info = strings.Replace(info, "\n", " ", -1) - // truncate - if len(info) > MaxCommentLength { - info = info[:MaxCommentLength] - } - return -} diff --git a/misc/dashboard/godashboard/_multiprocessing.py b/misc/dashboard/godashboard/_multiprocessing.py deleted file mode 100644 index 8c66c0659..000000000 --- a/misc/dashboard/godashboard/_multiprocessing.py +++ /dev/null @@ -1,5 +0,0 @@ -# Copyright 2009 The Go Authors. All rights reserved. -# Use of this source code is governed by a BSD-style -# license that can be found in the LICENSE file. - -import multiprocessing diff --git a/misc/dashboard/godashboard/app.yaml b/misc/dashboard/godashboard/app.yaml deleted file mode 100644 index 83611cf90..000000000 --- a/misc/dashboard/godashboard/app.yaml +++ /dev/null @@ -1,21 +0,0 @@ -application: godashboard -version: 7 -runtime: python -api_version: 1 - -handlers: -- url: /favicon\.ico - static_files: static/favicon.ico - upload: static/favicon\.ico - -- url: /static - static_dir: static - -- url: /package.* - script: package.py - -- url: /project.* - script: package.py - -- url: /.* - script: gobuild.py diff --git a/misc/dashboard/godashboard/auth.py b/misc/dashboard/godashboard/auth.py deleted file mode 100644 index 73a54c0d4..000000000 --- a/misc/dashboard/godashboard/auth.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2011 The Go Authors. All rights reserved. -# Use of this source code is governed by a BSD-style -# license that can be found in the LICENSE file. - -import hmac - -# local imports -import key - -def auth(req): - k = req.get('key') - return k == hmac.new(key.accessKey, req.get('builder')).hexdigest() or k == key.accessKey - diff --git a/misc/dashboard/godashboard/const.py b/misc/dashboard/godashboard/const.py deleted file mode 100644 index b0110c635..000000000 --- a/misc/dashboard/godashboard/const.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2011 The Go Authors. All rights reserved. -# Use of this source code is governed by a BSD-style -# license that can be found in the LICENSE file. - -mail_from = "Go Dashboard <builder@golang.org>" - -mail_submit_to = "adg@golang.org" -mail_submit_subject = "New Project Submitted" - -mail_fail_to = "golang-dev@googlegroups.com" -mail_fail_reply_to = "golang-dev@googlegroups.com" -mail_fail_subject = "%s broken by %s" - diff --git a/misc/dashboard/godashboard/fail-notify.txt b/misc/dashboard/godashboard/fail-notify.txt deleted file mode 100644 index a699005ea..000000000 --- a/misc/dashboard/godashboard/fail-notify.txt +++ /dev/null @@ -1,6 +0,0 @@ -Change {{node}} broke the {{builder}} build: -http://godashboard.appspot.com/log/{{loghash}} - -{{desc}} - -http://code.google.com/p/go/source/detail?r={{node}} diff --git a/misc/dashboard/godashboard/gobuild.py b/misc/dashboard/godashboard/gobuild.py deleted file mode 100644 index 685dc83a9..000000000 --- a/misc/dashboard/godashboard/gobuild.py +++ /dev/null @@ -1,558 +0,0 @@ -# Copyright 2009 The Go Authors. All rights reserved. -# Use of this source code is governed by a BSD-style -# license that can be found in the LICENSE file. - -# This is the server part of the continuous build system for Go. It must be run -# by AppEngine. - -from django.utils import simplejson -from google.appengine.api import mail -from google.appengine.api import memcache -from google.appengine.ext import db -from google.appengine.ext import webapp -from google.appengine.ext.webapp import template -from google.appengine.ext.webapp.util import run_wsgi_app -import datetime -import hashlib -import logging -import os -import re -import bz2 - -# local imports -from auth import auth -import const - -# The majority of our state are commit objects. One of these exists for each of -# the commits known to the build system. Their key names are of the form -# <commit number (%08x)> "-" <hg hash>. This means that a sorting by the key -# name is sufficient to order the commits. -# -# The commit numbers are purely local. They need not match up to the commit -# numbers in an hg repo. When inserting a new commit, the parent commit must be -# given and this is used to generate the new commit number. In order to create -# the first Commit object, a special command (/init) is used. -class Commit(db.Model): - num = db.IntegerProperty() # internal, monotonic counter. - node = db.StringProperty() # Hg hash - parentnode = db.StringProperty() # Hg hash - user = db.StringProperty() - date = db.DateTimeProperty() - desc = db.BlobProperty() - - # This is the list of builds. Each element is a string of the form <builder - # name> '`' <log hash>. If the log hash is empty, then the build was - # successful. - builds = db.StringListProperty() - - fail_notification_sent = db.BooleanProperty() - -# A CompressedLog contains the textual build log of a failed build. -# The key name is the hex digest of the SHA256 hash of the contents. -# The contents is bz2 compressed. -class CompressedLog(db.Model): - log = db.BlobProperty() - -N = 30 - -def builderInfo(b): - f = b.split('-', 3) - goos = f[0] - goarch = f[1] - note = "" - if len(f) > 2: - note = f[2] - return {'name': b, 'goos': goos, 'goarch': goarch, 'note': note} - -def builderset(): - q = Commit.all() - q.order('-__key__') - results = q.fetch(N) - builders = set() - for c in results: - builders.update(set(parseBuild(build)['builder'] for build in c.builds)) - return builders - -class MainPage(webapp.RequestHandler): - def get(self): - self.response.headers['Content-Type'] = 'text/html; charset=utf-8' - - try: - page = int(self.request.get('p', 1)) - if not page > 0: - raise - except: - page = 1 - - try: - num = int(self.request.get('n', N)) - if num <= 0 or num > 200: - raise - except: - num = N - - offset = (page-1) * num - - q = Commit.all() - q.order('-__key__') - results = q.fetch(num, offset) - - revs = [toRev(r) for r in results] - builders = {} - - for r in revs: - for b in r['builds']: - builders[b['builder']] = builderInfo(b['builder']) - - for r in revs: - have = set(x['builder'] for x in r['builds']) - need = set(builders.keys()).difference(have) - for n in need: - r['builds'].append({'builder': n, 'log':'', 'ok': False}) - r['builds'].sort(cmp = byBuilder) - - builders = list(builders.items()) - builders.sort() - values = {"revs": revs, "builders": [v for k,v in builders]} - - values['num'] = num - values['prev'] = page - 1 - if len(results) == num: - values['next'] = page + 1 - - path = os.path.join(os.path.dirname(__file__), 'main.html') - self.response.out.write(template.render(path, values)) - -# A DashboardHandler is a webapp.RequestHandler but provides -# authenticated_post - called by post after authenticating -# json - writes object in json format to response output -class DashboardHandler(webapp.RequestHandler): - def post(self): - if not auth(self.request): - self.response.set_status(403) - return - self.authenticated_post() - - def authenticated_post(self): - return - - def json(self, obj): - self.response.set_status(200) - simplejson.dump(obj, self.response.out) - return - -# Todo serves /todo. It tells the builder which commits need to be built. -class Todo(DashboardHandler): - def get(self): - builder = self.request.get('builder') - key = 'todo-%s' % builder - response = memcache.get(key) - if response is None: - # Fell out of memcache. Rebuild from datastore results. - # We walk the commit list looking for nodes that have not - # been built by this builder. - q = Commit.all() - q.order('-__key__') - todo = [] - first = None - for c in q.fetch(N+1): - if first is None: - first = c - if not built(c, builder): - todo.append({'Hash': c.node}) - response = simplejson.dumps(todo) - memcache.set(key, response, 3600) - self.response.set_status(200) - self.response.out.write(response) - -def built(c, builder): - for b in c.builds: - if b.startswith(builder+'`'): - return True - return False - -# Log serves /log/. It retrieves log data by content hash. -class LogHandler(DashboardHandler): - def get(self): - self.response.headers['Content-Type'] = 'text/plain; charset=utf-8' - hash = self.request.path[5:] - l = CompressedLog.get_by_key_name(hash) - if l is None: - self.response.set_status(404) - return - log = bz2.decompress(l.log) - self.response.set_status(200) - self.response.out.write(log) - -# Init creates the commit with id 0. Since this commit doesn't have a parent, -# it cannot be created by Build. -class Init(DashboardHandler): - def authenticated_post(self): - date = parseDate(self.request.get('date')) - node = self.request.get('node') - if not validNode(node) or date is None: - logging.error("Not valid node ('%s') or bad date (%s %s)", node, date, self.request.get('date')) - self.response.set_status(500) - return - - commit = Commit(key_name = '00000000-%s' % node) - commit.num = 0 - commit.node = node - commit.parentnode = '' - commit.user = self.request.get('user').encode('utf8') - commit.date = date - commit.desc = self.request.get('desc').encode('utf8') - - commit.put() - - self.response.set_status(200) - -# The last commit when we switched to using entity groups. -# This is the root of the new commit entity group. -RootCommitKeyName = '00000f26-f32c6f1038207c55d5780231f7484f311020747e' - -# CommitHandler serves /commit. -# A GET of /commit retrieves information about the specified commit. -# A POST of /commit creates a node for the given commit. -# If the commit already exists, the POST silently succeeds (like mkdir -p). -class CommitHandler(DashboardHandler): - def get(self): - node = self.request.get('node') - if not validNode(node): - return self.json({'Status': 'FAIL', 'Error': 'malformed node hash'}) - n = nodeByHash(node) - if n is None: - return self.json({'Status': 'FAIL', 'Error': 'unknown revision'}) - return self.json({'Status': 'OK', 'Node': nodeObj(n)}) - - def authenticated_post(self): - # Require auth with the master key, not a per-builder key. - if self.request.get('builder'): - self.response.set_status(403) - return - - node = self.request.get('node') - date = parseDate(self.request.get('date')) - user = self.request.get('user').encode('utf8') - desc = self.request.get('desc').encode('utf8') - parenthash = self.request.get('parent') - - if not validNode(node) or not validNode(parenthash) or date is None: - return self.json({'Status': 'FAIL', 'Error': 'malformed node, parent, or date'}) - - n = nodeByHash(node) - if n is None: - p = nodeByHash(parenthash) - if p is None: - return self.json({'Status': 'FAIL', 'Error': 'unknown parent'}) - - # Want to create new node in a transaction so that multiple - # requests creating it do not collide and so that multiple requests - # creating different nodes get different sequence numbers. - # All queries within a transaction must include an ancestor, - # but the original datastore objects we used for the dashboard - # have no common ancestor. Instead, we use a well-known - # root node - the last one before we switched to entity groups - - # as the as the common ancestor. - root = Commit.get_by_key_name(RootCommitKeyName) - - def add_commit(): - if nodeByHash(node, ancestor=root) is not None: - return - - # Determine number for this commit. - # Once we have created one new entry it will be lastRooted.num+1, - # but the very first commit created in this scheme will have to use - # last.num's number instead (last is likely not rooted). - q = Commit.all() - q.order('-__key__') - q.ancestor(root) - last = q.fetch(1)[0] - num = last.num+1 - - n = Commit(key_name = '%08x-%s' % (num, node), parent = root) - n.num = num - n.node = node - n.parentnode = parenthash - n.user = user - n.date = date - n.desc = desc - n.put() - db.run_in_transaction(add_commit) - n = nodeByHash(node) - if n is None: - return self.json({'Status': 'FAIL', 'Error': 'failed to create commit node'}) - - return self.json({'Status': 'OK', 'Node': nodeObj(n)}) - -# Build serves /build. -# A POST to /build records a new build result. -class Build(webapp.RequestHandler): - def post(self): - if not auth(self.request): - self.response.set_status(403) - return - - builder = self.request.get('builder') - log = self.request.get('log').encode('utf-8') - - loghash = '' - if len(log) > 0: - loghash = hashlib.sha256(log).hexdigest() - l = CompressedLog(key_name=loghash) - l.log = bz2.compress(log) - l.put() - - node = self.request.get('node') - if not validNode(node): - logging.error('Invalid node %s' % (node)) - self.response.set_status(500) - return - - n = nodeByHash(node) - if n is None: - logging.error('Cannot find node %s' % (node)) - self.response.set_status(404) - return - nn = n - - def add_build(): - n = nodeByHash(node, ancestor=nn) - if n is None: - logging.error('Cannot find hash in add_build: %s %s' % (builder, node)) - return - - s = '%s`%s' % (builder, loghash) - for i, b in enumerate(n.builds): - if b.split('`', 1)[0] == builder: - # logging.error('Found result for %s %s already' % (builder, node)) - n.builds[i] = s - break - else: - # logging.error('Added result for %s %s' % (builder, node)) - n.builds.append(s) - n.put() - - db.run_in_transaction(add_build) - - key = 'todo-%s' % builder - memcache.delete(key) - - c = getBrokenCommit(node, builder) - if c is not None and not c.fail_notification_sent: - notifyBroken(c, builder) - - self.response.set_status(200) - - -def getBrokenCommit(node, builder): - """ - getBrokenCommit returns a Commit that breaks the build. - The Commit will be either the one specified by node or the one after. - """ - - # Squelch mail if already fixed. - head = firstResult(builder) - if broken(head, builder) == False: - return - - # Get current node and node before, after. - cur = nodeByHash(node) - if cur is None: - return - before = nodeBefore(cur) - after = nodeAfter(cur) - - if broken(before, builder) == False and broken(cur, builder): - return cur - if broken(cur, builder) == False and broken(after, builder): - return after - - return - -def firstResult(builder): - q = Commit.all().order('-__key__') - for c in q.fetch(20): - for i, b in enumerate(c.builds): - p = b.split('`', 1) - if p[0] == builder: - return c - return None - -def nodeBefore(c): - return nodeByHash(c.parentnode) - -def nodeAfter(c): - return Commit.all().filter('parenthash', c.node).get() - -def notifyBroken(c, builder): - def send(): - n = Commit.get(c.key()) - if n is None: - logging.error("couldn't retrieve Commit '%s'" % c.key()) - return False - if n.fail_notification_sent: - return False - n.fail_notification_sent = True - return n.put() - if not db.run_in_transaction(send): - return - - subject = const.mail_fail_subject % (builder, c.desc.split('\n')[0]) - path = os.path.join(os.path.dirname(__file__), 'fail-notify.txt') - body = template.render(path, { - "builder": builder, - "node": c.node, - "user": c.user, - "desc": c.desc, - "loghash": logHash(c, builder) - }) - mail.send_mail( - sender=const.mail_from, - to=const.mail_fail_to, - subject=subject, - body=body - ) - -def logHash(c, builder): - for i, b in enumerate(c.builds): - p = b.split('`', 1) - if p[0] == builder: - return p[1] - return "" - -def broken(c, builder): - """ - broken returns True if commit c breaks the build for the specified builder, - False if it is a good build, and None if no results exist for this builder. - """ - if c is None: - return None - for i, b in enumerate(c.builds): - p = b.split('`', 1) - if p[0] == builder: - return len(p[1]) > 0 - return None - -def node(num): - q = Commit.all() - q.filter('num =', num) - n = q.get() - return n - -def nodeByHash(hash, ancestor=None): - q = Commit.all() - q.filter('node =', hash) - if ancestor is not None: - q.ancestor(ancestor) - n = q.get() - return n - -# nodeObj returns a JSON object (ready to be passed to simplejson.dump) describing node. -def nodeObj(n): - return { - 'Hash': n.node, - 'ParentHash': n.parentnode, - 'User': n.user, - 'Date': n.date.strftime('%Y-%m-%d %H:%M %z'), - 'Desc': n.desc, - } - -class FixedOffset(datetime.tzinfo): - """Fixed offset in minutes east from UTC.""" - - def __init__(self, offset): - self.__offset = datetime.timedelta(seconds = offset) - - def utcoffset(self, dt): - return self.__offset - - def tzname(self, dt): - return None - - def dst(self, dt): - return datetime.timedelta(0) - -def validNode(node): - if len(node) != 40: - return False - for x in node: - o = ord(x) - if (o < ord('0') or o > ord('9')) and (o < ord('a') or o > ord('f')): - return False - return True - -def parseDate(date): - if '-' in date: - (a, offset) = date.split('-', 1) - try: - return datetime.datetime.fromtimestamp(float(a), FixedOffset(0-int(offset))) - except ValueError: - return None - if '+' in date: - (a, offset) = date.split('+', 1) - try: - return datetime.datetime.fromtimestamp(float(a), FixedOffset(int(offset))) - except ValueError: - return None - try: - return datetime.datetime.utcfromtimestamp(float(date)) - except ValueError: - return None - -email_re = re.compile('^[^<]+<([^>]*)>$') - -def toUsername(user): - r = email_re.match(user) - if r is None: - return user - email = r.groups()[0] - return email.replace('@golang.org', '') - -def dateToShortStr(d): - return d.strftime('%a %b %d %H:%M') - -def parseBuild(build): - [builder, logblob] = build.split('`') - return {'builder': builder, 'log': logblob, 'ok': len(logblob) == 0} - -def nodeInfo(c): - return { - "node": c.node, - "user": toUsername(c.user), - "date": dateToShortStr(c.date), - "desc": c.desc, - "shortdesc": c.desc.split('\n', 2)[0] - } - -def toRev(c): - b = nodeInfo(c) - b['builds'] = [parseBuild(build) for build in c.builds] - return b - -def byBuilder(x, y): - return cmp(x['builder'], y['builder']) - -# Give old builders work; otherwise they pound on the web site. -class Hwget(DashboardHandler): - def get(self): - self.response.out.write("8000\n") - -# This is the URL map for the server. The first three entries are public, the -# rest are only used by the builders. -application = webapp.WSGIApplication( - [('/', MainPage), - ('/hw-get', Hwget), - ('/log/.*', LogHandler), - ('/commit', CommitHandler), - ('/init', Init), - ('/todo', Todo), - ('/build', Build), - ], debug=True) - -def main(): - run_wsgi_app(application) - -if __name__ == "__main__": - main() - diff --git a/misc/dashboard/godashboard/index.yaml b/misc/dashboard/godashboard/index.yaml deleted file mode 100644 index f39299d5d..000000000 --- a/misc/dashboard/godashboard/index.yaml +++ /dev/null @@ -1,51 +0,0 @@ -indexes: - -- kind: BenchmarkResult - ancestor: yes - properties: - - name: builder - - name: __key__ - direction: desc - -- kind: BenchmarkResult - ancestor: yes - properties: - - name: __key__ - direction: desc - -- kind: BenchmarkResults - properties: - - name: builder - - name: benchmark - -- kind: Commit - properties: - - name: __key__ - direction: desc - -- kind: Commit - ancestor: yes - properties: - - name: __key__ - direction: desc - -- kind: Project - properties: - - name: approved - - name: category - - name: name - -- kind: Project - properties: - - name: category - - name: name - -# AUTOGENERATED - -# This index.yaml is automatically updated whenever the dev_appserver -# detects that a new type of query is run. If you want to manage the -# index.yaml file manually, remove the above marker line (the line -# saying "# AUTOGENERATED"). If you want to manage some indexes -# manually, move them above the marker line. The index.yaml file is -# automatically uploaded to the admin console when you next deploy -# your application using appcfg.py. diff --git a/misc/dashboard/godashboard/key.py.dummy b/misc/dashboard/godashboard/key.py.dummy deleted file mode 100644 index 5b8bab186..000000000 --- a/misc/dashboard/godashboard/key.py.dummy +++ /dev/null @@ -1,10 +0,0 @@ -# Copyright 2009 The Go Authors. All rights reserved. -# Use of this source code is governed by a BSD-style -# license that can be found in the LICENSE file. - -# Copy this file to key.py after substituting the real key. - -# accessKey controls private access to the build server (i.e. to record new -# builds). It's tranmitted in the clear but, given the low value of the target, -# this should be sufficient. -accessKey = "this is not the real key" diff --git a/misc/dashboard/godashboard/main.html b/misc/dashboard/godashboard/main.html deleted file mode 100644 index 5390afce6..000000000 --- a/misc/dashboard/godashboard/main.html +++ /dev/null @@ -1,62 +0,0 @@ -<!DOCTYPE HTML> -<html> - <head> - <title>Build Status - Go Dashboard</title> - <link rel="stylesheet" type="text/css" href="static/style.css"> - </head> - - <body> - <a id="top"></a> - - <ul class="menu"> - <li>Build Status</li> - <li><a href="/package">Packages</a></li> - <li><a href="/project">Projects</a></li> - <li><a href="http://golang.org/">golang.org</a></li> - </ul> - - <h1>Go Dashboard</h1> - - <h2>Build Status</h2> - <table class="alternate" cellpadding="0" cellspacing="0"> - <tr> - <th></th> - {% for b in builders %} - <th class="builder">{{b.goos}}<br>{{b.goarch}}<br>{{b.note}}</th> - {% endfor %} - <th></th> - <th></th> - <th></th> - </tr> - - {% for r in revs %} - <tr> - <td class="revision"><span class="hash"><a href="https://code.google.com/p/go/source/detail?r={{r.node}}">{{r.node|slice:":12"}}</a></span></td> - - {% for b in r.builds %} - <td class="result"> - {% if b.ok %} - <span class="ok">ok</span> - {% else %} - {% if b.log %} - <a class="fail" href="/log/{{b.log}}">fail</a> - {% else %} - - {% endif %} - {% endif %} - </td> - {% endfor %} - - <td class="user">{{r.user|escape}}</td> - <td class="date">{{r.date|escape}}</td> - <td class="desc">{{r.shortdesc|escape}}</td> - </tr> - {% endfor %} - </table> - <div class="paginate"> - <a{% if prev %} href="?n={{num}}&p={{prev}}"{% else %} class="inactive"{% endif %}>prev</a> - <a{% if next %} href="?n={{num}}&p={{next}}"{% else %} class="inactive"{% endif %}>next</a> - <a{% if prev %} href="?n={{num}}&p=1"{% else %} class="inactive"{% endif %}>top</a> - </div> - </body> -</html> diff --git a/misc/dashboard/godashboard/package.html b/misc/dashboard/godashboard/package.html deleted file mode 100644 index 043080b5b..000000000 --- a/misc/dashboard/godashboard/package.html +++ /dev/null @@ -1,63 +0,0 @@ -<!DOCTYPE HTML> -<html> - <head> - <title>Packages - Go Dashboard</title> - <link rel="stylesheet" type="text/css" href="static/style.css"> - </head> - - <body> - <ul class="menu"> - <li><a href="/">Build Status</a></li> - <li>Packages</li> - <li><a href="/project">Projects</a></li> - <li><a href="http://golang.org/">golang.org</a></li> - </ul> - - <h1>Go Dashboard</h1> - - <p> - Packages listed on this page are written by third parties and - may or may not build or be safe to use. - </p> - - <p> - An "ok" in the <b>build</b> column indicates that the package is - <a href="http://golang.org/cmd/goinstall/">goinstallable</a> - with the latest - <a href="http://golang.org/doc/devel/release.html">release</a> of Go. - </p> - - <p> - The <b>info</b> column shows the first paragraph from the - <a href="http://blog.golang.org/2011/03/godoc-documenting-go-code.html">package doc comment</a>. - </p> - - <h2>Recently Installed Packages</h2> - <table class="alternate" cellpadding="0" cellspacing="0"> - <tr><th>last install</th><th>count</th><th>build</th><th>path</th><th>info</th></tr> - {% for r in by_time %} - <tr> - <td class="time">{{r.last_install|date:"Y-M-d H:i"}}</td> - <td class="count">{{r.count}}</td> - <td class="ok">{% if r.ok %}<a title="{{r.last_ok|date:"Y-M-d H:i"}}">ok</a>{% else %} {% endif %}</td> - <td class="path"><a href="{{r.web_url}}">{{r.path}}</a></td> - <td class="info">{% if r.info %}{{r.info|escape}}{% endif %}</td> - </tr> - {% endfor %} - </table> - - <h2>Most Installed Packages</h2> - <table class="alternate" cellpadding="0" cellspacing="0"> - <tr><th>last install</th><th>count</th><th>build</th><th>path</th><th>info</th></tr> - {% for r in by_count %} - <tr> - <td class="time">{{r.last_install|date:"Y-M-d H:i"}}</td> - <td class="count">{{r.count}}</td> - <td class="ok">{% if r.ok %}<a title="{{r.last_ok|date:"Y-M-d H:i"}}">ok</a>{% else %} {% endif %}</td> - <td class="path"><a href="{{r.web_url}}">{{r.path}}</a></td> - <td class="info">{% if r.info %}{{r.info|escape}}{% endif %}</td> - </tr> - {% endfor %} - </table> - </body> -</html> diff --git a/misc/dashboard/godashboard/package.py b/misc/dashboard/godashboard/package.py deleted file mode 100644 index 316f3867f..000000000 --- a/misc/dashboard/godashboard/package.py +++ /dev/null @@ -1,355 +0,0 @@ -# Copyright 2010 The Go Authors. All rights reserved. -# Use of this source code is governed by a BSD-style -# license that can be found in the LICENSE file. - -# This is the server part of the package dashboard. -# It must be run by App Engine. - -from google.appengine.api import memcache -from google.appengine.ext import db -from google.appengine.ext import webapp -from google.appengine.ext.webapp import template -from google.appengine.ext.webapp.util import run_wsgi_app -from google.appengine.api import users -from google.appengine.api import mail -from google.appengine.api import urlfetch -import datetime -import logging -import os -import re -import urllib2 -import sets - -# local imports -import toutf8 -import const -from auth import auth - -template.register_template_library('toutf8') - -# Storage model for package info recorded on server. -# Just path, count, and time of last install. -class Package(db.Model): - path = db.StringProperty() - web_url = db.StringProperty() # derived from path - count = db.IntegerProperty() - last_install = db.DateTimeProperty() - - # data contributed by gobuilder - info = db.StringProperty() - ok = db.BooleanProperty() - last_ok = db.DateTimeProperty() - -class Project(db.Model): - name = db.StringProperty(indexed=True) - descr = db.StringProperty() - web_url = db.StringProperty() - package = db.ReferenceProperty(Package) - category = db.StringProperty(indexed=True) - tags = db.ListProperty(str) - approved = db.BooleanProperty(indexed=True) - -re_bitbucket = re.compile(r'^(bitbucket\.org/[a-z0-9A-Z_.\-]+/[a-zA-Z0-9_.\-]+)(/[a-z0-9A-Z_.\-/]+)?$') -re_googlecode = re.compile(r'^[a-z0-9\-]+\.googlecode\.com/(svn|hg)(/[a-z0-9A-Z_.\-/]+)?$') -re_github = re.compile(r'^github\.com/[a-z0-9A-Z_.\-]+(/[a-z0-9A-Z_.\-]+)+$') -re_launchpad = re.compile(r'^launchpad\.net/([a-z0-9A-Z_.\-]+(/[a-z0-9A-Z_.\-]+)?|~[a-z0-9A-Z_.\-]+/(\+junk|[a-z0-9A-Z_.\-]+)/[a-z0-9A-Z_.\-]+)(/[a-z0-9A-Z_.\-/]+)?$') - -def vc_to_web(path): - if re_bitbucket.match(path): - m = re_bitbucket.match(path) - check_url = 'http://' + m.group(1) + '/?cmd=heads' - web = 'http://' + m.group(1) + '/' - elif re_github.match(path): - m = re_github_web.match(path) - check_url = 'https://raw.github.com/' + m.group(1) + '/' + m.group(2) + '/master/' - web = 'http://github.com/' + m.group(1) + '/' + m.group(2) + '/' - elif re_googlecode.match(path): - m = re_googlecode.match(path) - check_url = 'http://'+path - if not m.group(2): # append / after bare '/hg' - check_url += '/' - web = 'http://code.google.com/p/' + path[:path.index('.')] - elif re_launchpad.match(path): - check_url = web = 'https://'+path - else: - return False, False - return web, check_url - -re_bitbucket_web = re.compile(r'bitbucket\.org/([a-z0-9A-Z_.\-]+)/([a-z0-9A-Z_.\-]+)') -re_googlecode_web = re.compile(r'code.google.com/p/([a-z0-9\-]+)') -re_github_web = re.compile(r'github\.com/([a-z0-9A-Z_.\-]+)/([a-z0-9A-Z_.\-]+)') -re_launchpad_web = re.compile(r'launchpad\.net/([a-z0-9A-Z_.\-]+(/[a-z0-9A-Z_.\-]+)?|~[a-z0-9A-Z_.\-]+/(\+junk|[a-z0-9A-Z_.\-]+)/[a-z0-9A-Z_.\-]+)(/[a-z0-9A-Z_.\-/]+)?') -re_striphttp = re.compile(r'https?://(www\.)?') - -def web_to_vc(url): - url = re_striphttp.sub('', url) - m = re_bitbucket_web.match(url) - if m: - return 'bitbucket.org/'+m.group(1)+'/'+m.group(2) - m = re_github_web.match(url) - if m: - return 'github.com/'+m.group(1)+'/'+m.group(2) - m = re_googlecode_web.match(url) - if m: - path = m.group(1)+'.googlecode.com/' - # perform http request to path/hg to check if they're using mercurial - vcs = 'svn' - try: - response = urlfetch.fetch('http://'+path+'hg', deadline=1) - if response.status_code == 200: - vcs = 'hg' - except: pass - return path + vcs - m = re_launchpad_web.match(url) - if m: - return m.group(0) - return False - -MaxPathLength = 100 -CacheTimeout = 3600 - -class PackagePage(webapp.RequestHandler): - def get(self): - if self.request.get('fmt') == 'json': - return self.json() - - html = memcache.get('view-package') - if not html: - q = Package.all() - q.order('-last_install') - by_time = q.fetch(100) - - q = Package.all() - q.order('-count') - by_count = q.fetch(100) - - self.response.headers['Content-Type'] = 'text/html; charset=utf-8' - path = os.path.join(os.path.dirname(__file__), 'package.html') - html = template.render( - path, - {"by_time": by_time, "by_count": by_count} - ) - memcache.set('view-package', html, time=CacheTimeout) - - self.response.out.write(html) - - def json(self): - json = memcache.get('view-package-json') - if not json: - self.response.set_status(200) - self.response.headers['Content-Type'] = 'text/plain; charset=utf-8' - q = Package.all() - s = '{"packages": [' - sep = '' - for r in q.fetch(1000): - s += '%s\n\t{"path": "%s", "last_install": "%s", "count": "%s"}' % (sep, r.path, r.last_install, r.count) - sep = ',' - s += '\n]}\n' - json = s - memcache.set('view-package-json', json, time=CacheTimeout) - self.response.out.write(json) - - def can_get_url(self, url): - try: - urllib2.urlopen(urllib2.Request(url)) - return True - except: - return False - - def is_valid_package_path(self, path): - return (re_bitbucket.match(path) or - re_googlecode.match(path) or - re_github.match(path) or - re_launchpad.match(path)) - - def record_pkg(self, path): - # sanity check string - if not path or len(path) > MaxPathLength or not self.is_valid_package_path(path): - return False - - # look in datastore - key = 'pkg-' + path - p = Package.get_by_key_name(key) - if p is None: - # not in datastore - verify URL before creating - web, check_url = vc_to_web(path) - if not web: - logging.error('unrecognized path: %s', path) - return False - if not self.can_get_url(check_url): - logging.error('cannot get %s', check_url) - return False - p = Package(key_name = key, path = path, count = 0, web_url = web) - - # is this the builder updating package metadata? - if auth(self.request): - p.info = self.request.get('info') - p.ok = self.request.get('ok') == "true" - if p.ok: - p.last_ok = datetime.datetime.utcnow() - else: - p.count += 1 - p.last_install = datetime.datetime.utcnow() - - # update package object - p.put() - return True - - def post(self): - path = self.request.get('path') - ok = db.run_in_transaction(self.record_pkg, path) - if ok: - self.response.set_status(200) - self.response.out.write('ok') - else: - logging.error('invalid path in post: %s', path) - self.response.set_status(500) - self.response.out.write('not ok') - -class ProjectPage(webapp.RequestHandler): - - def get(self): - admin = users.is_current_user_admin() - if self.request.path == "/project/login": - self.redirect(users.create_login_url("/project")) - elif self.request.path == "/project/logout": - self.redirect(users.create_logout_url("/project")) - elif self.request.path == "/project/edit" and admin: - self.edit() - elif self.request.path == "/project/assoc" and admin: - self.assoc() - else: - self.list() - - def assoc(self): - projects = Project.all() - for p in projects: - if p.package: - continue - path = web_to_vc(p.web_url) - if not path: - continue - pkg = Package.get_by_key_name("pkg-"+path) - if not pkg: - self.response.out.write('no: %s %s<br>' % (p.web_url, path)) - continue - p.package = pkg - p.put() - self.response.out.write('yes: %s %s<br>' % (p.web_url, path)) - - def post(self): - if self.request.path == "/project/edit": - self.edit(True) - else: - data = dict(map(lambda x: (x, self.request.get(x)), ["name","descr","web_url"])) - if reduce(lambda x, y: x or not y, data.values(), False): - data["submitMsg"] = "You must complete all the fields." - self.list(data) - return - p = Project.get_by_key_name("proj-"+data["name"]) - if p is not None: - data["submitMsg"] = "A project by this name already exists." - self.list(data) - return - p = Project(key_name="proj-"+data["name"], **data) - p.put() - - path = os.path.join(os.path.dirname(__file__), 'project-notify.txt') - mail.send_mail( - sender=const.mail_from, - to=const.mail_submit_to, - subject=const.mail_submit_subject, - body=template.render(path, {'project': p})) - - self.list({"submitMsg": "Your project has been submitted."}) - - def list(self, additional_data={}): - cache_key = 'view-project-data' - tag = self.request.get('tag', None) - if tag: - cache_key += '-'+tag - data = memcache.get(cache_key) - admin = users.is_current_user_admin() - if admin or not data: - projects = Project.all().order('category').order('name') - if not admin: - projects = projects.filter('approved =', True) - projects = list(projects) - - tags = sets.Set() - for p in projects: - for t in p.tags: - tags.add(t) - - if tag: - projects = filter(lambda x: tag in x.tags, projects) - - data = {} - data['tag'] = tag - data['tags'] = tags - data['projects'] = projects - data['admin']= admin - if not admin: - memcache.set(cache_key, data, time=CacheTimeout) - - for k, v in additional_data.items(): - data[k] = v - - self.response.headers['Content-Type'] = 'text/html; charset=utf-8' - path = os.path.join(os.path.dirname(__file__), 'project.html') - self.response.out.write(template.render(path, data)) - - def edit(self, save=False): - if save: - name = self.request.get("orig_name") - else: - name = self.request.get("name") - - p = Project.get_by_key_name("proj-"+name) - if not p: - self.response.out.write("Couldn't find that Project.") - return - - if save: - if self.request.get("do") == "Delete": - p.delete() - else: - pkg_name = self.request.get("package", None) - if pkg_name: - pkg = Package.get_by_key_name("pkg-"+pkg_name) - if pkg: - p.package = pkg.key() - for f in ['name', 'descr', 'web_url', 'category']: - setattr(p, f, self.request.get(f, None)) - p.approved = self.request.get("approved") == "1" - p.tags = filter(lambda x: x, self.request.get("tags", "").split(",")) - p.put() - memcache.delete('view-project-data') - self.redirect('/project') - return - - # get all project categories and tags - cats, tags = sets.Set(), sets.Set() - for r in Project.all(): - cats.add(r.category) - for t in r.tags: - tags.add(t) - - self.response.headers['Content-Type'] = 'text/html; charset=utf-8' - path = os.path.join(os.path.dirname(__file__), 'project-edit.html') - self.response.out.write(template.render(path, { - "taglist": tags, "catlist": cats, "p": p, "tags": ",".join(p.tags) })) - - def redirect(self, url): - self.response.set_status(302) - self.response.headers.add_header("Location", url) - -def main(): - app = webapp.WSGIApplication([ - ('/package', PackagePage), - ('/project.*', ProjectPage), - ], debug=True) - run_wsgi_app(app) - -if __name__ == '__main__': - main() diff --git a/misc/dashboard/godashboard/project-edit.html b/misc/dashboard/godashboard/project-edit.html deleted file mode 100644 index ce18fb3fb..000000000 --- a/misc/dashboard/godashboard/project-edit.html +++ /dev/null @@ -1,47 +0,0 @@ -<html> -<head> -<link href="http://ajax.googleapis.com/ajax/libs/jqueryui/1.8/themes/base/jquery-ui.css" rel="stylesheet" type="text/css"/> -<script type="text/javascript" src="http://www.google.com/jsapi"></script> -<script> -google.load("jquery", "1"); -google.load("jqueryui", "1.8.2"); -</script> -</head> -<body> -<form action="/project/edit?orig_name={{p.name}}" method="POST"> -Name:<br/> -<input type="text" name="name" value="{{p.name|escape}}"><br/> -Description:<br/> -<input type="text" name="descr" value="{{p.descr|escape}}"><br/> -Category:<br/> -<input type="text" id="cats" name="category" value="{{p.category|escape}}"><br/> -Tags: (comma-separated)<br/> -<input type="text" id="tags" name="tags" value="{{tags}}"><br/> -Web URL:<br/> -<input type="text" name="web_url" value="{{p.web_url|escape}}"><br/> -Package URL: (to link to a goinstall'd package)<br/> -<input type="text" name="package" value="{{p.package.path|escape}}"><br/> -Approved: <input type="checkbox" name="approved" value="1" {% if p.approved %}checked{% endif %}><br/> -<br/> -<input type="submit" name="do" value="Save"> -<input type="submit" name="do" value="Delete" onClick="javascript:return confirm('Delete this?');"> -</form> -<script> -var tags = [ -{% for t in taglist %} - "{{t}}"{% if not forloop.last %},{% endif %} -{% endfor %} -]; -var cats = [ -{% for c in catlist %} - "{{c}}"{% if not forloop.last %},{% endif %} -{% endfor %} -]; - -google.setOnLoadCallback(function() { - $('#tags').autocomplete({source:tags}); - $('#cats').autocomplete({source:cats}); -}); -</script> -</body> -</html> diff --git a/misc/dashboard/godashboard/project-notify.txt b/misc/dashboard/godashboard/project-notify.txt deleted file mode 100644 index f55bf6421..000000000 --- a/misc/dashboard/godashboard/project-notify.txt +++ /dev/null @@ -1,9 +0,0 @@ -A new project has been submitted: - -Name: {{project.name}} -Description: {{project.descr}} -URL: {{project.web_url}} - -To edit/approve/delete: -http://godashboard.appspot.com/project/edit?name={{project.name|toutf8|urlencode}} - diff --git a/misc/dashboard/godashboard/project.html b/misc/dashboard/godashboard/project.html deleted file mode 100644 index 4fe1741c6..000000000 --- a/misc/dashboard/godashboard/project.html +++ /dev/null @@ -1,85 +0,0 @@ -<!DOCTYPE HTML> -<html> - <head> - <title>Projects - Go Dashboard</title> - <link rel="stylesheet" type="text/css" href="static/style.css"> - <style> - .unapproved a.name { color: red } - .tag { font-size: 0.8em; color: #666 } - </style> - </head> - - <body> - <ul class="menu"> - <li><a href="/">Build Status</a></li> - <li><a href="/package">Packages</a></li> - <li>Projects</li> - <li><a href="http://golang.org/">golang.org</a></li> - </ul> - - <h1>Go Dashboard</h1> - - <p> - These are external projects and not endorsed or supported by the Go project. - </p> - - <h2>Projects</h2> - - <div class="submit"> - <h3>Submit a Project</h3> - <p> - Using this form you can submit a project to be included in the list. - </p> - <form action="/project" method="POST"> - <table> - <tr><td>Name:<td><input type="text" name="name"> - <tr><td>Description:<td><input type="text" name="descr"> - <tr><td>URL:<td><input type="text" name="web_url"> - <tr><td> <td><input type="submit" value="Send"> - {% if submitMsg %} - <tr><td class="msg" colspan="2">{{ submitMsg }}</td></tr> - {% endif %} - </table> - </form> - </div> - - <p> - Filter by tag: - {% if tag %} - <a href="/project">all</a> - {% else %} - <b>all</b> - {% endif %} - {% for t in tags %} - {% ifequal t tag %} - <b>{{t}}</b> - {% else %} - <a href="?tag={{t}}">{{t}}</a> - {% endifequal %} - {% endfor %} - </p> - - {% for r in projects %} - {% ifchanged r.category %} - {% if not forloop.first %} - </ul> - {% endif %} - <h3>{{r.category}}</h3> - <ul> - {% endifchanged %} - <li{% if not r.approved %} class="unapproved"{% endif %}> - {% if admin %}[<a href="/project/edit?name={{r.name}}">edit</a>]{% endif %} - <a class="name" href="{{r.web_url}}">{{r.name}}</a> - {{r.descr}} - {% for tag in r.tags %} - <span class="tag">{{tag}}</span> - {% endfor %} - </li> - {% if forloop.last %} - </ul> - {% endif %} - {% endfor %} - </ul> - - - </body> -</html> diff --git a/misc/dashboard/godashboard/static/favicon.ico b/misc/dashboard/godashboard/static/favicon.ico Binary files differdeleted file mode 100644 index 48854ff3b..000000000 --- a/misc/dashboard/godashboard/static/favicon.ico +++ /dev/null diff --git a/misc/dashboard/godashboard/static/style.css b/misc/dashboard/godashboard/static/style.css deleted file mode 100644 index a7e61dda5..000000000 --- a/misc/dashboard/godashboard/static/style.css +++ /dev/null @@ -1,115 +0,0 @@ -body { - font-family: sans-serif; - margin: 0; - padding: 0; -} -h1, h2, h3, ul.menu, table, p { - padding: 0 0.5em; -} -h1, h2 { - margin: 0; - background: #eee; -} -h1 { - border-bottom: 1px solid #ccc; - font-size: 1em; - padding: 0.5em; - margin-bottom: 0.5em; - text-align: right; -} -h2 { - border-top: 1px solid #ccc; - padding-left: 0.2em; -} -.submit { - float: right; - border: 1px solid #ccc; - width: 350px; - padding-bottom: 1em; - margin: 0.5em; - background: #eee; -} -.submit table { - width: 100%; -} -.submit input[type=text] { - width: 200px; -} -.submit .msg { - text-align: center; - color: red; -} -table.alternate { - white-space: nowrap; - margin: 0.5em 0; -} -table.alternate td, -table.alternate th { - padding: 0.1em 0.25em; - font-size: small; -} -table.alternate tr td:last-child { - padding-right: 0; -} -table.alternate tr:nth-child(2n) { - background-color: #f0f0f0; -} -span.hash { - font-family: monospace; - font-size: small; - color: #aaa; -} -td.date { - color: #aaa; -} -td.ok { - text-align: center; - color: #060; - font-weight: bold; -} -td.ok a { - cursor: help; -} -th { - text-align: left; -} -th.builder { - text-align: center; - font-weight: bold; -} -a.fail { - color: #F00; -} -a.fail:visited { - color: #900; -} -ul.menu { - margin: 0; - padding: 0; - list-style-type: none; -} -ul.menu li { - float: left; - display: block; - font-size: 1em; - padding: 0.5em; - background: #EEF; - margin-left: 0.5em; - border-left: 1px solid #999; - border-right: 1px solid #999; -} -div.paginate { - padding: 0.5em; -} -div.paginate a { - padding: 0.5em; - margin-right: 0.5em; - background: #eee; - color: blue; -} -div.paginate a.inactive { - color: #999; -} -td.time { - font-family: monospace; -} diff --git a/misc/dashboard/godashboard/toutf8.py b/misc/dashboard/godashboard/toutf8.py deleted file mode 100644 index 544c681b6..000000000 --- a/misc/dashboard/godashboard/toutf8.py +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright 2010 The Go Authors. All rights reserved. -# Use of this source code is governed by a BSD-style -# license that can be found in the LICENSE file. - -# This is a Django custom template filter to work around the -# fact that GAE's urlencode filter doesn't handle unicode strings. - -from google.appengine.ext import webapp - -register = webapp.template.create_template_register() - -@register.filter -def toutf8(value): - return value.encode("utf-8") diff --git a/misc/dashboard/googlecode_upload.py b/misc/dashboard/googlecode_upload.py deleted file mode 100755 index e87db884a..000000000 --- a/misc/dashboard/googlecode_upload.py +++ /dev/null @@ -1,248 +0,0 @@ -#!/usr/bin/env python2 -# -# Copyright 2006, 2007 Google Inc. All Rights Reserved. -# Author: danderson@google.com (David Anderson) -# -# Script for uploading files to a Google Code project. -# -# This is intended to be both a useful script for people who want to -# streamline project uploads and a reference implementation for -# uploading files to Google Code projects. -# -# To upload a file to Google Code, you need to provide a path to the -# file on your local machine, a small summary of what the file is, a -# project name, and a valid account that is a member or owner of that -# project. You can optionally provide a list of labels that apply to -# the file. The file will be uploaded under the same name that it has -# in your local filesystem (that is, the "basename" or last path -# component). Run the script with '--help' to get the exact syntax -# and available options. -# -# Note that the upload script requests that you enter your -# googlecode.com password. This is NOT your Gmail account password! -# This is the password you use on googlecode.com for committing to -# Subversion and uploading files. You can find your password by going -# to http://code.google.com/hosting/settings when logged in with your -# Gmail account. If you have already committed to your project's -# Subversion repository, the script will automatically retrieve your -# credentials from there (unless disabled, see the output of '--help' -# for details). -# -# If you are looking at this script as a reference for implementing -# your own Google Code file uploader, then you should take a look at -# the upload() function, which is the meat of the uploader. You -# basically need to build a multipart/form-data POST request with the -# right fields and send it to https://PROJECT.googlecode.com/files . -# Authenticate the request using HTTP Basic authentication, as is -# shown below. -# -# Licensed under the terms of the Apache Software License 2.0: -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Questions, comments, feature requests and patches are most welcome. -# Please direct all of these to the Google Code users group: -# http://groups.google.com/group/google-code-hosting - -"""Google Code file uploader script. -""" - -__author__ = 'danderson@google.com (David Anderson)' - -import httplib -import os.path -import optparse -import getpass -import base64 -import sys - - -def upload(file, project_name, user_name, password, summary, labels=None): - """Upload a file to a Google Code project's file server. - - Args: - file: The local path to the file. - project_name: The name of your project on Google Code. - user_name: Your Google account name. - password: The googlecode.com password for your account. - Note that this is NOT your global Google Account password! - summary: A small description for the file. - labels: an optional list of label strings with which to tag the file. - - Returns: a tuple: - http_status: 201 if the upload succeeded, something else if an - error occurred. - http_reason: The human-readable string associated with http_status - file_url: If the upload succeeded, the URL of the file on Google - Code, None otherwise. - """ - # The login is the user part of user@gmail.com. If the login provided - # is in the full user@domain form, strip it down. - if user_name.endswith('@gmail.com'): - user_name = user_name[:user_name.index('@gmail.com')] - - form_fields = [('summary', summary)] - if labels is not None: - form_fields.extend([('label', l.strip()) for l in labels]) - - content_type, body = encode_upload_request(form_fields, file) - - upload_host = '%s.googlecode.com' % project_name - upload_uri = '/files' - auth_token = base64.b64encode('%s:%s'% (user_name, password)) - headers = { - 'Authorization': 'Basic %s' % auth_token, - 'User-Agent': 'Googlecode.com uploader v0.9.4', - 'Content-Type': content_type, - } - - server = httplib.HTTPSConnection(upload_host) - server.request('POST', upload_uri, body, headers) - resp = server.getresponse() - server.close() - - if resp.status == 201: - location = resp.getheader('Location', None) - else: - location = None - return resp.status, resp.reason, location - - -def encode_upload_request(fields, file_path): - """Encode the given fields and file into a multipart form body. - - fields is a sequence of (name, value) pairs. file is the path of - the file to upload. The file will be uploaded to Google Code with - the same file name. - - Returns: (content_type, body) ready for httplib.HTTP instance - """ - BOUNDARY = '----------Googlecode_boundary_reindeer_flotilla' - CRLF = '\r\n' - - body = [] - - # Add the metadata about the upload first - for key, value in fields: - body.extend( - ['--' + BOUNDARY, - 'Content-Disposition: form-data; name="%s"' % key, - '', - value, - ]) - - # Now add the file itself - file_name = os.path.basename(file_path) - f = open(file_path, 'rb') - file_content = f.read() - f.close() - - body.extend( - ['--' + BOUNDARY, - 'Content-Disposition: form-data; name="filename"; filename="%s"' - % file_name, - # The upload server determines the mime-type, no need to set it. - 'Content-Type: application/octet-stream', - '', - file_content, - ]) - - # Finalize the form body - body.extend(['--' + BOUNDARY + '--', '']) - - return 'multipart/form-data; boundary=%s' % BOUNDARY, CRLF.join(body) - - -def upload_find_auth(file_path, project_name, summary, labels=None, - user_name=None, password=None, tries=3): - """Find credentials and upload a file to a Google Code project's file server. - - file_path, project_name, summary, and labels are passed as-is to upload. - - Args: - file_path: The local path to the file. - project_name: The name of your project on Google Code. - summary: A small description for the file. - labels: an optional list of label strings with which to tag the file. - config_dir: Path to Subversion configuration directory, 'none', or None. - user_name: Your Google account name. - tries: How many attempts to make. - """ - - while tries > 0: - if user_name is None: - # Read username if not specified or loaded from svn config, or on - # subsequent tries. - sys.stdout.write('Please enter your googlecode.com username: ') - sys.stdout.flush() - user_name = sys.stdin.readline().rstrip() - if password is None: - # Read password if not loaded from svn config, or on subsequent tries. - print 'Please enter your googlecode.com password.' - print '** Note that this is NOT your Gmail account password! **' - print 'It is the password you use to access Subversion repositories,' - print 'and can be found here: http://code.google.com/hosting/settings' - password = getpass.getpass() - - status, reason, url = upload(file_path, project_name, user_name, password, - summary, labels) - # Returns 403 Forbidden instead of 401 Unauthorized for bad - # credentials as of 2007-07-17. - if status in [httplib.FORBIDDEN, httplib.UNAUTHORIZED]: - # Rest for another try. - user_name = password = None - tries = tries - 1 - else: - # We're done. - break - - return status, reason, url - - -def main(): - parser = optparse.OptionParser(usage='googlecode-upload.py -s SUMMARY ' - '-p PROJECT [options] FILE') - parser.add_option('-s', '--summary', dest='summary', - help='Short description of the file') - parser.add_option('-p', '--project', dest='project', - help='Google Code project name') - parser.add_option('-u', '--user', dest='user', - help='Your Google Code username') - parser.add_option('-w', '--password', dest='password', - help='Your Google Code password') - parser.add_option('-l', '--labels', dest='labels', - help='An optional list of comma-separated labels to attach ' - 'to the file') - - options, args = parser.parse_args() - - if not options.summary: - parser.error('File summary is missing.') - elif not options.project: - parser.error('Project name is missing.') - elif len(args) < 1: - parser.error('File to upload not provided.') - elif len(args) > 1: - parser.error('Only one file may be specified.') - - file_path = args[0] - - if options.labels: - labels = options.labels.split(',') - else: - labels = None - - status, reason, url = upload_find_auth(file_path, options.project, - options.summary, labels, - options.user, options.password) - if url: - print 'The file was uploaded successfully.' - print 'URL: %s' % url - return 0 - else: - print 'An error occurred. Your file was not uploaded.' - print 'Google Code upload server said: %s (%s)' % (reason, status) - return 1 - - -if __name__ == '__main__': - sys.exit(main()) |
