summaryrefslogtreecommitdiff
path: root/misc/dashboard/builder/main.go
diff options
context:
space:
mode:
Diffstat (limited to 'misc/dashboard/builder/main.go')
-rw-r--r--misc/dashboard/builder/main.go313
1 files changed, 129 insertions, 184 deletions
diff --git a/misc/dashboard/builder/main.go b/misc/dashboard/builder/main.go
index 85bb7ad4b..b2b8f43a6 100644
--- a/misc/dashboard/builder/main.go
+++ b/misc/dashboard/builder/main.go
@@ -6,16 +6,15 @@ package main
import (
"bytes"
- "encoding/xml"
"flag"
"fmt"
+ "io"
"io/ioutil"
"log"
"os"
"path/filepath"
"regexp"
"runtime"
- "strconv"
"strings"
"time"
)
@@ -32,6 +31,7 @@ const (
// These variables are copied from the gobuilder's environment
// to the envv of its subprocesses.
var extraEnv = []string{
+ "CC",
"GOARM",
"GOHOSTARCH",
"GOHOSTOS",
@@ -40,25 +40,27 @@ var extraEnv = []string{
}
type Builder struct {
+ goroot *Repo
name string
goos, goarch string
key string
}
var (
- buildroot = flag.String("buildroot", filepath.Join(os.TempDir(), "gobuilder"), "Directory under which to build")
- commitFlag = flag.Bool("commit", false, "upload information about new commits")
- dashboard = flag.String("dashboard", "build.golang.org", "Go Dashboard Host")
- buildRelease = flag.Bool("release", false, "Build and upload binary release archives")
- buildRevision = flag.String("rev", "", "Build specified revision and exit")
- buildCmd = flag.String("cmd", filepath.Join(".", allCmd), "Build command (specify relative to go/src/)")
- failAll = flag.Bool("fail", false, "fail all builds")
- parallel = flag.Bool("parallel", false, "Build multiple targets in parallel")
- verbose = flag.Bool("v", false, "verbose")
+ buildroot = flag.String("buildroot", defaultBuildRoot(), "Directory under which to build")
+ dashboard = flag.String("dashboard", "build.golang.org", "Go Dashboard Host")
+ buildRelease = flag.Bool("release", false, "Build and upload binary release archives")
+ buildRevision = flag.String("rev", "", "Build specified revision and exit")
+ buildCmd = flag.String("cmd", filepath.Join(".", allCmd), "Build command (specify relative to go/src/)")
+ failAll = flag.Bool("fail", false, "fail all builds")
+ parallel = flag.Bool("parallel", false, "Build multiple targets in parallel")
+ buildTimeout = flag.Duration("buildTimeout", 60*time.Minute, "Maximum time to wait for builds and tests")
+ cmdTimeout = flag.Duration("cmdTimeout", 5*time.Minute, "Maximum time to wait for an external command")
+ commitInterval = flag.Duration("commitInterval", 1*time.Minute, "Time to wait between polling for new commits (0 disables commit poller)")
+ verbose = flag.Bool("v", false, "verbose")
)
var (
- goroot string
binaryTagRe = regexp.MustCompile(`^(release\.r|weekly\.)[0-9\-.]+`)
releaseRe = regexp.MustCompile(`^release\.r[0-9\-.]+`)
allCmd = "all" + suffix
@@ -73,26 +75,15 @@ func main() {
os.Exit(2)
}
flag.Parse()
- if len(flag.Args()) == 0 && !*commitFlag {
+ if len(flag.Args()) == 0 {
flag.Usage()
}
- goroot = filepath.Join(*buildroot, "goroot")
- builders := make([]*Builder, len(flag.Args()))
- for i, builder := range flag.Args() {
- b, err := NewBuilder(builder)
- if err != nil {
- log.Fatal(err)
- }
- builders[i] = b
- }
-
- if *failAll {
- failMode(builders)
- return
+ goroot := &Repo{
+ Path: filepath.Join(*buildroot, "goroot"),
}
// set up work environment, use existing enviroment if possible
- if hgRepoExists(goroot) {
+ if goroot.Exists() || *failAll {
log.Print("Found old workspace, will use it")
} else {
if err := os.RemoveAll(*buildroot); err != nil {
@@ -101,22 +92,31 @@ func main() {
if err := os.Mkdir(*buildroot, mkdirPerm); err != nil {
log.Fatalf("Error making build root (%s): %s", *buildroot, err)
}
- if err := hgClone(hgUrl, goroot); err != nil {
+ var err error
+ goroot, err = RemoteRepo(hgUrl).Clone(goroot.Path, "tip")
+ if err != nil {
log.Fatal("Error cloning repository:", err)
}
}
- if *commitFlag {
- if len(flag.Args()) == 0 {
- commitWatcher()
- return
+ // set up builders
+ builders := make([]*Builder, len(flag.Args()))
+ for i, name := range flag.Args() {
+ b, err := NewBuilder(goroot, name)
+ if err != nil {
+ log.Fatal(err)
}
- go commitWatcher()
+ builders[i] = b
+ }
+
+ if *failAll {
+ failMode(builders)
+ return
}
// if specified, build revision and return
if *buildRevision != "" {
- hash, err := fullHash(goroot, *buildRevision)
+ hash, err := goroot.FullHash(*buildRevision)
if err != nil {
log.Fatal("Error finding revision: ", err)
}
@@ -128,7 +128,10 @@ func main() {
return
}
- // go continuous build mode (default)
+ // Start commit watcher
+ go commitWatcher(goroot)
+
+ // go continuous build mode
// check for new commits and build them
for {
built := false
@@ -175,15 +178,18 @@ func failMode(builders []*Builder) {
}
}
-func NewBuilder(builder string) (*Builder, error) {
- b := &Builder{name: builder}
+func NewBuilder(goroot *Repo, name string) (*Builder, error) {
+ b := &Builder{
+ goroot: goroot,
+ name: name,
+ }
// get goos/goarch from builder string
- s := strings.SplitN(builder, "-", 3)
+ s := strings.SplitN(b.name, "-", 3)
if len(s) >= 2 {
b.goos, b.goarch = s[0], s[1]
} else {
- return nil, fmt.Errorf("unsupported builder form: %s", builder)
+ return nil, fmt.Errorf("unsupported builder form: %s", name)
}
// read keys from keyfile
@@ -206,7 +212,7 @@ func NewBuilder(builder string) (*Builder, error) {
}
// build checks for a new commit for this builder
-// and builds it if one is found.
+// and builds it if one is found.
// It returns true if a build was attempted.
func (b *Builder) build() bool {
hash, err := b.todo("build-go-commit", "", "")
@@ -217,16 +223,8 @@ func (b *Builder) build() bool {
if hash == "" {
return false
}
- // Look for hash locally before running hg pull.
- if _, err := fullHash(goroot, hash[:12]); err != nil {
- // Don't have hash, so run hg pull.
- if err := run(nil, goroot, "hg", "pull"); err != nil {
- log.Println("hg pull failed:", err)
- return false
- }
- }
- err = b.buildHash(hash)
- if err != nil {
+
+ if err := b.buildHash(hash); err != nil {
log.Println(err)
}
return true
@@ -242,34 +240,49 @@ func (b *Builder) buildHash(hash string) error {
}
defer os.RemoveAll(workpath)
- // clone repo
- if err := run(nil, workpath, "hg", "clone", goroot, "go"); err != nil {
+ // pull before cloning to ensure we have the revision
+ if err := b.goroot.Pull(); err != nil {
return err
}
- // update to specified revision
- if err := run(nil, filepath.Join(workpath, "go"), "hg", "update", hash); err != nil {
+ // clone repo at specified revision
+ if _, err := b.goroot.Clone(filepath.Join(workpath, "go"), hash); err != nil {
return err
}
srcDir := filepath.Join(workpath, "go", "src")
// build
+ var buildlog bytes.Buffer
logfile := filepath.Join(workpath, "build.log")
+ f, err := os.Create(logfile)
+ if err != nil {
+ return err
+ }
+ defer f.Close()
+ w := io.MultiWriter(f, &buildlog)
+
cmd := *buildCmd
if !filepath.IsAbs(cmd) {
cmd = filepath.Join(srcDir, cmd)
}
startTime := time.Now()
- buildLog, status, err := runLog(b.envv(), logfile, srcDir, cmd)
+ ok, err := runOutput(*buildTimeout, b.envv(), w, srcDir, cmd)
runTime := time.Now().Sub(startTime)
- if err != nil {
- return fmt.Errorf("%s: %s", *buildCmd, err)
+ errf := func() string {
+ if err != nil {
+ return fmt.Sprintf("error: %v", err)
+ }
+ if !ok {
+ return "failed"
+ }
+ return "success"
}
+ fmt.Fprintf(w, "Build complete, duration %v. Result: %v\n", runTime, errf())
- if status != 0 {
+ if err != nil || !ok {
// record failure
- return b.recordResult(false, "", hash, "", buildLog, runTime)
+ return b.recordResult(false, "", hash, "", buildlog.String(), runTime)
}
// record success
@@ -278,13 +291,15 @@ func (b *Builder) buildHash(hash string) error {
}
// build Go sub-repositories
- b.buildSubrepos(filepath.Join(workpath, "go"), hash)
+ goRoot := filepath.Join(workpath, "go")
+ goPath := workpath
+ b.buildSubrepos(goRoot, goPath, hash)
return nil
}
// failBuild checks for a new commit for this builder
-// and fails it if one is found.
+// and fails it if one is found.
// It returns true if a build was "attempted".
func (b *Builder) failBuild() bool {
hash, err := b.todo("build-go-commit", "", "")
@@ -304,7 +319,7 @@ func (b *Builder) failBuild() bool {
return true
}
-func (b *Builder) buildSubrepos(goRoot, goHash string) {
+func (b *Builder) buildSubrepos(goRoot, goPath, goHash string) {
for _, pkg := range dashboardPackages("subrepo") {
// get the latest todo for this package
hash, err := b.todo("build-package", pkg, goHash)
@@ -320,7 +335,7 @@ func (b *Builder) buildSubrepos(goRoot, goHash string) {
if *verbose {
log.Printf("buildSubrepos %s: building %q", pkg, hash)
}
- buildLog, err := b.buildSubrepo(goRoot, pkg, hash)
+ buildLog, err := b.buildSubrepo(goRoot, goPath, pkg, hash)
if err != nil {
if buildLog == "" {
buildLog = err.Error()
@@ -338,45 +353,39 @@ func (b *Builder) buildSubrepos(goRoot, goHash string) {
// buildSubrepo fetches the given package, updates it to the specified hash,
// and runs 'go test -short pkg/...'. It returns the build log and any error.
-func (b *Builder) buildSubrepo(goRoot, pkg, hash string) (string, error) {
- goBin := filepath.Join(goRoot, "bin")
- goTool := filepath.Join(goBin, "go")
- env := append(b.envv(), "GOROOT="+goRoot)
+func (b *Builder) buildSubrepo(goRoot, goPath, pkg, hash string) (string, error) {
+ goTool := filepath.Join(goRoot, "bin", "go")
+ env := append(b.envv(), "GOROOT="+goRoot, "GOPATH="+goPath)
- // add goBin to PATH
+ // add $GOROOT/bin and $GOPATH/bin to PATH
for i, e := range env {
const p = "PATH="
if !strings.HasPrefix(e, p) {
continue
}
- env[i] = p + goBin + string(os.PathListSeparator) + e[len(p):]
+ sep := string(os.PathListSeparator)
+ env[i] = p + filepath.Join(goRoot, "bin") + sep + filepath.Join(goPath, "bin") + sep + e[len(p):]
}
// fetch package and dependencies
- log, status, err := runLog(env, "", goRoot, goTool, "get", "-d", pkg)
- if err == nil && status != 0 {
- err = fmt.Errorf("go exited with status %d", status)
+ log, ok, err := runLog(*cmdTimeout, env, goPath, goTool, "get", "-d", pkg+"/...")
+ if err == nil && !ok {
+ err = fmt.Errorf("go exited with status 1")
}
if err != nil {
- // 'go get -d' will fail for a subrepo because its top-level
- // directory does not contain a go package. No matter, just
- // check whether an hg directory exists and proceed.
- hgDir := filepath.Join(goRoot, "src/pkg", pkg, ".hg")
- if fi, e := os.Stat(hgDir); e != nil || !fi.IsDir() {
- return log, err
- }
+ return log, err
}
// hg update to the specified hash
- pkgPath := filepath.Join(goRoot, "src/pkg", pkg)
- if err := run(nil, pkgPath, "hg", "update", hash); err != nil {
+ repo := Repo{Path: filepath.Join(goPath, "src", pkg)}
+ if err := repo.UpdateTo(hash); err != nil {
return "", err
}
// test the package
- log, status, err = runLog(env, "", goRoot, goTool, "test", "-short", pkg+"/...")
- if err == nil && status != 0 {
- err = fmt.Errorf("go exited with status %d", status)
+ log, ok, err = runLog(*buildTimeout, env, goPath, goTool, "test", "-short", pkg+"/...")
+ if err == nil && !ok {
+ err = fmt.Errorf("go exited with status 1")
}
return log, err
}
@@ -449,9 +458,13 @@ func isFile(name string) bool {
}
// commitWatcher polls hg for new commits and tells the dashboard about them.
-func commitWatcher() {
+func commitWatcher(goroot *Repo) {
+ if *commitInterval == 0 {
+ log.Printf("commitInterval is %s, disabling commitWatcher", *commitInterval)
+ return
+ }
// Create builder just to get master key.
- b, err := NewBuilder("mercurial-commit")
+ b, err := NewBuilder(goroot, "mercurial-commit")
if err != nil {
log.Fatal(err)
}
@@ -462,104 +475,46 @@ func commitWatcher() {
log.Printf("poll...")
}
// Main Go repository.
- commitPoll(key, "")
+ commitPoll(goroot, "", key)
// Go sub-repositories.
for _, pkg := range dashboardPackages("subrepo") {
- commitPoll(key, pkg)
+ pkgroot := &Repo{
+ Path: filepath.Join(*buildroot, pkg),
+ }
+ commitPoll(pkgroot, pkg, key)
}
if *verbose {
log.Printf("sleep...")
}
- time.Sleep(60e9)
- }
-}
-
-func hgClone(url, path string) error {
- return run(nil, *buildroot, "hg", "clone", url, path)
-}
-
-func hgRepoExists(path string) bool {
- fi, err := os.Stat(filepath.Join(path, ".hg"))
- if err != nil {
- return false
+ time.Sleep(*commitInterval)
}
- return fi.IsDir()
-}
-
-// HgLog represents a single Mercurial revision.
-type HgLog struct {
- Hash string
- Author string
- Date string
- Desc string
- Parent string
-
- // Internal metadata
- added bool
}
// logByHash is a cache of all Mercurial revisions we know about,
// indexed by full hash.
var logByHash = map[string]*HgLog{}
-// xmlLogTemplate is a template to pass to Mercurial to make
-// hg log print the log in valid XML for parsing with xml.Unmarshal.
-const xmlLogTemplate = `
- <Log>
- <Hash>{node|escape}</Hash>
- <Parent>{parent|escape}</Parent>
- <Author>{author|escape}</Author>
- <Date>{date|rfc3339date}</Date>
- <Desc>{desc|escape}</Desc>
- </Log>
-`
-
// commitPoll pulls any new revisions from the hg server
// and tells the server about them.
-func commitPoll(key, pkg string) {
- pkgRoot := goroot
-
- if pkg != "" {
- pkgRoot = filepath.Join(*buildroot, pkg)
- if !hgRepoExists(pkgRoot) {
- if err := hgClone(repoURL(pkg), pkgRoot); err != nil {
- log.Printf("%s: hg clone failed: %v", pkg, err)
- if err := os.RemoveAll(pkgRoot); err != nil {
- log.Printf("%s: %v", pkg, err)
- }
- return
+func commitPoll(repo *Repo, pkg, key string) {
+ if !repo.Exists() {
+ var err error
+ repo, err = RemoteRepo(repoURL(pkg)).Clone(repo.Path, "tip")
+ if err != nil {
+ log.Printf("%s: hg clone failed: %v", pkg, err)
+ if err := os.RemoveAll(repo.Path); err != nil {
+ log.Printf("%s: %v", pkg, err)
}
}
- }
-
- if err := run(nil, pkgRoot, "hg", "pull"); err != nil {
- log.Printf("hg pull: %v", err)
return
}
- const N = 50 // how many revisions to grab
-
- data, _, err := runLog(nil, "", pkgRoot, "hg", "log",
- "--encoding=utf-8",
- "--limit="+strconv.Itoa(N),
- "--template="+xmlLogTemplate,
- )
+ logs, err := repo.Log() // repo.Log calls repo.Pull internally
if err != nil {
log.Printf("hg log: %v", err)
return
}
- var logStruct struct {
- Log []HgLog
- }
- err = xml.Unmarshal([]byte("<Top>"+data+"</Top>"), &logStruct)
- if err != nil {
- log.Printf("unmarshal hg log: %v", err)
- return
- }
-
- logs := logStruct.Log
-
// Pass 1. Fill in parents and add new log entries to logsByHash.
// Empty parent means take parent from next log entry.
// Non-empty parent has form 1234:hashhashhash; we want full hash.
@@ -568,7 +523,7 @@ func commitPoll(key, pkg string) {
if l.Parent == "" && i+1 < len(logs) {
l.Parent = logs[i+1].Hash
} else if l.Parent != "" {
- l.Parent, _ = fullHash(pkgRoot, l.Parent)
+ l.Parent, _ = repo.FullHash(l.Parent)
}
if *verbose {
log.Printf("hg log %s: %s < %s\n", pkg, l.Hash, l.Parent)
@@ -580,8 +535,7 @@ func commitPoll(key, pkg string) {
}
}
- for i := range logs {
- l := &logs[i]
+ for _, l := range logs {
addCommit(pkg, l.Hash, key)
}
}
@@ -625,28 +579,6 @@ func addCommit(pkg, hash, key string) bool {
return true
}
-// fullHash returns the full hash for the given Mercurial revision.
-func fullHash(root, rev string) (string, error) {
- s, _, err := runLog(nil, "", root,
- "hg", "log",
- "--encoding=utf-8",
- "--rev="+rev,
- "--limit=1",
- "--template={node}",
- )
- if err != nil {
- return "", nil
- }
- s = strings.TrimSpace(s)
- if s == "" {
- return "", fmt.Errorf("cannot find revision")
- }
- if len(s) != 40 {
- return "", fmt.Errorf("hg returned invalid hash " + s)
- }
- return s, nil
-}
-
var repoRe = regexp.MustCompile(`^code\.google\.com/p/([a-z0-9\-]+(\.[a-z0-9\-]+)?)(/[a-z0-9A-Z_.\-/]+)?$`)
// repoURL returns the repository URL for the supplied import path.
@@ -668,6 +600,19 @@ func defaultSuffix() string {
return ".bash"
}
+// defaultBuildRoot returns default buildroot directory.
+func defaultBuildRoot() string {
+ var d string
+ if runtime.GOOS == "windows" {
+ // will use c:\, otherwise absolute paths become too long
+ // during builder run, see http://golang.org/issue/3358.
+ d = `c:\`
+ } else {
+ d = os.TempDir()
+ }
+ return filepath.Join(d, "gobuilder")
+}
+
func getenvOk(k string) (v string, ok bool) {
v = os.Getenv(k)
if v != "" {
generated by cgit v1.2.3 (git 2.39.1) at 2025-07-04 19:27:17 +0000