diff options
Diffstat (limited to 'misc')
62 files changed, 1319 insertions, 1706 deletions
diff --git a/misc/cgo/gmp/Makefile b/misc/cgo/gmp/Makefile deleted file mode 100644 index d9390c146..000000000 --- a/misc/cgo/gmp/Makefile +++ /dev/null @@ -1,38 +0,0 @@ -# Copyright 2009 The Go Authors. All rights reserved. -# Use of this source code is governed by a BSD-style -# license that can be found in the LICENSE file. - -include ../../../src/Make.inc - -TARG=gmp - -# Can have plain GOFILES too, but this example doesn't. - -CGOFILES=\ - gmp.go - -CGO_LDFLAGS=-lgmp - -# To add flags necessary for locating the library or its include files, -# set CGO_CFLAGS or CGO_LDFLAGS. For example, to use an -# alternate installation of the library: -# CGO_CFLAGS=-I/home/rsc/gmp32/include -# CGO_LDFLAGS+=-L/home/rsc/gmp32/lib -# Note the += on the second line. - -CLEANFILES+=pi fib - -include ../../../src/Make.pkg - -# Simple test programs - -# Computes 1000 digits of pi; single-threaded. -pi: install pi.go - $(GC) $(GCFLAGS) $(GCIMPORTS) pi.go - $(LD) -o $@ pi.$O - -# Computes 200 Fibonacci numbers; multi-threaded. -fib: install fib.go - $(GC) $(GCFLAGS) $(GCIMPORTS) fib.go - $(LD) -o $@ fib.$O - diff --git a/misc/cgo/gmp/fib.go b/misc/cgo/gmp/fib.go index 3eda39e17..18434beaf 100644 --- a/misc/cgo/gmp/fib.go +++ b/misc/cgo/gmp/fib.go @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +// +build ignore + // Compute Fibonacci numbers with two goroutines // that pass integers back and forth. No actual // concurrency, just threads and synchronization @@ -10,7 +12,7 @@ package main import ( - big "gmp" + big "." "runtime" ) diff --git a/misc/cgo/gmp/gmp.go b/misc/cgo/gmp/gmp.go index 9325d8bfd..3bcf99151 100644 --- a/misc/cgo/gmp/gmp.go +++ b/misc/cgo/gmp/gmp.go @@ -98,8 +98,20 @@ Go to hang on to a reference to the pointer until C is done with it. */ package gmp -// #include <gmp.h> -// #include <stdlib.h> +/* +#cgo LDFLAGS: -lgmp +#include <gmp.h> +#include <stdlib.h> + +// gmp 5.0.0+ changed the type of the 3rd argument to mp_bitcnt_t, +// so, to support older versions, we wrap these two functions. +void _mpz_mul_2exp(mpz_ptr a, mpz_ptr b, unsigned long n) { + mpz_mul_2exp(a, b, n); +} +void _mpz_div_2exp(mpz_ptr a, mpz_ptr b, unsigned long n) { + mpz_div_2exp(a, b, n); +} +*/ import "C" import ( @@ -182,12 +194,12 @@ func (z *Int) SetInt64(x int64) *Int { func (z *Int) SetString(s string, base int) error { z.doinit() if base < 2 || base > 36 { - return os.EINVAL + return os.ErrInvalid } p := C.CString(s) defer C.free(unsafe.Pointer(p)) if C.mpz_set_str(&z.i[0], p, C.int(base)) < 0 { - return os.EINVAL + return os.ErrInvalid } return nil } @@ -265,7 +277,7 @@ func (z *Int) Mod(x, y *Int) *Int { func (z *Int) Lsh(x *Int, s uint) *Int { x.doinit() z.doinit() - C.mpz_mul_2exp(&z.i[0], &x.i[0], C.mp_bitcnt_t(s)) + C._mpz_mul_2exp(&z.i[0], &x.i[0], C.ulong(s)) return z } @@ -273,7 +285,7 @@ func (z *Int) Lsh(x *Int, s uint) *Int { func (z *Int) Rsh(x *Int, s uint) *Int { x.doinit() z.doinit() - C.mpz_div_2exp(&z.i[0], &x.i[0], C.mp_bitcnt_t(s)) + C._mpz_div_2exp(&z.i[0], &x.i[0], C.ulong(s)) return z } diff --git a/misc/cgo/gmp/pi.go b/misc/cgo/gmp/pi.go index 3e40624cf..1914cf214 100644 --- a/misc/cgo/gmp/pi.go +++ b/misc/cgo/gmp/pi.go @@ -1,3 +1,5 @@ +// +build ignore + /* Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -38,8 +40,8 @@ POSSIBILITY OF SUCH DAMAGE. package main import ( + big "." "fmt" - big "gmp" "runtime" ) @@ -100,5 +102,5 @@ func main() { } } - fmt.Printf("\n%d calls; bit sizes: %d %d %d\n", runtime.Cgocalls(), numer.Len(), accum.Len(), denom.Len()) + fmt.Printf("\n%d calls; bit sizes: %d %d %d\n", runtime.NumCgoCall(), numer.Len(), accum.Len(), denom.Len()) } diff --git a/misc/cgo/life/Makefile b/misc/cgo/life/Makefile deleted file mode 100644 index 1568a67f6..000000000 --- a/misc/cgo/life/Makefile +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright 2010 The Go Authors. All rights reserved. -# Use of this source code is governed by a BSD-style -# license that can be found in the LICENSE file. - -include ../../../src/Make.inc - -TARG=life - -CGOFILES=\ - life.go\ - -CGO_OFILES=\ - c-life.o\ - -ifeq ($(GOOS),windows) -ifeq ($(GOARCH),amd64) -CGO_OFILES+=\ - lib64_libmingwex_a-wassert.o\ - lib64_libmingw32_a-mingw_helpers.o\ - -lib64_libmingwex_a-wassert.o: - ar -x /mingw/x86_64-w64-mingw32/lib/libmingwex.a lib64_libmingwex_a-wassert.o - -lib64_libmingw32_a-mingw_helpers.o: - ar -x /mingw/x86_64-w64-mingw32/lib/libmingw32.a lib64_libmingw32_a-mingw_helpers.o -endif -endif - -CLEANFILES+=life - -include ../../../src/Make.pkg - -life: install main.go - $(GC) $(GCFLAGS) $(GCIMPORTS) main.go - $(LD) -o $@ main.$O diff --git a/misc/cgo/life/main.go b/misc/cgo/life/main.go index 9cfed434b..47ae0e18c 100644 --- a/misc/cgo/life/main.go +++ b/misc/cgo/life/main.go @@ -2,14 +2,16 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +// +build ignore + // Run the game of life in C using Go for parallelization. package main import ( + "." "flag" "fmt" - "life" ) const MAXDIM = 100 diff --git a/misc/cgo/life/test.bash b/misc/cgo/life/test.bash index 5c5fba1a9..bb483522c 100755 --- a/misc/cgo/life/test.bash +++ b/misc/cgo/life/test.bash @@ -4,8 +4,11 @@ # license that can be found in the LICENSE file. set -e -gomake life +go build -o life main.go + echo '*' life >run.out ./life >>run.out diff run.out golden.out -gomake clean + +rm -f life + diff --git a/misc/cgo/stdio/Makefile b/misc/cgo/stdio/Makefile deleted file mode 100644 index 586132b3c..000000000 --- a/misc/cgo/stdio/Makefile +++ /dev/null @@ -1,17 +0,0 @@ -# Copyright 2009 The Go Authors. All rights reserved. -# Use of this source code is governed by a BSD-style -# license that can be found in the LICENSE file. - -include ../../../src/Make.inc - -TARG=stdio -CGOFILES=\ - file.go\ - -CLEANFILES+=hello fib chain run.out - -include ../../../src/Make.pkg - -%: install %.go - $(GC) $(GCFLAGS) $(GCIMPORTS) $*.go - $(LD) -o $@ $*.$O diff --git a/misc/cgo/stdio/chain.go b/misc/cgo/stdio/chain.go index c188b2dd9..1cf0b1fe5 100644 --- a/misc/cgo/stdio/chain.go +++ b/misc/cgo/stdio/chain.go @@ -2,13 +2,15 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +// +build ignore + // Pass numbers along a chain of threads. package main import ( + "../stdio" "runtime" - "stdio" "strconv" ) diff --git a/misc/cgo/stdio/fib.go b/misc/cgo/stdio/fib.go index 431d9cefe..6d3ccfd52 100644 --- a/misc/cgo/stdio/fib.go +++ b/misc/cgo/stdio/fib.go @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +// +build ignore + // Compute Fibonacci numbers with two goroutines // that pass integers back and forth. No actual // concurrency, just threads and synchronization @@ -10,8 +12,8 @@ package main import ( + "../stdio" "runtime" - "stdio" "strconv" ) diff --git a/misc/cgo/stdio/file.go b/misc/cgo/stdio/file.go index ab1e88436..6e7d479ad 100644 --- a/misc/cgo/stdio/file.go +++ b/misc/cgo/stdio/file.go @@ -28,7 +28,7 @@ var Stderr = (*File)(C.stderr) // Test reference to library symbol. // Stdout and stderr are too special to be a reliable test. -var myerr = C.sys_errlist +//var = C.environ func (f *File) WriteString(s string) { p := C.CString(s) diff --git a/misc/cgo/stdio/hello.go b/misc/cgo/stdio/hello.go index 58fc6d574..4ab3c7447 100644 --- a/misc/cgo/stdio/hello.go +++ b/misc/cgo/stdio/hello.go @@ -2,9 +2,11 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +// +build ignore + package main -import "stdio" +import "../stdio" func main() { stdio.Stdout.WriteString(stdio.Greeting + "\n") diff --git a/misc/cgo/stdio/test.bash b/misc/cgo/stdio/test.bash index 82e3f7b45..21829fa31 100755 --- a/misc/cgo/stdio/test.bash +++ b/misc/cgo/stdio/test.bash @@ -4,7 +4,10 @@ # license that can be found in the LICENSE file. set -e -gomake hello fib chain +go build hello.go +go build fib.go +go build chain.go + echo '*' hello >run.out ./hello >>run.out echo '*' fib >>run.out @@ -12,4 +15,6 @@ echo '*' fib >>run.out echo '*' chain >>run.out ./chain >>run.out diff run.out golden.out -gomake clean + +rm -f hello fib chain + diff --git a/misc/cgo/test/backdoor/backdoor.go b/misc/cgo/test/backdoor/backdoor.go new file mode 100644 index 000000000..efe4f01f4 --- /dev/null +++ b/misc/cgo/test/backdoor/backdoor.go @@ -0,0 +1,7 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package backdoor + +func LockedOSThread() bool // in runtime.c diff --git a/misc/cgo/test/runtime.c b/misc/cgo/test/backdoor/runtime.c index e087c7622..54e6a1ef8 100644 --- a/misc/cgo/test/runtime.c +++ b/misc/cgo/test/backdoor/runtime.c @@ -3,6 +3,8 @@ // license that can be found in the LICENSE file. // Expose some runtime functions for testing. +// Must be in a non-cgo-using package so that +// the go command compiles this file with 6c, not gcc. typedef char bool; @@ -14,7 +16,7 @@ FLUSH(void*) } void -·lockedOSThread(bool b) +·LockedOSThread(bool b) { b = runtime·lockedOSThread(); FLUSH(&b); diff --git a/misc/cgo/test/basic.go b/misc/cgo/test/basic.go index cd6d88168..70ec5e43a 100644 --- a/misc/cgo/test/basic.go +++ b/misc/cgo/test/basic.go @@ -55,7 +55,7 @@ int add(int x, int y) { */ import "C" import ( - "os" + "syscall" "testing" "unsafe" ) @@ -110,7 +110,7 @@ func testErrno(t *testing.T) { C.fclose(f) t.Fatalf("C.fopen: should fail") } - if err != os.ENOENT { + if err != syscall.ENOENT { t.Fatalf("C.fopen: unexpected error: %v", err) } } diff --git a/misc/cgo/test/callback.go b/misc/cgo/test/callback.go index ef852561b..e6a1462b3 100644 --- a/misc/cgo/test/callback.go +++ b/misc/cgo/test/callback.go @@ -6,14 +6,12 @@ package cgotest /* void callback(void *f); -void callGoFoo(void) { - extern void goFoo(void); - goFoo(); -} +void callGoFoo(void); */ import "C" import ( + "./backdoor" "runtime" "testing" "unsafe" @@ -43,7 +41,7 @@ func testCallbackGC(t *testing.T) { nestedCall(runtime.GC) } -func lockedOSThread() bool // in runtime.c +var lockedOSThread = backdoor.LockedOSThread func testCallbackPanic(t *testing.T) { // Make sure panic during callback unwinds properly. diff --git a/misc/cgo/test/callback_c.c b/misc/cgo/test/callback_c.c index c296d70e0..47f07301b 100644 --- a/misc/cgo/test/callback_c.c +++ b/misc/cgo/test/callback_c.c @@ -15,3 +15,23 @@ callback(void *f) goCallback(f); data[sizeof(data)-1] = 0; } + +void +callGoFoo(void) +{ + extern void goFoo(void); + goFoo(); +} + +void +IntoC(void) +{ + BackIntoGo(); +} + +void +twoSleep(int n) +{ + BackgroundSleep(n); + sleep(n); +} diff --git a/misc/cgo/test/issue1328.go b/misc/cgo/test/issue1328.go index e01207dd9..e1796d6f7 100644 --- a/misc/cgo/test/issue1328.go +++ b/misc/cgo/test/issue1328.go @@ -7,7 +7,7 @@ package cgotest import "testing" // extern void BackIntoGo(void); -// void IntoC() { BackIntoGo(); } +// void IntoC(void); import "C" //export BackIntoGo diff --git a/misc/cgo/test/issue1560.go b/misc/cgo/test/issue1560.go index 833b14ae6..3faa966e7 100644 --- a/misc/cgo/test/issue1560.go +++ b/misc/cgo/test/issue1560.go @@ -10,10 +10,7 @@ package cgotest unsigned int sleep(unsigned int seconds); extern void BackgroundSleep(int); -void twoSleep(int n) { - BackgroundSleep(n); - sleep(n); -} +void twoSleep(int); */ import "C" diff --git a/misc/cgo/testso/Makefile b/misc/cgo/testso/Makefile deleted file mode 100644 index e472cf212..000000000 --- a/misc/cgo/testso/Makefile +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright 2011 The Go Authors. All rights reserved. -# Use of this source code is governed by a BSD-style -# license that can be found in the LICENSE file. - -include ../../../src/Make.inc - -TARG=cgosotest - -CGO_DEPS+=libcgoso.so -CGO_LDFLAGS+=-lcgoso -L. -CLEANFILES+=out libcgoso.so -CGOFILES=\ - cgoso.go\ - -include ../../../src/Make.pkg - -libcgoso.so: cgoso_c.c - gcc cgoso_c.c -fPIC -o $@ $(_CGO_CFLAGS_$(GOARCH)) $(_CGO_LDFLAGS_$(GOOS)) - -out: install main.go - $(GC) $(GCFLAGS) $(GCIMPORTS) main.go - $(LD) -o $@ main.$O diff --git a/misc/cgo/testso/cgoso.go b/misc/cgo/testso/cgoso.go index 6eb9f40e3..44fb616c1 100644 --- a/misc/cgo/testso/cgoso.go +++ b/misc/cgo/testso/cgoso.go @@ -4,7 +4,10 @@ package cgosotest -//void sofunc(void); +/* +#cgo LDFLAGS: -L. -lcgosotest +void sofunc(void); +*/ import "C" func Test() { diff --git a/misc/cgo/testso/cgoso_c.c b/misc/cgo/testso/cgoso_c.c index e29f7e807..8c15a6b9f 100644 --- a/misc/cgo/testso/cgoso_c.c +++ b/misc/cgo/testso/cgoso_c.c @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +// +build ignore + void sofunc(void) { extern void goCallback(void); diff --git a/misc/cgo/testso/main.go b/misc/cgo/testso/main.go index 672ab262b..88aa4322d 100644 --- a/misc/cgo/testso/main.go +++ b/misc/cgo/testso/main.go @@ -2,9 +2,11 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +// +build ignore + package main -import "cgosotest" +import "." func main() { cgosotest.Test() diff --git a/misc/cgo/testso/test.bash b/misc/cgo/testso/test.bash index f275eb572..ecef873c8 100755 --- a/misc/cgo/testso/test.bash +++ b/misc/cgo/testso/test.bash @@ -4,6 +4,7 @@ # license that can be found in the LICENSE file. set -e -gomake out -LD_LIBRARY_PATH=. ./out -gomake clean +gcc $(go env GOGCCFLAGS) -shared -o libcgosotest.so cgoso_c.c +go build main.go +LD_LIBRARY_PATH=. ./main +rm -f libcgosotest.so main diff --git a/misc/dashboard/app/app.yaml b/misc/dashboard/app/app.yaml index 47b7d822a..6e19db09c 100644 --- a/misc/dashboard/app/app.yaml +++ b/misc/dashboard/app/app.yaml @@ -13,8 +13,8 @@ handlers: static_dir: static - url: /log/.+ script: _go_app -- url: /(|commit|install|packages|result|tag|todo) +- url: /(|commit|packages|result|tag|todo) script: _go_app -- url: /(init|buildtest|key|_ah/queue/go/delay|install/cron) +- url: /(init|buildtest|key|_ah/queue/go/delay) script: _go_app login: admin diff --git a/misc/dashboard/app/build/build.go b/misc/dashboard/app/build/build.go index a7b08321b..c49fa8bb2 100644 --- a/misc/dashboard/app/build/build.go +++ b/misc/dashboard/app/build/build.go @@ -12,7 +12,6 @@ import ( "fmt" "io" "io/ioutil" - "strconv" "strings" "time" @@ -28,10 +27,6 @@ type Package struct { Name string Path string // (empty for the main Go tree) NextNum int // Num of the next head Commit - - Installs int // All-time total install count - InstallsByDay []string `datastore:",noindex"` // "yyyy-mm-dd,n" - InstallsThisWeek int // Rolling weekly count } func (p *Package) String() string { @@ -46,53 +41,6 @@ func (p *Package) Key(c appengine.Context) *datastore.Key { return datastore.NewKey(c, "Package", key, 0, nil) } -const day = time.Hour * 24 - -// IncrementInstalls increments the total install count and today's install count. -// Daily install counts for dates older than 30 days are discarded. -func (p *Package) IncrementInstalls() { - c := p.dayCounts() - s := []string{} - now := time.Now() - for i := 0; i < 30; i++ { - d := now.Add(-day * time.Duration(i)).Format("2006-01-02") - n := c[d] - if i == 0 { - n++ // increment today's count - } - if n > 0 { // no need to store zeroes in the datastore - s = append(s, fmt.Sprintf("%s,%d", d, n)) - } - } - p.InstallsByDay = s - p.Installs++ - p.UpdateInstallsThisWeek() -} - -// UpdateInstallsThisWeek updates the package's InstallsThisWeek field using data -// from the InstallsByDay list. -func (p *Package) UpdateInstallsThisWeek() { - c := p.dayCounts() - n := 0 - now := time.Now() - for i := 0; i < 7; i++ { - d := now.Add(-day * time.Duration(i)).Format("2006-01-02") - n += c[d] - } - p.InstallsThisWeek = n -} - -// dayCounts explodes InstallsByDay into a map of dates to install counts. -func (p *Package) dayCounts() map[string]int { - c := make(map[string]int) - for _, d := range p.InstallsByDay { - p := strings.SplitN(d, ",", 2) - n, _ := strconv.Atoi(p[1]) - c[p[0]] = n - } - return c -} - // LastCommit returns the most recent Commit for this Package. func (p *Package) LastCommit(c appengine.Context) (*Commit, error) { var commits []*Commit diff --git a/misc/dashboard/app/build/init.go b/misc/dashboard/app/build/init.go index 494585b0e..5311688b7 100644 --- a/misc/dashboard/app/build/init.go +++ b/misc/dashboard/app/build/init.go @@ -15,7 +15,7 @@ import ( // defaultPackages specifies the Package records to be created by initHandler. var defaultPackages = []*Package{ - &Package{Name: "Go", Kind: "go"}, + {Name: "Go", Kind: "go"}, } // subRepos specifies the Go project sub-repositories. diff --git a/misc/dashboard/app/build/pkg.go b/misc/dashboard/app/build/pkg.go deleted file mode 100644 index 076fafe4f..000000000 --- a/misc/dashboard/app/build/pkg.go +++ /dev/null @@ -1,201 +0,0 @@ -// Copyright 2012 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package build - -import ( - "net/http" - "regexp" - "strings" - - "appengine" - "appengine/datastore" - "appengine/delay" - "appengine/urlfetch" -) - -func init() { - http.HandleFunc("/install", installHandler) - http.HandleFunc("/install/cron", installCronHandler) -} - -// installHandler serves requests from the go tool to increment the install -// count for a given package. -func installHandler(w http.ResponseWriter, r *http.Request) { - installLater.Call(appengine.NewContext(r), r.FormValue("packagePath")) -} - -// installCronHandler starts a task to update the weekly install counts for -// every external package. -func installCronHandler(w http.ResponseWriter, r *http.Request) { - c := appengine.NewContext(r) - q := datastore.NewQuery("Package").Filter("Kind=", "external").KeysOnly() - for t := q.Run(c); ; { - key, err := t.Next(nil) - if err == datastore.Done { - break - } else if err != nil { - c.Errorf("%v", err) - return - } - updateWeeklyLater.Call(c, key) - } -} - -var ( - installLater = delay.Func("install", install) - updateWeeklyLater = delay.Func("updateWeekly", updateWeekly) -) - -// install validates the provided package path, increments its install count, -// and creates the Package record if it doesn't exist. -func install(c appengine.Context, path string) { - if !validPath(c, path) { - return - } - tx := func(c appengine.Context) error { - p := &Package{Path: path, Kind: "external"} - err := datastore.Get(c, p.Key(c), p) - if err != nil && err != datastore.ErrNoSuchEntity { - return err - } - p.IncrementInstalls() - _, err = datastore.Put(c, p.Key(c), p) - return err - } - if err := datastore.RunInTransaction(c, tx, nil); err != nil { - c.Errorf("install(%q): %v", path, err) - } -} - -// updateWeekly updates the weekly count for the specified Package. -func updateWeekly(c appengine.Context, key *datastore.Key) { - tx := func(c appengine.Context) error { - p := new(Package) - if err := datastore.Get(c, key, p); err != nil { - return err - } - p.UpdateInstallsThisWeek() - _, err := datastore.Put(c, key, p) - return err - } - if err := datastore.RunInTransaction(c, tx, nil); err != nil { - c.Errorf("updateWeekly: %v", err) - } -} - -// validPath validates the specified import path by matching it against the -// vcsPath regexen and validating its existence by making an HTTP GET request -// to the remote repository. -func validPath(c appengine.Context, path string) bool { - for _, p := range vcsPaths { - if !strings.HasPrefix(path, p.prefix) { - continue - } - m := p.regexp.FindStringSubmatch(path) - if m == nil { - continue - } - if p.check == nil { - // no check function, so just say OK - return true - } - match := make(map[string]string) - for i, name := range p.regexp.SubexpNames() { - if name != "" { - match[name] = m[i] - } - } - return p.check(c, match) - } - c.Debugf("validPath(%q): matching vcsPath not found", path) - return false -} - -// A vcsPath describes how to convert an import path into a version control -// system and repository name. -// -// This is a cut down and modified version of the data structure from -// $GOROOT/src/cmd/go/vcs.go. -type vcsPath struct { - prefix string // prefix this description applies to - re string // pattern for import path - - // check should perform an HTTP request to validate the import path - check func(c appengine.Context, match map[string]string) bool - - regexp *regexp.Regexp // cached compiled form of re -} - -// vcsPaths lists the known vcs paths. -// -// This is a cut down version of the data from $GOROOT/src/cmd/go/vcs.go. -var vcsPaths = []*vcsPath{ - // Google Code - new syntax - { - prefix: "code.google.com/", - re: `^(?P<root>code\.google\.com/p/(?P<project>[a-z0-9\-]+)(\.(?P<subrepo>[a-z0-9\-]+))?)(/[A-Za-z0-9_.\-]+)*$`, - check: googleCodeVCS, - }, - - // Github - { - prefix: "github.com/", - re: `^(?P<root>github\.com/[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+)(/[A-Za-z0-9_.\-]+)*$`, - check: checkRoot, - }, - - // Bitbucket - { - prefix: "bitbucket.org/", - re: `^(?P<root>bitbucket\.org/(?P<bitname>[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+))(/[A-Za-z0-9_.\-]+)*$`, - check: checkRoot, - }, - - // Launchpad - { - prefix: "launchpad.net/", - re: `^(?P<root>launchpad\.net/((?P<project>[A-Za-z0-9_.\-]+)(?P<series>/[A-Za-z0-9_.\-]+)?|~[A-Za-z0-9_.\-]+/(\+junk|[A-Za-z0-9_.\-]+)/[A-Za-z0-9_.\-]+))(/[A-Za-z0-9_.\-]+)*$`, - // TODO(adg): write check function for Launchpad - }, -} - -func init() { - // Compile the regular expressions. - for i := range vcsPaths { - vcsPaths[i].regexp = regexp.MustCompile(vcsPaths[i].re) - } -} - -// googleCodeVCS performs an HTTP GET to verify that a Google Code project -// (and, optionally, a sub-repository) exists. -func googleCodeVCS(c appengine.Context, match map[string]string) bool { - u := "https://code.google.com/p/" + match["project"] - if match["subrepo"] != "" { - u += "/source/checkout?repo=" + match["subrepo"] - } - return checkURL(c, u) -} - -// checkRoot performs an HTTP GET to verify that a specific repository root -// exists (for github and bitbucket both). -func checkRoot(c appengine.Context, match map[string]string) bool { - return checkURL(c, "https://"+match["root"]) -} - -// checkURL performs an HTTP GET to the specified URL and returns whether the -// remote server returned a 2xx response. -func checkURL(c appengine.Context, u string) bool { - client := urlfetch.Client(c) - resp, err := client.Get(u) - if err != nil { - c.Errorf("checkURL(%q): %v", u, err) - return false - } - if resp.StatusCode/100 != 2 { - c.Debugf("checkURL(%q): HTTP status: %s", u, resp.Status) - return false - } - return true -} diff --git a/misc/dashboard/app/build/test.go b/misc/dashboard/app/build/test.go index 4114c25c5..d8470fec1 100644 --- a/misc/dashboard/app/build/test.go +++ b/misc/dashboard/app/build/test.go @@ -37,7 +37,7 @@ const testPkg = "code.google.com/p/go.test" var testPackage = &Package{Name: "Test", Kind: "subrepo", Path: testPkg} var testPackages = []*Package{ - &Package{Name: "Go", Path: ""}, + {Name: "Go", Path: ""}, testPackage, } diff --git a/misc/dashboard/app/cron.yaml b/misc/dashboard/app/cron.yaml deleted file mode 100644 index b54c24cef..000000000 --- a/misc/dashboard/app/cron.yaml +++ /dev/null @@ -1,4 +0,0 @@ -cron: -- description: update rolling package install counts - url: /install/cron - schedule: every 24 hours diff --git a/misc/dashboard/builder/http.go b/misc/dashboard/builder/http.go index f5a1fcf9b..e50ae5724 100644 --- a/misc/dashboard/builder/http.go +++ b/misc/dashboard/builder/http.go @@ -125,41 +125,6 @@ func (b *Builder) recordResult(ok bool, pkg, hash, goHash, buildLog string, runT return dash("POST", "result", args, req, nil) } -// packages fetches a list of package paths from the dashboard -func packages() (pkgs []string, err error) { - return nil, nil - /* TODO(adg): un-stub this once the new package builder design is done - var resp struct { - Packages []struct { - Path string - } - } - err = dash("GET", "package", &resp, param{"fmt": "json"}) - if err != nil { - return - } - for _, p := range resp.Packages { - pkgs = append(pkgs, p.Path) - } - return - */ -} - -// updatePackage sends package build results and info to the dashboard -func (b *Builder) updatePackage(pkg string, ok bool, buildLog, info string) error { - return nil - /* TODO(adg): un-stub this once the new package builder design is done - return dash("POST", "package", nil, param{ - "builder": b.name, - "key": b.key, - "path": pkg, - "ok": strconv.FormatBool(ok), - "log": buildLog, - "info": info, - }) - */ -} - func postCommit(key, pkg string, l *HgLog) error { t, err := time.Parse(time.RFC3339, l.Date) if err != nil { diff --git a/misc/dashboard/builder/main.go b/misc/dashboard/builder/main.go index 4fe65b7a5..85bb7ad4b 100644 --- a/misc/dashboard/builder/main.go +++ b/misc/dashboard/builder/main.go @@ -5,8 +5,8 @@ package main import ( + "bytes" "encoding/xml" - "errors" "flag" "fmt" "io/ioutil" @@ -23,7 +23,7 @@ import ( const ( codeProject = "go" codePyScript = "misc/dashboard/googlecode_upload.py" - hgUrl = "https://go.googlecode.com/hg/" + hgUrl = "https://code.google.com/p/go/" mkdirPerm = 0750 waitInterval = 30 * time.Second // time to wait before checking for new revs pkgBuildInterval = 24 * time.Hour // rebuild packages every 24 hours @@ -43,8 +43,6 @@ type Builder struct { name string goos, goarch string key string - codeUsername string - codePassword string } var ( @@ -55,7 +53,6 @@ var ( buildRevision = flag.String("rev", "", "Build specified revision and exit") buildCmd = flag.String("cmd", filepath.Join(".", allCmd), "Build command (specify relative to go/src/)") failAll = flag.Bool("fail", false, "fail all builds") - external = flag.Bool("external", false, "Build external packages") parallel = flag.Bool("parallel", false, "Build multiple targets in parallel") verbose = flag.Bool("v", false, "verbose") ) @@ -131,14 +128,6 @@ func main() { return } - // external package build mode - if *external { - if len(builders) != 1 { - log.Fatal("only one goos-goarch should be specified with -external") - } - builders[0].buildExternal() - } - // go continuous build mode (default) // check for new commits and build them for { @@ -212,53 +201,10 @@ func NewBuilder(builder string) (*Builder, error) { if err != nil { return nil, fmt.Errorf("readKeys %s (%s): %s", b.name, fn, err) } - v := strings.Split(string(c), "\n") - b.key = v[0] - if len(v) >= 3 { - b.codeUsername, b.codePassword = v[1], v[2] - } - + b.key = string(bytes.TrimSpace(bytes.SplitN(c, []byte("\n"), 2)[0])) return b, nil } -// buildExternal downloads and builds external packages, and -// reports their build status to the dashboard. -// It will re-build all packages after pkgBuildInterval nanoseconds or -// a new release tag is found. -func (b *Builder) buildExternal() { - var prevTag string - var nextBuild time.Time - for { - time.Sleep(waitInterval) - err := run(nil, goroot, "hg", "pull", "-u") - if err != nil { - log.Println("hg pull failed:", err) - continue - } - hash, tag, err := firstTag(releaseRe) - if err != nil { - log.Println(err) - continue - } - if *verbose { - log.Println("latest release:", tag) - } - // don't rebuild if there's no new release - // and it's been less than pkgBuildInterval - // nanoseconds since the last build. - if tag == prevTag && time.Now().Before(nextBuild) { - continue - } - // build will also build the packages - if err := b.buildHash(hash); err != nil { - log.Println(err) - continue - } - prevTag = tag - nextBuild = time.Now().Add(pkgBuildInterval) - } -} - // build checks for a new commit for this builder // and builds it if one is found. // It returns true if a build was attempted. @@ -321,14 +267,6 @@ func (b *Builder) buildHash(hash string) error { return fmt.Errorf("%s: %s", *buildCmd, err) } - // if we're in external mode, build all packages and return - if *external { - if status != 0 { - return errors.New("go build failed") - } - return b.buildExternalPackages(workpath, hash) - } - if status != 0 { // record failure return b.recordResult(false, "", hash, "", buildLog, runTime) @@ -342,36 +280,6 @@ func (b *Builder) buildHash(hash string) error { // build Go sub-repositories b.buildSubrepos(filepath.Join(workpath, "go"), hash) - // finish here if codeUsername and codePassword aren't set - if b.codeUsername == "" || b.codePassword == "" || !*buildRelease { - return nil - } - - // if this is a release, create tgz and upload to google code - releaseHash, release, err := firstTag(binaryTagRe) - if hash == releaseHash { - // clean out build state - cmd := filepath.Join(srcDir, cleanCmd) - if err := run(b.envv(), srcDir, cmd, "--nopkg"); err != nil { - return fmt.Errorf("%s: %s", cleanCmd, err) - } - // upload binary release - fn := fmt.Sprintf("go.%s.%s-%s.tar.gz", release, b.goos, b.goarch) - if err := run(nil, workpath, "tar", "czf", fn, "go"); err != nil { - return fmt.Errorf("tar: %s", err) - } - err := run(nil, workpath, filepath.Join(goroot, codePyScript), - "-s", release, - "-p", codeProject, - "-u", b.codeUsername, - "-w", b.codePassword, - "-l", fmt.Sprintf("%s,%s", b.goos, b.goarch), - fn) - if err != nil { - return fmt.Errorf("%s: %s", codePyScript, err) - } - } - return nil } @@ -429,7 +337,7 @@ func (b *Builder) buildSubrepos(goRoot, goHash string) { } // buildSubrepo fetches the given package, updates it to the specified hash, -// and runs 'go test pkg/...'. It returns the build log and any error. +// and runs 'go test -short pkg/...'. It returns the build log and any error. func (b *Builder) buildSubrepo(goRoot, pkg, hash string) (string, error) { goBin := filepath.Join(goRoot, "bin") goTool := filepath.Join(goBin, "go") @@ -466,7 +374,7 @@ func (b *Builder) buildSubrepo(goRoot, pkg, hash string) (string, error) { } // test the package - log, status, err = runLog(env, "", goRoot, goTool, "test", pkg+"/...") + log, status, err = runLog(env, "", goRoot, goTool, "test", "-short", pkg+"/...") if err == nil && status != 0 { err = fmt.Errorf("go exited with status %d", status) } @@ -739,31 +647,6 @@ func fullHash(root, rev string) (string, error) { return s, nil } -var revisionRe = regexp.MustCompile(`^([^ ]+) +[0-9]+:([0-9a-f]+)$`) - -// firstTag returns the hash and tag of the most recent tag matching re. -func firstTag(re *regexp.Regexp) (hash string, tag string, err error) { - o, _, err := runLog(nil, "", goroot, "hg", "tags") - for _, l := range strings.Split(o, "\n") { - if l == "" { - continue - } - s := revisionRe.FindStringSubmatch(l) - if s == nil { - err = errors.New("couldn't find revision number") - return - } - if !re.MatchString(s[1]) { - continue - } - tag = s[1] - hash, err = fullHash(goroot, s[2]) - return - } - err = errors.New("no matching tag found") - return -} - var repoRe = regexp.MustCompile(`^code\.google\.com/p/([a-z0-9\-]+(\.[a-z0-9\-]+)?)(/[a-z0-9A-Z_.\-/]+)?$`) // repoURL returns the repository URL for the supplied import path. diff --git a/misc/dashboard/builder/package.go b/misc/dashboard/builder/package.go deleted file mode 100644 index dcd449ab8..000000000 --- a/misc/dashboard/builder/package.go +++ /dev/null @@ -1,121 +0,0 @@ -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package main - -import ( - "errors" - "fmt" - "go/doc" - "go/parser" - "go/token" - "log" - "os" - "path/filepath" - "strings" -) - -const MaxCommentLength = 500 // App Engine won't store more in a StringProperty. - -func (b *Builder) buildExternalPackages(workpath string, hash string) error { - logdir := filepath.Join(*buildroot, "log") - if err := os.Mkdir(logdir, 0755); err != nil { - return err - } - pkgs, err := packages() - if err != nil { - return err - } - for _, p := range pkgs { - goroot := filepath.Join(workpath, "go") - gobin := filepath.Join(goroot, "bin") - goinstall := filepath.Join(gobin, "goinstall") - envv := append(b.envv(), "GOROOT="+goroot) - - // add GOBIN to path - for i, v := range envv { - if strings.HasPrefix(v, "PATH=") { - p := filepath.SplitList(v[5:]) - p = append([]string{gobin}, p...) - s := strings.Join(p, string(filepath.ListSeparator)) - envv[i] = "PATH=" + s - } - } - - // goinstall - buildLog, code, err := runLog(envv, "", goroot, goinstall, "-dashboard=false", p) - if err != nil { - log.Printf("goinstall %v: %v", p, err) - } - - // get doc comment from package source - var info string - pkgPath := filepath.Join(goroot, "src", "pkg", p) - if _, err := os.Stat(pkgPath); err == nil { - info, err = packageComment(p, pkgPath) - if err != nil { - log.Printf("packageComment %v: %v", p, err) - } - } - - // update dashboard with build state + info - err = b.updatePackage(p, code == 0, buildLog, info) - if err != nil { - log.Printf("updatePackage %v: %v", p, err) - } - - if code == 0 { - log.Println("Build succeeded:", p) - } else { - log.Println("Build failed:", p) - fn := filepath.Join(logdir, strings.Replace(p, "/", "_", -1)) - if f, err := os.Create(fn); err != nil { - log.Printf("creating %s: %v", fn, err) - } else { - fmt.Fprint(f, buildLog) - f.Close() - } - } - } - return nil -} - -func isGoFile(fi os.FileInfo) bool { - return !fi.IsDir() && // exclude directories - !strings.HasPrefix(fi.Name(), ".") && // ignore .files - !strings.HasSuffix(fi.Name(), "_test.go") && // ignore tests - filepath.Ext(fi.Name()) == ".go" -} - -func packageComment(pkg, pkgpath string) (info string, err error) { - fset := token.NewFileSet() - pkgs, err := parser.ParseDir(fset, pkgpath, isGoFile, parser.PackageClauseOnly|parser.ParseComments) - if err != nil { - return - } - for name := range pkgs { - if name == "main" { - continue - } - pdoc := doc.New(pkgs[name], pkg, doc.AllDecls) - if pdoc.Doc == "" { - continue - } - if info != "" { - return "", errors.New("multiple packages with docs") - } - info = pdoc.Doc - } - // grab only first paragraph - if parts := strings.SplitN(info, "\n\n", 2); len(parts) > 1 { - info = parts[0] - } - // replace newlines with spaces - info = strings.Replace(info, "\n", " ", -1) - // truncate - if len(info) > MaxCommentLength { - info = info[:MaxCommentLength] - } - return -} diff --git a/misc/dashboard/godashboard/_multiprocessing.py b/misc/dashboard/godashboard/_multiprocessing.py deleted file mode 100644 index 8c66c0659..000000000 --- a/misc/dashboard/godashboard/_multiprocessing.py +++ /dev/null @@ -1,5 +0,0 @@ -# Copyright 2009 The Go Authors. All rights reserved. -# Use of this source code is governed by a BSD-style -# license that can be found in the LICENSE file. - -import multiprocessing diff --git a/misc/dashboard/godashboard/app.yaml b/misc/dashboard/godashboard/app.yaml index 8c7670437..ae2892497 100644 --- a/misc/dashboard/godashboard/app.yaml +++ b/misc/dashboard/godashboard/app.yaml @@ -11,16 +11,5 @@ handlers: - url: /static static_dir: static -- url: /package - script: package.py - -- url: /package/daily - script: package.py - login: admin - -- url: /project.* - script: package.py - -- url: / - static_files: main.html - upload: main.html +- url: /(|project(|/login|/edit)) + script: project.py diff --git a/misc/dashboard/godashboard/auth.py b/misc/dashboard/godashboard/auth.py deleted file mode 100644 index 73a54c0d4..000000000 --- a/misc/dashboard/godashboard/auth.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2011 The Go Authors. All rights reserved. -# Use of this source code is governed by a BSD-style -# license that can be found in the LICENSE file. - -import hmac - -# local imports -import key - -def auth(req): - k = req.get('key') - return k == hmac.new(key.accessKey, req.get('builder')).hexdigest() or k == key.accessKey - diff --git a/misc/dashboard/godashboard/const.py b/misc/dashboard/godashboard/const.py index b0110c635..d92d32aa9 100644 --- a/misc/dashboard/godashboard/const.py +++ b/misc/dashboard/godashboard/const.py @@ -3,11 +3,5 @@ # license that can be found in the LICENSE file. mail_from = "Go Dashboard <builder@golang.org>" - mail_submit_to = "adg@golang.org" mail_submit_subject = "New Project Submitted" - -mail_fail_to = "golang-dev@googlegroups.com" -mail_fail_reply_to = "golang-dev@googlegroups.com" -mail_fail_subject = "%s broken by %s" - diff --git a/misc/dashboard/godashboard/cron.yaml b/misc/dashboard/godashboard/cron.yaml deleted file mode 100644 index 953b6a1cd..000000000 --- a/misc/dashboard/godashboard/cron.yaml +++ /dev/null @@ -1,4 +0,0 @@ -cron: -- description: daily package maintenance - url: /package/daily - schedule: every 24 hours diff --git a/misc/dashboard/godashboard/fail-notify.txt b/misc/dashboard/godashboard/fail-notify.txt deleted file mode 100644 index f75d09aa2..000000000 --- a/misc/dashboard/godashboard/fail-notify.txt +++ /dev/null @@ -1,9 +0,0 @@ -Change {{node}} broke the {{builder}} build: -http://godashboard.appspot.com/log/{{loghash}} - -{{desc}} - -http://code.google.com/p/go/source/detail?r={{node}} - -$ tail -n 100 < log -{{log}} diff --git a/misc/dashboard/godashboard/key.py.dummy b/misc/dashboard/godashboard/key.py.dummy deleted file mode 100644 index 5b8bab186..000000000 --- a/misc/dashboard/godashboard/key.py.dummy +++ /dev/null @@ -1,10 +0,0 @@ -# Copyright 2009 The Go Authors. All rights reserved. -# Use of this source code is governed by a BSD-style -# license that can be found in the LICENSE file. - -# Copy this file to key.py after substituting the real key. - -# accessKey controls private access to the build server (i.e. to record new -# builds). It's tranmitted in the clear but, given the low value of the target, -# this should be sufficient. -accessKey = "this is not the real key" diff --git a/misc/dashboard/godashboard/main.html b/misc/dashboard/godashboard/main.html deleted file mode 100644 index 4cd98d851..000000000 --- a/misc/dashboard/godashboard/main.html +++ /dev/null @@ -1,23 +0,0 @@ -<!DOCTYPE HTML> -<html> - <head> - <title>Build Status - Go Dashboard</title> - <link rel="stylesheet" type="text/css" href="static/style.css"> - </head> - - <body> - <ul class="menu"> - <li>Build Status</li> - <li><a href="/package">Packages</a></li> - <li><a href="/project">Projects</a></li> - <li><a href="http://golang.org/">golang.org</a></li> - </ul> - - <h1>Go Dashboard</h1> - - <h2>Build Status</h2> - - <p class="notice">The build status dashboard has moved to <a href="http://build.golang.org">build.golang.org</a>.</p> - - </body> -</html> diff --git a/misc/dashboard/godashboard/package.html b/misc/dashboard/godashboard/package.html deleted file mode 100644 index b688af9e2..000000000 --- a/misc/dashboard/godashboard/package.html +++ /dev/null @@ -1,79 +0,0 @@ -<!DOCTYPE HTML> -<html> - <head> - <title>Packages - Go Dashboard</title> - <link rel="stylesheet" type="text/css" href="static/style.css"> - </head> - - <body> - <ul class="menu"> - <li><a href="/">Build Status</a></li> - <li>Packages</li> - <li><a href="/project">Projects</a></li> - <li><a href="http://golang.org/">golang.org</a></li> - </ul> - - <h1>Go Dashboard</h1> - - <p> - Packages listed on this page are written by third parties and - may or may not build or be safe to use. - </p> - -<!-- - <p> - An "ok" in the <b>build</b> column indicates that the package is - <a href="http://golang.org/cmd/goinstall/">goinstallable</a> - with the latest - <a href="http://golang.org/doc/devel/release.html">release</a> of Go. - </p> ---> - - <p> - The <b>info</b> column shows the first paragraph from the - <a href="http://blog.golang.org/2011/03/godoc-documenting-go-code.html">package doc comment</a>. - </p> - - <h2>Most Installed Packages (this week)</h2> - <table class="alternate" cellpadding="0" cellspacing="0"> - <tr><th>last install</th><th>count</th><th>build</th><th>path</th><th>info</th></tr> - {% for r in by_week_count %} - <tr> - <td class="time">{{r.last_install|date:"Y-M-d H:i"}}</td> - <td class="count">{{r.week_count}}</td> -<!-- <td class="ok">{% if r.ok %}<a title="{{r.last_ok|date:"Y-M-d H:i"}}">ok</a>{% else %} {% endif %}</td> --> - <td class="path"><a href="{{r.web_url}}">{{r.path}}</a></td> - <td class="info">{% if r.info %}{{r.info|escape}}{% endif %}</td> - </tr> - {% endfor %} - </table> - - <h2>Recently Installed Packages</h2> - <table class="alternate" cellpadding="0" cellspacing="0"> - <tr><th>last install</th><th>count</th><th>build</th><th>path</th><th>info</th></tr> - {% for r in by_time %} - <tr> - <td class="time">{{r.last_install|date:"Y-M-d H:i"}}</td> - <td class="count">{{r.count}}</td> -<!-- <td class="ok">{% if r.ok %}<a title="{{r.last_ok|date:"Y-M-d H:i"}}">ok</a>{% else %} {% endif %}</td> --> - <td class="path"><a href="{{r.web_url}}">{{r.path}}</a></td> - <td class="info">{% if r.info %}{{r.info|escape}}{% endif %}</td> - </tr> - {% endfor %} - </table> - - <h2>Most Installed Packages (all time)</h2> - <table class="alternate" cellpadding="0" cellspacing="0"> - <tr><th>last install</th><th>count</th><th>build</th><th>path</th><th>info</th></tr> - {% for r in by_count %} - <tr> - <td class="time">{{r.last_install|date:"Y-M-d H:i"}}</td> - <td class="count">{{r.count}}</td> -<!-- <td class="ok">{% if r.ok %}<a title="{{r.last_ok|date:"Y-M-d H:i"}}">ok</a>{% else %} {% endif %}</td> --> - <td class="path"><a href="{{r.web_url}}">{{r.path}}</a></td> - <td class="info">{% if r.info %}{{r.info|escape}}{% endif %}</td> - </tr> - {% endfor %} - </table> - </body> -</html> diff --git a/misc/dashboard/godashboard/package.py b/misc/dashboard/godashboard/package.py deleted file mode 100644 index 7feac8286..000000000 --- a/misc/dashboard/godashboard/package.py +++ /dev/null @@ -1,429 +0,0 @@ -# Copyright 2010 The Go Authors. All rights reserved. -# Use of this source code is governed by a BSD-style -# license that can be found in the LICENSE file. - -# This is the server part of the package dashboard. -# It must be run by App Engine. - -from google.appengine.api import mail -from google.appengine.api import memcache -from google.appengine.api import taskqueue -from google.appengine.api import urlfetch -from google.appengine.api import users -from google.appengine.ext import db -from google.appengine.ext import webapp -from google.appengine.ext.webapp import template -from google.appengine.ext.webapp.util import run_wsgi_app -import datetime -import logging -import os -import re -import sets -import urllib2 - -# local imports -from auth import auth -import toutf8 -import const - -template.register_template_library('toutf8') - -# Storage model for package info recorded on server. -class Package(db.Model): - path = db.StringProperty() - web_url = db.StringProperty() # derived from path - count = db.IntegerProperty() # grand total - week_count = db.IntegerProperty() # rolling weekly count - day_count = db.TextProperty(default='') # daily count - last_install = db.DateTimeProperty() - - # data contributed by gobuilder - info = db.StringProperty() - ok = db.BooleanProperty() - last_ok = db.DateTimeProperty() - - def get_day_count(self): - counts = {} - if not self.day_count: - return counts - for d in str(self.day_count).split('\n'): - date, count = d.split(' ') - counts[date] = int(count) - return counts - - def set_day_count(self, count): - days = [] - for day, count in count.items(): - days.append('%s %d' % (day, count)) - days.sort(reverse=True) - days = days[:28] - self.day_count = '\n'.join(days) - - def inc(self): - count = self.get_day_count() - today = str(datetime.date.today()) - count[today] = count.get(today, 0) + 1 - self.set_day_count(count) - self.update_week_count(count) - self.count += 1 - - def update_week_count(self, count=None): - if count is None: - count = self.get_day_count() - total = 0 - today = datetime.date.today() - for i in range(7): - day = str(today - datetime.timedelta(days=i)) - if day in count: - total += count[day] - self.week_count = total - - -# PackageDaily kicks off the daily package maintenance cron job -# and serves the associated task queue. -class PackageDaily(webapp.RequestHandler): - - def get(self): - # queue a task to update each package with a week_count > 0 - keys = Package.all(keys_only=True).filter('week_count >', 0) - for key in keys: - taskqueue.add(url='/package/daily', params={'key': key.name()}) - - def post(self): - # update a single package (in a task queue) - def update(key): - p = Package.get_by_key_name(key) - if not p: - return - p.update_week_count() - p.put() - key = self.request.get('key') - if not key: - return - db.run_in_transaction(update, key) - - -class Project(db.Model): - name = db.StringProperty(indexed=True) - descr = db.StringProperty() - web_url = db.StringProperty() - package = db.ReferenceProperty(Package) - category = db.StringProperty(indexed=True) - tags = db.ListProperty(str) - approved = db.BooleanProperty(indexed=True) - - -re_bitbucket = re.compile(r'^(bitbucket\.org/[a-z0-9A-Z_.\-]+/[a-zA-Z0-9_.\-]+)(/[a-z0-9A-Z_.\-/]+)?$') -re_googlecode = re.compile(r'^[a-z0-9\-]+\.googlecode\.com/(svn|hg|git)(/[a-z0-9A-Z_.\-/]+)?$') -re_github = re.compile(r'^github\.com/[a-z0-9A-Z_.\-]+(/[a-z0-9A-Z_.\-]+)+$') -re_launchpad = re.compile(r'^launchpad\.net/([a-z0-9A-Z_.\-]+(/[a-z0-9A-Z_.\-]+)?|~[a-z0-9A-Z_.\-]+/(\+junk|[a-z0-9A-Z_.\-]+)/[a-z0-9A-Z_.\-]+)(/[a-z0-9A-Z_.\-/]+)?$') - -def vc_to_web(path): - if re_bitbucket.match(path): - m = re_bitbucket.match(path) - check_url = 'http://' + m.group(1) + '/?cmd=heads' - web = 'http://' + m.group(1) + '/' - elif re_github.match(path): - m = re_github_web.match(path) - check_url = 'https://raw.github.com/' + m.group(1) + '/' + m.group(2) + '/master/' - web = 'http://github.com/' + m.group(1) + '/' + m.group(2) + '/' - elif re_googlecode.match(path): - m = re_googlecode.match(path) - check_url = 'http://'+path - if not m.group(2): # append / after bare '/hg' or '/git' - check_url += '/' - web = 'http://code.google.com/p/' + path[:path.index('.')] - elif re_launchpad.match(path): - check_url = web = 'https://'+path - else: - return False, False - return web, check_url - -re_bitbucket_web = re.compile(r'bitbucket\.org/([a-z0-9A-Z_.\-]+)/([a-z0-9A-Z_.\-]+)') -re_googlecode_web = re.compile(r'code.google.com/p/([a-z0-9\-]+)') -re_github_web = re.compile(r'github\.com/([a-z0-9A-Z_.\-]+)/([a-z0-9A-Z_.\-]+)') -re_launchpad_web = re.compile(r'launchpad\.net/([a-z0-9A-Z_.\-]+(/[a-z0-9A-Z_.\-]+)?|~[a-z0-9A-Z_.\-]+/(\+junk|[a-z0-9A-Z_.\-]+)/[a-z0-9A-Z_.\-]+)(/[a-z0-9A-Z_.\-/]+)?') -re_striphttp = re.compile(r'https?://(www\.)?') - -def find_googlecode_vcs(path): - # Perform http request to path/hg or path/git to check if they're - # using mercurial or git. Otherwise, assume svn. - for vcs in ['git', 'hg']: - try: - response = urlfetch.fetch('http://'+path+vcs, deadline=1) - if response.status_code == 200: - return vcs - except: pass - return 'svn' - -def web_to_vc(url): - url = re_striphttp.sub('', url) - m = re_bitbucket_web.match(url) - if m: - return 'bitbucket.org/'+m.group(1)+'/'+m.group(2) - m = re_github_web.match(url) - if m: - return 'github.com/'+m.group(1)+'/'+m.group(2) - m = re_googlecode_web.match(url) - if m: - path = m.group(1)+'.googlecode.com/' - vcs = find_googlecode_vcs(path) - return path + vcs - m = re_launchpad_web.match(url) - if m: - return m.group(0) - return False - -MaxPathLength = 100 -CacheTimeout = 3600 - -class PackagePage(webapp.RequestHandler): - def get(self): - if self.request.get('fmt') == 'json': - return self.json() - - html = memcache.get('view-package') - if not html: - tdata = {} - - q = Package.all().filter('week_count >', 0) - q.order('-week_count') - tdata['by_week_count'] = q.fetch(50) - - q = Package.all() - q.order('-last_install') - tdata['by_time'] = q.fetch(20) - - q = Package.all() - q.order('-count') - tdata['by_count'] = q.fetch(100) - - path = os.path.join(os.path.dirname(__file__), 'package.html') - html = template.render(path, tdata) - memcache.set('view-package', html, time=CacheTimeout) - - self.response.headers['Content-Type'] = 'text/html; charset=utf-8' - self.response.out.write(html) - - def json(self): - json = memcache.get('view-package-json') - if not json: - q = Package.all() - s = '{"packages": [' - sep = '' - for r in q: - s += '%s\n\t{"path": "%s", "last_install": "%s", "count": "%s"}' % (sep, r.path, r.last_install, r.count) - sep = ',' - s += '\n]}\n' - json = s - memcache.set('view-package-json', json, time=CacheTimeout) - self.response.set_status(200) - self.response.headers['Content-Type'] = 'text/plain; charset=utf-8' - self.response.out.write(json) - - def can_get_url(self, url): - try: - urllib2.urlopen(urllib2.Request(url)) - return True - except: - return False - - def is_valid_package_path(self, path): - return (re_bitbucket.match(path) or - re_googlecode.match(path) or - re_github.match(path) or - re_launchpad.match(path)) - - def record_pkg(self, path): - # sanity check string - if not path or len(path) > MaxPathLength or not self.is_valid_package_path(path): - return False - - # look in datastore - key = 'pkg-' + path - p = Package.get_by_key_name(key) - if p is None: - # not in datastore - verify URL before creating - web, check_url = vc_to_web(path) - if not web: - logging.error('unrecognized path: %s', path) - return False - if not self.can_get_url(check_url): - logging.error('cannot get %s', check_url) - return False - p = Package(key_name = key, path = path, count = 0, web_url = web) - - if auth(self.request): - # builder updating package metadata - p.info = self.request.get('info') - p.ok = self.request.get('ok') == "true" - if p.ok: - p.last_ok = datetime.datetime.utcnow() - else: - # goinstall reporting an install - p.inc() - p.last_install = datetime.datetime.utcnow() - - # update package object - p.put() - return True - - def post(self): - path = self.request.get('path') - ok = db.run_in_transaction(self.record_pkg, path) - if ok: - self.response.set_status(200) - self.response.out.write('ok') - else: - logging.error('invalid path in post: %s', path) - self.response.set_status(500) - self.response.out.write('not ok') - -class ProjectPage(webapp.RequestHandler): - - def get(self): - admin = users.is_current_user_admin() - if self.request.path == "/project/login": - self.redirect(users.create_login_url("/project")) - elif self.request.path == "/project/logout": - self.redirect(users.create_logout_url("/project")) - elif self.request.path == "/project/edit" and admin: - self.edit() - elif self.request.path == "/project/assoc" and admin: - self.assoc() - else: - self.list() - - def assoc(self): - projects = Project.all() - for p in projects: - if p.package: - continue - path = web_to_vc(p.web_url) - if not path: - continue - pkg = Package.get_by_key_name("pkg-"+path) - if not pkg: - self.response.out.write('no: %s %s<br>' % (p.web_url, path)) - continue - p.package = pkg - p.put() - self.response.out.write('yes: %s %s<br>' % (p.web_url, path)) - - def post(self): - if self.request.path == "/project/edit": - self.edit(True) - else: - data = dict(map(lambda x: (x, self.request.get(x)), ["name","descr","web_url"])) - if reduce(lambda x, y: x or not y, data.values(), False): - data["submitMsg"] = "You must complete all the fields." - self.list(data) - return - p = Project.get_by_key_name("proj-"+data["name"]) - if p is not None: - data["submitMsg"] = "A project by this name already exists." - self.list(data) - return - p = Project(key_name="proj-"+data["name"], **data) - p.put() - - path = os.path.join(os.path.dirname(__file__), 'project-notify.txt') - mail.send_mail( - sender=const.mail_from, - to=const.mail_submit_to, - subject=const.mail_submit_subject, - body=template.render(path, {'project': p})) - - self.list({"submitMsg": "Your project has been submitted."}) - - def list(self, additional_data={}): - cache_key = 'view-project-data' - tag = self.request.get('tag', None) - if tag: - cache_key += '-'+tag - data = memcache.get(cache_key) - admin = users.is_current_user_admin() - if admin or not data: - projects = Project.all().order('category').order('name') - if not admin: - projects = projects.filter('approved =', True) - projects = list(projects) - - tags = sets.Set() - for p in projects: - for t in p.tags: - tags.add(t) - - if tag: - projects = filter(lambda x: tag in x.tags, projects) - - data = {} - data['tag'] = tag - data['tags'] = tags - data['projects'] = projects - data['admin']= admin - if not admin: - memcache.set(cache_key, data, time=CacheTimeout) - - for k, v in additional_data.items(): - data[k] = v - - self.response.headers['Content-Type'] = 'text/html; charset=utf-8' - path = os.path.join(os.path.dirname(__file__), 'project.html') - self.response.out.write(template.render(path, data)) - - def edit(self, save=False): - if save: - name = self.request.get("orig_name") - else: - name = self.request.get("name") - - p = Project.get_by_key_name("proj-"+name) - if not p: - self.response.out.write("Couldn't find that Project.") - return - - if save: - if self.request.get("do") == "Delete": - p.delete() - else: - pkg_name = self.request.get("package", None) - if pkg_name: - pkg = Package.get_by_key_name("pkg-"+pkg_name) - if pkg: - p.package = pkg.key() - for f in ['name', 'descr', 'web_url', 'category']: - setattr(p, f, self.request.get(f, None)) - p.approved = self.request.get("approved") == "1" - p.tags = filter(lambda x: x, self.request.get("tags", "").split(",")) - p.put() - memcache.delete('view-project-data') - self.redirect('/project') - return - - # get all project categories and tags - cats, tags = sets.Set(), sets.Set() - for r in Project.all(): - cats.add(r.category) - for t in r.tags: - tags.add(t) - - self.response.headers['Content-Type'] = 'text/html; charset=utf-8' - path = os.path.join(os.path.dirname(__file__), 'project-edit.html') - self.response.out.write(template.render(path, { - "taglist": tags, "catlist": cats, "p": p, "tags": ",".join(p.tags) })) - - def redirect(self, url): - self.response.set_status(302) - self.response.headers.add_header("Location", url) - -def main(): - app = webapp.WSGIApplication([ - ('/package', PackagePage), - ('/package/daily', PackageDaily), - ('/project.*', ProjectPage), - ], debug=True) - run_wsgi_app(app) - -if __name__ == '__main__': - main() diff --git a/misc/dashboard/godashboard/project-edit.html b/misc/dashboard/godashboard/project-edit.html index ce18fb3fb..8ca5b22a3 100644 --- a/misc/dashboard/godashboard/project-edit.html +++ b/misc/dashboard/godashboard/project-edit.html @@ -19,8 +19,6 @@ Tags: (comma-separated)<br/> <input type="text" id="tags" name="tags" value="{{tags}}"><br/> Web URL:<br/> <input type="text" name="web_url" value="{{p.web_url|escape}}"><br/> -Package URL: (to link to a goinstall'd package)<br/> -<input type="text" name="package" value="{{p.package.path|escape}}"><br/> Approved: <input type="checkbox" name="approved" value="1" {% if p.approved %}checked{% endif %}><br/> <br/> <input type="submit" name="do" value="Save"> diff --git a/misc/dashboard/godashboard/project.html b/misc/dashboard/godashboard/project.html index 4fe1741c6..2d2429461 100644 --- a/misc/dashboard/godashboard/project.html +++ b/misc/dashboard/godashboard/project.html @@ -1,23 +1,13 @@ <!DOCTYPE HTML> <html> <head> - <title>Projects - Go Dashboard</title> + <title>Go Projects</title> <link rel="stylesheet" type="text/css" href="static/style.css"> - <style> - .unapproved a.name { color: red } - .tag { font-size: 0.8em; color: #666 } - </style> </head> <body> - <ul class="menu"> - <li><a href="/">Build Status</a></li> - <li><a href="/package">Packages</a></li> - <li>Projects</li> - <li><a href="http://golang.org/">golang.org</a></li> - </ul> - - <h1>Go Dashboard</h1> + <ul class="menu"><li><a href="http://golang.org/">golang.org</a></li></ul> + <h1>Go Projects</h1> <p> These are external projects and not endorsed or supported by the Go project. @@ -80,6 +70,5 @@ {% endfor %} </ul> - </body> </html> diff --git a/misc/dashboard/godashboard/project.py b/misc/dashboard/godashboard/project.py new file mode 100644 index 000000000..b13599d5e --- /dev/null +++ b/misc/dashboard/godashboard/project.py @@ -0,0 +1,151 @@ +# Copyright 2010 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +from google.appengine.api import mail +from google.appengine.api import memcache +from google.appengine.api import users +from google.appengine.ext import db +from google.appengine.ext import webapp +from google.appengine.ext.webapp import template +from google.appengine.ext.webapp.util import run_wsgi_app +import os +import sets + +# local imports +import toutf8 +import const + +template.register_template_library('toutf8') + +class Project(db.Model): + name = db.StringProperty(indexed=True) + descr = db.StringProperty() + web_url = db.StringProperty() + category = db.StringProperty(indexed=True) + tags = db.ListProperty(str) + approved = db.BooleanProperty(indexed=True) + +CacheTimeout = 3600 + +class ProjectPage(webapp.RequestHandler): + + def get(self): + admin = users.is_current_user_admin() + if self.request.path == "/project/login": + self.redirect(users.create_login_url("/project")) + elif self.request.path == "/project/edit" and admin: + self.edit() + else: + self.list() + + def post(self): + if self.request.path == "/project/edit": + self.edit(True) + else: + data = dict(map(lambda x: (x, self.request.get(x)), ["name","descr","web_url"])) + if reduce(lambda x, y: x or not y, data.values(), False): + data["submitMsg"] = "You must complete all the fields." + self.list(data) + return + p = Project.get_by_key_name("proj-"+data["name"]) + if p is not None: + data["submitMsg"] = "A project by this name already exists." + self.list(data) + return + p = Project(key_name="proj-"+data["name"], **data) + p.put() + + path = os.path.join(os.path.dirname(__file__), 'project-notify.txt') + mail.send_mail( + sender=const.mail_from, + to=const.mail_submit_to, + subject=const.mail_submit_subject, + body=template.render(path, {'project': p})) + + self.list({"submitMsg": "Your project has been submitted."}) + + def list(self, additional_data={}): + cache_key = 'view-project-data' + tag = self.request.get('tag', None) + if tag: + cache_key += '-'+tag + data = memcache.get(cache_key) + admin = users.is_current_user_admin() + if admin or not data: + projects = Project.all().order('category').order('name') + if not admin: + projects = projects.filter('approved =', True) + projects = list(projects) + + tags = sets.Set() + for p in projects: + for t in p.tags: + tags.add(t) + + if tag: + projects = filter(lambda x: tag in x.tags, projects) + + data = {} + data['tag'] = tag + data['tags'] = tags + data['projects'] = projects + data['admin']= admin + if not admin: + memcache.set(cache_key, data, time=CacheTimeout) + + for k, v in additional_data.items(): + data[k] = v + + self.response.headers['Content-Type'] = 'text/html; charset=utf-8' + path = os.path.join(os.path.dirname(__file__), 'project.html') + self.response.out.write(template.render(path, data)) + + def edit(self, save=False): + if save: + name = self.request.get("orig_name") + else: + name = self.request.get("name") + + p = Project.get_by_key_name("proj-"+name) + if not p: + self.response.out.write("Couldn't find that Project.") + return + + if save: + if self.request.get("do") == "Delete": + p.delete() + else: + for f in ['name', 'descr', 'web_url', 'category']: + setattr(p, f, self.request.get(f, None)) + p.approved = self.request.get("approved") == "1" + p.tags = filter(lambda x: x, self.request.get("tags", "").split(",")) + p.put() + memcache.delete('view-project-data') + self.redirect('/project') + return + + # get all project categories and tags + cats, tags = sets.Set(), sets.Set() + for r in Project.all(): + cats.add(r.category) + for t in r.tags: + tags.add(t) + + self.response.headers['Content-Type'] = 'text/html; charset=utf-8' + path = os.path.join(os.path.dirname(__file__), 'project-edit.html') + self.response.out.write(template.render(path, { + "taglist": tags, "catlist": cats, "p": p, "tags": ",".join(p.tags) })) + + def redirect(self, url): + self.response.set_status(302) + self.response.headers.add_header("Location", url) + +def main(): + app = webapp.WSGIApplication([ + ('/.*', ProjectPage), + ], debug=True) + run_wsgi_app(app) + +if __name__ == '__main__': + main() diff --git a/misc/dashboard/godashboard/static/style.css b/misc/dashboard/godashboard/static/style.css index 0ce583a54..ad7038c3d 100644 --- a/misc/dashboard/godashboard/static/style.css +++ b/misc/dashboard/godashboard/static/style.css @@ -127,3 +127,10 @@ td.time { .notice a { color: #FF6; } +.unapproved a.name { + color: red; +} +.tag { + font-size: 0.8em; + color: #666; +} diff --git a/misc/dashboard/googlecode_upload.py b/misc/dashboard/googlecode_upload.py deleted file mode 100755 index e87db884a..000000000 --- a/misc/dashboard/googlecode_upload.py +++ /dev/null @@ -1,248 +0,0 @@ -#!/usr/bin/env python2 -# -# Copyright 2006, 2007 Google Inc. All Rights Reserved. -# Author: danderson@google.com (David Anderson) -# -# Script for uploading files to a Google Code project. -# -# This is intended to be both a useful script for people who want to -# streamline project uploads and a reference implementation for -# uploading files to Google Code projects. -# -# To upload a file to Google Code, you need to provide a path to the -# file on your local machine, a small summary of what the file is, a -# project name, and a valid account that is a member or owner of that -# project. You can optionally provide a list of labels that apply to -# the file. The file will be uploaded under the same name that it has -# in your local filesystem (that is, the "basename" or last path -# component). Run the script with '--help' to get the exact syntax -# and available options. -# -# Note that the upload script requests that you enter your -# googlecode.com password. This is NOT your Gmail account password! -# This is the password you use on googlecode.com for committing to -# Subversion and uploading files. You can find your password by going -# to http://code.google.com/hosting/settings when logged in with your -# Gmail account. If you have already committed to your project's -# Subversion repository, the script will automatically retrieve your -# credentials from there (unless disabled, see the output of '--help' -# for details). -# -# If you are looking at this script as a reference for implementing -# your own Google Code file uploader, then you should take a look at -# the upload() function, which is the meat of the uploader. You -# basically need to build a multipart/form-data POST request with the -# right fields and send it to https://PROJECT.googlecode.com/files . -# Authenticate the request using HTTP Basic authentication, as is -# shown below. -# -# Licensed under the terms of the Apache Software License 2.0: -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Questions, comments, feature requests and patches are most welcome. -# Please direct all of these to the Google Code users group: -# http://groups.google.com/group/google-code-hosting - -"""Google Code file uploader script. -""" - -__author__ = 'danderson@google.com (David Anderson)' - -import httplib -import os.path -import optparse -import getpass -import base64 -import sys - - -def upload(file, project_name, user_name, password, summary, labels=None): - """Upload a file to a Google Code project's file server. - - Args: - file: The local path to the file. - project_name: The name of your project on Google Code. - user_name: Your Google account name. - password: The googlecode.com password for your account. - Note that this is NOT your global Google Account password! - summary: A small description for the file. - labels: an optional list of label strings with which to tag the file. - - Returns: a tuple: - http_status: 201 if the upload succeeded, something else if an - error occurred. - http_reason: The human-readable string associated with http_status - file_url: If the upload succeeded, the URL of the file on Google - Code, None otherwise. - """ - # The login is the user part of user@gmail.com. If the login provided - # is in the full user@domain form, strip it down. - if user_name.endswith('@gmail.com'): - user_name = user_name[:user_name.index('@gmail.com')] - - form_fields = [('summary', summary)] - if labels is not None: - form_fields.extend([('label', l.strip()) for l in labels]) - - content_type, body = encode_upload_request(form_fields, file) - - upload_host = '%s.googlecode.com' % project_name - upload_uri = '/files' - auth_token = base64.b64encode('%s:%s'% (user_name, password)) - headers = { - 'Authorization': 'Basic %s' % auth_token, - 'User-Agent': 'Googlecode.com uploader v0.9.4', - 'Content-Type': content_type, - } - - server = httplib.HTTPSConnection(upload_host) - server.request('POST', upload_uri, body, headers) - resp = server.getresponse() - server.close() - - if resp.status == 201: - location = resp.getheader('Location', None) - else: - location = None - return resp.status, resp.reason, location - - -def encode_upload_request(fields, file_path): - """Encode the given fields and file into a multipart form body. - - fields is a sequence of (name, value) pairs. file is the path of - the file to upload. The file will be uploaded to Google Code with - the same file name. - - Returns: (content_type, body) ready for httplib.HTTP instance - """ - BOUNDARY = '----------Googlecode_boundary_reindeer_flotilla' - CRLF = '\r\n' - - body = [] - - # Add the metadata about the upload first - for key, value in fields: - body.extend( - ['--' + BOUNDARY, - 'Content-Disposition: form-data; name="%s"' % key, - '', - value, - ]) - - # Now add the file itself - file_name = os.path.basename(file_path) - f = open(file_path, 'rb') - file_content = f.read() - f.close() - - body.extend( - ['--' + BOUNDARY, - 'Content-Disposition: form-data; name="filename"; filename="%s"' - % file_name, - # The upload server determines the mime-type, no need to set it. - 'Content-Type: application/octet-stream', - '', - file_content, - ]) - - # Finalize the form body - body.extend(['--' + BOUNDARY + '--', '']) - - return 'multipart/form-data; boundary=%s' % BOUNDARY, CRLF.join(body) - - -def upload_find_auth(file_path, project_name, summary, labels=None, - user_name=None, password=None, tries=3): - """Find credentials and upload a file to a Google Code project's file server. - - file_path, project_name, summary, and labels are passed as-is to upload. - - Args: - file_path: The local path to the file. - project_name: The name of your project on Google Code. - summary: A small description for the file. - labels: an optional list of label strings with which to tag the file. - config_dir: Path to Subversion configuration directory, 'none', or None. - user_name: Your Google account name. - tries: How many attempts to make. - """ - - while tries > 0: - if user_name is None: - # Read username if not specified or loaded from svn config, or on - # subsequent tries. - sys.stdout.write('Please enter your googlecode.com username: ') - sys.stdout.flush() - user_name = sys.stdin.readline().rstrip() - if password is None: - # Read password if not loaded from svn config, or on subsequent tries. - print 'Please enter your googlecode.com password.' - print '** Note that this is NOT your Gmail account password! **' - print 'It is the password you use to access Subversion repositories,' - print 'and can be found here: http://code.google.com/hosting/settings' - password = getpass.getpass() - - status, reason, url = upload(file_path, project_name, user_name, password, - summary, labels) - # Returns 403 Forbidden instead of 401 Unauthorized for bad - # credentials as of 2007-07-17. - if status in [httplib.FORBIDDEN, httplib.UNAUTHORIZED]: - # Rest for another try. - user_name = password = None - tries = tries - 1 - else: - # We're done. - break - - return status, reason, url - - -def main(): - parser = optparse.OptionParser(usage='googlecode-upload.py -s SUMMARY ' - '-p PROJECT [options] FILE') - parser.add_option('-s', '--summary', dest='summary', - help='Short description of the file') - parser.add_option('-p', '--project', dest='project', - help='Google Code project name') - parser.add_option('-u', '--user', dest='user', - help='Your Google Code username') - parser.add_option('-w', '--password', dest='password', - help='Your Google Code password') - parser.add_option('-l', '--labels', dest='labels', - help='An optional list of comma-separated labels to attach ' - 'to the file') - - options, args = parser.parse_args() - - if not options.summary: - parser.error('File summary is missing.') - elif not options.project: - parser.error('Project name is missing.') - elif len(args) < 1: - parser.error('File to upload not provided.') - elif len(args) > 1: - parser.error('Only one file may be specified.') - - file_path = args[0] - - if options.labels: - labels = options.labels.split(',') - else: - labels = None - - status, reason, url = upload_find_auth(file_path, options.project, - options.summary, labels, - options.user, options.password) - if url: - print 'The file was uploaded successfully.' - print 'URL: %s' % url - return 0 - else: - print 'An error occurred. Your file was not uploaded.' - print 'Google Code upload server said: %s (%s)' % (reason, status) - return 1 - - -if __name__ == '__main__': - sys.exit(main()) diff --git a/misc/dist/bindist.go b/misc/dist/bindist.go index f307d9b76..b03fd706d 100644 --- a/misc/dist/bindist.go +++ b/misc/dist/bindist.go @@ -7,7 +7,11 @@ package main import ( + "archive/tar" + "archive/zip" + "bufio" "bytes" + "compress/gzip" "encoding/base64" "errors" "flag" @@ -20,12 +24,15 @@ import ( "os" "os/exec" "path/filepath" + "runtime" "strings" ) var ( - tag = flag.String("tag", "weekly", "mercurial tag to check out") - repo = flag.String("repo", "https://code.google.com/p/go", "repo URL") + tag = flag.String("tag", "weekly", "mercurial tag to check out") + repo = flag.String("repo", "https://code.google.com/p/go", "repo URL") + verbose = flag.Bool("v", false, "verbose output") + upload = flag.Bool("upload", true, "upload resulting files to Google Code") username, password string // for Google Code upload ) @@ -35,6 +42,13 @@ const ( uploadURL = "https://go.googlecode.com/files" ) +var preBuildCleanFiles = []string{ + "src/cmd/cov", + "src/cmd/prof", + "src/pkg/exp", + "src/pkg/old", +} + var cleanFiles = []string{ ".hg", ".hgtags", @@ -42,6 +56,11 @@ var cleanFiles = []string{ "VERSION.cache", } +var sourceCleanFiles = []string{ + "bin", + "pkg", +} + func main() { flag.Usage = func() { fmt.Fprintf(os.Stderr, "usage: %s [flags] targets...\n", os.Args[0]) @@ -52,14 +71,28 @@ func main() { if flag.NArg() == 0 { flag.Usage() } - readCredentials() + if runtime.GOOS == "windows" { + checkWindowsDeps() + } + + if *upload { + if err := readCredentials(); err != nil { + log.Println("readCredentials:", err) + } + } for _, targ := range flag.Args() { - p := strings.SplitN(targ, "-", 2) - if len(p) != 2 { - log.Println("Ignoring unrecognized target:", targ) - continue + var b Build + if targ == "source" { + b.Source = true + } else { + p := strings.SplitN(targ, "-", 2) + if len(p) != 2 { + log.Println("Ignoring unrecognized target:", targ) + continue + } + b.OS = p[0] + b.Arch = p[1] } - b := Build{OS: p[0], Arch: p[1]} if err := b.Do(); err != nil { log.Printf("%s: %v", targ, err) } @@ -67,9 +100,10 @@ func main() { } type Build struct { - OS string - Arch string - root string + Source bool // if true, OS and Arch must be empty + OS string + Arch string + root string } func (b *Build) Do() error { @@ -90,41 +124,80 @@ func (b *Build) Do() error { return err } - // Build. - _, err = b.run(filepath.Join(work, "go/src"), "bash", "make.bash") + // Remove exp and old packages. + if err := b.clean(preBuildCleanFiles); err != nil { + return err + } + + src := filepath.Join(b.root, "src") + if b.Source { + if runtime.GOOS == "windows" { + log.Print("Warning: running make.bash on Windows; source builds are intended to be run on a Unix machine") + } + // Build dist tool only. + _, err = b.run(src, "bash", "make.bash", "--dist-tool") + } else { + // Build. + if b.OS == "windows" { + _, err = b.run(src, "cmd", "/C", "make.bat") + } else { + _, err = b.run(src, "bash", "make.bash") + } + } if err != nil { return err } - // Get version string. - version, err := b.run("", filepath.Join(b.root, "bin/go"), "version") + // Get version strings. + var ( + version string // "weekly.2012-03-04" + fullVersion []byte // "weekly.2012-03-04 9353aa1efdf3" + ) + pat := filepath.Join(b.root, "pkg/tool/*/dist*") // trailing * for .exe + m, err := filepath.Glob(pat) if err != nil { return err } - v := bytes.SplitN(version, []byte(" "), 4) - version = bytes.Join(v[2:], []byte(" ")) + if len(m) == 0 { + return fmt.Errorf("couldn't find dist in %q", pat) + } + fullVersion, err = b.run("", m[0], "version") + if err != nil { + return err + } + fullVersion = bytes.TrimSpace(fullVersion) + v := bytes.SplitN(fullVersion, []byte(" "), 2) + version = string(v[0]) // Write VERSION file. - err = ioutil.WriteFile(filepath.Join(b.root, "VERSION"), version, 0644) + err = ioutil.WriteFile(filepath.Join(b.root, "VERSION"), fullVersion, 0644) if err != nil { return err } // Clean goroot. - for _, name := range cleanFiles { - err = os.RemoveAll(filepath.Join(b.root, name)) - if err != nil { + if err := b.clean(cleanFiles); err != nil { + return err + } + if b.Source { + if err := b.clean(sourceCleanFiles); err != nil { return err } } // Create packages. - targ := fmt.Sprintf("go.%s.%s-%s", v[2], b.OS, b.Arch) + base := fmt.Sprintf("go.%s.%s-%s", version, b.OS, b.Arch) + var targs []string switch b.OS { - case "linux", "freebsd": + case "linux", "freebsd", "": // build tarball + targ := base + if b.Source { + targ = fmt.Sprintf("go.%s.src", version) + } targ += ".tar.gz" - _, err = b.run("", "tar", "czf", targ, "-C", work, "go") + err = makeTar(targ, work) + targs = append(targs, targ) case "darwin": // arrange work so it's laid out as the dest filesystem etc := filepath.Join(b.root, "misc/dist/darwin/etc") @@ -149,7 +222,7 @@ func (b *Build) Do() error { return errors.New("couldn't find PackageMaker") } } - targ += ".pkg" + targ := base + ".pkg" scripts := filepath.Join(work, "usr/local/go/misc/dist/darwin/scripts") _, err = b.run("", pm, "-v", "-r", work, @@ -158,19 +231,88 @@ func (b *Build) Do() error { "--id", "com.googlecode.go", "--title", "Go", "--version", "1.0", - "--target", "10.5") + "--target", "10.6") + targs = append(targs, targ) + case "windows": + // Create ZIP file. + zip := filepath.Join(work, base+".zip") + err = makeZip(zip, work) + // Copy zip to target file. + targ := base + ".zip" + err = cp(targ, zip) + if err != nil { + return err + } + targs = append(targs, targ) + + // Create MSI installer. + win := filepath.Join(b.root, "misc/dist/windows") + installer := filepath.Join(win, "installer.wxs") + appfiles := filepath.Join(work, "AppFiles.wxs") + msi := filepath.Join(work, "installer.msi") + // Gather files. + _, err = b.run(work, "heat", "dir", "go", + "-nologo", + "-gg", "-g1", "-srd", "-sfrag", + "-cg", "AppFiles", + "-template", "fragment", + "-dr", "INSTALLDIR", + "-var", "var.SourceDir", + "-out", appfiles) + if err != nil { + return err + } + // Build package. + _, err = b.run(work, "candle", + "-nologo", + "-dVersion="+version, + "-dArch="+b.Arch, + "-dSourceDir=go", + installer, appfiles) + if err != nil { + return err + } + appfiles = filepath.Join(work, "AppFiles.wixobj") + installer = filepath.Join(work, "installer.wixobj") + _, err = b.run(win, "light", + "-nologo", + "-ext", "WixUIExtension", + "-ext", "WixUtilExtension", + installer, appfiles, + "-o", msi) + if err != nil { + return err + } + // Copy installer to target file. + targ = base + ".msi" + err = cp(targ, msi) + targs = append(targs, targ) } - if err == nil && password != "" { - err = b.upload(string(v[2]), targ) + if err == nil && *upload { + for _, targ := range targs { + err = b.upload(version, targ) + if err != nil { + return err + } + } } return err } func (b *Build) run(dir, name string, args ...string) ([]byte, error) { buf := new(bytes.Buffer) - cmd := exec.Command(name, args...) - cmd.Stdout = buf - cmd.Stderr = buf + absName, err := lookPath(name) + if err != nil { + return nil, err + } + cmd := exec.Command(absName, args...) + var output io.Writer = buf + if *verbose { + log.Printf("Running %q %q", absName, args) + output = io.MultiWriter(buf, os.Stdout) + } + cmd.Stdout = output + cmd.Stderr = output cmd.Dir = dir cmd.Env = b.env() if err := cmd.Run(); err != nil { @@ -199,20 +341,24 @@ func (b *Build) env() []string { } } } + final := "/usr/local/go" + if b.OS == "windows" { + final = `c:\go` + } env = append(env, "GOARCH="+b.Arch, "GOHOSTARCH="+b.Arch, "GOHOSTOS="+b.OS, "GOOS="+b.OS, "GOROOT="+b.root, - "GOROOT_FINAL=/usr/local/go", + "GOROOT_FINAL="+final, ) return env } func (b *Build) upload(version string, filename string) error { // Prepare upload metadata. - labels := []string{"Arch-" + b.Arch} + var labels []string os_, arch := b.OS, b.Arch switch b.Arch { case "386": @@ -220,6 +366,9 @@ func (b *Build) upload(version string, filename string) error { case "amd64": arch = "64-bit" } + if arch != "" { + labels = append(labels, "Arch-"+b.Arch) + } switch b.OS { case "linux": os_ = "Linux" @@ -230,8 +379,25 @@ func (b *Build) upload(version string, filename string) error { case "darwin": os_ = "Mac OS X" labels = append(labels, "Type-Installer", "OpSys-OSX") + case "windows": + os_ = "Windows" + labels = append(labels, "OpSys-Windows") } summary := fmt.Sprintf("Go %s %s (%s)", version, os_, arch) + if b.OS == "windows" { + switch { + case strings.HasSuffix(filename, ".msi"): + labels = append(labels, "Type-Installer") + summary += " MSI installer" + case strings.HasSuffix(filename, ".zip"): + labels = append(labels, "Type-Archive") + summary += " ZIP archive" + } + } + if b.Source { + labels = append(labels, "Type-Source") + summary = fmt.Sprintf("Go %s (source only)", version) + } // Open file to upload. f, err := os.Open(filename) @@ -285,20 +451,311 @@ func (b *Build) upload(version string, filename string) error { return nil } +func (b *Build) clean(files []string) error { + for _, name := range files { + err := os.RemoveAll(filepath.Join(b.root, name)) + if err != nil { + return err + } + } + return nil +} + func exists(path string) bool { _, err := os.Stat(path) return err == nil } -func readCredentials() { - name := filepath.Join(os.Getenv("HOME"), ".gobuildkey") - c, err := ioutil.ReadFile(name) +func readCredentials() error { + name := os.Getenv("HOME") + if runtime.GOOS == "windows" { + name = os.Getenv("HOMEDRIVE") + os.Getenv("HOMEPATH") + } + name = filepath.Join(name, ".gobuildkey") + f, err := os.Open(name) + if err != nil { + return err + } + defer f.Close() + r := bufio.NewReader(f) + for i := 0; i < 3; i++ { + b, _, err := r.ReadLine() + if err != nil { + return err + } + b = bytes.TrimSpace(b) + switch i { + case 1: + username = string(b) + case 2: + password = string(b) + } + } + return nil +} + +func cp(dst, src string) error { + sf, err := os.Open(src) + if err != nil { + return err + } + defer sf.Close() + df, err := os.Create(dst) + if err != nil { + return err + } + defer df.Close() + _, err = io.Copy(df, sf) + return err +} + +func makeTar(targ, workdir string) error { + f, err := os.Create(targ) + if err != nil { + return err + } + zout := gzip.NewWriter(f) + tw := tar.NewWriter(zout) + + filepath.Walk(workdir, filepath.WalkFunc(func(path string, fi os.FileInfo, err error) error { + if !strings.HasPrefix(path, workdir) { + log.Panicf("walked filename %q doesn't begin with workdir %q", path, workdir) + } + name := path[len(workdir):] + + // Chop of any leading / from filename, leftover from removing workdir. + if strings.HasPrefix(name, "/") { + name = name[1:] + } + // Don't include things outside of the go subdirectory (for instance, + // the zip file that we're currently writing here.) + if !strings.HasPrefix(name, "go/") { + return nil + } + if *verbose { + log.Printf("adding to tar: %s", name) + } + if fi.IsDir() { + return nil + } + hdr, err := tarFileInfoHeader(fi, path) + if err != nil { + return err + } + hdr.Name = name + hdr.Uname = "root" + hdr.Gname = "root" + hdr.Uid = 0 + hdr.Gid = 0 + + // Force permissions to 0755 for executables, 0644 for everything else. + if fi.Mode().Perm()&0111 != 0 { + hdr.Mode = hdr.Mode&^0777 | 0755 + } else { + hdr.Mode = hdr.Mode&^0777 | 0644 + } + + err = tw.WriteHeader(hdr) + if err != nil { + return fmt.Errorf("Error writing file %q: %v", name, err) + } + r, err := os.Open(path) + if err != nil { + return err + } + defer r.Close() + _, err = io.Copy(tw, r) + return err + })) + + if err := tw.Close(); err != nil { + return err + } + if err := zout.Close(); err != nil { + return err + } + return f.Close() +} + +func makeZip(targ, workdir string) error { + f, err := os.Create(targ) if err != nil { - log.Println("readCredentials:", err) + return err + } + zw := zip.NewWriter(f) + + filepath.Walk(workdir, filepath.WalkFunc(func(path string, fi os.FileInfo, err error) error { + if fi.IsDir() { + return nil + } + if !strings.HasPrefix(path, workdir) { + log.Panicf("walked filename %q doesn't begin with workdir %q", path, workdir) + } + name := path[len(workdir):] + + // Convert to Unix-style named paths, as that's the + // type of zip file that archive/zip creates. + name = strings.Replace(name, "\\", "/", -1) + // Chop of any leading / from filename, leftover from removing workdir. + if strings.HasPrefix(name, "/") { + name = name[1:] + } + // Don't include things outside of the go subdirectory (for instance, + // the zip file that we're currently writing here.) + if !strings.HasPrefix(name, "go/") { + return nil + } + if *verbose { + log.Printf("adding to zip: %s", name) + } + fh, err := zip.FileInfoHeader(fi) + if err != nil { + return err + } + fh.Name = name + fh.Method = zip.Deflate + w, err := zw.CreateHeader(fh) + if err != nil { + return err + } + r, err := os.Open(path) + if err != nil { + return err + } + defer r.Close() + _, err = io.Copy(w, r) + return err + })) + + if err := zw.Close(); err != nil { + return err + } + return f.Close() +} + +type tool struct { + name string + commonDirs []string +} + +var wixTool = tool{ + "http://wix.sourceforge.net/, version 3.5", + []string{`C:\Program Files\Windows Installer XML v3.5\bin`, + `C:\Program Files (x86)\Windows Installer XML v3.5\bin`}, +} + +var hgTool = tool{ + "http://mercurial.selenic.com/wiki/WindowsInstall", + []string{`C:\Program Files\Mercurial`, + `C:\Program Files (x86)\Mercurial`, + }, +} + +var gccTool = tool{ + "Mingw gcc; http://sourceforge.net/projects/mingw/files/Installer/mingw-get-inst/", + []string{`C:\Mingw\bin`}, +} + +var windowsDeps = map[string]tool{ + "gcc": gccTool, + "heat": wixTool, + "candle": wixTool, + "light": wixTool, + "cmd": {"Windows cmd.exe", nil}, + "hg": hgTool, +} + +func checkWindowsDeps() { + for prog, help := range windowsDeps { + absPath, err := lookPath(prog) + if err != nil { + log.Fatalf("Failed to find necessary binary %q in path or common locations; %s", prog, help) + } + if *verbose { + log.Printf("found windows dep %s at %s", prog, absPath) + } + } +} + +func lookPath(prog string) (absPath string, err error) { + absPath, err = exec.LookPath(prog) + if err == nil { + return + } + t, ok := windowsDeps[prog] + if !ok { return } - v := bytes.Split(c, []byte("\n")) - if len(v) >= 3 { - username, password = string(v[1]), string(v[2]) + for _, dir := range t.commonDirs { + for _, ext := range []string{"exe", "bat"} { + absPath = filepath.Join(dir, prog+"."+ext) + if _, err1 := os.Stat(absPath); err1 == nil { + err = nil + os.Setenv("PATH", os.Getenv("PATH")+";"+dir) + return + } + } + } + return +} + +// sysStat, if non-nil, populates h from system-dependent fields of fi. +var sysStat func(fi os.FileInfo, h *tar.Header) error + +// Mode constants from the tar spec. +const ( + c_ISDIR = 040000 + c_ISFIFO = 010000 + c_ISREG = 0100000 + c_ISLNK = 0120000 + c_ISBLK = 060000 + c_ISCHR = 020000 + c_ISSOCK = 0140000 +) + +// tarFileInfoHeader creates a partially-populated Header from an os.FileInfo. +// The filename parameter is used only in the case of symlinks, to call os.Readlink. +// If fi is a symlink but filename is empty, an error is returned. +func tarFileInfoHeader(fi os.FileInfo, filename string) (*tar.Header, error) { + h := &tar.Header{ + Name: fi.Name(), + ModTime: fi.ModTime(), + Mode: int64(fi.Mode().Perm()), // or'd with c_IS* constants later + } + switch { + case fi.Mode()&os.ModeType == 0: + h.Mode |= c_ISREG + h.Typeflag = tar.TypeReg + h.Size = fi.Size() + case fi.IsDir(): + h.Typeflag = tar.TypeDir + h.Mode |= c_ISDIR + case fi.Mode()&os.ModeSymlink != 0: + h.Typeflag = tar.TypeSymlink + h.Mode |= c_ISLNK + if filename == "" { + return h, fmt.Errorf("archive/tar: unable to populate Header.Linkname of symlinks") + } + targ, err := os.Readlink(filename) + if err != nil { + return h, err + } + h.Linkname = targ + case fi.Mode()&os.ModeDevice != 0: + if fi.Mode()&os.ModeCharDevice != 0 { + h.Mode |= c_ISCHR + h.Typeflag = tar.TypeChar + } else { + h.Mode |= c_ISBLK + h.Typeflag = tar.TypeBlock + } + case fi.Mode()&os.ModeSocket != 0: + h.Mode |= c_ISSOCK + default: + return nil, fmt.Errorf("archive/tar: unknown file mode %v", fi.Mode()) + } + if sysStat != nil { + return h, sysStat(fi, h) } + return h, nil } diff --git a/misc/dist/stat_darwin.go b/misc/dist/stat_darwin.go new file mode 100644 index 000000000..eb3f76a27 --- /dev/null +++ b/misc/dist/stat_darwin.go @@ -0,0 +1,32 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build darwin + +package main + +import ( + "archive/tar" + "os" + "syscall" + "time" +) + +func init() { + sysStat = func(fi os.FileInfo, h *tar.Header) error { + sys, ok := fi.Sys().(*syscall.Stat_t) + if !ok { + return nil + } + h.Uid = int(sys.Uid) + h.Gid = int(sys.Gid) + // TODO(bradfitz): populate username & group. os/user + // doesn't cache LookupId lookups, and lacks group + // lookup functions. + h.AccessTime = time.Unix(sys.Atimespec.Unix()) + h.ChangeTime = time.Unix(sys.Ctimespec.Unix()) + // TODO(bradfitz): major/minor device numbers? + return nil + } +} diff --git a/misc/dist/stat_linux.go b/misc/dist/stat_linux.go new file mode 100644 index 000000000..0ddb8a3bf --- /dev/null +++ b/misc/dist/stat_linux.go @@ -0,0 +1,32 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build linux + +package main + +import ( + "archive/tar" + "os" + "syscall" + "time" +) + +func init() { + sysStat = func(fi os.FileInfo, h *tar.Header) error { + sys, ok := fi.Sys().(*syscall.Stat_t) + if !ok { + return nil + } + h.Uid = int(sys.Uid) + h.Gid = int(sys.Gid) + // TODO(bradfitz): populate username & group. os/user + // doesn't cache LookupId lookups, and lacks group + // lookup functions. + h.AccessTime = time.Unix(sys.Atim.Unix()) + h.ChangeTime = time.Unix(sys.Ctim.Unix()) + // TODO(bradfitz): major/minor device numbers? + return nil + } +} diff --git a/misc/dist/windows/README.txt b/misc/dist/windows/README.txt index b2f00d427..0cf828b24 100644 --- a/misc/dist/windows/README.txt +++ b/misc/dist/windows/README.txt @@ -1,34 +1,25 @@ -dist.bat packages the Go toolchain for Windows in both zip
-and installer (msi) format.
-
-Dependencies
-============
-- Windows Installer XML (WiX) toolset: http://wix.sourceforge.net/
-- 7Zip (command line version): http://www.7-zip.org/download.html
-- Mercurial (hg): http://mercurial.selenic.com/
-
-
-Packaging
-=========
-The dependencies must be callable from dist.bat, therefore,
-they'll need to be in/added to the system's search PATH.
-
-The packaging needs to be done from within a tracked Go folder.
-Packages are built by cloning the same version of the source tree
-that the Go tools were built from.
-
-Run dist.bat from a command prompt or click on the batch file.
-
-TODO
----- -- Write a Go program for dist.bat functionality
-- Documentation server shortcut checkbox option
-
-Misc
-----
-WiX box sizes:
- - banner size: 493x58
- - left side of dialog: 164x312
- - full dialog size: 493x312
-
-
+ +Windows build dependencies + +- Mercurial (hg): http://mercurial.selenic.com/ +- MinGW: http://www.mingw.org/ +- Windows Installer XML (WiX) toolset: http://wix.sourceforge.net/ + +Packaging + +The dependencies must be in/added to the system's search PATH. + +Run bindist as normal, eg: + bindist windows-386 + +TODO + +- Documentation server shortcut checkbox option + +Misc + +WiX box sizes: + - banner size: 493x58 + - left side of dialog: 164x312 + - full dialog size: 493x312 + diff --git a/misc/dist/windows/dist.bat b/misc/dist/windows/dist.bat deleted file mode 100644 index 4ae2df58a..000000000 --- a/misc/dist/windows/dist.bat +++ /dev/null @@ -1,61 +0,0 @@ -:: Copyright 2012 The Go Authors. All rights reserved.
-:: Use of this source code is governed by a BSD-style
-:: license that can be found in the LICENSE file.
-@echo off
-
-setlocal
-
-:: Requires Windows Installer XML (WiX), 7zip, and Mercurial (hg)
-
-echo # Cleaning previous WiX output files
-del /F /Q /S *.wixobj AppFiles.wxs *.wixpdb>NUL
-
-echo # Setting some variables
-for /f %%i in ('hg.exe root') do set ROOT=%%i
-for /f "tokens=3" %%i in ('%ROOT%\bin\go.exe version') do set VER=%%i
-for /f "tokens=4" %%i in ('%ROOT%\bin\go.exe version') do set ID=%%i
-set ID=%ID:+=%
-if errorlevel 1 goto end
-
-echo # Getting GOARCH
-%ROOT%\bin\go tool dist env -w>env.bat
-call env.bat
-del /F /Q /S env.bat>NUL
-if errorlevel 1 goto end
-
-
-rmdir /S /Q go>NUL
-mkdir go
-
-echo # Cloning the Go tree
-hg clone -r %ID% %ROOT% go
-if errorlevel 1 goto end
-
-rmdir /S /Q go\.hg>NUL
-del /F /Q /S go\.hgignore go\.hgtags>NUL
-
-echo # Copying pkg, bin, and src/pkg/runtime/z*
-xcopy %ROOT%\pkg go\pkg /V /E /Y /I
-xcopy %ROOT%\bin go\bin /V /E /Y /I
-xcopy %ROOT%\src\pkg\runtime\z*.c go\src\pkg\runtime /V /E /Y
-xcopy %ROOT%\src\pkg\runtime\z*.go go\src\pkg\runtime /V /E /Y
-xcopy %ROOT%\src\pkg\runtime\z*.h go\src\pkg\runtime /V /E /Y
-
-echo # Starting zip packaging
-7za a -tzip -mx=9 go.%VER%.windows-%GOARCH%.zip "go/"
-if errorlevel 1 goto end
-
-
-echo # Starting Go directory file harvesting
-heat dir go -nologo -cg AppFiles -gg -g1 -srd -sfrag -template fragment -dr INSTALLDIR -var var.SourceDir -out AppFiles.wxs
-if errorlevel 1 goto end
-
-echo # Starting installer packaging
-candle -nologo -dVersion=%VER% -dArch=%GOARCH% -dSourceDir=go installer.wxs AppFiles.wxs
-light -nologo -ext WixUIExtension -ext WixUtilExtension installer.wixobj AppFiles.wixobj -o go.%VER%.windows-%GOARCH%.msi
-if errorlevel 1 goto end
-
-del /F /Q /S *.wixobj AppFiles.wxs *.wixpdb>NUL
-
-:end
-endlocal
diff --git a/misc/dist/windows/images/gopher.ico b/misc/dist/windows/images/gopher.ico Binary files differindex 39a6c47c4..2e861ebe0 100644 --- a/misc/dist/windows/images/gopher.ico +++ b/misc/dist/windows/images/gopher.ico diff --git a/misc/dist/windows/installer.wxs b/misc/dist/windows/installer.wxs index faece29b0..20f261f9d 100644 --- a/misc/dist/windows/installer.wxs +++ b/misc/dist/windows/installer.wxs @@ -7,11 +7,15 @@ -->
<?if $(var.Arch) = 386 ?>
- <?define SYSFOLDER=SystemFolder ?>
- <?define Win64=no ?>
+ <?define ProdId = {FF5B30B2-08C2-11E1-85A2-6ACA4824019B} ?>
+ <?define UpgradeCode = {1C3114EA-08C3-11E1-9095-7FCA4824019B} ?>
+ <?define SysFolder=SystemFolder ?>
+ <?define IsX64Target = no ?>
<?else?>
- <?define SYSFOLDER=System64Folder ?>
- <?define Win64=yes ?>
+ <?define ProdId = {716c3eaa-9302-48d2-8e5e-5cfec5da2fab} ?>
+ <?define UpgradeCode = {22ea7650-4ac6-4001-bf29-f4b8775db1c0} ?>
+ <?define SysFolder=System64Folder ?>
+ <?define IsX64Target = yes ?>
<?endif?>
<Product
@@ -21,7 +25,7 @@ Codepage="1252"
Version="0.0.0.0"
Manufacturer="http://golang.org"
- UpgradeCode="1C3114EA-08C3-11E1-9095-7FCA4824019B" >
+ UpgradeCode="$(var.UpgradeCode)" >
<!-- Version="$(var.Version)" TODO: Version requires X.X.X.X format -->
<Package
@@ -41,12 +45,12 @@ <Property Id="ARPHELPLINK" Value="golang.org/doc/community.html" />
<Property Id="ARPREADME" Value="golang.org" />
<Property Id="ARPURLINFOABOUT" Value="golang.org" />
+<Property Id="LicenseAccepted">1</Property>
<Icon Id="gopher.ico" SourceFile="images\gopher.ico"/>
<Property Id="ARPPRODUCTICON" Value="gopher.ico" />
<Media Id='1' Cabinet="go.cab" EmbedCab="yes" CompressionLevel="high" />
<Condition Message="Windows 2000 or greater required."> VersionNT >= 500</Condition>
<MajorUpgrade AllowDowngrades="yes" />
-<SetDirectory Id="INSTALLDIRROOT" Value="C:\"/>
<CustomAction
Id="SetApplicationRootDirectory"
@@ -64,34 +68,24 @@ <Directory Id="EnvironmentEntries">
<Directory Id="GoEnvironmentEntries" Name="Go Programming Language"/>
</Directory>
- <Directory Id="DesktopFolder" Name="Desktop"/>
</Directory>
-<!-- Programs Menu & Desktop Shortcuts -->
+<!-- Programs Menu Shortcuts -->
<DirectoryRef Id="GoProgramShortcutsDir">
- <Component Id="Component_GoShortCuts" Guid="f5fbfb5e-6c5c-423b-9298-21b0e3c98f4b">
+ <Component Id="Component_GoProgramShortCuts" Guid="{f5fbfb5e-6c5c-423b-9298-21b0e3c98f4b}" Win64="$(var.IsX64Target)">
<Shortcut
Id="GoDocServerStartMenuShortcut"
Name="GoDocServer"
Description="Starts the Go documentation server (http://localhost:6060)"
Show="minimized"
- Arguments='/c "start /d[INSTALLDIR]bin godoc.exe -http=:6060 && start http://localhost:6060"'
+ Arguments='/c start "Godoc Server http://localhost:6060" "[INSTALLDIR]bin\godoc.exe" -http=localhost:6060 -goroot="[INSTALLDIR]." && start http://localhost:6060'
Icon="gopher.ico"
Target="[%ComSpec]" />
<Shortcut
- Id="GoDocServerDesktopShortcut"
- Directory="DesktopFolder"
- Name="GoDocServer"
- Description="Starts the godoc server (http://localhost:6060)"
- Show="minimized"
- Icon="gopher.ico"
- Arguments='/c "start /d[INSTALLDIR]bin godoc.exe -http=:6060 && start http://localhost:6060"'
- Target="[%ComSpec]" />
- <Shortcut
Id="UninstallShortcut"
Name="Uninstall Go"
Description="Uninstalls Go and all of its components"
- Target="[$(var.SYSFOLDER)]msiexec.exe"
+ Target="[$(var.SysFolder)]msiexec.exe"
Arguments="/x [ProductCode]" />
<RemoveFolder
Id="GoProgramShortcutsDir"
@@ -108,7 +102,7 @@ <!-- Registry & Environment Settings -->
<DirectoryRef Id="GoEnvironmentEntries">
- <Component Id="Component_GoEnvironment" Guid="3ec7a4d5-eb08-4de7-9312-2df392c45993">
+ <Component Id="Component_GoEnvironment" Guid="{3ec7a4d5-eb08-4de7-9312-2df392c45993}" Win64="$(var.IsX64Target)">
<RegistryKey
Root="HKCU"
Key="Software\GoProgrammingLanguage"
@@ -124,21 +118,27 @@ Value="[INSTALLDIR]" />
</RegistryKey>
<Environment
- Id="Environment"
+ Id="GoPathEntry"
Action="set"
Part="last"
Name="PATH"
Permanent="no"
System="yes"
Value="[INSTALLDIR]bin" />
+ <Environment
+ Id="GoRoot"
+ Action="set"
+ Part="all"
+ Name="GOROOT"
+ Permanent="no"
+ System="yes"
+ Value="[INSTALLDIR]" />
<RemoveFolder
Id="GoEnvironmentEntries"
On="uninstall" />
</Component>
</DirectoryRef>
-<!-- Components -->
-
<!-- Install the files -->
<Feature
Id="GoTools"
@@ -146,7 +146,7 @@ Level="1">
<ComponentRef Id="Component_GoEnvironment" />
<ComponentGroupRef Id="AppFiles" />
- <ComponentRef Id="Component_GoShortCuts" />
+ <ComponentRef Id="Component_GoProgramShortCuts" />
</Feature>
<!-- Update the environment -->
diff --git a/misc/emacs/go-mode.el b/misc/emacs/go-mode.el index 7fe0934ff..1af38e3d0 100644 --- a/misc/emacs/go-mode.el +++ b/misc/emacs/go-mode.el @@ -406,6 +406,22 @@ token on the line." (when (/= (skip-chars-backward "[:word:]_") 0) (not (looking-at go-mode-non-terminating-keywords-regexp))))))) +(defun go-mode-backward-skip-comments () + "Skip backward over comments and whitespace." + (when (not (bobp)) + (backward-char)) + (while (and (not (bobp)) + (or (eq 32 (char-syntax (char-after (point)))) + (go-mode-cs))) + (skip-syntax-backward "-") + (when (and (not (bobp)) (eq 32 (char-syntax (char-after (point))))) + (backward-char)) + (when (go-mode-cs) + (let ((pos (previous-single-property-change (point) 'go-mode-cs))) + (if pos (goto-char pos) (goto-char (point-min)))))) + (when (and (not (go-mode-cs)) (eq 32 (char-syntax (char-after (1+ (point)))))) + (forward-char 1))) + (defun go-mode-indentation () "Compute the ideal indentation level of the current line. @@ -451,7 +467,8 @@ indented one level." (incf indent tab-width)) ((?\() (goto-char (car nest)) - (forward-comment (- (buffer-size))) + (beginning-of-line) + (go-mode-backward-skip-comments) ;; Really just want the token before (when (looking-back "\\<import\\|const\\|var\\|type" (max (- (point) 7) (point-min))) @@ -465,7 +482,8 @@ indented one level." (decf indent tab-width)) ;; Continuation lines are indented 1 level - (forward-comment (- (buffer-size))) + (beginning-of-line) + (go-mode-backward-skip-comments) (when (case (char-before) ((nil ?\{ ?:) ;; At the beginning of a block or the statement diff --git a/misc/vim/ftdetect/gofiletype.vim b/misc/vim/ftdetect/gofiletype.vim index 096d05244..b658f6b0e 100644 --- a/misc/vim/ftdetect/gofiletype.vim +++ b/misc/vim/ftdetect/gofiletype.vim @@ -1 +1,23 @@ -au BufRead,BufNewFile *.go set filetype=go fileencoding=utf-8 fileencodings=utf-8 +" We take care to preserve the user's fileencodings and fileformats, +" because those settings are global (not buffer local), yet we want +" to override them for loading Go files, which are defined to be UTF-8. +let s:current_fileformats = '' +let s:current_fileencodings = '' + +" define fileencodings to open as utf-8 encoding even if it's ascii. +function! s:gofiletype_pre() + let s:current_fileformats = &g:fileformats + let s:current_fileencodings = &g:fileencodings + set fileencodings=utf-8 fileformats=unix + setlocal filetype=go +endfunction + +" restore fileencodings as others +function! s:gofiletype_post() + let &g:fileformats = s:current_fileformats + let &g:fileencodings = s:current_fileencodings +endfunction + +au BufNewFile *.go setlocal filetype=go fileencoding=utf-8 fileformat=unix +au BufRead *.go call s:gofiletype_pre() +au BufReadPost *.go call s:gofiletype_post() diff --git a/misc/xcode/3/README b/misc/xcode/3/README index c4cb915dc..69f371c2c 100644 --- a/misc/xcode/3/README +++ b/misc/xcode/3/README @@ -1,3 +1,3 @@ This directory contains files for Go syntax highlighting in Xcode 3.x. -See the comments go.pbfilespec and go.xclangspec for installation +See the comments in go.pbfilespec and go.xclangspec for installation instructions. diff --git a/misc/xcode/4/README b/misc/xcode/4/README new file mode 100644 index 000000000..09576d6d5 --- /dev/null +++ b/misc/xcode/4/README @@ -0,0 +1,2 @@ +This directory contains files for Go syntax highlighting in Xcode 4.x. +For installation, read, edit, and run go4xcode.sh .
\ No newline at end of file diff --git a/misc/xcode/4/go.xclangspec b/misc/xcode/4/go.xclangspec new file mode 100644 index 000000000..96edc8000 --- /dev/null +++ b/misc/xcode/4/go.xclangspec @@ -0,0 +1,290 @@ +/* + Copyright 2012 The Go Authors. All rights reserved. + Use of this source code is governed by a BSD-style + license that can be found in the LICENSE file. + + go.xclangspec + Go language specification for Xcode 4.x. + + This is a preliminary version that supports basic syntax coloring + (such as keywords, literals, and comments) and an attempt to provide + some structure information (incomplete). + + There is not much documentation available regarding the format + of .xclangspec files. As a starting point, see for instance the + outdated documentation at: + + http://maxao.free.fr/xcode-plugin-interface/specifications.html + +*/ + +( + +// ---------------------------------------------------------------------------- +// Keywords + +// TODO How do we get general Unicode identifiers? + + { + Identifier = "xcode.lang.go.identifier"; + Syntax = { + StartChars = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ_"; + Chars = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_"; + Words = ( + "break", + "case", + "chan", + "const", + "continue", + + "default", + "defer", + "else", + "fallthrough", + "for", + + "func", + "go", + "goto", + "if", + "import", + + "interface", + "map", + "package", + "range", + "return", + + "select", + "struct", + "switch", + "type", + "var", + ); + Type = "xcode.syntax.keyword"; + AltType = "xcode.syntax.identifier"; // non-keywords are identifiers + }; + }, + +// TODO decide what should go here, if anything + { + Identifier = "xcode.lang.go.interestingOperators"; + Syntax = { + Words = ( + "...", + ".", + "*", + ",", + ":", + ); + Type = "xcode.syntax.plain"; + }; + }, + + { + Identifier = "xcode.lang.go.rawstring"; + Syntax = { + Start = "`"; + End = "`"; + Type = "xcode.syntax.string"; + }; + }, + +// ---------------------------------------------------------------------------- +// Syntax Coloring + + { + Identifier = "xcode.lang.go"; + Description = "Go Coloring"; + BasedOn = "xcode.lang.simpleColoring"; + IncludeInMenu = YES; + Name = "Go"; + Syntax = { + Tokenizer = "xcode.lang.go.lexer.toplevel"; + IncludeRules = ( + "xcode.lang.go.block", + "xcode.lang.go.bracketexpr", + "xcode.lang.go.parenexpr", + ); + Type = "xcode.syntax.plain"; + }; + }, + + // The following rule returns tokens to the other rules + { + Identifier = "xcode.lang.go.lexer"; + Syntax = { + IncludeRules = ( + "xcode.lang.go.comment", + "xcode.lang.go.comment.singleline", + "xcode.lang.string", + "xcode.lang.character", + "xcode.lang.go.rawstring", + "xcode.lang.go.identifier", + "xcode.lang.number", + "xcode.lang.go.interestingOperators", + ); + }; + }, + + { + Identifier = "xcode.lang.go.lexer.toplevel"; + Syntax = { + IncludeRules = ( + "xcode.lang.go.comment", + "xcode.lang.go.comment.singleline", + "xcode.lang.string", + "xcode.lang.character", + "xcode.lang.go.rawstring", + "xcode.lang.go.type.declaration", + "xcode.lang.go.method.declaration", + "xcode.lang.go.function.declaration", + "xcode.lang.go.identifier", + "xcode.lang.number", + ); + }; + }, + + { + Identifier = "xcode.lang.go.method.declaration"; + Syntax = { + Tokenizer = "xcode.lang.go.lexer"; + Rules = ( + "func", + "xcode.lang.go.parenexpr", + "xcode.lang.go.identifier", + "xcode.lang.go.parenexpr", + ); + Type = "xcode.syntax.declaration.method"; + }; + }, + + { + Identifier = "xcode.lang.go.type.declaration"; + Syntax = { + Tokenizer = "xcode.lang.go.lexer"; + Rules = ( + "type", + "xcode.lang.go.identifier", + ); + Type = "xcode.syntax.typedef"; + }; + }, + + { + Identifier = "xcode.lang.go.function.declaration"; + Syntax = { + Tokenizer = "xcode.lang.go.lexer"; + Rules = ( + "func", + "xcode.lang.go.identifier", + "xcode.lang.go.parenexpr", + ); + Type = "xcode.syntax.declaration.function"; + }; + }, + +// ---------------------------------------------------------------------------- +// Blocks + + { + Identifier = "xcode.lang.go.block"; + Syntax = { + Tokenizer = "xcode.lang.go.lexer"; + Start = "{"; + End = "}"; + Foldable = YES; + Recursive = YES; + IncludeRules = ( + "xcode.lang.go.bracketexpr", + "xcode.lang.go.parenexpr", + ); + }; + }, + + { + Identifier = "xcode.lang.go.parenexpr"; + Syntax = { + Tokenizer = "xcode.lang.go.lexer"; + Start = "("; + End = ")"; + Recursive = YES; + IncludeRules = ( + "xcode.lang.go.bracketexpr", + "xcode.lang.go.block", + ); + }; + }, + + { + Identifier = "xcode.lang.go.bracketexpr"; + Syntax = { + Tokenizer = "xcode.lang.go.lexer"; + Start = "["; + End = "]"; + Recursive = YES; + IncludeRules = ( + "xcode.lang.go.parenexpr", + ); + }; + }, + + { + Identifier = "xcode.lang.go.comment"; + Syntax = { + Start = "/*"; + End = "*/"; + Foldable = YES; + IncludeRules = ( + "xcode.lang.url", + "xcode.lang.url.mail", + "xcode.lang.comment.mark", + ); + Type = "xcode.syntax.comment"; + }; + }, + + { + Identifier = "xcode.lang.go.comment.singleline"; + Syntax = { + Start = "//"; + End = "\n"; + IncludeRules = ( + "xcode.lang.url", + "xcode.lang.url.mail", + "xcode.lang.comment.mark", + ); + Type = "xcode.syntax.comment"; + }; + }, + + // This rule recognizes special comments markers and adds them + // to the list of file markers at the top of the editor window. + // This overrides the markers specified in + // /Developer/Library/PrivateFrameworks/XcodeEdit.framework/Versions/A/Resources/BaseSupport.xclangspec + // and appears to apply them to all languages. Thus, for now + // "inherit" the existing markers here for backward-compatibility. + { + Identifier = "xcode.lang.comment.mark"; + Syntax = { + StartChars = "BMTF!?"; + Match = ( + // Go-specific markers + "^\(BUG.*$\)$", // include "BUG" in the markers list + "^\(TODO.*$\)$", // include "TODO" in the markers list + // inherited markers + "^MARK:[ \t]+\(.*\)$", + "^\(TODO:[ \t]+.*\)$", // include "TODO: " in the markers list + "^\(FIXME:[ \t]+.*\)$", // include "FIXME: " in the markers list + "^\(!!!:.*\)$", // include "!!!:" in the markers list + "^\(\\?\\?\\?:.*\)$" // include "???:" in the markers list + ); + // This is the order of captures. All of the match strings above need the same order. + CaptureTypes = ( + "xcode.syntax.mark" + ); + Type = "xcode.syntax.comment"; + }; + }, + +) diff --git a/misc/xcode/4/go4xcode.sh b/misc/xcode/4/go4xcode.sh new file mode 100755 index 000000000..fbdd0cfed --- /dev/null +++ b/misc/xcode/4/go4xcode.sh @@ -0,0 +1,100 @@ +#!/usr/bin/env bash +# Copyright 2012 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +# Illustrates how a Go language specification can be installed for Xcode 4.x., +# to enable syntax coloring, by adding an entry to a plugindata file. +# +# FIXME: Write a decent Xcode plugin to handle the file type association and +# language specification properly instead of altering Xcode library files. + +set -e + +# Assumes Xcode 4+. +XCODE_MAJOR_VERSION=`xcodebuild -version | awk 'NR == 1 {print substr($2,1,1)}'` +if [ "$XCODE_MAJOR_VERSION" -lt "4" ]; then + echo "Xcode 4.x not found." + exit 1 +fi + +# DVTFOUNDATION_DIR may vary depending on Xcode setup. Change it to reflect +# your current Xcode setup. Find suitable path with e.g.: +# +# find / -type f -name 'DVTFoundation.xcplugindata' 2> /dev/null +# +# Example of DVTFOUNDATION_DIR's from "default" Xcode 4.x setups; +# +# Xcode 4.1: /Developer/Library/PrivateFrameworks/DVTFoundation.framework/Versions/A/Resources/ +# Xcode 4.3: /Applications/Xcode.app/Contents/SharedFrameworks/DVTFoundation.framework/Versions/A/Resources/ +# +DVTFOUNDATION_DIR="/Applications/Xcode.app/Contents/SharedFrameworks/DVTFoundation.framework/Versions/A/Resources/" +PLUGINDATA_FILE="DVTFoundation.xcplugindata" + +PLISTBUDDY=/usr/libexec/PlistBuddy +PLIST_FILE=tmp.plist + +# Provide means of deleting the Go entry from the plugindata file. +if [ "$1" = "--delete-entry" ]; then + echo "Removing Go language specification entry." + $PLISTBUDDY -c "Delete :plug-in:extensions:Xcode.SourceCodeLanguage.Go" $DVTFOUNDATION_DIR/$PLUGINDATA_FILE + echo "Run 'sudo rm -rf /var/folders/*' and restart Xcode to update change immediately." + exit 0 +fi + +GO_VERSION="`go version`" + +GO_LANG_ENTRY=" + <?xml version=\"1.0\" encoding=\"UTF-8\"?> + <!DOCTYPE plist PUBLIC \"-//Apple//DTD PLIST 1.0//EN\" \"http://www.apple.com/DTDs/PropertyList-1.0.dtd\"> + <plist version=\"1.0\"> + <dict> + <key>Xcode.SourceCodeLanguage.Go</key> + <dict> + <key>conformsTo</key> + <array> + <dict> + <key>identifier</key> + <string>Xcode.SourceCodeLanguage.Generic</string> + </dict> + </array> + <key>documentationAbbreviation</key> + <string>go</string> + <key>fileDataType</key> + <array> + <dict> + <key>identifier</key> + <string>com.apple.xcode.go-source</string> + </dict> + </array> + <key>id</key> + <string>Xcode.SourceCodeLanguage.Go</string> + <key>languageName</key> + <string>Go</string> + <key>languageSpecification</key> + <string>xcode.lang.go</string> + <key>name</key> + <string>The Go Programming Language</string> + <key>point</key> + <string>Xcode.SourceCodeLanguage</string> + <key>version</key> + <string>$GO_VERSION</string> + </dict> + </dict> + </plist> +" + +echo "Backing up plugindata file." +cp $DVTFOUNDATION_DIR/$PLUGINDATA_FILE $DVTFOUNDATION_DIR/$PLUGINDATA_FILE.bak + +echo "Adding Go language specification entry." +echo $GO_LANG_ENTRY > $PLIST_FILE +$PLISTBUDDY -c "Merge $PLIST_FILE plug-in:extensions" $DVTFOUNDATION_DIR/$PLUGINDATA_FILE + +rm -f $PLIST_FILE + +echo "Installing Go language specification file for Xcode." +cp $GOROOT/misc/xcode/4/go.xclangspec $DVTFOUNDATION_DIR + +echo "Run 'sudo rm -rf /var/folders/*' and restart Xcode to update change immediately." +echo "Syntax coloring must be manually selected from the Editor - Syntax Coloring menu in Xcode." |