diff options
author | Ondřej Surý <ondrej@sury.org> | 2012-03-26 16:50:58 +0200 |
---|---|---|
committer | Ondřej Surý <ondrej@sury.org> | 2012-03-26 16:50:58 +0200 |
commit | 519725bb3c075ee2462c929f5997cb068e18466a (patch) | |
tree | 5b162e8488ad147a645048c073577821b4a2bee9 /src/cmd | |
parent | 842623c5dd2819d980ca9c58048d6bc6ed82475f (diff) | |
download | golang-upstream-weekly/2012.03.22.tar.gz |
Imported Upstream version 2012.03.22upstream-weekly/2012.03.22
Diffstat (limited to 'src/cmd')
65 files changed, 2670 insertions, 1522 deletions
diff --git a/src/cmd/5l/noop.c b/src/cmd/5l/noop.c index eb44344f4..004f9f2fa 100644 --- a/src/cmd/5l/noop.c +++ b/src/cmd/5l/noop.c @@ -226,8 +226,7 @@ noops(void) p->as = AMOVW; p->scond = C_SCOND_LO; p->from.type = D_CONST; - /* 160 comes from 3 calls (3*8) 4 safes (4*8) and 104 guard */ - p->from.offset = autosize+160; + p->from.offset = autosize; p->to.type = D_REG; p->to.reg = 1; diff --git a/src/cmd/6l/pass.c b/src/cmd/6l/pass.c index 2357a7f77..c9b477627 100644 --- a/src/cmd/6l/pass.c +++ b/src/cmd/6l/pass.c @@ -501,10 +501,17 @@ dostkoff(void) q = p; } - /* 160 comes from 3 calls (3*8) 4 safes (4*8) and 104 guard */ + // If we ask for more stack, we'll get a minimum of StackMin bytes. + // We need a stack frame large enough to hold the top-of-stack data, + // the function arguments+results, our caller's PC, our frame, + // a word for the return PC of the next call, and then the StackLimit bytes + // that must be available on entry to any function called from a function + // that did a stack check. If StackMin is enough, don't ask for a specific + // amount: then we can use the custom functions and save a few + // instructions. moreconst1 = 0; - if(autoffset+160+textarg > 4096) - moreconst1 = (autoffset+160) & ~7LL; + if(StackTop + textarg + PtrSize + autoffset + PtrSize + StackLimit >= StackMin) + moreconst1 = autoffset; moreconst2 = textarg; // 4 varieties varieties (const1==0 cross const2==0) diff --git a/src/cmd/8l/pass.c b/src/cmd/8l/pass.c index b900a5f79..9034fdf3a 100644 --- a/src/cmd/8l/pass.c +++ b/src/cmd/8l/pass.c @@ -527,10 +527,18 @@ dostkoff(void) p = appendp(p); // save frame size in DX p->as = AMOVL; p->to.type = D_DX; - /* 160 comes from 3 calls (3*8) 4 safes (4*8) and 104 guard */ p->from.type = D_CONST; - if(autoffset+160+cursym->text->to.offset2 > 4096) - p->from.offset = (autoffset+160) & ~7LL; + + // If we ask for more stack, we'll get a minimum of StackMin bytes. + // We need a stack frame large enough to hold the top-of-stack data, + // the function arguments+results, our caller's PC, our frame, + // a word for the return PC of the next call, and then the StackLimit bytes + // that must be available on entry to any function called from a function + // that did a stack check. If StackMin is enough, don't ask for a specific + // amount: then we can use the custom functions and save a few + // instructions. + if(StackTop + cursym->text->to.offset2 + PtrSize + autoffset + PtrSize + StackLimit >= StackMin) + p->from.offset = (autoffset+7) & ~7LL; p = appendp(p); // save arg size in AX p->as = AMOVL; diff --git a/src/cmd/api/goapi.go b/src/cmd/api/goapi.go index fe9c862f4..7363f6d82 100644 --- a/src/cmd/api/goapi.go +++ b/src/cmd/api/goapi.go @@ -52,6 +52,12 @@ var contexts = []*build.Context{ {GOOS: "windows", GOARCH: "386"}, } +func init() { + for _, c := range contexts { + c.Compiler = build.Default.Compiler + } +} + func contextName(c *build.Context) string { s := c.GOOS + "-" + c.GOARCH if c.CgoEnabled { @@ -125,7 +131,7 @@ func main() { if err != nil { log.Fatalf("Error reading file %s: %v", *checkFile, err) } - v1 := strings.Split(string(bs), "\n") + v1 := strings.Split(strings.TrimSpace(string(bs)), "\n") sort.Strings(v1) v2 := features take := func(sl *[]string) string { @@ -133,17 +139,24 @@ func main() { *sl = (*sl)[1:] return s } + changes := false for len(v1) > 0 || len(v2) > 0 { switch { case len(v2) == 0 || v1[0] < v2[0]: fmt.Fprintf(bw, "-%s\n", take(&v1)) + changes = true case len(v1) == 0 || v1[0] > v2[0]: fmt.Fprintf(bw, "+%s\n", take(&v2)) + changes = true default: take(&v1) take(&v2) } } + if changes { + bw.Flush() + os.Exit(1) + } } else { for _, f := range features { fmt.Fprintf(bw, "%s\n", f) @@ -278,7 +291,9 @@ func (w *Walker) WalkPackage(name string) { } } - log.Printf("package %s", name) + if *verbose { + log.Printf("package %s", name) + } pop := w.pushScope("pkg " + name) defer pop() @@ -573,7 +588,14 @@ func (w *Walker) varValueType(vi interface{}) (string, error) { } } // maybe a function call; maybe a conversion. Need to lookup type. - return "", fmt.Errorf("not a known function %q", w.nodeString(v.Fun)) + // TODO(bradfitz): this is a hack, but arguably most of this tool is, + // until the Go AST has type information. + nodeStr := w.nodeString(v.Fun) + switch nodeStr { + case "string", "[]byte": + return nodeStr, nil + } + return "", fmt.Errorf("not a known function %q", nodeStr) default: return "", fmt.Errorf("unknown const value type %T", vi) } diff --git a/src/cmd/api/testdata/src/pkg/p1/golden.txt b/src/cmd/api/testdata/src/pkg/p1/golden.txt index 3a1b3f535..e334e5776 100644 --- a/src/cmd/api/testdata/src/pkg/p1/golden.txt +++ b/src/cmd/api/testdata/src/pkg/p1/golden.txt @@ -58,14 +58,16 @@ pkg p1, type T struct pkg p1, type TPtrExported struct pkg p1, type TPtrExported struct, embedded *Embedded pkg p1, type TPtrUnexported struct +pkg p1, var ByteConv []byte pkg p1, var ChecksumError error pkg p1, var SIPtr *SI pkg p1, var SIPtr2 *SI pkg p1, var SIVal SI +pkg p1, var StrConv string pkg p1, var V string -pkg p1, var VError Error pkg p1, var V1 uint64 pkg p1, var V2 p2.Twoer +pkg p1, var VError Error pkg p1, var X I pkg p1, var X int64 pkg p1, var Y int diff --git a/src/cmd/api/testdata/src/pkg/p1/p1.go b/src/cmd/api/testdata/src/pkg/p1/p1.go index 9d2afa913..d965bb75e 100644 --- a/src/cmd/api/testdata/src/pkg/p1/p1.go +++ b/src/cmd/api/testdata/src/pkg/p1/p1.go @@ -27,6 +27,12 @@ var ( V2 = ptwo.G() ) +// Variables with conversions: +var ( + StrConv = string("foo") + ByteConv = []byte("foo") +) + var ChecksumError = ptwo.NewError("gzip checksum error") const B = 2 diff --git a/src/cmd/cgo/doc.go b/src/cmd/cgo/doc.go index f6a14ae08..1bb48f44e 100644 --- a/src/cmd/cgo/doc.go +++ b/src/cmd/cgo/doc.go @@ -44,6 +44,11 @@ For example: // #include <png.h> import "C" +The CGO_CFLAGS and CGO_LDFLAGS environment variables are added +to the flags derived from these directives. Package-specific flags should +be set using the directives, not the environment variables, so that builds +work in unmodified environments. + Within the Go file, C identifiers or field names that are keywords in Go can be accessed by prefixing them with an underscore: if x points at a C struct with a field named "type", x._type accesses the field. @@ -111,13 +116,13 @@ Not all Go types can be mapped to C types in a useful way. Cgo transforms the input file into four output files: two Go source files, a C file for 6c (or 8c or 5c), and a C file for gcc. -The standard package makefile rules in Make.pkg automate the -process of using cgo. See $GOROOT/misc/cgo/stdio and -$GOROOT/misc/cgo/gmp for examples. +The standard package construction rules of the go command +automate the process of using cgo. See $GOROOT/misc/cgo/stdio +and $GOROOT/misc/cgo/gmp for examples. Cgo does not yet work with gccgo. See "C? Go? Cgo!" for an introduction to using cgo: -http://blog.golang.org/2011/03/c-go-cgo.html +http://golang.org/doc/articles/c_go_cgo.html */ package documentation diff --git a/src/cmd/cgo/main.go b/src/cmd/cgo/main.go index a8be7be7d..7449f04c4 100644 --- a/src/cmd/cgo/main.go +++ b/src/cmd/cgo/main.go @@ -136,6 +136,7 @@ var cdefs = flag.Bool("cdefs", false, "for bootstrap: write C definitions for C var objDir = flag.String("objdir", "", "object directory") var gccgo = flag.Bool("gccgo", false, "generate files for use with gccgo") +var gccgoprefix = flag.String("gccgoprefix", "go", "prefix of symbols generated by gccgo") var importRuntimeCgo = flag.Bool("import_runtime_cgo", true, "import runtime/cgo in generated code") var goarch, goos string @@ -147,7 +148,7 @@ func main() { // cgo -dynimport is essentially a separate helper command // built into the cgo binary. It scans a gcc-produced executable // and dumps information about the imported symbols and the - // imported libraries. The Make.pkg rules for cgo prepare an + // imported libraries. The 'go build' rules for cgo prepare an // appropriate executable and then use its import information // instead of needing to make the linkers duplicate all the // specialized knowledge gcc has about where to look for imported diff --git a/src/cmd/cgo/out.go b/src/cmd/cgo/out.go index 4dc0f8454..814250c2e 100644 --- a/src/cmd/cgo/out.go +++ b/src/cmd/cgo/out.go @@ -107,7 +107,11 @@ func (p *Package) writeDefs() { } } - p.writeExports(fgo2, fc, fm) + if *gccgo { + p.writeGccgoExports(fgo2, fc, fm) + } else { + p.writeExports(fgo2, fc, fm) + } fgo2.Close() fc.Close() @@ -280,8 +284,13 @@ func (p *Package) writeDefsFunc(fc, fgo2 *os.File, n *Name) { } conf.Fprint(fgo2, fset, d) fmt.Fprintf(fgo2, "{\n") + fmt.Fprintf(fgo2, "\tsyscall.SetErrno(0)\n") fmt.Fprintf(fgo2, "\tr := %s(%s)\n", cname, strings.Join(paramnames, ", ")) - fmt.Fprintf(fgo2, "\treturn r, syscall.GetErrno()\n") + fmt.Fprintf(fgo2, "\te := syscall.GetErrno()\n") + fmt.Fprintf(fgo2, "\tif e != 0 {\n") + fmt.Fprintf(fgo2, "\t\treturn r, e\n") + fmt.Fprintf(fgo2, "\t}\n") + fmt.Fprintf(fgo2, "\treturn r, nil\n") fmt.Fprintf(fgo2, "}\n") // declare the C function. fmt.Fprintf(fgo2, "//extern %s\n", n.C) @@ -411,10 +420,20 @@ func (p *Package) writeOutputFunc(fgcc *os.File, n *Name) { } } fmt.Fprintf(fgcc, "%s(", n.C) - for i := range n.FuncType.Params { + for i, t := range n.FuncType.Params { if i > 0 { fmt.Fprintf(fgcc, ", ") } + // We know the type params are correct, because + // the Go equivalents had good type params. + // However, our version of the type omits the magic + // words const and volatile, which can provoke + // C compiler warnings. Silence them by casting + // all pointers to void*. (Eventually that will produce + // other warnings.) + if c := t.C.String(); c[len(c)-1] == '*' { + fmt.Fprintf(fgcc, "(void*)") + } fmt.Fprintf(fgcc, "a->p%d", i) } fmt.Fprintf(fgcc, ");\n") @@ -563,8 +582,9 @@ func (p *Package) writeExports(fgo2, fc, fm *os.File) { goname = "_cgoexpwrap" + cPrefix + "_" + fn.Recv.List[0].Names[0].Name + "_" + goname } fmt.Fprintf(fc, "#pragma dynexport %s %s\n", goname, goname) - fmt.Fprintf(fc, "extern void ·%s();\n", goname) - fmt.Fprintf(fc, "\nvoid\n") + fmt.Fprintf(fc, "extern void ·%s();\n\n", goname) + fmt.Fprintf(fc, "#pragma textflag 7\n") // no split stack, so no use of m or g + fmt.Fprintf(fc, "void\n") fmt.Fprintf(fc, "_cgoexp%s_%s(void *a, int32 n)\n", cPrefix, exp.ExpName) fmt.Fprintf(fc, "{\n") fmt.Fprintf(fc, "\truntime·cgocallback(·%s, a, n);\n", goname) @@ -613,6 +633,83 @@ func (p *Package) writeExports(fgo2, fc, fm *os.File) { } } +// Write out the C header allowing C code to call exported gccgo functions. +func (p *Package) writeGccgoExports(fgo2, fc, fm *os.File) { + fgcc := creat(*objDir + "_cgo_export.c") + fgcch := creat(*objDir + "_cgo_export.h") + _ = fgcc + + fmt.Fprintf(fgcch, "/* Created by cgo - DO NOT EDIT. */\n") + fmt.Fprintf(fgcch, "%s\n", p.Preamble) + fmt.Fprintf(fgcch, "%s\n", gccExportHeaderProlog) + fmt.Fprintf(fm, "#include \"_cgo_export.h\"\n") + + clean := func(r rune) rune { + switch { + case 'A' <= r && r <= 'Z', 'a' <= r && r <= 'z', + '0' <= r && r <= '9': + return r + } + return '_' + } + gccgoSymbolPrefix := strings.Map(clean, *gccgoprefix) + + for _, exp := range p.ExpFunc { + // TODO: support functions with receivers. + fn := exp.Func + fntype := fn.Type + + if !ast.IsExported(fn.Name.Name) { + fatalf("cannot export unexported function %s with gccgo", fn.Name) + } + + cdeclBuf := new(bytes.Buffer) + resultCount := 0 + forFieldList(fntype.Results, + func(i int, atype ast.Expr) { resultCount++ }) + switch resultCount { + case 0: + fmt.Fprintf(cdeclBuf, "void") + case 1: + forFieldList(fntype.Results, + func(i int, atype ast.Expr) { + t := p.cgoType(atype) + fmt.Fprintf(cdeclBuf, "%s", t.C) + }) + default: + // Declare a result struct. + fmt.Fprintf(fgcch, "struct %s_result {\n", exp.ExpName) + forFieldList(fntype.Results, + func(i int, atype ast.Expr) { + t := p.cgoType(atype) + fmt.Fprintf(fgcch, "\t%s r%d;\n", t.C, i) + }) + fmt.Fprintf(fgcch, "};\n") + fmt.Fprintf(cdeclBuf, "struct %s_result", exp.ExpName) + } + + // The function name. + fmt.Fprintf(cdeclBuf, " "+exp.ExpName) + gccgoSymbol := fmt.Sprintf("%s.%s.%s", gccgoSymbolPrefix, p.PackageName, exp.Func.Name) + fmt.Fprintf(cdeclBuf, " (") + // Function parameters. + forFieldList(fntype.Params, + func(i int, atype ast.Expr) { + if i > 0 { + fmt.Fprintf(cdeclBuf, ", ") + } + t := p.cgoType(atype) + fmt.Fprintf(cdeclBuf, "%s p%d", t.C, i) + }) + fmt.Fprintf(cdeclBuf, ")") + cdecl := cdeclBuf.String() + + fmt.Fprintf(fgcch, "extern %s __asm__(\"%s\");\n", cdecl, gccgoSymbol) + // Dummy declaration for _cgo_main.c + fmt.Fprintf(fm, "%s {}\n", cdecl) + } +} + // Call a function for each entry in an ast.FieldList, passing the // index into the list and the type. func forFieldList(fl *ast.FieldList, fn func(int, ast.Expr)) { diff --git a/src/cmd/dist/build.c b/src/cmd/dist/build.c index a40853fad..3936f7621 100644 --- a/src/cmd/dist/build.c +++ b/src/cmd/dist/build.c @@ -209,7 +209,7 @@ findgoversion(void) // What are the tags along the current branch? tag = ""; rev = "."; - run(&b, goroot, CheckExit, "hg", "log", "-b", bstr(&branch), "--template", "{tags} + ", nil); + run(&b, goroot, CheckExit, "hg", "log", "-b", bstr(&branch), "-r", ".:0", "--template", "{tags} + ", nil); splitfields(&tags, bstr(&b)); nrev = 0; for(i=0; i<tags.len; i++) { @@ -1214,6 +1214,8 @@ clean(void) vinit(&dir); for(i=0; i<nelem(cleantab); i++) { + if((streq(cleantab[i], "cmd/cov") || streq(cleantab[i], "cmd/prof")) && !isdir(cleantab[i])) + continue; bpathf(&path, "%s/src/%s", goroot, cleantab[i]); xreaddir(&dir, bstr(&path)); // Remove generated files. @@ -1351,6 +1353,9 @@ cmdbootstrap(int argc, char **argv) goversion = findgoversion(); setup(); + xsetenv("GOROOT", goroot); + xsetenv("GOROOT_FINAL", goroot_final); + // For the main bootstrap, building for host os/arch. oldgoos = goos; oldgoarch = goarch; diff --git a/src/cmd/fix/go1rename.go b/src/cmd/fix/go1rename.go index 4b666720b..9266c749c 100644 --- a/src/cmd/fix/go1rename.go +++ b/src/cmd/fix/go1rename.go @@ -74,4 +74,94 @@ var go1renameReplace = []rename{ Old: "runtime.Goroutines", New: "runtime.NumGoroutine", }, + { + OldImport: "net/http", + NewImport: "net/http/httputil", + Old: "http.ErrPersistEOF", + New: "httputil.ErrPersistEOF", + }, + { + OldImport: "net/http", + NewImport: "net/http/httputil", + Old: "http.ErrPipeline", + New: "httputil.ErrPipeline", + }, + { + OldImport: "net/http", + NewImport: "net/http/httputil", + Old: "http.ErrClosed", + New: "httputil.ErrClosed", + }, + { + OldImport: "net/http", + NewImport: "net/http/httputil", + Old: "http.ServerConn", + New: "httputil.ServerConn", + }, + { + OldImport: "net/http", + NewImport: "net/http/httputil", + Old: "http.ClientConn", + New: "httputil.ClientConn", + }, + { + OldImport: "net/http", + NewImport: "net/http/httputil", + Old: "http.NewChunkedReader", + New: "httputil.NewChunkedReader", + }, + { + OldImport: "net/http", + NewImport: "net/http/httputil", + Old: "http.NewChunkedWriter", + New: "httputil.NewChunkedWriter", + }, + { + OldImport: "net/http", + NewImport: "net/http/httputil", + Old: "http.ReverseProxy", + New: "httputil.ReverseProxy", + }, + { + OldImport: "net/http", + NewImport: "net/http/httputil", + Old: "http.NewSingleHostReverseProxy", + New: "httputil.NewSingleHostReverseProxy", + }, + { + OldImport: "net/http", + NewImport: "net/http/httputil", + Old: "http.DumpRequest", + New: "httputil.DumpRequest", + }, + { + OldImport: "net/http", + NewImport: "net/http/httputil", + Old: "http.DumpRequestOut", + New: "httputil.DumpRequestOut", + }, + { + OldImport: "net/http", + NewImport: "net/http/httputil", + Old: "http.DumpResponse", + New: "httputil.DumpResponse", + }, + { + OldImport: "net/http", + NewImport: "net/http/httputil", + Old: "http.NewClientConn", + New: "httputil.NewClientConn", + }, + { + OldImport: "net/http", + NewImport: "net/http/httputil", + Old: "http.NewServerConn", + New: "httputil.NewServerConn", + }, + { + OldImport: "net/http", + NewImport: "net/http/httputil", + Old: "http.NewProxyClientConn", + New: "httputil.NewProxyClientConn", + }, } diff --git a/src/cmd/fix/go1rename_test.go b/src/cmd/fix/go1rename_test.go index 481ebea8e..90219ba71 100644 --- a/src/cmd/fix/go1rename_test.go +++ b/src/cmd/fix/go1rename_test.go @@ -17,6 +17,7 @@ import ( "crypto/aes" "crypto/des" "encoding/json" + "net/http" "net/url" "os" "runtime" @@ -34,6 +35,15 @@ var ( _ = os.Exec _ = runtime.Cgocalls _ = runtime.Goroutines + _ = http.ErrPersistEOF + _ = http.ErrPipeline + _ = http.ErrClosed + _ = http.NewSingleHostReverseProxy + _ = http.NewChunkedReader + _ = http.NewChunkedWriter + _ *http.ReverseProxy + _ *http.ClientConn + _ *http.ServerConn ) `, Out: `package main @@ -42,6 +52,7 @@ import ( "crypto/aes" "crypto/cipher" "encoding/json" + "net/http/httputil" "net/url" "runtime" "syscall" @@ -59,7 +70,126 @@ var ( _ = syscall.Exec _ = runtime.NumCgoCall _ = runtime.NumGoroutine + _ = httputil.ErrPersistEOF + _ = httputil.ErrPipeline + _ = httputil.ErrClosed + _ = httputil.NewSingleHostReverseProxy + _ = httputil.NewChunkedReader + _ = httputil.NewChunkedWriter + _ *httputil.ReverseProxy + _ *httputil.ClientConn + _ *httputil.ServerConn ) `, }, + { + Name: "httputil.0", + In: `package main + +import "net/http" + +func f() { + http.DumpRequest(nil, false) + http.DumpRequestOut(nil, false) + http.DumpResponse(nil, false) + http.NewChunkedReader(nil) + http.NewChunkedWriter(nil) + http.NewClientConn(nil, nil) + http.NewProxyClientConn(nil, nil) + http.NewServerConn(nil, nil) + http.NewSingleHostReverseProxy(nil) +} +`, + Out: `package main + +import "net/http/httputil" + +func f() { + httputil.DumpRequest(nil, false) + httputil.DumpRequestOut(nil, false) + httputil.DumpResponse(nil, false) + httputil.NewChunkedReader(nil) + httputil.NewChunkedWriter(nil) + httputil.NewClientConn(nil, nil) + httputil.NewProxyClientConn(nil, nil) + httputil.NewServerConn(nil, nil) + httputil.NewSingleHostReverseProxy(nil) +} +`, + }, + { + Name: "httputil.1", + In: `package main + +import "net/http" + +func f() { + http.DumpRequest(nil, false) + http.DumpRequestOut(nil, false) + http.DumpResponse(nil, false) + http.NewChunkedReader(nil) + http.NewChunkedWriter(nil) + http.NewClientConn(nil, nil) + http.NewProxyClientConn(nil, nil) + http.NewServerConn(nil, nil) + http.NewSingleHostReverseProxy(nil) +} +`, + Out: `package main + +import "net/http/httputil" + +func f() { + httputil.DumpRequest(nil, false) + httputil.DumpRequestOut(nil, false) + httputil.DumpResponse(nil, false) + httputil.NewChunkedReader(nil) + httputil.NewChunkedWriter(nil) + httputil.NewClientConn(nil, nil) + httputil.NewProxyClientConn(nil, nil) + httputil.NewServerConn(nil, nil) + httputil.NewSingleHostReverseProxy(nil) +} +`, + }, + { + Name: "httputil.2", + In: `package main + +import "net/http" + +func f() { + http.DumpRequest(nil, false) + http.DumpRequestOut(nil, false) + http.DumpResponse(nil, false) + http.NewChunkedReader(nil) + http.NewChunkedWriter(nil) + http.NewClientConn(nil, nil) + http.NewProxyClientConn(nil, nil) + http.NewServerConn(nil, nil) + http.NewSingleHostReverseProxy(nil) + http.Get("") +} +`, + Out: `package main + +import ( + "net/http" + "net/http/httputil" +) + +func f() { + httputil.DumpRequest(nil, false) + httputil.DumpRequestOut(nil, false) + httputil.DumpResponse(nil, false) + httputil.NewChunkedReader(nil) + httputil.NewChunkedWriter(nil) + httputil.NewClientConn(nil, nil) + httputil.NewProxyClientConn(nil, nil) + httputil.NewServerConn(nil, nil) + httputil.NewSingleHostReverseProxy(nil) + http.Get("") +} +`, + }, } diff --git a/src/cmd/fix/httputil.go b/src/cmd/fix/httputil.go deleted file mode 100644 index 86c42e160..000000000 --- a/src/cmd/fix/httputil.go +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package main - -import "go/ast" - -func init() { - register(httputilFix) -} - -var httputilFix = fix{ - "httputil", - "2011-11-18", - httputil, - `Move some functions in http package into httputil package. - -http://codereview.appspot.com/5336049 -`, -} - -var httputilFuncs = []string{ - "DumpRequest", - "DumpRequestOut", - "DumpResponse", - "NewChunkedReader", - "NewChunkedWriter", - "NewClientConn", - "NewProxyClientConn", - "NewServerConn", - "NewSingleHostReverseProxy", -} - -func httputil(f *ast.File) bool { - if imports(f, "net/http/httputil") { - return false - } - - fixed := false - - walk(f, func(n interface{}) { - // Rename package name. - if expr, ok := n.(ast.Expr); ok { - for _, s := range httputilFuncs { - if isPkgDot(expr, "http", s) { - if !fixed { - addImport(f, "net/http/httputil") - fixed = true - } - expr.(*ast.SelectorExpr).X.(*ast.Ident).Name = "httputil" - } - } - } - }) - - // Remove the net/http import if no longer needed. - if fixed && !usesImport(f, "net/http") { - deleteImport(f, "net/http") - } - - return fixed -} diff --git a/src/cmd/fix/httputil_test.go b/src/cmd/fix/httputil_test.go deleted file mode 100644 index 83e9f6dfb..000000000 --- a/src/cmd/fix/httputil_test.go +++ /dev/null @@ -1,122 +0,0 @@ -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package main - -func init() { - addTestCases(httputilTests, httputil) -} - -var httputilTests = []testCase{ - { - Name: "httputil.0", - In: `package main - -import "net/http" - -func f() { - http.DumpRequest(nil, false) - http.DumpRequestOut(nil, false) - http.DumpResponse(nil, false) - http.NewChunkedReader(nil) - http.NewChunkedWriter(nil) - http.NewClientConn(nil, nil) - http.NewProxyClientConn(nil, nil) - http.NewServerConn(nil, nil) - http.NewSingleHostReverseProxy(nil) -} -`, - Out: `package main - -import "net/http/httputil" - -func f() { - httputil.DumpRequest(nil, false) - httputil.DumpRequestOut(nil, false) - httputil.DumpResponse(nil, false) - httputil.NewChunkedReader(nil) - httputil.NewChunkedWriter(nil) - httputil.NewClientConn(nil, nil) - httputil.NewProxyClientConn(nil, nil) - httputil.NewServerConn(nil, nil) - httputil.NewSingleHostReverseProxy(nil) -} -`, - }, - { - Name: "httputil.1", - In: `package main - -import "net/http" - -func f() { - http.DumpRequest(nil, false) - http.DumpRequestOut(nil, false) - http.DumpResponse(nil, false) - http.NewChunkedReader(nil) - http.NewChunkedWriter(nil) - http.NewClientConn(nil, nil) - http.NewProxyClientConn(nil, nil) - http.NewServerConn(nil, nil) - http.NewSingleHostReverseProxy(nil) -} -`, - Out: `package main - -import "net/http/httputil" - -func f() { - httputil.DumpRequest(nil, false) - httputil.DumpRequestOut(nil, false) - httputil.DumpResponse(nil, false) - httputil.NewChunkedReader(nil) - httputil.NewChunkedWriter(nil) - httputil.NewClientConn(nil, nil) - httputil.NewProxyClientConn(nil, nil) - httputil.NewServerConn(nil, nil) - httputil.NewSingleHostReverseProxy(nil) -} -`, - }, - { - Name: "httputil.2", - In: `package main - -import "net/http" - -func f() { - http.DumpRequest(nil, false) - http.DumpRequestOut(nil, false) - http.DumpResponse(nil, false) - http.NewChunkedReader(nil) - http.NewChunkedWriter(nil) - http.NewClientConn(nil, nil) - http.NewProxyClientConn(nil, nil) - http.NewServerConn(nil, nil) - http.NewSingleHostReverseProxy(nil) - http.Get("") -} -`, - Out: `package main - -import ( - "net/http" - "net/http/httputil" -) - -func f() { - httputil.DumpRequest(nil, false) - httputil.DumpRequestOut(nil, false) - httputil.DumpResponse(nil, false) - httputil.NewChunkedReader(nil) - httputil.NewChunkedWriter(nil) - httputil.NewClientConn(nil, nil) - httputil.NewProxyClientConn(nil, nil) - httputil.NewServerConn(nil, nil) - httputil.NewSingleHostReverseProxy(nil) - http.Get("") -} -`, - }, -} diff --git a/src/cmd/gc/Makefile b/src/cmd/gc/Makefile index df34c05b2..58e25faaf 100644 --- a/src/cmd/gc/Makefile +++ b/src/cmd/gc/Makefile @@ -6,7 +6,7 @@ include ../../Make.dist install: y.tab.h builtin.c -y.tab.h: go.y +y.tab.h: go.y go.errors bisonerrors bison -v -y -d go.y # make yystate global, yytname mutable cat y.tab.c | sed '/ int yystate;/d; s/int yychar;/int yychar, yystate;/; s/static const char \*const yytname/const char *yytname/; s/char const \*yymsgp/char *yymsgp/' >y1.tab.c diff --git a/src/cmd/gc/dcl.c b/src/cmd/gc/dcl.c index 4a0e7430a..4121a45ab 100644 --- a/src/cmd/gc/dcl.c +++ b/src/cmd/gc/dcl.c @@ -1168,21 +1168,22 @@ methodsym(Sym *nsym, Type *t0, int iface) char *p; Type *t; char *suffix; + Pkg *spkg; + static Pkg *toppkg; t = t0; if(t == T) goto bad; s = t->sym; - if(s == S) { - if(!isptr[t->etype]) - goto bad; + if(s == S && isptr[t->etype]) { t = t->type; if(t == T) goto bad; s = t->sym; - if(s == S) - goto bad; } + spkg = nil; + if(s != S) + spkg = s->pkg; // if t0 == *t and t0 has a sym, // we want to see *t, not t0, in the method name. @@ -1195,11 +1196,23 @@ methodsym(Sym *nsym, Type *t0, int iface) if(t0->width < types[tptr]->width) suffix = "·i"; } - if(t0->sym == S && isptr[t0->etype]) - p = smprint("(%-hT).%s%s", t0, nsym->name, suffix); - else - p = smprint("%-hT.%s%s", t0, nsym->name, suffix); - s = pkglookup(p, s->pkg); + if((spkg == nil || nsym->pkg != spkg) && !exportname(nsym->name)) { + if(t0->sym == S && isptr[t0->etype]) + p = smprint("(%-hT).%s.%s%s", t0, nsym->pkg->prefix, nsym->name, suffix); + else + p = smprint("%-hT.%s.%s%s", t0, nsym->pkg->prefix, nsym->name, suffix); + } else { + if(t0->sym == S && isptr[t0->etype]) + p = smprint("(%-hT).%s%s", t0, nsym->name, suffix); + else + p = smprint("%-hT.%s%s", t0, nsym->name, suffix); + } + if(spkg == nil) { + if(toppkg == nil) + toppkg = mkpkg(strlit("go")); + spkg = toppkg; + } + s = pkglookup(p, spkg); free(p); return s; @@ -1268,7 +1281,7 @@ addmethod(Sym *sf, Type *t, int local) } pa = pa->type; - f = methtype(pa); + f = methtype(pa, 1); if(f == T) { t = pa; if(t != T) { diff --git a/src/cmd/gc/doc.go b/src/cmd/gc/doc.go index 5a2977eab..163d3862c 100644 --- a/src/cmd/gc/doc.go +++ b/src/cmd/gc/doc.go @@ -26,7 +26,7 @@ package P to read the files of P's dependencies, only the compiled output of P. Usage: - 6g [flags] file... + go tool 6g [flags] file... The specified files must be Go source files and all part of the same package. Substitute 6g with 8g or 5g where appropriate. diff --git a/src/cmd/gc/go.h b/src/cmd/gc/go.h index 753360e46..8c4fff15a 100644 --- a/src/cmd/gc/go.h +++ b/src/cmd/gc/go.h @@ -1154,7 +1154,7 @@ int cplxsubtype(int et); int eqtype(Type *t1, Type *t2); int eqtypenoname(Type *t1, Type *t2); void errorexit(void); -void expandmeth(Sym *s, Type *t); +void expandmeth(Type *t); void fatal(char *fmt, ...); void flusherrors(void); void frame(int context); @@ -1192,7 +1192,7 @@ NodeList* listtreecopy(NodeList *l); Sym* lookup(char *name); void* mal(int32 n); Type* maptype(Type *key, Type *val); -Type* methtype(Type *t); +Type* methtype(Type *t, int mustname); Pkg* mkpkg(Strlit *path); Sym* ngotype(Node *n); int noconv(Type *t1, Type *t2); diff --git a/src/cmd/gc/inl.c b/src/cmd/gc/inl.c index 96080cbfa..efce56057 100644 --- a/src/cmd/gc/inl.c +++ b/src/cmd/gc/inl.c @@ -182,6 +182,8 @@ ishairy(Node *n, int *budget) case OCALLFUNC: case OCALLINTER: case OCALLMETH: + case OPANIC: + case ORECOVER: if(debug['l'] < 4) return 1; break; diff --git a/src/cmd/gc/lex.c b/src/cmd/gc/lex.c index 96786b5e6..e71fd3848 100644 --- a/src/cmd/gc/lex.c +++ b/src/cmd/gc/lex.c @@ -514,7 +514,7 @@ addidir(char* dir) static int islocalname(Strlit *name) { - if(!windows && name->len >= 1 && name->s[0] == '/') + if(name->len >= 1 && name->s[0] == '/') return 1; if(windows && name->len >= 3 && yy_isalpha(name->s[0]) && name->s[1] == ':' && name->s[2] == '/') diff --git a/src/cmd/gc/obj.c b/src/cmd/gc/obj.c index aae566dbb..e45b4e0d4 100644 --- a/src/cmd/gc/obj.c +++ b/src/cmd/gc/obj.c @@ -126,10 +126,37 @@ outhist(Biobuf *b) { Hist *h; char *p, ds[] = {'c', ':', '/', 0}; + char *tofree; + int n; + static int first = 1; + static char *goroot, *goroot_final; + if(first) { + // Decide whether we need to rewrite paths from $GOROOT to $GOROOT_FINAL. + first = 0; + goroot = getenv("GOROOT"); + goroot_final = getenv("GOROOT_FINAL"); + if(goroot == nil) + goroot = ""; + if(goroot_final == nil) + goroot_final = goroot; + if(strcmp(goroot, goroot_final) == 0) { + goroot = nil; + goroot_final = nil; + } + } + + tofree = nil; for(h = hist; h != H; h = h->link) { p = h->name; if(p) { + if(goroot != nil) { + n = strlen(goroot); + if(strncmp(p, goroot, strlen(goroot)) == 0 && p[n] == '/') { + tofree = smprint("%s%s", goroot_final, p+n); + p = tofree; + } + } if(windows) { // if windows variable is set, then, we know already, // pathname is started with windows drive specifier @@ -161,9 +188,12 @@ outhist(Biobuf *b) outzfile(b, p); } } - } zhist(b, h->line, h->offset); + if(tofree) { + free(tofree); + tofree = nil; + } } } diff --git a/src/cmd/gc/reflect.c b/src/cmd/gc/reflect.c index 0847e9a3f..07b426508 100644 --- a/src/cmd/gc/reflect.c +++ b/src/cmd/gc/reflect.c @@ -144,11 +144,11 @@ methods(Type *t) Sig *a, *b; Sym *method; - // named method type - mt = methtype(t); + // method type + mt = methtype(t, 0); if(mt == T) return nil; - expandmeth(mt->sym, mt); + expandmeth(mt); // type stored in interface word it = t; diff --git a/src/cmd/gc/runtime.go b/src/cmd/gc/runtime.go index 000b2328f..15a61d9ef 100644 --- a/src/cmd/gc/runtime.go +++ b/src/cmd/gc/runtime.go @@ -3,7 +3,7 @@ // license that can be found in the LICENSE file. // NOTE: If you change this file you must run "./mkbuiltin" -// to update builtin.c.boot. This is not done automatically +// to update builtin.c. This is not done automatically // to avoid depending on having a working compiler binary. // +build ignore diff --git a/src/cmd/gc/subr.c b/src/cmd/gc/subr.c index 6eb7734f0..681c023a0 100644 --- a/src/cmd/gc/subr.c +++ b/src/cmd/gc/subr.c @@ -944,7 +944,7 @@ isideal(Type *t) * return type to hang methods off (r). */ Type* -methtype(Type *t) +methtype(Type *t, int mustname) { if(t == T) return T; @@ -959,7 +959,7 @@ methtype(Type *t) } // need a type name - if(t->sym == S) + if(t->sym == S && (mustname || t->etype != TSTRUCT)) return T; // check types @@ -2101,7 +2101,7 @@ lookdot0(Sym *s, Type *t, Type **save, int ignorecase) c++; } } - u = methtype(t); + u = methtype(t, 0); if(u != T) { for(f=u->method; f!=T; f=f->down) if(f->embedded == 0 && (f->sym == s || (ignorecase && ucistrcmp(f->sym->name, s->name) == 0))) { @@ -2251,7 +2251,7 @@ expand0(Type *t, int followptr) return; } - u = methtype(t); + u = methtype(t, 0); if(u != T) { for(f=u->method; f!=T; f=f->down) { if(f->sym->flags & SymUniq) @@ -2301,14 +2301,12 @@ out: } void -expandmeth(Sym *s, Type *t) +expandmeth(Type *t) { Symlink *sl; Type *f; int c, d; - if(s == S) - return; if(t == T || t->xmethod != nil) return; @@ -3021,9 +3019,9 @@ implements(Type *t, Type *iface, Type **m, Type **samename, int *ptr) return 1; } - t = methtype(t); + t = methtype(t, 0); if(t != T) - expandmeth(t->sym, t); + expandmeth(t); for(im=iface->type; im; im=im->down) { imtype = methodfunc(im->type, 0); tm = ifacelookdot(im->sym, t, &followptr, 0); @@ -3626,23 +3624,23 @@ isbadimport(Strlit *path) while(*s) { s += chartorune(&r, s); if(r == Runeerror) { - yyerror("import path contains invalid UTF-8 sequence"); + yyerror("import path contains invalid UTF-8 sequence: \"%Z\"", path); return 1; } if(r < 0x20 || r == 0x7f) { - yyerror("import path contains control character"); + yyerror("import path contains control character: \"%Z\"", path); return 1; } if(r == '\\') { - yyerror("import path contains backslash; use slash"); + yyerror("import path contains backslash; use slash: \"%Z\"", path); return 1; } if(isspacerune(r)) { - yyerror("import path contains space character"); + yyerror("import path contains space character: \"%Z\"", path); return 1; } - if(utfrune("!\"#$%&'()*,:;<=>?[]^`{|}~", r)) { - yyerror("import path contains invalid character '%C'", r); + if(utfrune("!\"#$%&'()*,:;<=>?[]^`{|}", r)) { + yyerror("import path contains invalid character '%C': \"%Z\"", r, path); return 1; } } diff --git a/src/cmd/gc/typecheck.c b/src/cmd/gc/typecheck.c index 90bd24964..e98d53857 100644 --- a/src/cmd/gc/typecheck.c +++ b/src/cmd/gc/typecheck.c @@ -190,6 +190,46 @@ typecheck(Node **np, int top) return n; } +/* + * does n contain a call or receive operation? + */ +static int callrecvlist(NodeList*); + +static int +callrecv(Node *n) +{ + if(n == nil) + return 0; + + switch(n->op) { + case OCALL: + case OCALLMETH: + case OCALLINTER: + case OCALLFUNC: + case ORECV: + return 1; + } + + return callrecv(n->left) || + callrecv(n->right) || + callrecv(n->ntest) || + callrecv(n->nincr) || + callrecvlist(n->ninit) || + callrecvlist(n->nbody) || + callrecvlist(n->nelse) || + callrecvlist(n->list) || + callrecvlist(n->rlist); +} + +static int +callrecvlist(NodeList *l) +{ + for(; l; l=l->next) + if(callrecv(l->n)) + return 1; + return 0; +} + static void typecheck1(Node **np, int top) { @@ -995,12 +1035,14 @@ reswitch: } break; case TARRAY: - if(t->bound >= 0 && l->op == ONAME) { - r = nod(OXXX, N, N); - nodconst(r, types[TINT], t->bound); - r->orig = n; - n = r; - } + if(t->bound < 0) // slice + break; + if(callrecv(l)) // has call or receive + break; + r = nod(OXXX, N, N); + nodconst(r, types[TINT], t->bound); + r->orig = n; + n = r; break; } n->type = types[TINT]; @@ -1664,11 +1706,11 @@ looktypedot(Node *n, Type *t, int dostrcmp) if(t->sym == S && isptr[t->etype]) tt = t->type; - f2 = methtype(tt); + f2 = methtype(tt, 0); if(f2 == T) return 0; - expandmeth(f2->sym, f2); + expandmeth(f2); f2 = lookdot1(n, s, f2, f2->xmethod, dostrcmp); if(f2 == T) return 0; @@ -1712,7 +1754,7 @@ lookdot(Node *n, Type *t, int dostrcmp) f2 = T; if(n->left->type == t || n->left->type->sym == S) { - f2 = methtype(t); + f2 = methtype(t, 0); if(f2 != T) { // Use f2->method, not f2->xmethod: adddot has // already inserted all the necessary embedded dots. @@ -1964,7 +2006,7 @@ keydup(Node *n, Node *hash[], ulong nhash) b = cmp.val.u.bval; if(b) { // too lazy to print the literal - yyerror("duplicate key in map literal"); + yyerror("duplicate key %N in map literal", n); return; } } diff --git a/src/cmd/gc/walk.c b/src/cmd/gc/walk.c index 74298e126..7dfd34a7a 100644 --- a/src/cmd/gc/walk.c +++ b/src/cmd/gc/walk.c @@ -646,12 +646,6 @@ walkexpr(Node **np, NodeList **init) n->ninit = nil; l = n->list->n; r = n->list->next->n; - if(n->right != N) { - // TODO: Remove once two-element map assigment is gone. - l = safeexpr(l, init); - r = safeexpr(r, init); - safeexpr(n->right, init); // cause side effects from n->right - } t = l->type; n = mkcall1(mapfndel("mapdelete", t), t->down, init, typename(t), l, r); goto ret; @@ -2364,6 +2358,12 @@ append(Node *n, NodeList **init) walkexprlistsafe(n->list, init); + // walkexprlistsafe will leave OINDEX (s[n]) alone if both s + // and n are name or literal, but those may index the slice we're + // modifying here. Fix explicitly. + for(l=n->list; l; l=l->next) + l->n = cheapexpr(l->n, init); + nsrc = n->list->n; argc = count(n->list) - 1; if (argc < 1) { @@ -2520,6 +2520,7 @@ walkcompare(Node **np, NodeList **init) expr = nodbool(n->op == OEQ); typecheck(&expr, Erv); walkexpr(&expr, init); + expr->type = n->type; *np = expr; return; } @@ -2540,6 +2541,7 @@ walkcompare(Node **np, NodeList **init) expr = nodbool(n->op == OEQ); typecheck(&expr, Erv); walkexpr(&expr, init); + expr->type = n->type; *np = expr; return; } diff --git a/src/cmd/go/bootstrap.go b/src/cmd/go/bootstrap.go index bc9a3dbbc..32941404c 100644 --- a/src/cmd/go/bootstrap.go +++ b/src/cmd/go/bootstrap.go @@ -10,8 +10,21 @@ package main -import "errors" +import ( + "errors" + "io" +) + +var errHTTP = errors.New("no http in bootstrap go command") func httpGET(url string) ([]byte, error) { - return nil, errors.New("no http in bootstrap go command") + return nil, errHTTP +} + +func httpsOrHTTP(importPath string) (string, io.ReadCloser, error) { + return "", nil, errHTTP +} + +func parseMetaGoImports(r io.Reader) (imports []metaImport) { + panic("unreachable") } diff --git a/src/cmd/go/build.go b/src/cmd/go/build.go index c330bd5de..16177c127 100644 --- a/src/cmd/go/build.go +++ b/src/cmd/go/build.go @@ -12,6 +12,7 @@ import ( "go/build" "io" "io/ioutil" + "log" "os" "os/exec" "path" @@ -20,6 +21,7 @@ import ( "runtime" "strings" "sync" + "time" ) var cmdBuild = &Command{ @@ -58,6 +60,8 @@ The build flags are shared by the build, install, run, and test commands: -x print the commands. + -compiler name + name of compiler to use, as in runtime.Compiler (gccgo or gc) -gccgoflags 'arg list' arguments to pass on each gccgo compiler/linker invocation -gcflags 'arg list' @@ -97,9 +101,42 @@ var buildLdflags []string // -ldflags flag var buildGccgoflags []string // -gccgoflags flag var buildContext = build.Default +var buildToolchain toolchain = noToolchain{} + +// buildCompiler implements flag.Var. +// It implements Set by updating both +// buildToolchain and buildContext.Compiler. +type buildCompiler struct{} + +func (c buildCompiler) Set(value string) error { + switch value { + case "gc": + buildToolchain = gcToolchain{} + case "gccgo": + buildToolchain = gccgcToolchain{} + default: + return fmt.Errorf("unknown compiler %q", value) + } + buildContext.Compiler = value + return nil +} + +func (c buildCompiler) String() string { + return buildContext.Compiler +} + +func init() { + switch build.Default.Compiler { + case "gc": + buildToolchain = gcToolchain{} + case "gccgo": + buildToolchain = gccgcToolchain{} + } +} // addBuildFlags adds the flags common to the build and install commands. func addBuildFlags(cmd *Command) { + // NOTE: If you add flags here, also add them to testflag.go. cmd.Flag.BoolVar(&buildA, "a", false, "") cmd.Flag.BoolVar(&buildN, "n", false, "") cmd.Flag.IntVar(&buildP, "p", buildP, "") @@ -110,6 +147,7 @@ func addBuildFlags(cmd *Command) { cmd.Flag.Var((*stringsFlag)(&buildLdflags), "ldflags", "") cmd.Flag.Var((*stringsFlag)(&buildGccgoflags), "gccgoflags", "") cmd.Flag.Var((*stringsFlag)(&buildContext.BuildTags), "tags", "") + cmd.Flag.Var(buildCompiler{}, "compiler", "") } type stringsFlag []string @@ -131,9 +169,7 @@ func runBuild(cmd *Command, args []string) { if len(pkgs) == 1 && pkgs[0].Name == "main" && *buildO == "" { _, *buildO = path.Split(pkgs[0].ImportPath) - if goos == "windows" { - *buildO += ".exe" - } + *buildO += exeSuffix } if *buildO != "" { @@ -348,6 +384,7 @@ func goFilesPackage(gofiles []string) *Package { bp, err := ctxt.ImportDir(dir, 0) pkg := new(Package) + pkg.local = true pkg.load(&stk, bp, err) pkg.localPrefix = dirToImportPath(dir) pkg.ImportPath = "command-line-arguments" @@ -355,7 +392,7 @@ func goFilesPackage(gofiles []string) *Package { if *buildO == "" { if pkg.Name == "main" { _, elem := filepath.Split(gofiles[0]) - *buildO = elem[:len(elem)-len(".go")] + *buildO = elem[:len(elem)-len(".go")] + exeSuffix } else { *buildO = pkg.Name + ".a" } @@ -412,7 +449,7 @@ func (b *builder) action(mode buildMode, depMode buildMode, p *Package) *action return a } // gccgo standard library is "fake" too. - if _, ok := buildToolchain.(gccgoToolchain); ok { + if _, ok := buildToolchain.(gccgcToolchain); ok { // the target name is needed for cgo. a.target = p.target return a @@ -568,7 +605,12 @@ func (b *builder) do(root *action) { } // build is the action for building a single package or command. -func (b *builder) build(a *action) error { +func (b *builder) build(a *action) (err error) { + defer func() { + if err != nil && err != errPrintedOutput { + err = fmt.Errorf("go build %s: %v", a.p.ImportPath, err) + } + }() if buildN { // In -n mode, print a banner between packages. // The banner is five lines so that when changes to @@ -620,7 +662,7 @@ func (b *builder) build(a *action) error { } cgoExe := tool("cgo") - if a.cgo != nil { + if a.cgo != nil && a.cgo.target != "" { cgoExe = a.cgo.target } outGo, outObj, err := b.cgo(a.p, cgoExe, obj, gccfiles) @@ -692,6 +734,11 @@ func (b *builder) build(a *action) error { // http://golang.org/issue/2601 objects = append(objects, cgoObjects...) + // Add system object files. + for _, syso := range a.p.SysoFiles { + objects = append(objects, filepath.Join(a.p.Dir, syso)) + } + // Pack into archive in obj directory if err := buildToolchain.pack(b, a.p, obj, a.objpkg, objects); err != nil { return err @@ -712,7 +759,12 @@ func (b *builder) build(a *action) error { } // install is the action for installing a single package or executable. -func (b *builder) install(a *action) error { +func (b *builder) install(a *action) (err error) { + defer func() { + if err != nil && err != errPrintedOutput { + err = fmt.Errorf("go install %s: %v", a.p.ImportPath, err) + } + }() a1 := a.deps[0] perm := os.FileMode(0666) if a1.link { @@ -767,7 +819,7 @@ func (b *builder) includeArgs(flag string, all []*action) []string { for _, a1 := range all { if dir := a1.pkgdir; dir == a1.p.build.PkgRoot && !incMap[dir] { incMap[dir] = true - if _, ok := buildToolchain.(gccgoToolchain); ok { + if _, ok := buildToolchain.(gccgcToolchain); ok { dir = filepath.Join(dir, "gccgo") } else { dir = filepath.Join(dir, goos+"_"+goarch) @@ -833,7 +885,7 @@ func (b *builder) copyFile(a *action, dst, src string, perm os.FileMode) error { df.Close() if err != nil { os.Remove(dst) - return err + return fmt.Errorf("copying %s to %s: %v", src, dst, err) } return nil } @@ -888,8 +940,6 @@ func (b *builder) fmtcmd(dir string, format string, args ...interface{}) string if b.work != "" { cmd = strings.Replace(cmd, b.work, "$WORK", -1) } - cmd = strings.Replace(cmd, gobin, "$GOBIN", -1) - cmd = strings.Replace(cmd, goroot, "$GOROOT", -1) return cmd } @@ -998,14 +1048,66 @@ func (b *builder) runOut(dir string, desc string, cmdargs ...interface{}) ([]byt } } - var buf bytes.Buffer - cmd := exec.Command(cmdline[0], cmdline[1:]...) - cmd.Stdout = &buf - cmd.Stderr = &buf - cmd.Dir = dir - // TODO: cmd.Env - err := cmd.Run() - return buf.Bytes(), err + nbusy := 0 + for { + var buf bytes.Buffer + cmd := exec.Command(cmdline[0], cmdline[1:]...) + cmd.Stdout = &buf + cmd.Stderr = &buf + cmd.Dir = dir + // TODO: cmd.Env + err := cmd.Run() + + // cmd.Run will fail on Unix if some other process has the binary + // we want to run open for writing. This can happen here because + // we build and install the cgo command and then run it. + // If another command was kicked off while we were writing the + // cgo binary, the child process for that command may be holding + // a reference to the fd, keeping us from running exec. + // + // But, you might reasonably wonder, how can this happen? + // The cgo fd, like all our fds, is close-on-exec, so that we need + // not worry about other processes inheriting the fd accidentally. + // The answer is that running a command is fork and exec. + // A child forked while the cgo fd is open inherits that fd. + // Until the child has called exec, it holds the fd open and the + // kernel will not let us run cgo. Even if the child were to close + // the fd explicitly, it would still be open from the time of the fork + // until the time of the explicit close, and the race would remain. + // + // On Unix systems, this results in ETXTBSY, which formats + // as "text file busy". Rather than hard-code specific error cases, + // we just look for that string. If this happens, sleep a little + // and try again. We let this happen three times, with increasing + // sleep lengths: 100+200+400 ms = 0.7 seconds. + // + // An alternate solution might be to split the cmd.Run into + // separate cmd.Start and cmd.Wait, and then use an RWLock + // to make sure that copyFile only executes when no cmd.Start + // call is in progress. However, cmd.Start (really syscall.forkExec) + // only guarantees that when it returns, the exec is committed to + // happen and succeed. It uses a close-on-exec file descriptor + // itself to determine this, so we know that when cmd.Start returns, + // at least one close-on-exec file descriptor has been closed. + // However, we cannot be sure that all of them have been closed, + // so the program might still encounter ETXTBSY even with such + // an RWLock. The race window would be smaller, perhaps, but not + // guaranteed to be gone. + // + // Sleeping when we observe the race seems to be the most reliable + // option we have. + // + // http://golang.org/issue/3001 + // + if err != nil && nbusy < 3 && strings.Contains(err.Error(), "text file busy") { + time.Sleep(100 * time.Millisecond << uint(nbusy)) + nbusy++ + continue + } + + return buf.Bytes(), err + } + panic("unreachable") } // mkdir makes the named directory. @@ -1072,32 +1174,60 @@ type toolchain interface { linker() string } -type goToolchain struct{} -type gccgoToolchain struct{} +type noToolchain struct{} + +func noCompiler() error { + log.Fatalf("unknown compiler %q", buildContext.Compiler) + return nil +} -var buildToolchain toolchain +func (noToolchain) compiler() string { + noCompiler() + return "" +} -func init() { - // TODO(rsc): Decide how to trigger gccgo. Issue 3157. - if os.Getenv("GC") == "gccgo" { - buildContext.Gccgo = true - buildToolchain = gccgoToolchain{} - } else { - buildToolchain = goToolchain{} - } +func (noToolchain) linker() string { + noCompiler() + return "" +} + +func (noToolchain) gc(b *builder, p *Package, obj string, importArgs []string, gofiles []string) (ofile string, err error) { + return "", noCompiler() +} + +func (noToolchain) asm(b *builder, p *Package, obj, ofile, sfile string) error { + return noCompiler() +} + +func (noToolchain) pkgpath(basedir string, p *Package) string { + noCompiler() + return "" +} + +func (noToolchain) pack(b *builder, p *Package, objDir, afile string, ofiles []string) error { + return noCompiler() +} + +func (noToolchain) ld(b *builder, p *Package, out string, allactions []*action, mainpkg string, ofiles []string) error { + return noCompiler() +} + +func (noToolchain) cc(b *builder, p *Package, objdir, ofile, cfile string) error { + return noCompiler() } // The Go toolchain. +type gcToolchain struct{} -func (goToolchain) compiler() string { +func (gcToolchain) compiler() string { return tool(archChar + "g") } -func (goToolchain) linker() string { +func (gcToolchain) linker() string { return tool(archChar + "l") } -func (goToolchain) gc(b *builder, p *Package, obj string, importArgs []string, gofiles []string) (ofile string, err error) { +func (gcToolchain) gc(b *builder, p *Package, obj string, importArgs []string, gofiles []string) (ofile string, err error) { out := "_go_." + archChar ofile = obj + out gcargs := []string{"-p", p.ImportPath} @@ -1114,17 +1244,17 @@ func (goToolchain) gc(b *builder, p *Package, obj string, importArgs []string, g return ofile, b.run(p.Dir, p.ImportPath, args) } -func (goToolchain) asm(b *builder, p *Package, obj, ofile, sfile string) error { +func (gcToolchain) asm(b *builder, p *Package, obj, ofile, sfile string) error { sfile = mkAbs(p.Dir, sfile) return b.run(p.Dir, p.ImportPath, tool(archChar+"a"), "-I", obj, "-o", ofile, "-DGOOS_"+goos, "-DGOARCH_"+goarch, sfile) } -func (goToolchain) pkgpath(basedir string, p *Package) string { +func (gcToolchain) pkgpath(basedir string, p *Package) string { end := filepath.FromSlash(p.ImportPath + ".a") return filepath.Join(basedir, end) } -func (goToolchain) pack(b *builder, p *Package, objDir, afile string, ofiles []string) error { +func (gcToolchain) pack(b *builder, p *Package, objDir, afile string, ofiles []string) error { var absOfiles []string for _, f := range ofiles { absOfiles = append(absOfiles, mkAbs(objDir, f)) @@ -1132,12 +1262,12 @@ func (goToolchain) pack(b *builder, p *Package, objDir, afile string, ofiles []s return b.run(p.Dir, p.ImportPath, tool("pack"), "grc", mkAbs(objDir, afile), absOfiles) } -func (goToolchain) ld(b *builder, p *Package, out string, allactions []*action, mainpkg string, ofiles []string) error { +func (gcToolchain) ld(b *builder, p *Package, out string, allactions []*action, mainpkg string, ofiles []string) error { importArgs := b.includeArgs("-L", allactions) - return b.run(p.Dir, p.ImportPath, tool(archChar+"l"), "-o", out, importArgs, buildLdflags, mainpkg) + return b.run(".", p.ImportPath, tool(archChar+"l"), "-o", out, importArgs, buildLdflags, mainpkg) } -func (goToolchain) cc(b *builder, p *Package, objdir, ofile, cfile string) error { +func (gcToolchain) cc(b *builder, p *Package, objdir, ofile, cfile string) error { inc := filepath.Join(goroot, "pkg", fmt.Sprintf("%s_%s", goos, goarch)) cfile = mkAbs(p.Dir, cfile) return b.run(p.Dir, p.ImportPath, tool(archChar+"c"), "-FVw", @@ -1146,27 +1276,24 @@ func (goToolchain) cc(b *builder, p *Package, objdir, ofile, cfile string) error } // The Gccgo toolchain. +type gccgcToolchain struct{} var gccgoBin, _ = exec.LookPath("gccgo") -func (gccgoToolchain) compiler() string { +func (gccgcToolchain) compiler() string { return gccgoBin } -func (gccgoToolchain) linker() string { +func (gccgcToolchain) linker() string { return gccgoBin } -func (gccgoToolchain) gc(b *builder, p *Package, obj string, importArgs []string, gofiles []string) (ofile string, err error) { +func (gccgcToolchain) gc(b *builder, p *Package, obj string, importArgs []string, gofiles []string) (ofile string, err error) { out := p.Name + ".o" ofile = obj + out gcargs := []string{"-g"} - if p.Name != "main" { - if p.fake { - gcargs = append(gcargs, "-fgo-prefix=fake_"+p.ImportPath) - } else { - gcargs = append(gcargs, "-fgo-prefix=go_"+p.ImportPath) - } + if prefix := gccgoPrefix(p); prefix != "" { + gcargs = append(gcargs, "-fgo-prefix="+gccgoPrefix(p)) } args := stringList("gccgo", importArgs, "-c", gcargs, "-o", ofile, buildGccgoflags) for _, f := range gofiles { @@ -1175,19 +1302,19 @@ func (gccgoToolchain) gc(b *builder, p *Package, obj string, importArgs []string return ofile, b.run(p.Dir, p.ImportPath, args) } -func (gccgoToolchain) asm(b *builder, p *Package, obj, ofile, sfile string) error { +func (gccgcToolchain) asm(b *builder, p *Package, obj, ofile, sfile string) error { sfile = mkAbs(p.Dir, sfile) return b.run(p.Dir, p.ImportPath, "gccgo", "-I", obj, "-o", ofile, "-DGOOS_"+goos, "-DGOARCH_"+goarch, sfile) } -func (gccgoToolchain) pkgpath(basedir string, p *Package) string { +func (gccgcToolchain) pkgpath(basedir string, p *Package) string { end := filepath.FromSlash(p.ImportPath + ".a") afile := filepath.Join(basedir, end) // add "lib" to the final element return filepath.Join(filepath.Dir(afile), "lib"+filepath.Base(afile)) } -func (gccgoToolchain) pack(b *builder, p *Package, objDir, afile string, ofiles []string) error { +func (gccgcToolchain) pack(b *builder, p *Package, objDir, afile string, ofiles []string) error { var absOfiles []string for _, f := range ofiles { absOfiles = append(absOfiles, mkAbs(objDir, f)) @@ -1195,7 +1322,7 @@ func (gccgoToolchain) pack(b *builder, p *Package, objDir, afile string, ofiles return b.run(p.Dir, p.ImportPath, "ar", "cru", mkAbs(objDir, afile), absOfiles) } -func (tools gccgoToolchain) ld(b *builder, p *Package, out string, allactions []*action, mainpkg string, ofiles []string) error { +func (tools gccgcToolchain) ld(b *builder, p *Package, out string, allactions []*action, mainpkg string, ofiles []string) error { // gccgo needs explicit linking with all package dependencies, // and all LDFLAGS from cgo dependencies. afiles := make(map[*Package]string) @@ -1215,10 +1342,10 @@ func (tools gccgoToolchain) ld(b *builder, p *Package, out string, allactions [] ldflags = append(ldflags, afile) } ldflags = append(ldflags, cgoldflags...) - return b.run(p.Dir, p.ImportPath, "gccgo", "-o", out, buildGccgoflags, ofiles, "-Wl,-(", ldflags, "-Wl,-)") + return b.run(".", p.ImportPath, "gccgo", "-o", out, buildGccgoflags, ofiles, "-Wl,-(", ldflags, "-Wl,-)") } -func (gccgoToolchain) cc(b *builder, p *Package, objdir, ofile, cfile string) error { +func (gccgcToolchain) cc(b *builder, p *Package, objdir, ofile, cfile string) error { inc := filepath.Join(goroot, "pkg", fmt.Sprintf("%s_%s", goos, goarch)) cfile = mkAbs(p.Dir, cfile) return b.run(p.Dir, p.ImportPath, "gcc", "-Wall", "-g", @@ -1226,6 +1353,16 @@ func (gccgoToolchain) cc(b *builder, p *Package, objdir, ofile, cfile string) er "-DGOOS_"+goos, "-DGOARCH_"+goarch, "-c", cfile) } +func gccgoPrefix(p *Package) string { + switch { + case p.build.IsCommand() && !p.forceLibrary: + return "" + case p.fake: + return "fake_" + p.ImportPath + } + return "go_" + p.ImportPath +} + // gcc runs the gcc C compiler to create an object from a single C file. func (b *builder) gcc(p *Package, out string, flags []string, cfile string) error { cfile = mkAbs(p.Dir, cfile) @@ -1239,6 +1376,9 @@ func (b *builder) gccld(p *Package, out string, flags []string, obj []string) er // gccCmd returns a gcc command line prefix func (b *builder) gccCmd(objdir string) []string { + // NOTE: env.go's mkEnv knows that the first three + // strings returned are "gcc", "-I", objdir (and cuts them off). + // TODO: HOST_CC? a := []string{"gcc", "-I", objdir, "-g", "-O2"} @@ -1263,6 +1403,14 @@ func (b *builder) gccCmd(objdir string) []string { a = append(a, "-pthread") } } + + // On OS X, some of the compilers behave as if -fno-common + // is always set, and the Mach-O linker in 6l/8l assumes this. + // See http://golang.org/issue/3253. + if goos == "darwin" { + a = append(a, "-fno-common") + } + return a } @@ -1318,11 +1466,17 @@ func (b *builder) cgo(p *Package, cgoExe, obj string, gccfiles []string) (outGo, cgoflags := []string{} // TODO: make cgo not depend on $GOARCH? + objExt := archChar + if p.Standard && p.ImportPath == "runtime/cgo" { cgoflags = append(cgoflags, "-import_runtime_cgo=false") } - if _, ok := buildToolchain.(gccgoToolchain); ok { + if _, ok := buildToolchain.(gccgcToolchain); ok { cgoflags = append(cgoflags, "-gccgo") + if prefix := gccgoPrefix(p); prefix != "" { + cgoflags = append(cgoflags, "-gccgoprefix="+gccgoPrefix(p)) + } + objExt = "o" } if err := b.run(p.Dir, p.ImportPath, cgoExe, "-objdir", obj, cgoflags, "--", cgoCFLAGS, p.CgoFiles); err != nil { return nil, nil, err @@ -1330,7 +1484,7 @@ func (b *builder) cgo(p *Package, cgoExe, obj string, gccfiles []string) (outGo, outGo = append(outGo, gofiles...) // cc _cgo_defun.c - defunObj := obj + "_cgo_defun." + archChar + defunObj := obj + "_cgo_defun." + objExt if err := buildToolchain.cc(b, p, obj, defunObj, defunC); err != nil { return nil, nil, err } @@ -1361,7 +1515,7 @@ func (b *builder) cgo(p *Package, cgoExe, obj string, gccfiles []string) (outGo, return nil, nil, err } - if _, ok := buildToolchain.(gccgoToolchain); ok { + if _, ok := buildToolchain.(gccgcToolchain); ok { // we don't use dynimport when using gccgo. return outGo, outObj, nil } @@ -1373,7 +1527,7 @@ func (b *builder) cgo(p *Package, cgoExe, obj string, gccfiles []string) (outGo, } // cc _cgo_import.ARCH - importObj := obj + "_cgo_import." + archChar + importObj := obj + "_cgo_import." + objExt if err := buildToolchain.cc(b, p, obj, importObj, importC); err != nil { return nil, nil, err } diff --git a/src/cmd/go/clean.go b/src/cmd/go/clean.go index 809e0f0e4..773951826 100644 --- a/src/cmd/go/clean.go +++ b/src/cmd/go/clean.go @@ -110,7 +110,7 @@ func clean(p *Package) { } dirs, err := ioutil.ReadDir(p.Dir) if err != nil { - errorf("%v", err) + errorf("go clean %s: %v", p.Dir, err) return } diff --git a/src/cmd/go/discovery.go b/src/cmd/go/discovery.go new file mode 100644 index 000000000..d9f930867 --- /dev/null +++ b/src/cmd/go/discovery.go @@ -0,0 +1,63 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !cmd_go_bootstrap + +// This code is compiled into the real 'go' binary, but it is not +// compiled into the binary that is built during all.bash, so as +// to avoid needing to build net (and thus use cgo) during the +// bootstrap process. + +package main + +import ( + "encoding/xml" + "io" + "strings" +) + +// parseMetaGoImports returns meta imports from the HTML in r. +// Parsing ends at the end of the <head> section or the beginning of the <body>. +func parseMetaGoImports(r io.Reader) (imports []metaImport) { + d := xml.NewDecoder(r) + d.Strict = false + for { + t, err := d.Token() + if err != nil { + return + } + if e, ok := t.(xml.StartElement); ok && strings.EqualFold(e.Name.Local, "body") { + return + } + if e, ok := t.(xml.EndElement); ok && strings.EqualFold(e.Name.Local, "head") { + return + } + e, ok := t.(xml.StartElement) + if !ok || !strings.EqualFold(e.Name.Local, "meta") { + continue + } + if attrValue(e.Attr, "name") != "go-import" { + continue + } + if f := strings.Fields(attrValue(e.Attr, "content")); len(f) == 3 { + imports = append(imports, metaImport{ + Prefix: f[0], + VCS: f[1], + RepoRoot: f[2], + }) + } + } + return +} + +// attrValue returns the attribute value for the case-insensitive key +// `name', or the empty string if nothing is found. +func attrValue(attrs []xml.Attr, name string) string { + for _, a := range attrs { + if strings.EqualFold(a.Name.Local, name) { + return a.Value + } + } + return "" +} diff --git a/src/cmd/go/doc.go b/src/cmd/go/doc.go index 8df57ff38..775f305d2 100644 --- a/src/cmd/go/doc.go +++ b/src/cmd/go/doc.go @@ -14,6 +14,7 @@ The commands are: build compile packages and dependencies clean remove object files doc run godoc on package sources + env print Go environment information fix run go tool fix on packages fmt run gofmt on package sources get download and install packages and dependencies @@ -76,6 +77,8 @@ The build flags are shared by the build, install, run, and test commands: -x print the commands. + -compiler name + name of compiler to use, as in runtime.Compiler (gccgo or gc) -gccgoflags 'arg list' arguments to pass on each gccgo compiler/linker invocation -gcflags 'arg list' @@ -153,6 +156,20 @@ To run godoc with specific options, run godoc itself. See also: go fix, go fmt, go vet. +Print Go environment information + +Usage: + + go env [var ...] + +Env prints Go environment information. + +By default env prints information as a shell script +(on Windows, a batch file). If one or more variable +names is given as arguments, env prints the value of +each named variable on its own line. + + Run go tool fix on packages Usage: @@ -196,7 +213,7 @@ Get downloads and installs the packages named by the import paths, along with their dependencies. The -a, -n, -v, -x, and -p flags have the same meaning as in 'go build' -and 'go install'. See 'go help install'. +and 'go install'. See 'go help build'. The -d flag instructs get to stop after downloading the packages; that is, it instructs get not to install the packages. @@ -253,21 +270,28 @@ is equivalent to -f '{{.ImportPath}}'. The struct being passed to the template is: type Package struct { + Dir string // directory containing package sources + ImportPath string // import path of package in dir Name string // package name Doc string // package documentation string - ImportPath string // import path of package in dir - Dir string // directory containing package sources - Version string // version of installed package (TODO) + Target string // install path + Goroot bool // is this package in the Go root? + Standard bool // is this package part of the standard Go library? Stale bool // would 'go install' do anything for this package? + Root string // Go root or Go path dir containing this package // Source files - GoFiles []string // .go source files (excluding CgoFiles, TestGoFiles, and XTestGoFiles) - TestGoFiles []string // _test.go source files internal to the package they are testing - XTestGoFiles []string // _test.go source files external to the package they are testing - CFiles []string // .c source files - HFiles []string // .h source files - SFiles []string // .s source files - CgoFiles []string // .go sources files that import "C" + GoFiles []string // .go source files (excluding CgoFiles, TestGoFiles, XTestGoFiles) + CgoFiles []string // .go sources files that import "C" + CFiles []string // .c source files + HFiles []string // .h source files + SFiles []string // .s source files + SysoFiles []string // .syso object files to add to archive + + // Cgo directives + CgoCFLAGS []string // cgo: flags for C compiler + CgoLDFLAGS []string // cgo: flags for linker + CgoPkgConfig []string // cgo: pkg-config names // Dependency information Imports []string // import paths used by this package @@ -275,8 +299,13 @@ being passed to the template is: // Error information Incomplete bool // this package or a dependency has an error - Error *PackageError // error loading package + Error *PackageError // error loading package DepsErrors []*PackageError // errors loading dependencies + + TestGoFiles []string // _test.go files in package + TestImports []string // imports from TestGoFiles + XTestGoFiles []string // _test.go files outside package + XTestImports []string // imports from XTestGoFiles } The -json flag causes the package data to be printed in JSON format @@ -479,9 +508,8 @@ An import path is a pattern if it includes one or more "..." wildcards, each of which can match any string, including the empty string and strings containing slashes. Such a pattern expands to all package directories found in the GOPATH trees with names matching the -patterns. For example, encoding/... expands to all packages -in subdirectories of the encoding tree, while net... expands to -net and all its subdirectories. +patterns. As a special case, x/... matches x as well as x's subdirectories. +For example, net/... expands to net and packages in its subdirectories. An import path can also name a package to be downloaded from a remote repository. Run 'go help remote' for details. @@ -535,7 +563,12 @@ A few common code hosting sites have special syntax: import "launchpad.net/~user/project/branch" import "launchpad.net/~user/project/branch/sub/directory" -For code hosted on other servers, an import path of the form +For code hosted on other servers, import paths may either be qualified +with the version control type, or the go tool can dynamically fetch +the import path over https/http and discover where the code resides +from a <meta> tag in the HTML. + +To declare the code location, an import path of the form repository.vcs/path @@ -564,6 +597,42 @@ When a version control system supports multiple protocols, each is tried in turn when downloading. For example, a Git download tries git://, then https://, then http://. +If the import path is not a known code hosting site and also lacks a +version control qualifier, the go tool attempts to fetch the import +over https/http and looks for a <meta> tag in the document's HTML +<head>. + +The meta tag has the form: + + <meta name="go-import" content="import-prefix vcs repo-root"> + +The import-prefix is the import path correponding to the repository +root. It must be a prefix or an exact match of the package being +fetched with "go get". If it's not an exact match, another http +request is made at the prefix to verify the <meta> tags match. + +The vcs is one of "git", "hg", "svn", etc, + +The repo-root is the root of the version control system +containing a scheme and not containing a .vcs qualifier. + +For example, + + import "example.org/pkg/foo" + +will result in the following request(s): + + https://example.org/pkg/foo?go-get=1 (preferred) + http://example.org/pkg/foo?go-get=1 (fallback) + +If that page contains the meta tag + + <meta name="go-import" content="example.org git https://code.org/r/p/exproj"> + +the go tool will verify that https://example.org/?go-get=1 contains the +same meta tag and then git clone https://code.org/r/p/exproj into +GOPATH/src/example.org. + New downloaded packages are written to the first directory listed in the GOPATH environment variable (see 'go help gopath'). diff --git a/src/cmd/go/env.go b/src/cmd/go/env.go new file mode 100644 index 000000000..d5b034809 --- /dev/null +++ b/src/cmd/go/env.go @@ -0,0 +1,89 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "fmt" + "runtime" + "strings" +) + +var cmdEnv = &Command{ + Run: runEnv, + UsageLine: "env [var ...]", + Short: "print Go environment information", + Long: ` +Env prints Go environment information. + +By default env prints information as a shell script +(on Windows, a batch file). If one or more variable +names is given as arguments, env prints the value of +each named variable on its own line. + `, +} + +type envVar struct { + name, value string +} + +func mkEnv() []envVar { + var b builder + b.init() + + env := []envVar{ + {"GOROOT", goroot}, + {"GOBIN", gobin}, + {"GOARCH", goarch}, + {"GOCHAR", archChar}, + {"GOOS", goos}, + {"GOEXE", exeSuffix}, + {"GOHOSTARCH", runtime.GOARCH}, + {"GOHOSTOS", runtime.GOOS}, + {"GOTOOLDIR", toolDir}, + {"GOGCCFLAGS", strings.Join(b.gccCmd(".")[3:], " ")}, + } + + if buildContext.CgoEnabled { + env = append(env, envVar{"CGO_ENABLED", "1"}) + } else { + env = append(env, envVar{"CGO_ENABLED", "0"}) + } + + return env +} + +func findEnv(env []envVar, name string) string { + for _, e := range env { + if e.name == name { + return e.value + } + } + return "" +} + +func runEnv(cmd *Command, args []string) { + env := mkEnv() + if len(args) > 0 { + for _, name := range args { + fmt.Printf("%s\n", findEnv(env, name)) + } + return + } + + switch runtime.GOOS { + default: + for _, e := range env { + fmt.Printf("%s=\"%s\"\n", e.name, e.value) + } + case "plan9": + for _, e := range env { + fmt.Printf("%s='%s'\n", e.name, strings.Replace(e.value, "'", "''", -1)) + } + case "windows": + for _, e := range env { + fmt.Printf("set %s=%s\n", e.name, e.value) + } + } +} diff --git a/src/cmd/go/get.go b/src/cmd/go/get.go index 0ad22adb0..abaf5ffa0 100644 --- a/src/cmd/go/get.go +++ b/src/cmd/go/get.go @@ -8,6 +8,7 @@ package main import ( "fmt" + "go/build" "os" "path/filepath" "runtime" @@ -23,7 +24,7 @@ Get downloads and installs the packages named by the import paths, along with their dependencies. The -a, -n, -v, -x, and -p flags have the same meaning as in 'go build' -and 'go install'. See 'go help install'. +and 'go install'. See 'go help build'. The -d flag instructs get to stop after downloading the packages; that is, it instructs get not to install the packages. @@ -57,19 +58,13 @@ func init() { func runGet(cmd *Command, args []string) { // Phase 1. Download/update. - args = importPaths(args) var stk importStack - for _, arg := range args { + for _, arg := range downloadPaths(args) { download(arg, &stk) } exitIfErrors() - if *getD { - // download only - return - } - - // Phase 2. Install. + // Phase 2. Rescan packages and reevaluate args list. // Code we downloaded and all code that depends on it // needs to be evicted from the package cache so that @@ -80,9 +75,48 @@ func runGet(cmd *Command, args []string) { delete(packageCache, name) } + args = importPaths(args) + + // Phase 3. Install. + if *getD { + // Download only. + // Check delayed until now so that importPaths + // has a chance to print errors. + return + } + runInstall(cmd, args) } +// downloadPath prepares the list of paths to pass to download. +// It expands ... patterns that can be expanded. If there is no match +// for a particular pattern, downloadPaths leaves it in the result list, +// in the hope that we can figure out the repository from the +// initial ...-free prefix. +func downloadPaths(args []string) []string { + args = importPathsNoDotExpansion(args) + var out []string + for _, a := range args { + if strings.Contains(a, "...") { + var expand []string + // Use matchPackagesInFS to avoid printing + // warnings. They will be printed by the + // eventual call to importPaths instead. + if build.IsLocalImport(a) { + expand = matchPackagesInFS(a) + } else { + expand = matchPackages(a) + } + if len(expand) > 0 { + out = append(out, expand...) + continue + } + } + out = append(out, a) + } + return out +} + // downloadCache records the import paths we have already // considered during the download, to avoid duplicate work when // there is more than one dependency sequence leading to @@ -112,38 +146,73 @@ func download(arg string, stk *importStack) { } downloadCache[arg] = true + pkgs := []*Package{p} + wildcardOkay := len(*stk) == 0 + // Download if the package is missing, or update if we're using -u. if p.Dir == "" || *getU { // The actual download. stk.push(p.ImportPath) - defer stk.pop() - if err := downloadPackage(p); err != nil { + err := downloadPackage(p) + if err != nil { errorf("%s", &PackageError{ImportStack: stk.copy(), Err: err.Error()}) + stk.pop() return } - // Reread the package information from the updated files. - p = reloadPackage(arg, stk) - if p.Error != nil { - errorf("%s", p.Error) - return + args := []string{arg} + // If the argument has a wildcard in it, re-evaluate the wildcard. + // We delay this until after reloadPackage so that the old entry + // for p has been replaced in the package cache. + if wildcardOkay && strings.Contains(arg, "...") { + if build.IsLocalImport(arg) { + args = matchPackagesInFS(arg) + } else { + args = matchPackages(arg) + } } - } - if *getFix { - run(stringList(tool("fix"), relPaths(p.gofiles))) + // Clear all relevant package cache entries before + // doing any new loads. + for _, arg := range args { + p := packageCache[arg] + if p != nil { + delete(packageCache, p.Dir) + delete(packageCache, p.ImportPath) + } + } - // The imports might have changed, so reload again. - p = reloadPackage(arg, stk) - if p.Error != nil { - errorf("%s", p.Error) - return + pkgs = pkgs[:0] + for _, arg := range args { + stk.push(arg) + p := loadPackage(arg, stk) + stk.pop() + if p.Error != nil { + errorf("%s", p.Error) + continue + } + pkgs = append(pkgs, p) } } - // Process dependencies, now that we know what they are. - for _, dep := range p.deps { - download(dep.ImportPath, stk) + // Process package, which might now be multiple packages + // due to wildcard expansion. + for _, p := range pkgs { + if *getFix { + run(stringList(tool("fix"), relPaths(p.gofiles))) + + // The imports might have changed, so reload again. + p = reloadPackage(arg, stk) + if p.Error != nil { + errorf("%s", p.Error) + return + } + } + + // Process dependencies, now that we know what they are. + for _, dep := range p.deps { + download(dep.ImportPath, stk) + } } } @@ -162,10 +231,11 @@ func downloadPackage(p *Package) error { } else { // Analyze the import path to determine the version control system, // repository, and the import path for the root of the repository. - vcs, repo, rootPath, err = vcsForImportPath(p.ImportPath) - } - if err != nil { - return err + rr, err := repoRootForImportPath(p.ImportPath) + if err != nil { + return err + } + vcs, repo, rootPath = rr.vcs, rr.repo, rr.root } if p.build.SrcRoot == "" { diff --git a/src/cmd/go/help.go b/src/cmd/go/help.go index 60654a272..26640d833 100644 --- a/src/cmd/go/help.go +++ b/src/cmd/go/help.go @@ -36,9 +36,8 @@ An import path is a pattern if it includes one or more "..." wildcards, each of which can match any string, including the empty string and strings containing slashes. Such a pattern expands to all package directories found in the GOPATH trees with names matching the -patterns. For example, encoding/... expands to all packages -in subdirectories of the encoding tree, while net... expands to -net and all its subdirectories. +patterns. As a special case, x/... matches x as well as x's subdirectories. +For example, net/... expands to net and packages in its subdirectories. An import path can also name a package to be downloaded from a remote repository. Run 'go help remote' for details. @@ -96,7 +95,12 @@ A few common code hosting sites have special syntax: import "launchpad.net/~user/project/branch" import "launchpad.net/~user/project/branch/sub/directory" -For code hosted on other servers, an import path of the form +For code hosted on other servers, import paths may either be qualified +with the version control type, or the go tool can dynamically fetch +the import path over https/http and discover where the code resides +from a <meta> tag in the HTML. + +To declare the code location, an import path of the form repository.vcs/path @@ -125,6 +129,42 @@ When a version control system supports multiple protocols, each is tried in turn when downloading. For example, a Git download tries git://, then https://, then http://. +If the import path is not a known code hosting site and also lacks a +version control qualifier, the go tool attempts to fetch the import +over https/http and looks for a <meta> tag in the document's HTML +<head>. + +The meta tag has the form: + + <meta name="go-import" content="import-prefix vcs repo-root"> + +The import-prefix is the import path correponding to the repository +root. It must be a prefix or an exact match of the package being +fetched with "go get". If it's not an exact match, another http +request is made at the prefix to verify the <meta> tags match. + +The vcs is one of "git", "hg", "svn", etc, + +The repo-root is the root of the version control system +containing a scheme and not containing a .vcs qualifier. + +For example, + + import "example.org/pkg/foo" + +will result in the following request(s): + + https://example.org/pkg/foo?go-get=1 (preferred) + http://example.org/pkg/foo?go-get=1 (fallback) + +If that page contains the meta tag + + <meta name="go-import" content="example.org git https://code.org/r/p/exproj"> + +the go tool will verify that https://example.org/?go-get=1 contains the +same meta tag and then git clone https://code.org/r/p/exproj into +GOPATH/src/example.org. + New downloaded packages are written to the first directory listed in the GOPATH environment variable (see 'go help gopath'). diff --git a/src/cmd/go/http.go b/src/cmd/go/http.go index 8d9b2a165..6de9a3e1e 100644 --- a/src/cmd/go/http.go +++ b/src/cmd/go/http.go @@ -13,8 +13,11 @@ package main import ( "fmt" + "io" "io/ioutil" + "log" "net/http" + "net/url" ) // httpGET returns the data from an HTTP GET request for the given URL. @@ -33,3 +36,52 @@ func httpGET(url string) ([]byte, error) { } return b, nil } + +// httpClient is the default HTTP client, but a variable so it can be +// changed by tests, without modifying http.DefaultClient. +var httpClient = http.DefaultClient + +// httpsOrHTTP returns the body of either the importPath's +// https resource or, if unavailable, the http resource. +func httpsOrHTTP(importPath string) (urlStr string, body io.ReadCloser, err error) { + fetch := func(scheme string) (urlStr string, res *http.Response, err error) { + u, err := url.Parse(scheme + "://" + importPath) + if err != nil { + return "", nil, err + } + u.RawQuery = "go-get=1" + urlStr = u.String() + if buildV { + log.Printf("Fetching %s", urlStr) + } + res, err = httpClient.Get(urlStr) + return + } + closeBody := func(res *http.Response) { + if res != nil { + res.Body.Close() + } + } + urlStr, res, err := fetch("https") + if err != nil || res.StatusCode != 200 { + if buildV { + if err != nil { + log.Printf("https fetch failed.") + } else { + log.Printf("ignoring https fetch with status code %d", res.StatusCode) + } + } + closeBody(res) + urlStr, res, err = fetch("http") + } + if err != nil { + closeBody(res) + return "", nil, err + } + // Note: accepting a non-200 OK here, so people can serve a + // meta import in their http 404 page. + if buildV { + log.Printf("Parsing meta tags from %s (status code %d)", urlStr, res.StatusCode) + } + return urlStr, res.Body, nil +} diff --git a/src/cmd/go/list.go b/src/cmd/go/list.go index fa3f5d330..edb59aa79 100644 --- a/src/cmd/go/list.go +++ b/src/cmd/go/list.go @@ -30,30 +30,42 @@ is equivalent to -f '{{.ImportPath}}'. The struct being passed to the template is: type Package struct { + Dir string // directory containing package sources + ImportPath string // import path of package in dir Name string // package name Doc string // package documentation string - ImportPath string // import path of package in dir - Dir string // directory containing package sources - Version string // version of installed package (TODO) + Target string // install path + Goroot bool // is this package in the Go root? + Standard bool // is this package part of the standard Go library? Stale bool // would 'go install' do anything for this package? + Root string // Go root or Go path dir containing this package // Source files - GoFiles []string // .go source files (excluding CgoFiles, TestGoFiles, and XTestGoFiles) - TestGoFiles []string // _test.go source files internal to the package they are testing - XTestGoFiles []string // _test.go source files external to the package they are testing - CFiles []string // .c source files - HFiles []string // .h source files - SFiles []string // .s source files - CgoFiles []string // .go sources files that import "C" + GoFiles []string // .go source files (excluding CgoFiles, TestGoFiles, XTestGoFiles) + CgoFiles []string // .go sources files that import "C" + CFiles []string // .c source files + HFiles []string // .h source files + SFiles []string // .s source files + SysoFiles []string // .syso object files to add to archive + + // Cgo directives + CgoCFLAGS []string // cgo: flags for C compiler + CgoLDFLAGS []string // cgo: flags for linker + CgoPkgConfig []string // cgo: pkg-config names // Dependency information Imports []string // import paths used by this package Deps []string // all (recursively) imported dependencies - + // Error information Incomplete bool // this package or a dependency has an error - Error *PackageError // error loading package + Error *PackageError // error loading package DepsErrors []*PackageError // errors loading dependencies + + TestGoFiles []string // _test.go files in package + TestImports []string // imports from TestGoFiles + XTestGoFiles []string // _test.go files outside package + XTestImports []string // imports from XTestGoFiles } The -json flag causes the package data to be printed in JSON format @@ -75,6 +87,7 @@ For more about specifying packages, see 'go help packages'. func init() { cmdList.Run = runList // break init cycle + cmdList.Flag.Var(buildCompiler{}, "compiler", "") } var listE = cmdList.Flag.Bool("e", false, "") diff --git a/src/cmd/go/main.go b/src/cmd/go/main.go index 3a0f7a089..2f8209c86 100644 --- a/src/cmd/go/main.go +++ b/src/cmd/go/main.go @@ -76,6 +76,7 @@ var commands = []*Command{ cmdBuild, cmdClean, cmdDoc, + cmdEnv, cmdFix, cmdFmt, cmdGet, @@ -246,8 +247,9 @@ func help(args []string) { os.Exit(2) // failed at 'go help cmd' } -// importPaths returns the import paths to use for the given command line. -func importPaths(args []string) []string { +// importPathsNoDotExpansion returns the import paths to use for the given +// command line, but it does no ... expansion. +func importPathsNoDotExpansion(args []string) []string { if len(args) == 0 { return []string{"."} } @@ -269,13 +271,26 @@ func importPaths(args []string) []string { } else { a = path.Clean(a) } - - if build.IsLocalImport(a) && strings.Contains(a, "...") { - out = append(out, allPackagesInFS(a)...) + if a == "all" || a == "std" { + out = append(out, allPackages(a)...) continue } - if a == "all" || a == "std" || strings.Contains(a, "...") { - out = append(out, allPackages(a)...) + out = append(out, a) + } + return out +} + +// importPaths returns the import paths to use for the given command line. +func importPaths(args []string) []string { + args = importPathsNoDotExpansion(args) + var out []string + for _, a := range args { + if strings.Contains(a, "...") { + if build.IsLocalImport(a) { + out = append(out, allPackagesInFS(a)...) + } else { + out = append(out, allPackages(a)...) + } continue } out = append(out, a) @@ -344,6 +359,10 @@ func runOut(dir string, cmdargs ...interface{}) []byte { func matchPattern(pattern string) func(name string) bool { re := regexp.QuoteMeta(pattern) re = strings.Replace(re, `\.\.\.`, `.*`, -1) + // Special case: foo/... matches foo too. + if strings.HasSuffix(re, `/.*`) { + re = re[:len(re)-len(`/.*`)] + `(/.*)?` + } reg := regexp.MustCompile(`^` + re + `$`) return func(name string) bool { return reg.MatchString(name) @@ -355,6 +374,14 @@ func matchPattern(pattern string) func(name string) bool { // The pattern is either "all" (all packages), "std" (standard packages) // or a path including "...". func allPackages(pattern string) []string { + pkgs := matchPackages(pattern) + if len(pkgs) == 0 { + fmt.Fprintf(os.Stderr, "warning: %q matched no packages\n", pattern) + } + return pkgs +} + +func matchPackages(pattern string) []string { match := func(string) bool { return true } if pattern != "all" && pattern != "std" { match = matchPattern(pattern) @@ -431,10 +458,6 @@ func allPackages(pattern string) []string { return nil }) } - - if len(pkgs) == 0 { - fmt.Fprintf(os.Stderr, "warning: %q matched no packages\n", pattern) - } return pkgs } @@ -442,6 +465,14 @@ func allPackages(pattern string) []string { // beginning ./ or ../, meaning it should scan the tree rooted // at the given directory. There are ... in the pattern too. func allPackagesInFS(pattern string) []string { + pkgs := matchPackagesInFS(pattern) + if len(pkgs) == 0 { + fmt.Fprintf(os.Stderr, "warning: %q matched no packages\n", pattern) + } + return pkgs +} + +func matchPackagesInFS(pattern string) []string { // Find directory to begin the scan. // Could be smarter but this one optimization // is enough for now, since ... is usually at the @@ -481,10 +512,6 @@ func allPackagesInFS(pattern string) []string { pkgs = append(pkgs, name) return nil }) - - if len(pkgs) == 0 { - fmt.Fprintf(os.Stderr, "warning: %q matched no packages\n", pattern) - } return pkgs } diff --git a/src/cmd/go/match_test.go b/src/cmd/go/match_test.go new file mode 100644 index 000000000..f058f235a --- /dev/null +++ b/src/cmd/go/match_test.go @@ -0,0 +1,36 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import "testing" + +var matchTests = []struct { + pattern string + path string + match bool +}{ + {"...", "foo", true}, + {"net", "net", true}, + {"net", "net/http", false}, + {"net/http", "net", false}, + {"net/http", "net/http", true}, + {"net...", "netchan", true}, + {"net...", "net", true}, + {"net...", "net/http", true}, + {"net...", "not/http", false}, + {"net/...", "netchan", false}, + {"net/...", "net", true}, + {"net/...", "net/http", true}, + {"net/...", "not/http", false}, +} + +func TestMatchPattern(t *testing.T) { + for _, tt := range matchTests { + match := matchPattern(tt.pattern)(tt.path) + if match != tt.match { + t.Errorf("matchPattern(%q)(%q) = %v, want %v", tt.pattern, tt.path, match, tt.match) + } + } +} diff --git a/src/cmd/go/pkg.go b/src/cmd/go/pkg.go index 3763000c6..44dbd6798 100644 --- a/src/cmd/go/pkg.go +++ b/src/cmd/go/pkg.go @@ -17,6 +17,7 @@ import ( "sort" "strings" "time" + "unicode" ) // A Package describes a single package found in a directory. @@ -24,25 +25,23 @@ type Package struct { // Note: These fields are part of the go command's public API. // See list.go. It is okay to add fields, but not to change or // remove existing ones. Keep in sync with list.go - Dir string `json:",omitempty"` // directory containing package sources - ImportPath string `json:",omitempty"` // import path of package in dir - Name string `json:",omitempty"` // package name - Doc string `json:",omitempty"` // package documentation string - Target string `json:",omitempty"` // install path - Goroot bool `json:",omitempty"` // is this package found in the Go root? - Standard bool `json:",omitempty"` // is this package part of the standard Go library? - Stale bool `json:",omitempty"` // would 'go install' do anything for this package? - Incomplete bool `json:",omitempty"` // was there an error loading this package or dependencies? - Error *PackageError `json:",omitempty"` // error loading this package (not dependencies) - - Root string `json:",omitempty"` // root dir of tree this package belongs to + Dir string `json:",omitempty"` // directory containing package sources + ImportPath string `json:",omitempty"` // import path of package in dir + Name string `json:",omitempty"` // package name + Doc string `json:",omitempty"` // package documentation string + Target string `json:",omitempty"` // install path + Goroot bool `json:",omitempty"` // is this package found in the Go root? + Standard bool `json:",omitempty"` // is this package part of the standard Go library? + Stale bool `json:",omitempty"` // would 'go install' do anything for this package? + Root string `json:",omitempty"` // Go root or Go path dir containing this package // Source files - GoFiles []string `json:",omitempty"` // .go source files (excluding CgoFiles, TestGoFiles XTestGoFiles) - CgoFiles []string `json:",omitempty"` // .go sources files that import "C" - CFiles []string `json:",omitempty"` // .c source files - HFiles []string `json:",omitempty"` // .h source files - SFiles []string `json:",omitempty"` // .s source files + GoFiles []string `json:",omitempty"` // .go source files (excluding CgoFiles, TestGoFiles, XTestGoFiles) + CgoFiles []string `json:",omitempty"` // .go sources files that import "C" + CFiles []string `json:",omitempty"` // .c source files + HFiles []string `json:",omitempty"` // .h source files + SFiles []string `json:",omitempty"` // .s source files + SysoFiles []string `json:",omitempty"` // .syso system object files added to package // Cgo directives CgoCFLAGS []string `json:",omitempty"` // cgo: flags for C compiler @@ -50,8 +49,12 @@ type Package struct { CgoPkgConfig []string `json:",omitempty"` // cgo: pkg-config names // Dependency information - Imports []string `json:",omitempty"` // import paths used by this package - Deps []string `json:",omitempty"` // all (recursively) imported dependencies + Imports []string `json:",omitempty"` // import paths used by this package + Deps []string `json:",omitempty"` // all (recursively) imported dependencies + + // Error information + Incomplete bool `json:",omitempty"` // was there an error loading this package or dependencies? + Error *PackageError `json:",omitempty"` // error loading this package (not dependencies) DepsErrors []*PackageError `json:",omitempty"` // errors loading dependencies // Test information @@ -61,16 +64,17 @@ type Package struct { XTestImports []string `json:",omitempty"` // imports from XTestGoFiles // Unexported fields are not part of the public API. - build *build.Package - pkgdir string // overrides build.PkgDir - imports []*Package - deps []*Package - gofiles []string // GoFiles+CgoFiles+TestGoFiles+XTestGoFiles files, absolute paths - target string // installed file for this package (may be executable) - fake bool // synthesized package - forceBuild bool // this package must be rebuilt - local bool // imported via local path (./ or ../) - localPrefix string // interpret ./ and ../ imports relative to this prefix + build *build.Package + pkgdir string // overrides build.PkgDir + imports []*Package + deps []*Package + gofiles []string // GoFiles+CgoFiles+TestGoFiles+XTestGoFiles files, absolute paths + target string // installed file for this package (may be executable) + fake bool // synthesized package + forceBuild bool // this package must be rebuilt + forceLibrary bool // this package is a library (even if named "main") + local bool // imported via local path (./ or ../) + localPrefix string // interpret ./ and ../ imports relative to this prefix } func (p *Package) copyBuild(pp *build.Package) { @@ -89,6 +93,7 @@ func (p *Package) copyBuild(pp *build.Package) { p.CFiles = pp.CFiles p.HFiles = pp.HFiles p.SFiles = pp.SFiles + p.SysoFiles = pp.SysoFiles p.CgoCFLAGS = pp.CgoCFLAGS p.CgoLDFLAGS = pp.CgoLDFLAGS p.CgoPkgConfig = pp.CgoPkgConfig @@ -171,7 +176,16 @@ func reloadPackage(arg string, stk *importStack) *Package { // a special case, so that all the code to deal with ordinary imports works // automatically. func dirToImportPath(dir string) string { - return pathpkg.Join("_", strings.Replace(filepath.ToSlash(dir), ":", "_", -1)) + return pathpkg.Join("_", strings.Map(makeImportValid, filepath.ToSlash(dir))) +} + +func makeImportValid(r rune) rune { + // Should match Go spec, compilers, and ../../pkg/go/parser/parser.go:/isValidImport. + const illegalChars = `!"#$%&'()*,:;<=>?[\]^{|}` + "`\uFFFD" + if !unicode.IsGraphic(r) || unicode.IsSpace(r) || strings.ContainsRune(illegalChars, r) { + return '_' + } + return r } // loadImport scans the directory named by path, which must be an import path, @@ -276,9 +290,8 @@ func (p *Package) load(stk *importStack, bp *build.Package, err error) *Package p.copyBuild(bp) // The localPrefix is the path we interpret ./ imports relative to. - // Now that we've fixed the import path, it's just the import path. // Synthesized main packages sometimes override this. - p.localPrefix = p.ImportPath + p.localPrefix = dirToImportPath(p.Dir) if err != nil { p.Incomplete = true @@ -340,6 +353,16 @@ func (p *Package) load(stk *importStack, bp *build.Package, err error) *Package } p1 := loadImport(path, p.Dir, stk, p.build.ImportPos[path]) if p1.local { + if !p.local && p.Error == nil { + p.Error = &PackageError{ + ImportStack: stk.copy(), + Err: fmt.Sprintf("local import %q in non-local package", path), + } + pos := p.build.ImportPos[path] + if len(pos) > 0 { + p.Error.Pos = pos[0].String() + } + } path = p1.ImportPath importPaths[i] = path } @@ -371,7 +394,7 @@ func (p *Package) load(stk *importStack, bp *build.Package, err error) *Package } // unsafe is a fake package. - if p.Standard && p.ImportPath == "unsafe" { + if p.Standard && (p.ImportPath == "unsafe" || buildContext.Compiler == "gccgo") { p.target = "" } @@ -416,7 +439,7 @@ func computeStale(pkgs ...*Package) { // isStale reports whether package p needs to be rebuilt. func isStale(p *Package, topRoot map[string]bool) bool { - if p.Standard && p.ImportPath == "unsafe" { + if p.Standard && (p.ImportPath == "unsafe" || buildContext.Compiler == "gccgo") { // fake, builtin package return false } @@ -486,7 +509,7 @@ func isStale(p *Package, topRoot map[string]bool) bool { return false } - srcs := stringList(p.GoFiles, p.CFiles, p.HFiles, p.SFiles, p.CgoFiles) + srcs := stringList(p.GoFiles, p.CFiles, p.HFiles, p.SFiles, p.CgoFiles, p.SysoFiles) for _, src := range srcs { if olderThan(filepath.Join(p.Dir, src)) { return true diff --git a/src/cmd/go/run.go b/src/cmd/go/run.go index 2976d5c8d..94cd59296 100644 --- a/src/cmd/go/run.go +++ b/src/cmd/go/run.go @@ -26,9 +26,7 @@ See also: go build. func init() { cmdRun.Run = runRun // break init loop - cmdRun.Flag.BoolVar(&buildA, "a", false, "") - cmdRun.Flag.BoolVar(&buildN, "n", false, "") - cmdRun.Flag.BoolVar(&buildX, "x", false, "") + addBuildFlags(cmdRun) } func printStderr(args ...interface{}) (int, error) { @@ -44,12 +42,15 @@ func runRun(cmd *Command, args []string) { i++ } files, cmdArgs := args[:i], args[i:] + if len(files) == 0 { + fatalf("go run: no go files listed") + } p := goFilesPackage(files) if p.Error != nil { fatalf("%s", p.Error) } if p.Name != "main" { - fatalf("cannot run non-main package") + fatalf("go run: cannot run non-main package") } p.target = "" // must build - not up to date a1 := b.action(modeBuild, modeBuild, p) diff --git a/src/cmd/go/test.bash b/src/cmd/go/test.bash index daca144ee..541535101 100755 --- a/src/cmd/go/test.bash +++ b/src/cmd/go/test.bash @@ -22,37 +22,50 @@ do done # Test local (./) imports. -./testgo build -o hello testdata/local/easy.go -./hello >hello.out -if ! grep -q '^easysub\.Hello' hello.out; then - echo "testdata/local/easy.go did not generate expected output" - cat hello.out - ok=false -fi - -./testgo build -o hello testdata/local/easysub/main.go -./hello >hello.out -if ! grep -q '^easysub\.Hello' hello.out; then - echo "testdata/local/easysub/main.go did not generate expected output" - cat hello.out - ok=false -fi - -./testgo build -o hello testdata/local/hard.go -./hello >hello.out -if ! grep -q '^sub\.Hello' hello.out || ! grep -q '^subsub\.Hello' hello.out ; then - echo "testdata/local/hard.go did not generate expected output" - cat hello.out - ok=false -fi +testlocal() { + local="$1" + ./testgo build -o hello "testdata/$local/easy.go" + ./hello >hello.out + if ! grep -q '^easysub\.Hello' hello.out; then + echo "testdata/$local/easy.go did not generate expected output" + cat hello.out + ok=false + fi + + ./testgo build -o hello "testdata/$local/easysub/main.go" + ./hello >hello.out + if ! grep -q '^easysub\.Hello' hello.out; then + echo "testdata/$local/easysub/main.go did not generate expected output" + cat hello.out + ok=false + fi + + ./testgo build -o hello "testdata/$local/hard.go" + ./hello >hello.out + if ! grep -q '^sub\.Hello' hello.out || ! grep -q '^subsub\.Hello' hello.out ; then + echo "testdata/$local/hard.go did not generate expected output" + cat hello.out + ok=false + fi + + rm -f err.out hello.out hello + + # Test that go install x.go fails. + if ./testgo install "testdata/$local/easy.go" >/dev/null 2>&1; then + echo "go install testdata/$local/easy.go succeeded" + ok=false + fi +} -rm -f err.out hello.out hello +# Test local imports +testlocal local -# Test that go install x.go fails. -if ./testgo install testdata/local/easy.go >/dev/null 2>&1; then - echo "go install testdata/local/easy.go succeeded" - ok=false -fi +# Test local imports again, with bad characters in the directory name. +bad='#$%:, &()*;<=>?\^{}' +rm -rf "testdata/$bad" +cp -R testdata/local "testdata/$bad" +testlocal "$bad" +rm -rf "testdata/$bad" # Test tests with relative imports. if ! ./testgo test ./testdata/testimport; then diff --git a/src/cmd/go/test.go b/src/cmd/go/test.go index 6ca49d10f..870ab190f 100644 --- a/src/cmd/go/test.go +++ b/src/cmd/go/test.go @@ -192,8 +192,6 @@ See the documentation of the testing package for more information. var ( testC bool // -c flag testI bool // -i flag - testP int // -p flag - testX bool // -x flag testV bool // -v flag testFiles []string // -file flag(s) TODO: not respected testTimeout string // -timeout flag @@ -241,11 +239,6 @@ func runTest(cmd *Command, args []string) { testStreamOutput = len(pkgArgs) == 0 || testBench || (len(pkgs) <= 1 && testShowPass) - buildX = testX - if testP > 0 { - buildP = testP - } - var b builder b.init() @@ -265,6 +258,9 @@ func runTest(cmd *Command, args []string) { for _, path := range p.TestImports { deps[path] = true } + for _, path := range p.XTestImports { + deps[path] = true + } } // translate C to runtime/cgo @@ -450,6 +446,7 @@ func (b *builder) test(p *Package) (buildAction, runAction, printAction *action, ptest.imports = append(append([]*Package{}, p.imports...), imports...) ptest.pkgdir = testDir ptest.fake = true + ptest.forceLibrary = true ptest.Stale = true ptest.build = new(build.Package) *ptest.build = *p.build @@ -461,12 +458,6 @@ func (b *builder) test(p *Package) (buildAction, runAction, printAction *action, m[k] = append(m[k], v...) } ptest.build.ImportPos = m - computeStale(ptest) - a := b.action(modeBuild, modeBuild, ptest) - a.objdir = testDir + string(filepath.Separator) - a.objpkg = ptestObj - a.target = ptestObj - a.link = false } else { ptest = p } @@ -477,6 +468,7 @@ func (b *builder) test(p *Package) (buildAction, runAction, printAction *action, Name: p.Name + "_test", ImportPath: p.ImportPath + "_test", localPrefix: p.localPrefix, + Root: p.Root, Dir: p.Dir, GoFiles: p.XTestGoFiles, Imports: p.XTestImports, @@ -488,11 +480,6 @@ func (b *builder) test(p *Package) (buildAction, runAction, printAction *action, fake: true, Stale: true, } - computeStale(pxtest) - a := b.action(modeBuild, modeBuild, pxtest) - a.objdir = testDir + string(filepath.Separator) - a.objpkg = buildToolchain.pkgpath(testDir, pxtest) - a.target = a.objpkg } // Action for building pkg.test. @@ -501,8 +488,9 @@ func (b *builder) test(p *Package) (buildAction, runAction, printAction *action, Dir: testDir, GoFiles: []string{"_testmain.go"}, ImportPath: "testmain", + Root: p.Root, imports: []*Package{ptest}, - build: &build.Package{}, + build: &build.Package{Name: "main"}, fake: true, Stale: true, } @@ -523,6 +511,21 @@ func (b *builder) test(p *Package) (buildAction, runAction, printAction *action, pmain.imports = append(pmain.imports, ptesting, pregexp) computeStale(pmain) + if ptest != p { + a := b.action(modeBuild, modeBuild, ptest) + a.objdir = testDir + string(filepath.Separator) + a.objpkg = ptestObj + a.target = ptestObj + a.link = false + } + + if pxtest != nil { + a := b.action(modeBuild, modeBuild, pxtest) + a.objdir = testDir + string(filepath.Separator) + a.objpkg = buildToolchain.pkgpath(testDir, pxtest) + a.target = a.objpkg + } + a := b.action(modeBuild, modeBuild, pmain) a.objdir = testDir + string(filepath.Separator) a.objpkg = filepath.Join(testDir, "main.a") @@ -639,6 +642,9 @@ func (b *builder) runTest(a *action) error { // cleanTest is the action for cleaning up after a test. func (b *builder) cleanTest(a *action) error { + if buildWork { + return nil + } run := a.deps[0] testDir := filepath.Join(b.work, filepath.FromSlash(run.p.ImportPath+"/_test")) os.RemoveAll(testDir) diff --git a/src/cmd/go/testflag.go b/src/cmd/go/testflag.go index 7c9b7f16d..ecf5bf456 100644 --- a/src/cmd/go/testflag.go +++ b/src/cmd/go/testflag.go @@ -47,7 +47,7 @@ func testUsage() { // testFlagSpec defines a flag we know about. type testFlagSpec struct { name string - isBool bool + boolVar *bool passToTest bool // pass to Test multiOK bool // OK to have multiple instances present bool // flag has been seen @@ -56,11 +56,21 @@ type testFlagSpec struct { // testFlagDefn is the set of flags we process. var testFlagDefn = []*testFlagSpec{ // local. - {name: "c", isBool: true}, + {name: "c", boolVar: &testC}, {name: "file", multiOK: true}, - {name: "i", isBool: true}, + {name: "i", boolVar: &testI}, + + // build flags. + {name: "a", boolVar: &buildA}, + {name: "n", boolVar: &buildN}, {name: "p"}, - {name: "x", isBool: true}, + {name: "x", boolVar: &buildX}, + {name: "work", boolVar: &buildWork}, + {name: "gcflags"}, + {name: "ldflags"}, + {name: "gccgoflags"}, + {name: "tags"}, + {name: "compiler"}, // passed to 6.out, adding a "test." prefix to the name if necessary: -v becomes -test.v. {name: "bench", passToTest: true}, @@ -71,9 +81,9 @@ var testFlagDefn = []*testFlagSpec{ {name: "memprofilerate", passToTest: true}, {name: "parallel", passToTest: true}, {name: "run", passToTest: true}, - {name: "short", isBool: true, passToTest: true}, + {name: "short", boolVar: new(bool), passToTest: true}, {name: "timeout", passToTest: true}, - {name: "v", isBool: true, passToTest: true}, + {name: "v", boolVar: &testV, passToTest: true}, } // testFlags processes the command line, grabbing -x and -c, rewriting known flags @@ -118,16 +128,21 @@ func testFlags(args []string) (packageNames, passToTest []string) { continue } switch f.name { - case "c": - setBoolFlag(&testC, value) - case "i": - setBoolFlag(&testI, value) + // bool flags. + case "a", "c", "i", "n", "x", "v", "work": + setBoolFlag(f.boolVar, value) case "p": - setIntFlag(&testP, value) - case "x": - setBoolFlag(&testX, value) - case "v": - setBoolFlag(&testV, value) + setIntFlag(&buildP, value) + case "gcflags": + buildGcflags = strings.Fields(value) + case "ldflags": + buildLdflags = strings.Fields(value) + case "gccgoflags": + buildGccgoflags = strings.Fields(value) + case "tags": + buildContext.BuildTags = strings.Fields(value) + case "compiler": + buildCompiler{}.Set(value) case "file": testFiles = append(testFiles, value) case "bench": @@ -172,7 +187,7 @@ func testFlag(args []string, i int) (f *testFlagSpec, value string, extra bool) for _, f = range testFlagDefn { if name == f.name { // Booleans are special because they have modes -x, -x=true, -x=false. - if f.isBool { + if f.boolVar != nil { if equals < 0 { // otherwise, it's been set and will be verified in setBoolFlag value = "true" } else { diff --git a/src/cmd/go/vcs.go b/src/cmd/go/vcs.go index cf3410242..3634b606c 100644 --- a/src/cmd/go/vcs.go +++ b/src/cmd/go/vcs.go @@ -7,7 +7,9 @@ package main import ( "bytes" "encoding/json" + "errors" "fmt" + "log" "os" "os/exec" "path/filepath" @@ -102,7 +104,7 @@ var vcsGit = &vcsCmd{ tagSyncCmd: "checkout {tag}", tagSyncDefault: "checkout origin/master", - scheme: []string{"git", "https", "http"}, + scheme: []string{"git", "https", "http", "git+ssh"}, pingCmd: "ls-remote {scheme}://{repo}", } @@ -121,7 +123,7 @@ var vcsBzr = &vcsCmd{ tagSyncCmd: "update -r {tag}", tagSyncDefault: "update -r revno:-1", - scheme: []string{"https", "http", "bzr"}, + scheme: []string{"https", "http", "bzr", "bzr+ssh"}, pingCmd: "info {scheme}://{repo}", } @@ -136,7 +138,7 @@ var vcsSvn = &vcsCmd{ // There is no tag command in subversion. // The branch information is all in the path names. - scheme: []string{"https", "http", "svn"}, + scheme: []string{"https", "http", "svn", "svn+ssh"}, pingCmd: "info {scheme}://{repo}", } @@ -302,12 +304,58 @@ func vcsForDir(p *Package) (vcs *vcsCmd, root string, err error) { return nil, "", fmt.Errorf("directory %q is not using a known version control system", dir) } -// vcsForImportPath analyzes importPath to determine the +// repoRoot represents a version control system, a repo, and a root of +// where to put it on disk. +type repoRoot struct { + vcs *vcsCmd + + // repo is the repository URL, including scheme + repo string + + // root is the import path corresponding to the root of the + // repository + root string +} + +// repoRootForImportPath analyzes importPath to determine the // version control system, and code repository to use. -// On return, repo is the repository URL and root is the -// import path corresponding to the root of the repository -// (thus root is a prefix of importPath). -func vcsForImportPath(importPath string) (vcs *vcsCmd, repo, root string, err error) { +func repoRootForImportPath(importPath string) (*repoRoot, error) { + rr, err := repoRootForImportPathStatic(importPath, "") + if err == errUnknownSite { + rr, err = repoRootForImportDynamic(importPath) + + // repoRootForImportDynamic returns error detail + // that is irrelevant if the user didn't intend to use a + // dynamic import in the first place. + // Squelch it. + if err != nil { + if buildV { + log.Printf("import %q: %v", importPath, err) + } + err = fmt.Errorf("unrecognized import path %q", importPath) + } + } + + if err == nil && strings.Contains(importPath, "...") && strings.Contains(rr.root, "...") { + // Do not allow wildcards in the repo root. + rr = nil + err = fmt.Errorf("cannot expand ... in %q", importPath) + } + return rr, err +} + +var errUnknownSite = errors.New("dynamic lookup required to find mapping") + +// repoRootForImportPathStatic attempts to map importPath to a +// repoRoot using the commonly-used VCS hosting sites in vcsPaths +// (github.com/user/dir), or from a fully-qualified importPath already +// containing its VCS type (foo.com/repo.git/dir) +// +// If scheme is non-empty, that scheme is forced. +func repoRootForImportPathStatic(importPath, scheme string) (*repoRoot, error) { + if strings.Contains(importPath, "://") { + return nil, fmt.Errorf("invalid import path %q", importPath) + } for _, srv := range vcsPaths { if !strings.HasPrefix(importPath, srv.prefix) { continue @@ -315,7 +363,7 @@ func vcsForImportPath(importPath string) (vcs *vcsCmd, repo, root string, err er m := srv.regexp.FindStringSubmatch(importPath) if m == nil { if srv.prefix != "" { - return nil, "", "", fmt.Errorf("invalid %s import path %q", srv.prefix, importPath) + return nil, fmt.Errorf("invalid %s import path %q", srv.prefix, importPath) } continue } @@ -338,24 +386,127 @@ func vcsForImportPath(importPath string) (vcs *vcsCmd, repo, root string, err er } if srv.check != nil { if err := srv.check(match); err != nil { - return nil, "", "", err + return nil, err } } vcs := vcsByCmd(match["vcs"]) if vcs == nil { - return nil, "", "", fmt.Errorf("unknown version control system %q", match["vcs"]) + return nil, fmt.Errorf("unknown version control system %q", match["vcs"]) } if srv.ping { - for _, scheme := range vcs.scheme { - if vcs.ping(scheme, match["repo"]) == nil { - match["repo"] = scheme + "://" + match["repo"] - break + if scheme != "" { + match["repo"] = scheme + "://" + match["repo"] + } else { + for _, scheme := range vcs.scheme { + if vcs.ping(scheme, match["repo"]) == nil { + match["repo"] = scheme + "://" + match["repo"] + break + } } } } - return vcs, match["repo"], match["root"], nil + rr := &repoRoot{ + vcs: vcs, + repo: match["repo"], + root: match["root"], + } + return rr, nil + } + return nil, errUnknownSite +} + +// repoRootForImportDynamic finds a *repoRoot for a custom domain that's not +// statically known by repoRootForImportPathStatic. +// +// This handles "vanity import paths" like "name.tld/pkg/foo". +func repoRootForImportDynamic(importPath string) (*repoRoot, error) { + slash := strings.Index(importPath, "/") + if slash < 0 { + return nil, fmt.Errorf("missing / in import %q", importPath) + } + urlStr, body, err := httpsOrHTTP(importPath) + if err != nil { + return nil, fmt.Errorf("http/https fetch for import %q: %v", importPath, err) + } + defer body.Close() + metaImport, err := matchGoImport(parseMetaGoImports(body), importPath) + if err != nil { + if err != errNoMatch { + return nil, fmt.Errorf("parse %s: %v", urlStr, err) + } + return nil, fmt.Errorf("parse %s: no go-import meta tags", urlStr) + } + if buildV { + log.Printf("get %q: found meta tag %#v at %s", importPath, metaImport, urlStr) + } + // If the import was "uni.edu/bob/project", which said the + // prefix was "uni.edu" and the RepoRoot was "evilroot.com", + // make sure we don't trust Bob and check out evilroot.com to + // "uni.edu" yet (possibly overwriting/preempting another + // non-evil student). Instead, first verify the root and see + // if it matches Bob's claim. + if metaImport.Prefix != importPath { + if buildV { + log.Printf("get %q: verifying non-authoritative meta tag", importPath) + } + urlStr0 := urlStr + urlStr, body, err = httpsOrHTTP(metaImport.Prefix) + if err != nil { + return nil, fmt.Errorf("fetch %s: %v", urlStr, err) + } + imports := parseMetaGoImports(body) + if len(imports) == 0 { + return nil, fmt.Errorf("fetch %s: no go-import meta tag", urlStr) + } + metaImport2, err := matchGoImport(imports, importPath) + if err != nil || metaImport != metaImport2 { + return nil, fmt.Errorf("%s and %s disagree about go-import for %s", urlStr0, urlStr, metaImport.Prefix) + } + } + + if !strings.Contains(metaImport.RepoRoot, "://") { + return nil, fmt.Errorf("%s: invalid repo root %q; no scheme", urlStr, metaImport.RepoRoot) + } + rr := &repoRoot{ + vcs: vcsByCmd(metaImport.VCS), + repo: metaImport.RepoRoot, + root: metaImport.Prefix, + } + if rr.vcs == nil { + return nil, fmt.Errorf("%s: unknown vcs %q", urlStr, metaImport.VCS) + } + return rr, nil +} + +// metaImport represents the parsed <meta name="go-import" +// content="prefix vcs reporoot" /> tags from HTML files. +type metaImport struct { + Prefix, VCS, RepoRoot string +} + +// errNoMatch is returned from matchGoImport when there's no applicable match. +var errNoMatch = errors.New("no import match") + +// matchGoImport returns the metaImport from imports matching importPath. +// An error is returned if there are multiple matches. +// errNoMatch is returned if none match. +func matchGoImport(imports []metaImport, importPath string) (_ metaImport, err error) { + match := -1 + for i, im := range imports { + if !strings.HasPrefix(importPath, im.Prefix) { + continue + } + if match != -1 { + err = fmt.Errorf("multiple meta tags match import path %q", importPath) + return + } + match = i + } + if match == -1 { + err = errNoMatch + return } - return nil, "", "", fmt.Errorf("unrecognized import path %q", importPath) + return imports[match], nil } // expand rewrites s to replace {k} with match[k] for each key k in match. diff --git a/src/cmd/godoc/appinit.go b/src/cmd/godoc/appinit.go index e65be4094..4096a4f22 100644 --- a/src/cmd/godoc/appinit.go +++ b/src/cmd/godoc/appinit.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build ignore +// +build appengine package main @@ -42,8 +42,7 @@ func init() { log.Fatalf("%s: %s\n", zipfile, err) } // rc is never closed (app running forever) - fs = NewZipFS(rc) - fsHttp = NewHttpZipFS(rc, *goroot) + fs.Bind("/", NewZipFS(rc, zipFilename), *goroot, bindReplace) // initialize http handlers readTemplates() @@ -53,9 +52,6 @@ func init() { // initialize default directory tree with corresponding timestamp. initFSTree() - // initialize directory trees for user-defined file systems (-path flag). - initDirTrees() - // Immediately update metadata. updateMetadata() diff --git a/src/cmd/godoc/codewalk.go b/src/cmd/godoc/codewalk.go index 018259f7d..f7f51d0a0 100644 --- a/src/cmd/godoc/codewalk.go +++ b/src/cmd/godoc/codewalk.go @@ -31,7 +31,7 @@ import ( // Handler for /doc/codewalk/ and below. func codewalk(w http.ResponseWriter, r *http.Request) { relpath := r.URL.Path[len("/doc/codewalk/"):] - abspath := absolutePath(r.URL.Path[1:], *goroot) + abspath := r.URL.Path r.ParseForm() if f := r.FormValue("fileprint"); f != "" { @@ -53,7 +53,9 @@ func codewalk(w http.ResponseWriter, r *http.Request) { } // Otherwise append .xml and hope to find - // a codewalk description. + // a codewalk description, but before trim + // the trailing /. + abspath = strings.TrimRight(abspath, "/") cw, err := loadCodewalk(abspath + ".xml") if err != nil { log.Print(err) @@ -67,7 +69,7 @@ func codewalk(w http.ResponseWriter, r *http.Request) { } b := applyTemplate(codewalkHTML, "codewalk", cw) - servePage(w, "Codewalk: "+cw.Title, "", "", b) + servePage(w, cw.Title, "Codewalk: "+cw.Title, "", "", b) } // A Codewalk represents a single codewalk read from an XML file. @@ -130,7 +132,7 @@ func loadCodewalk(filename string) (*Codewalk, error) { i = len(st.Src) } filename := st.Src[0:i] - data, err := ReadFile(fs, absolutePath(filename, *goroot)) + data, err := ReadFile(fs, filename) if err != nil { st.Err = err continue @@ -198,7 +200,7 @@ func codewalkDir(w http.ResponseWriter, r *http.Request, relpath, abspath string } b := applyTemplate(codewalkdirHTML, "codewalkdir", v) - servePage(w, "Codewalks", "", "", b) + servePage(w, "", "Codewalks", "", "", b) } // codewalkFileprint serves requests with ?fileprint=f&lo=lo&hi=hi. @@ -208,7 +210,7 @@ func codewalkDir(w http.ResponseWriter, r *http.Request, relpath, abspath string // of the codewalk pages. It is a separate iframe and does not get // the usual godoc HTML wrapper. func codewalkFileprint(w http.ResponseWriter, r *http.Request, f string) { - abspath := absolutePath(f, *goroot) + abspath := f data, err := ReadFile(fs, abspath) if err != nil { log.Print(err) diff --git a/src/cmd/godoc/dirtrees.go b/src/cmd/godoc/dirtrees.go index 1acde99bd..b9b529f87 100644 --- a/src/cmd/godoc/dirtrees.go +++ b/src/cmd/godoc/dirtrees.go @@ -13,7 +13,7 @@ import ( "go/token" "log" "os" - "path/filepath" + pathpkg "path" "strings" ) @@ -35,7 +35,7 @@ func isGoFile(fi os.FileInfo) bool { name := fi.Name() return !fi.IsDir() && len(name) > 0 && name[0] != '.' && // ignore .files - filepath.Ext(name) == ".go" + pathpkg.Ext(name) == ".go" } func isPkgFile(fi os.FileInfo) bool { @@ -50,12 +50,11 @@ func isPkgDir(fi os.FileInfo) bool { } type treeBuilder struct { - pathFilter func(string) bool - maxDepth int + maxDepth int } func (b *treeBuilder) newDirTree(fset *token.FileSet, path, name string, depth int) *Directory { - if b.pathFilter != nil && !b.pathFilter(path) || name == testdataDirName { + if name == testdataDirName { return nil } @@ -70,13 +69,7 @@ func (b *treeBuilder) newDirTree(fset *token.FileSet, path, name string, depth i } } - list, err := fs.ReadDir(path) - if err != nil { - // newDirTree is called with a path that should be a package - // directory; errors here should not happen, but if they do, - // we want to know about them - log.Printf("ReadDir(%s): %s", path, err) - } + list, _ := fs.ReadDir(path) // determine number of subdirectories and if there are package files ndirs := 0 @@ -92,7 +85,7 @@ func (b *treeBuilder) newDirTree(fset *token.FileSet, path, name string, depth i // though the directory doesn't contain any real package files - was bug) if synopses[0] == "" { // no "optimal" package synopsis yet; continue to collect synopses - file, err := parseFile(fset, filepath.Join(path, d.Name()), + file, err := parseFile(fset, pathpkg.Join(path, d.Name()), parser.ParseComments|parser.PackageClauseOnly) if err == nil { hasPkgFiles = true @@ -126,7 +119,7 @@ func (b *treeBuilder) newDirTree(fset *token.FileSet, path, name string, depth i for _, d := range list { if isPkgDir(d) { name := d.Name() - dd := b.newDirTree(fset, filepath.Join(path, name), name, depth+1) + dd := b.newDirTree(fset, pathpkg.Join(path, name), name, depth+1) if dd != nil { dirs[i] = dd i++ @@ -170,7 +163,7 @@ func (b *treeBuilder) newDirTree(fset *token.FileSet, path, name string, depth i // are assumed to contain package files even if their contents are not known // (i.e., in this case the tree may contain directories w/o any package files). // -func newDirectory(root string, pathFilter func(string) bool, maxDepth int) *Directory { +func newDirectory(root string, maxDepth int) *Directory { // The root could be a symbolic link so use Stat not Lstat. d, err := fs.Stat(root) // If we fail here, report detailed error messages; otherwise @@ -186,7 +179,7 @@ func newDirectory(root string, pathFilter func(string) bool, maxDepth int) *Dire if maxDepth < 0 { maxDepth = 1e6 // "infinity" } - b := treeBuilder{pathFilter, maxDepth} + b := treeBuilder{maxDepth} // the file set provided is only for local parsing, no position // information escapes and thus we don't need to save the set return b.newDirTree(token.NewFileSet(), root, d.Name(), 0) @@ -235,10 +228,20 @@ func (dir *Directory) lookupLocal(name string) *Directory { return nil } +func splitPath(p string) []string { + if strings.HasPrefix(p, "/") { + p = p[1:] + } + if p == "" { + return nil + } + return strings.Split(p, "/") +} + // lookup looks for the *Directory for a given path, relative to dir. func (dir *Directory) lookup(path string) *Directory { - d := strings.Split(dir.Path, string(filepath.Separator)) - p := strings.Split(path, string(filepath.Separator)) + d := splitPath(dir.Path) + p := splitPath(path) i := 0 for i < len(d) { if i >= len(p) || d[i] != p[i] { @@ -311,8 +314,8 @@ func (root *Directory) listing(skipRoot bool) *DirList { if strings.HasPrefix(d.Path, root.Path) { path = d.Path[len(root.Path):] } - // remove trailing separator if any - path must be relative - if len(path) > 0 && path[0] == filepath.Separator { + // remove leading separator if any - path must be relative + if len(path) > 0 && path[0] == '/' { path = path[1:] } p.Path = path diff --git a/src/cmd/godoc/doc.go b/src/cmd/godoc/doc.go index 15c393cd7..39ecc6e63 100644 --- a/src/cmd/godoc/doc.go +++ b/src/cmd/godoc/doc.go @@ -25,7 +25,7 @@ In command-line mode, the -q flag enables search queries against a godoc running as a webserver. If no explicit server address is specified with the -server flag, godoc first tries localhost:6060 and then http://golang.org. - godoc -q Reader Writer + godoc -q Reader godoc -q math.Sin godoc -server=:6060 -q sin @@ -77,27 +77,22 @@ The flags are: HTTP service address (e.g., '127.0.0.1:6060' or just ':6060') -server=addr webserver address for command line searches - -sync="command" - if this and -sync_minutes are set, run the argument as a - command every sync_minutes; it is intended to update the - repository holding the source files. - -sync_minutes=0 - sync interval in minutes; sync is disabled if <= 0 -templates="" directory containing alternate template files; if set, the directory may provide alternative template files for the files in $GOROOT/lib/godoc - -filter="" - filter file containing permitted package directory paths - -filter_minutes=0 - filter file update interval in minutes; update is disabled if <= 0 + -url=path + print to standard output the data that would be served by + an HTTP request for path -zip="" zip file providing the file system to serve; disabled if empty -The -path flag accepts a list of colon-separated paths; unrooted paths are relative -to the current working directory. Each path is considered as an additional root for -packages in order of appearance. The last (absolute) path element is the prefix for -the package path. For instance, given the flag value: +By default, godoc looks at the packages it finds via $GOROOT and $GOPATH (if set). +Additional directories may be specified via the -path flag which accepts a list +of colon-separated paths; unrooted paths are relative to the current working +directory. Each path is considered as an additional root for packages in order +of appearance. The last (absolute) path element is the prefix for the package +path. For instance, given the flag value: path=".:/home/bar:/public" @@ -108,23 +103,8 @@ as follows: /home/bar/x -> bar/x /public/x -> public/x -Paths provided via -path may point to very large file systems that contain -non-Go files. Creating the subtree of directories with Go packages may take -a long amount of time. A file containing newline-separated directory paths -may be provided with the -filter flag; if it exists, only directories -on those paths are considered. If -filter_minutes is set, the filter_file is -updated regularly by walking the entire directory tree. - When godoc runs as a web server and -index is set, a search index is maintained. -The index is created at startup and is automatically updated every time the --sync command terminates with exit status 0, indicating that files have changed. - -If the sync exit status is 1, godoc assumes that it succeeded without errors -but that no files changed; the index is not updated in this case. - -In all other cases, sync is assumed to have failed and godoc backs off running -sync exponentially (up to 1 day). As soon as sync succeeds again (exit status 0 -or 1), the normal sync rhythm is re-established. +The index is created at startup. The index contains both identifier and full text search information (searchable via regular expressions). The maximum number of full text search results shown @@ -158,6 +138,7 @@ one may run godoc as follows: godoc -http=:6060 -zip=go.zip -goroot=$HOME/go See "Godoc: documenting Go code" for how to write good comments for godoc: -http://blog.golang.org/2011/03/godoc-documenting-go-code.html +http://golang.org/doc/articles/godoc_documenting_go_code.html + */ package documentation diff --git a/src/cmd/godoc/filesystem.go b/src/cmd/godoc/filesystem.go index 4e48c9e68..09d7b2463 100644 --- a/src/cmd/godoc/filesystem.go +++ b/src/cmd/godoc/filesystem.go @@ -12,16 +12,56 @@ import ( "fmt" "io" "io/ioutil" + "net/http" "os" + pathpkg "path" + "path/filepath" + "sort" + "strings" + "time" ) +// fs is the file system that godoc reads from and serves. +// It is a virtual file system that operates on slash-separated paths, +// and its root corresponds to the Go distribution root: /src/pkg +// holds the source tree, and so on. This means that the URLs served by +// the godoc server are the same as the paths in the virtual file +// system, which helps keep things simple. +// +// New file trees - implementations of FileSystem - can be added to +// the virtual file system using nameSpace's Bind method. +// The usual setup is to bind OS(runtime.GOROOT) to the root +// of the name space and then bind any GOPATH/src directories +// on top of /src/pkg, so that all sources are in /src/pkg. +// +// For more about name spaces, see the nameSpace type's +// documentation below. +// +// The use of this virtual file system means that most code processing +// paths can assume they are slash-separated and should be using +// package path (often imported as pathpkg) to manipulate them, +// even on Windows. +// +var fs = nameSpace{} // the underlying file system for godoc + +// Setting debugNS = true will enable debugging prints about +// name space translations. +const debugNS = false + // The FileSystem interface specifies the methods godoc is using // to access the file system for which it serves documentation. type FileSystem interface { - Open(path string) (io.ReadCloser, error) + Open(path string) (readSeekCloser, error) Lstat(path string) (os.FileInfo, error) Stat(path string) (os.FileInfo, error) ReadDir(path string) ([]os.FileInfo, error) + String() string +} + +type readSeekCloser interface { + io.Reader + io.Seeker + io.Closer } // ReadFile reads the file named by path from fs and returns the contents. @@ -34,16 +74,31 @@ func ReadFile(fs FileSystem, path string) ([]byte, error) { return ioutil.ReadAll(rc) } -// ---------------------------------------------------------------------------- -// OS-specific FileSystem implementation +// OS returns an implementation of FileSystem reading from the +// tree rooted at root. Recording a root is convenient everywhere +// but necessary on Windows, because the slash-separated path +// passed to Open has no way to specify a drive letter. Using a root +// lets code refer to OS(`c:\`), OS(`d:\`) and so on. +func OS(root string) FileSystem { + return osFS(root) +} + +type osFS string -var OS FileSystem = osFS{} +func (root osFS) String() string { return "os(" + string(root) + ")" } -// osFS is the OS-specific implementation of FileSystem -type osFS struct{} +func (root osFS) resolve(path string) string { + // Clean the path so that it cannot possibly begin with ../. + // If it did, the result of filepath.Join would be outside the + // tree rooted at root. We probably won't ever see a path + // with .. in it, but be safe anyway. + path = pathpkg.Clean("/" + path) -func (osFS) Open(path string) (io.ReadCloser, error) { - f, err := os.Open(path) + return filepath.Join(string(root), path) +} + +func (root osFS) Open(path string) (readSeekCloser, error) { + f, err := os.Open(root.resolve(path)) if err != nil { return nil, err } @@ -57,14 +112,453 @@ func (osFS) Open(path string) (io.ReadCloser, error) { return f, nil } -func (osFS) Lstat(path string) (os.FileInfo, error) { - return os.Lstat(path) +func (root osFS) Lstat(path string) (os.FileInfo, error) { + return os.Lstat(root.resolve(path)) +} + +func (root osFS) Stat(path string) (os.FileInfo, error) { + return os.Stat(root.resolve(path)) +} + +func (root osFS) ReadDir(path string) ([]os.FileInfo, error) { + return ioutil.ReadDir(root.resolve(path)) // is sorted +} + +// hasPathPrefix returns true if x == y or x == y + "/" + more +func hasPathPrefix(x, y string) bool { + return x == y || strings.HasPrefix(x, y) && (strings.HasSuffix(y, "/") || strings.HasPrefix(x[len(y):], "/")) +} + +// A nameSpace is a file system made up of other file systems +// mounted at specific locations in the name space. +// +// The representation is a map from mount point locations +// to the list of file systems mounted at that location. A traditional +// Unix mount table would use a single file system per mount point, +// but we want to be able to mount multiple file systems on a single +// mount point and have the system behave as if the union of those +// file systems were present at the mount point. +// For example, if the OS file system has a Go installation in +// c:\Go and additional Go path trees in d:\Work1 and d:\Work2, then +// this name space creates the view we want for the godoc server: +// +// nameSpace{ +// "/": { +// {old: "/", fs: OS(`c:\Go`), new: "/"}, +// }, +// "/src/pkg": { +// {old: "/src/pkg", fs: OS(`c:\Go`), new: "/src/pkg"}, +// {old: "/src/pkg", fs: OS(`d:\Work1`), new: "/src"}, +// {old: "/src/pkg", fs: OS(`d:\Work2`), new: "/src"}, +// }, +// } +// +// This is created by executing: +// +// ns := nameSpace{} +// ns.Bind("/", OS(`c:\Go`), "/", bindReplace) +// ns.Bind("/src/pkg", OS(`d:\Work1`), "/src", bindAfter) +// ns.Bind("/src/pkg", OS(`d:\Work2`), "/src", bindAfter) +// +// A particular mount point entry is a triple (old, fs, new), meaning that to +// operate on a path beginning with old, replace that prefix (old) with new +// and then pass that path to the FileSystem implementation fs. +// +// Given this name space, a ReadDir of /src/pkg/code will check each prefix +// of the path for a mount point (first /src/pkg/code, then /src/pkg, then /src, +// then /), stopping when it finds one. For the above example, /src/pkg/code +// will find the mount point at /src/pkg: +// +// {old: "/src/pkg", fs: OS(`c:\Go`), new: "/src/pkg"}, +// {old: "/src/pkg", fs: OS(`d:\Work1`), new: "/src"}, +// {old: "/src/pkg", fs: OS(`d:\Work2`), new: "/src"}, +// +// ReadDir will when execute these three calls and merge the results: +// +// OS(`c:\Go`).ReadDir("/src/pkg/code") +// OS(`d:\Work1').ReadDir("/src/code") +// OS(`d:\Work2').ReadDir("/src/code") +// +// Note that the "/src/pkg" in "/src/pkg/code" has been replaced by +// just "/src" in the final two calls. +// +// OS is itself an implementation of a file system: it implements +// OS(`c:\Go`).ReadDir("/src/pkg/code") as ioutil.ReadDir(`c:\Go\src\pkg\code`). +// +// Because the new path is evaluated by fs (here OS(root)), another way +// to read the mount table is to mentally combine fs+new, so that this table: +// +// {old: "/src/pkg", fs: OS(`c:\Go`), new: "/src/pkg"}, +// {old: "/src/pkg", fs: OS(`d:\Work1`), new: "/src"}, +// {old: "/src/pkg", fs: OS(`d:\Work2`), new: "/src"}, +// +// reads as: +// +// "/src/pkg" -> c:\Go\src\pkg +// "/src/pkg" -> d:\Work1\src +// "/src/pkg" -> d:\Work2\src +// +// An invariant (a redundancy) of the name space representation is that +// ns[mtpt][i].old is always equal to mtpt (in the example, ns["/src/pkg"]'s +// mount table entries always have old == "/src/pkg"). The 'old' field is +// useful to callers, because they receive just a []mountedFS and not any +// other indication of which mount point was found. +// +type nameSpace map[string][]mountedFS + +// A mountedFS handles requests for path by replacing +// a prefix 'old' with 'new' and then calling the fs methods. +type mountedFS struct { + old string + fs FileSystem + new string +} + +// translate translates path for use in m, replacing old with new. +// +// mountedFS{"/src/pkg", fs, "/src"}.translate("/src/pkg/code") == "/src/code". +func (m mountedFS) translate(path string) string { + path = pathpkg.Clean("/" + path) + if !hasPathPrefix(path, m.old) { + panic("translate " + path + " but old=" + m.old) + } + return pathpkg.Join(m.new, path[len(m.old):]) +} + +func (nameSpace) String() string { + return "ns" +} + +// Fprint writes a text representation of the name space to w. +func (ns nameSpace) Fprint(w io.Writer) { + fmt.Fprint(w, "name space {\n") + var all []string + for mtpt := range ns { + all = append(all, mtpt) + } + sort.Strings(all) + for _, mtpt := range all { + fmt.Fprintf(w, "\t%s:\n", mtpt) + for _, m := range ns[mtpt] { + fmt.Fprintf(w, "\t\t%s %s\n", m.fs, m.new) + } + } + fmt.Fprint(w, "}\n") +} + +// clean returns a cleaned, rooted path for evaluation. +// It canonicalizes the path so that we can use string operations +// to analyze it. +func (nameSpace) clean(path string) string { + return pathpkg.Clean("/" + path) +} + +// Bind causes references to old to redirect to the path new in newfs. +// If mode is bindReplace, old redirections are discarded. +// If mode is bindBefore, this redirection takes priority over existing ones, +// but earlier ones are still consulted for paths that do not exist in newfs. +// If mode is bindAfter, this redirection happens only after existing ones +// have been tried and failed. + +const ( + bindReplace = iota + bindBefore + bindAfter +) + +func (ns nameSpace) Bind(old string, newfs FileSystem, new string, mode int) { + old = ns.clean(old) + new = ns.clean(new) + m := mountedFS{old, newfs, new} + var mtpt []mountedFS + switch mode { + case bindReplace: + mtpt = append(mtpt, m) + case bindAfter: + mtpt = append(mtpt, ns.resolve(old)...) + mtpt = append(mtpt, m) + case bindBefore: + mtpt = append(mtpt, m) + mtpt = append(mtpt, ns.resolve(old)...) + } + + // Extend m.old, m.new in inherited mount point entries. + for i := range mtpt { + m := &mtpt[i] + if m.old != old { + if !hasPathPrefix(old, m.old) { + // This should not happen. If it does, panic so + // that we can see the call trace that led to it. + panic(fmt.Sprintf("invalid Bind: old=%q m={%q, %s, %q}", old, m.old, m.fs.String(), m.new)) + } + suffix := old[len(m.old):] + m.old = pathpkg.Join(m.old, suffix) + m.new = pathpkg.Join(m.new, suffix) + } + } + + ns[old] = mtpt +} + +// resolve resolves a path to the list of mountedFS to use for path. +func (ns nameSpace) resolve(path string) []mountedFS { + path = ns.clean(path) + for { + if m := ns[path]; m != nil { + if debugNS { + fmt.Printf("resolve %s: %v\n", path, m) + } + return m + } + if path == "/" { + break + } + path = pathpkg.Dir(path) + } + return nil +} + +// Open implements the FileSystem Open method. +func (ns nameSpace) Open(path string) (readSeekCloser, error) { + var err error + for _, m := range ns.resolve(path) { + if debugNS { + fmt.Printf("tx %s: %v\n", path, m.translate(path)) + } + r, err1 := m.fs.Open(m.translate(path)) + if err1 == nil { + return r, nil + } + if err == nil { + err = err1 + } + } + if err == nil { + err = &os.PathError{Op: "open", Path: path, Err: os.ErrNotExist} + } + return nil, err +} + +// stat implements the FileSystem Stat and Lstat methods. +func (ns nameSpace) stat(path string, f func(FileSystem, string) (os.FileInfo, error)) (os.FileInfo, error) { + var err error + for _, m := range ns.resolve(path) { + fi, err1 := f(m.fs, m.translate(path)) + if err1 == nil { + return fi, nil + } + if err == nil { + err = err1 + } + } + if err == nil { + err = &os.PathError{Op: "stat", Path: path, Err: os.ErrNotExist} + } + return nil, err +} + +func (ns nameSpace) Stat(path string) (os.FileInfo, error) { + return ns.stat(path, FileSystem.Stat) +} + +func (ns nameSpace) Lstat(path string) (os.FileInfo, error) { + return ns.stat(path, FileSystem.Lstat) +} + +// dirInfo is a trivial implementation of os.FileInfo for a directory. +type dirInfo string + +func (d dirInfo) Name() string { return string(d) } +func (d dirInfo) Size() int64 { return 0 } +func (d dirInfo) Mode() os.FileMode { return os.ModeDir | 0555 } +func (d dirInfo) ModTime() time.Time { return startTime } +func (d dirInfo) IsDir() bool { return true } +func (d dirInfo) Sys() interface{} { return nil } + +var startTime = time.Now() + +// ReadDir implements the FileSystem ReadDir method. It's where most of the magic is. +// (The rest is in resolve.) +// +// Logically, ReadDir must return the union of all the directories that are named +// by path. In order to avoid misinterpreting Go packages, of all the directories +// that contain Go source code, we only include the files from the first, +// but we include subdirectories from all. +// +// ReadDir must also return directory entries needed to reach mount points. +// If the name space looks like the example in the type nameSpace comment, +// but c:\Go does not have a src/pkg subdirectory, we still want to be able +// to find that subdirectory, because we've mounted d:\Work1 and d:\Work2 +// there. So if we don't see "src" in the directory listing for c:\Go, we add an +// entry for it before returning. +// +func (ns nameSpace) ReadDir(path string) ([]os.FileInfo, error) { + path = ns.clean(path) + + var ( + haveGo = false + haveName = map[string]bool{} + all []os.FileInfo + err error + first []os.FileInfo + ) + + for _, m := range ns.resolve(path) { + dir, err1 := m.fs.ReadDir(m.translate(path)) + if err1 != nil { + if err == nil { + err = err1 + } + continue + } + + if dir == nil { + dir = []os.FileInfo{} + } + + if first == nil { + first = dir + } + + // If we don't yet have Go files in 'all' and this directory + // has some, add all the files from this directory. + // Otherwise, only add subdirectories. + useFiles := false + if !haveGo { + for _, d := range dir { + if strings.HasSuffix(d.Name(), ".go") { + useFiles = true + haveGo = true + break + } + } + } + + for _, d := range dir { + name := d.Name() + if (d.IsDir() || useFiles) && !haveName[name] { + haveName[name] = true + all = append(all, d) + } + } + } + + // We didn't find any directories containing Go files. + // If some directory returned successfully, use that. + if !haveGo { + for _, d := range first { + if !haveName[d.Name()] { + haveName[d.Name()] = true + all = append(all, d) + } + } + } + + // Built union. Add any missing directories needed to reach mount points. + for old := range ns { + if hasPathPrefix(old, path) && old != path { + // Find next element after path in old. + elem := old[len(path):] + if strings.HasPrefix(elem, "/") { + elem = elem[1:] + } + if i := strings.Index(elem, "/"); i >= 0 { + elem = elem[:i] + } + if !haveName[elem] { + haveName[elem] = true + all = append(all, dirInfo(elem)) + } + } + } + + if len(all) == 0 { + return nil, err + } + + sort.Sort(byName(all)) + return all, nil +} + +// byName implements sort.Interface. +type byName []os.FileInfo + +func (f byName) Len() int { return len(f) } +func (f byName) Less(i, j int) bool { return f[i].Name() < f[j].Name() } +func (f byName) Swap(i, j int) { f[i], f[j] = f[j], f[i] } + +// An httpFS implements http.FileSystem using a FileSystem. +type httpFS struct { + fs FileSystem +} + +func (h *httpFS) Open(name string) (http.File, error) { + fi, err := h.fs.Stat(name) + if err != nil { + return nil, err + } + if fi.IsDir() { + return &httpDir{h.fs, name, nil}, nil + } + f, err := h.fs.Open(name) + if err != nil { + return nil, err + } + return &httpFile{h.fs, f, name}, nil +} + +// httpDir implements http.File for a directory in a FileSystem. +type httpDir struct { + fs FileSystem + name string + pending []os.FileInfo +} + +func (h *httpDir) Close() error { return nil } +func (h *httpDir) Stat() (os.FileInfo, error) { return h.fs.Stat(h.name) } +func (h *httpDir) Read([]byte) (int, error) { + return 0, fmt.Errorf("cannot Read from directory %s", h.name) +} + +func (h *httpDir) Seek(offset int64, whence int) (int64, error) { + if offset == 0 && whence == 0 { + h.pending = nil + return 0, nil + } + return 0, fmt.Errorf("unsupported Seek in directory %s", h.name) +} + +func (h *httpDir) Readdir(count int) ([]os.FileInfo, error) { + if h.pending == nil { + d, err := h.fs.ReadDir(h.name) + if err != nil { + return nil, err + } + if d == nil { + d = []os.FileInfo{} // not nil + } + h.pending = d + } + + if len(h.pending) == 0 && count > 0 { + return nil, io.EOF + } + if count <= 0 || count > len(h.pending) { + count = len(h.pending) + } + d := h.pending[:count] + h.pending = h.pending[count:] + return d, nil } -func (osFS) Stat(path string) (os.FileInfo, error) { - return os.Stat(path) +// httpFile implements http.File for a file (not directory) in a FileSystem. +type httpFile struct { + fs FileSystem + readSeekCloser + name string } -func (osFS) ReadDir(path string) ([]os.FileInfo, error) { - return ioutil.ReadDir(path) // is sorted +func (h *httpFile) Stat() (os.FileInfo, error) { return h.fs.Stat(h.name) } +func (h *httpFile) Readdir(int) ([]os.FileInfo, error) { + return nil, fmt.Errorf("cannot Readdir from file %s", h.name) } diff --git a/src/cmd/godoc/godoc.go b/src/cmd/godoc/godoc.go index e5f7a73d4..26814d2fa 100644 --- a/src/cmd/godoc/godoc.go +++ b/src/cmd/godoc/godoc.go @@ -20,7 +20,7 @@ import ( "net/http" "net/url" "os" - "path" + pathpkg "path" "path/filepath" "regexp" "runtime" @@ -55,12 +55,9 @@ var ( // file system roots // TODO(gri) consider the invariant that goroot always end in '/' - goroot = flag.String("goroot", runtime.GOROOT(), "Go root directory") - testDir = flag.String("testdir", "", "Go root subdirectory - for testing only (faster startups)") - pkgPath = flag.String("path", "", "additional package directories (colon-separated)") - filter = flag.String("filter", "", "filter file containing permitted package directory paths") - filterMin = flag.Int("filter_minutes", 0, "filter file update interval in minutes; disabled if <= 0") - filterDelay delayTime // actual filter update interval in minutes; usually filterDelay == filterMin, but filterDelay may back off exponentially + goroot = flag.String("goroot", runtime.GOROOT(), "Go root directory") + testDir = flag.String("testdir", "", "Go root subdirectory - for testing only (faster startups)") + pkgPath = flag.String("path", "", "additional package directories (colon-separated)") // layout control tabwidth = flag.Int("tabwidth", 4, "tab width") @@ -70,44 +67,42 @@ var ( // search index indexEnabled = flag.Bool("index", false, "enable search index") indexFiles = flag.String("index_files", "", "glob pattern specifying index files;"+ - "if not empty, the index is read from these files in sorted order") + "if not empty, the index is read from these files in sorted order") maxResults = flag.Int("maxresults", 10000, "maximum number of full text search results shown") indexThrottle = flag.Float64("index_throttle", 0.75, "index throttle value; 0.0 = no time allocated, 1.0 = full throttle") - // file system mapping - fs FileSystem // the underlying file system for godoc - fsHttp http.FileSystem // the underlying file system for http - fsMap Mapping // user-defined mapping - fsTree RWValue // *Directory tree of packages, updated with each sync - pathFilter RWValue // filter used when building fsMap directory trees - fsModified RWValue // timestamp of last call to invalidateIndex - docMetadata RWValue // mapping from paths to *Metadata + // file system information + fsTree RWValue // *Directory tree of packages, updated with each sync (but sync code is removed now) + fsModified RWValue // timestamp of last call to invalidateIndex + docMetadata RWValue // mapping from paths to *Metadata // http handlers fileServer http.Handler // default file server - cmdHandler httpHandler - pkgHandler httpHandler + cmdHandler docServer + pkgHandler docServer ) func initHandlers() { - paths := filepath.SplitList(*pkgPath) - gorootSrc := filepath.Join(build.Default.GOROOT, "src", "pkg") - for _, p := range build.Default.SrcDirs() { - if p != gorootSrc { - paths = append(paths, p) + // Add named directories in -path argument as + // subdirectories of src/pkg. + for _, p := range filepath.SplitList(*pkgPath) { + _, elem := filepath.Split(p) + if elem == "" { + log.Fatalf("invalid -path argument: %q has no final element", p) } + fs.Bind("/src/pkg/"+elem, OS(p), "/", bindReplace) } - fsMap.Init(paths) - fileServer = http.FileServer(fsHttp) - cmdHandler = httpHandler{"/cmd/", filepath.Join(*goroot, "src", "cmd"), false} - pkgHandler = httpHandler{"/pkg/", filepath.Join(*goroot, "src", "pkg"), true} + fileServer = http.FileServer(&httpFS{fs}) + cmdHandler = docServer{"/cmd/", "/src/cmd", false} + pkgHandler = docServer{"/pkg/", "/src/pkg", true} } func registerPublicHandlers(mux *http.ServeMux) { mux.Handle(cmdHandler.pattern, &cmdHandler) mux.Handle(pkgHandler.pattern, &pkgHandler) mux.HandleFunc("/doc/codewalk/", codewalk) + mux.Handle("/doc/play/", fileServer) mux.HandleFunc("/search", search) mux.Handle("/robots.txt", fileServer) mux.HandleFunc("/opensearch.xml", serveSearchDesc) @@ -115,7 +110,7 @@ func registerPublicHandlers(mux *http.ServeMux) { } func initFSTree() { - dir := newDirectory(filepath.Join(*goroot, *testDir), nil, -1) + dir := newDirectory(pathpkg.Join("/", *testDir), -1) if dir == nil { log.Println("Warning: FSTree is nil") return @@ -125,177 +120,6 @@ func initFSTree() { } // ---------------------------------------------------------------------------- -// Directory filters - -// isParentOf returns true if p is a parent of (or the same as) q -// where p and q are directory paths. -func isParentOf(p, q string) bool { - n := len(p) - return strings.HasPrefix(q, p) && (len(q) <= n || q[n] == '/') -} - -func setPathFilter(list []string) { - if len(list) == 0 { - pathFilter.set(nil) - return - } - - // len(list) > 0 - pathFilter.set(func(path string) bool { - // list is sorted in increasing order and for each path all its children are removed - i := sort.Search(len(list), func(i int) bool { return list[i] > path }) - // Now we have list[i-1] <= path < list[i]. - // Path may be a child of list[i-1] or a parent of list[i]. - return i > 0 && isParentOf(list[i-1], path) || i < len(list) && isParentOf(path, list[i]) - }) -} - -func getPathFilter() func(string) bool { - f, _ := pathFilter.get() - if f != nil { - return f.(func(string) bool) - } - return nil -} - -// readDirList reads a file containing a newline-separated list -// of directory paths and returns the list of paths. -func readDirList(filename string) ([]string, error) { - contents, err := ReadFile(fs, filename) - if err != nil { - return nil, err - } - // create a sorted list of valid directory names - filter := func(path string) bool { - d, e := fs.Lstat(path) - if e != nil && err == nil { - // remember first error and return it from readDirList - // so we have at least some information if things go bad - err = e - } - return e == nil && isPkgDir(d) - } - list := canonicalizePaths(strings.Split(string(contents), "\n"), filter) - // for each parent path, remove all its children q - // (requirement for binary search to work when filtering) - i := 0 - for _, q := range list { - if i == 0 || !isParentOf(list[i-1], q) { - list[i] = q - i++ - } - } - return list[0:i], err -} - -// updateMappedDirs computes the directory tree for -// each user-defined file system mapping. If a filter -// is provided, it is used to filter directories. -// -func updateMappedDirs(filter func(string) bool) { - if !fsMap.IsEmpty() { - fsMap.Iterate(func(path string, value *RWValue) bool { - value.set(newDirectory(path, filter, -1)) - return true - }) - invalidateIndex() - } -} - -func updateFilterFile() { - updateMappedDirs(nil) // no filter for accuracy - - // collect directory tree leaf node paths - var buf bytes.Buffer - fsMap.Iterate(func(_ string, value *RWValue) bool { - v, _ := value.get() - if v != nil && v.(*Directory) != nil { - v.(*Directory).writeLeafs(&buf) - } - return true - }) - - // update filter file - if err := writeFileAtomically(*filter, buf.Bytes()); err != nil { - log.Printf("writeFileAtomically(%s): %s", *filter, err) - filterDelay.backoff(24 * time.Hour) // back off exponentially, but try at least once a day - } else { - filterDelay.set(*filterMin) // revert to regular filter update schedule - } -} - -func initDirTrees() { - // setup initial path filter - if *filter != "" { - list, err := readDirList(*filter) - if err != nil { - log.Printf("readDirList(%s): %s", *filter, err) - } - if *verbose || len(list) == 0 { - log.Printf("found %d directory paths in file %s", len(list), *filter) - } - setPathFilter(list) - } - - go updateMappedDirs(getPathFilter()) // use filter for speed - - // start filter update goroutine, if enabled. - if *filter != "" && *filterMin > 0 { - filterDelay.set(time.Duration(*filterMin) * time.Minute) // initial filter update delay - go func() { - for { - if *verbose { - log.Printf("start update of %s", *filter) - } - updateFilterFile() - delay, _ := filterDelay.get() - dt := delay.(time.Duration) - if *verbose { - log.Printf("next filter update in %s", dt) - } - time.Sleep(dt) - } - }() - } -} - -// ---------------------------------------------------------------------------- -// Path mapping - -// Absolute paths are file system paths (backslash-separated on Windows), -// but relative paths are always slash-separated. - -func absolutePath(relpath, defaultRoot string) string { - abspath := fsMap.ToAbsolute(relpath) - if abspath == "" { - // no user-defined mapping found; use default mapping - abspath = filepath.Join(defaultRoot, filepath.FromSlash(relpath)) - } - return abspath -} - -func relativeURL(abspath string) string { - relpath := fsMap.ToRelative(abspath) - if relpath == "" { - // prefix must end in a path separator - prefix := *goroot - if len(prefix) > 0 && prefix[len(prefix)-1] != filepath.Separator { - prefix += string(filepath.Separator) - } - if strings.HasPrefix(abspath, prefix) { - // no user-defined mapping found; use default mapping - relpath = filepath.ToSlash(abspath[len(prefix):]) - } - } - // Only if path is an invalid absolute path is relpath == "" - // at this point. This should never happen since absolute paths - // are only created via godoc for files that do exist. However, - // it is ok to return ""; it will simply provide a link to the - // top of the pkg or src directories. - return relpath -} - -// ---------------------------------------------------------------------------- // Tab conversion var spaces = []byte(" ") // 32 spaces seems like a good number @@ -391,7 +215,7 @@ func writeNode(w io.Writer, fset *token.FileSet, x interface{}) { } func filenameFunc(path string) string { - _, localname := filepath.Split(path) + _, localname := pathpkg.Split(path) return localname } @@ -581,7 +405,7 @@ func splitExampleName(s string) (name, suffix string) { } func pkgLinkFunc(path string) string { - relpath := relativeURL(path) + relpath := path[1:] // because of the irregular mapping under goroot // we need to correct certain relative paths if strings.HasPrefix(relpath, "src/pkg/") { @@ -597,7 +421,7 @@ func posLink_urlFunc(node ast.Node, fset *token.FileSet) string { if p := node.Pos(); p.IsValid() { pos := fset.Position(p) - relpath = relativeURL(pos.Filename) + relpath = pos.Filename line = pos.Line low = pos.Offset } @@ -626,6 +450,10 @@ func posLink_urlFunc(node ast.Node, fset *token.FileSet) string { return buf.String() } +func srcLinkFunc(s string) string { + return pathpkg.Clean("/" + s) +} + // fmap describes the template functions installed with all godoc templates. // Convention: template function names ending in "_html" or "_url" produce // HTML- or URL-escaped strings; all other function results may @@ -652,7 +480,7 @@ var fmap = template.FuncMap{ // support for URL attributes "pkgLink": pkgLinkFunc, - "srcLink": relativeURL, + "srcLink": srcLinkFunc, "posLink_url": posLink_urlFunc, // formatting of Examples @@ -662,10 +490,10 @@ var fmap = template.FuncMap{ } func readTemplate(name string) *template.Template { - path := filepath.Join(*goroot, "lib", "godoc", name) + path := "lib/godoc/" + name if *templateDir != "" { defaultpath := path - path = filepath.Join(*templateDir, name) + path = pathpkg.Join(*templateDir, name) if _, err := fs.Stat(path); err != nil { log.Print("readTemplate:", err) path = defaultpath @@ -718,20 +546,23 @@ func readTemplates() { // ---------------------------------------------------------------------------- // Generic HTML wrapper -func servePage(w http.ResponseWriter, title, subtitle, query string, content []byte) { +func servePage(w http.ResponseWriter, tabtitle, title, subtitle, query string, content []byte) { + if tabtitle == "" { + tabtitle = title + } d := struct { + Tabtitle string Title string Subtitle string - PkgRoots []string SearchBox bool Query string Version string Menu []byte Content []byte }{ + tabtitle, title, subtitle, - fsMap.PrefixList(), *indexEnabled, query, runtime.Version(), @@ -780,6 +611,23 @@ func serveHTMLDoc(w http.ResponseWriter, r *http.Request, abspath, relpath strin log.Printf("decoding metadata %s: %v", relpath, err) } + // evaluate as template if indicated + if meta.Template { + tmpl, err := template.New("main").Funcs(templateFuncs).Parse(string(src)) + if err != nil { + log.Printf("parsing template %s: %v", relpath, err) + serveError(w, r, relpath, err) + return + } + var buf bytes.Buffer + if err := tmpl.Execute(&buf, nil); err != nil { + log.Printf("executing template %s: %v", relpath, err) + serveError(w, r, relpath, err) + return + } + src = buf.Bytes() + } + // if it's the language spec, add tags to EBNF productions if strings.HasSuffix(abspath, "go_spec.html") { var buf bytes.Buffer @@ -787,7 +635,7 @@ func serveHTMLDoc(w http.ResponseWriter, r *http.Request, abspath, relpath strin src = buf.Bytes() } - servePage(w, meta.Title, meta.Subtitle, "", src) + servePage(w, "", meta.Title, meta.Subtitle, "", src) } func applyTemplate(t *template.Template, name string, data interface{}) []byte { @@ -799,7 +647,7 @@ func applyTemplate(t *template.Template, name string, data interface{}) []byte { } func redirect(w http.ResponseWriter, r *http.Request) (redirected bool) { - canonical := path.Clean(r.URL.Path) + canonical := pathpkg.Clean(r.URL.Path) if !strings.HasSuffix("/", canonical) { canonical += "/" } @@ -820,10 +668,10 @@ func serveTextFile(w http.ResponseWriter, r *http.Request, abspath, relpath, tit var buf bytes.Buffer buf.WriteString("<pre>") - FormatText(&buf, src, 1, filepath.Ext(abspath) == ".go", r.FormValue("h"), rangeSelection(r.FormValue("s"))) + FormatText(&buf, src, 1, pathpkg.Ext(abspath) == ".go", r.FormValue("h"), rangeSelection(r.FormValue("s"))) buf.WriteString("</pre>") - servePage(w, title+" "+relpath, "", "", buf.Bytes()) + servePage(w, relpath, title+" "+relpath, "", "", buf.Bytes()) } func serveDirectory(w http.ResponseWriter, r *http.Request, abspath, relpath string) { @@ -833,13 +681,12 @@ func serveDirectory(w http.ResponseWriter, r *http.Request, abspath, relpath str list, err := fs.ReadDir(abspath) if err != nil { - log.Printf("ReadDir: %s", err) serveError(w, r, relpath, err) return } contents := applyTemplate(dirlistHTML, "dirlistHTML", list) - servePage(w, "Directory "+relpath, "", "", contents) + servePage(w, relpath, "Directory "+relpath, "", "", contents) } func serveFile(w http.ResponseWriter, r *http.Request) { @@ -856,10 +703,10 @@ func serveFile(w http.ResponseWriter, r *http.Request) { relpath = m.filePath } + abspath := relpath relpath = relpath[1:] // strip leading slash - abspath := absolutePath(relpath, *goroot) - switch path.Ext(relpath) { + switch pathpkg.Ext(relpath) { case ".html": if strings.HasSuffix(relpath, "/index.html") { // We'll show index.html for the directory. @@ -886,8 +733,8 @@ func serveFile(w http.ResponseWriter, r *http.Request) { if redirect(w, r) { return } - if index := filepath.Join(abspath, "index.html"); isTextFile(index) { - serveHTMLDoc(w, r, index, relativeURL(index)) + if index := pathpkg.Join(abspath, "index.html"); isTextFile(index) { + serveHTMLDoc(w, r, index, index) return } serveDirectory(w, r, abspath, relpath) @@ -992,7 +839,7 @@ func (info *PageInfo) IsEmpty() bool { return info.Err != nil || info.PAst == nil && info.PDoc == nil && info.Dirs == nil } -type httpHandler struct { +type docServer struct { pattern string // url pattern; e.g. "/pkg/" fsRoot string // file system root to which the pattern is mapped isPkg bool // true if this handler serves real package documentation (as opposed to command documentation) @@ -1029,7 +876,7 @@ func inList(name string, list []string) bool { // directories, PageInfo.Dirs is nil. If a directory read error occurred, // PageInfo.Err is set to the respective error but the error is not logged. // -func (h *httpHandler) getPageInfo(abspath, relpath, pkgname string, mode PageInfoMode) PageInfo { +func (h *docServer) getPageInfo(abspath, relpath, pkgname string, mode PageInfoMode) PageInfo { var pkgFiles []string // If we're showing the default package, restrict to the ones @@ -1043,7 +890,7 @@ func (h *httpHandler) getPageInfo(abspath, relpath, pkgname string, mode PageInf // to choose, set ctxt.GOOS and ctxt.GOARCH before // calling ctxt.ScanDir. ctxt := build.Default - ctxt.IsAbsPath = path.IsAbs + ctxt.IsAbsPath = pathpkg.IsAbs ctxt.ReadDir = fsReadDir ctxt.OpenFile = fsOpenFile dir, err := ctxt.ImportDir(abspath, 0) @@ -1091,13 +938,13 @@ func (h *httpHandler) getPageInfo(abspath, relpath, pkgname string, mode PageInf // the package with dirname, and the 3rd choice is a package // that is not called "main" if there is exactly one such // package. Otherwise, don't select a package. - dirpath, dirname := filepath.Split(abspath) + dirpath, dirname := pathpkg.Split(abspath) // If the dirname is "go" we might be in a sub-directory for // .go files - use the outer directory name instead for better // results. if dirname == "go" { - _, dirname = filepath.Split(filepath.Clean(dirpath)) + _, dirname = pathpkg.Split(pathpkg.Clean(dirpath)) } var choice3 *ast.Package @@ -1161,7 +1008,7 @@ func (h *httpHandler) getPageInfo(abspath, relpath, pkgname string, mode PageInf if mode&allMethods != 0 { m |= doc.AllMethods } - pdoc = doc.New(pkg, path.Clean(relpath), m) // no trailing '/' in importpath + pdoc = doc.New(pkg, pathpkg.Clean(relpath), m) // no trailing '/' in importpath } else { // show source code // TODO(gri) Consider eliminating export filtering in this mode, @@ -1184,34 +1031,11 @@ func (h *httpHandler) getPageInfo(abspath, relpath, pkgname string, mode PageInf timestamp = ts } if dir == nil { - // the path may refer to a user-specified file system mapped - // via fsMap; lookup that mapping and corresponding RWValue - // if any - var v *RWValue - fsMap.Iterate(func(path string, value *RWValue) bool { - if isParentOf(path, abspath) { - // mapping found - v = value - return false - } - return true - }) - if v != nil { - // found a RWValue associated with a user-specified file - // system; a non-nil RWValue stores a (possibly out-of-date) - // directory tree for that file system - if tree, ts := v.get(); tree != nil && tree.(*Directory) != nil { - dir = tree.(*Directory).lookup(abspath) - timestamp = ts - } - } - } - if dir == nil { // no directory tree present (too early after startup or // command-line mode); compute one level for this page // note: cannot use path filter here because in general // it doesn't contain the fsTree path - dir = newDirectory(abspath, nil, 1) + dir = newDirectory(abspath, 1) timestamp = time.Now() } @@ -1230,13 +1054,13 @@ func (h *httpHandler) getPageInfo(abspath, relpath, pkgname string, mode PageInf } } -func (h *httpHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { +func (h *docServer) ServeHTTP(w http.ResponseWriter, r *http.Request) { if redirect(w, r) { return } - relpath := path.Clean(r.URL.Path[len(h.pattern):]) - abspath := absolutePath(relpath, h.fsRoot) + relpath := pathpkg.Clean(r.URL.Path[len(h.pattern):]) + abspath := pathpkg.Join(h.fsRoot, relpath) mode := getPageInfoMode(r) if relpath == builtinPkgPath { mode = noFiltering @@ -1254,30 +1078,41 @@ func (h *httpHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } - var title, subtitle string + var tabtitle, title, subtitle string switch { case info.PAst != nil: - title = "Package " + info.PAst.Name.Name + tabtitle = info.PAst.Name.Name + title = "Package " + tabtitle case info.PDoc != nil: - switch { - case info.IsPkg: - title = "Package " + info.PDoc.Name - case info.PDoc.Name == fakePkgName: + if info.PDoc.Name == fakePkgName { // assume that the directory name is the command name - _, pkgname := path.Split(relpath) - title = "Command " + pkgname - default: - title = "Command " + info.PDoc.Name + _, tabtitle = pathpkg.Split(relpath) + } else { + tabtitle = info.PDoc.Name + } + if info.IsPkg { + title = "Package " + tabtitle + } else { + title = "Command " + tabtitle } default: - title = "Directory " + relativeURL(info.Dirname) + tabtitle = info.Dirname + title = "Directory " + tabtitle if *showTimestamps { subtitle = "Last update: " + info.DirTime.String() } } + // special cases for top-level package/command directories + switch tabtitle { + case "/src/pkg": + tabtitle = "Packages" + case "/src/cmd": + tabtitle = "Commands" + } + contents := applyTemplate(packageHTML, "packageHTML", info) - servePage(w, title, subtitle, "", contents) + servePage(w, tabtitle, title, subtitle, "", contents) } // ---------------------------------------------------------------------------- @@ -1367,7 +1202,7 @@ func search(w http.ResponseWriter, r *http.Request) { } contents := applyTemplate(searchHTML, "searchHTML", result) - servePage(w, title, "", query, contents) + servePage(w, query, title, "", query, contents) } // ---------------------------------------------------------------------------- @@ -1376,6 +1211,7 @@ func search(w http.ResponseWriter, r *http.Request) { type Metadata struct { Title string Subtitle string + Template bool // execute as template Path string // canonical path for this page filePath string // filesystem path relative to goroot } @@ -1414,7 +1250,7 @@ func updateMetadata() { return } for _, fi := range fis { - name := filepath.Join(dir, fi.Name()) + name := pathpkg.Join(dir, fi.Name()) if fi.IsDir() { scan(name) // recurse continue @@ -1434,7 +1270,7 @@ func updateMetadata() { continue } // Store relative filesystem path in Metadata. - meta.filePath = filepath.Join("/", name[len(*goroot):]) + meta.filePath = name if meta.Path == "" { // If no Path, canonical path is actual path. meta.Path = meta.filePath @@ -1444,7 +1280,7 @@ func updateMetadata() { metadata[meta.filePath] = &meta } } - scan(filepath.Join(*goroot, "doc")) + scan("/doc") docMetadata.set(metadata) } @@ -1519,13 +1355,9 @@ func feedDirnames(root *RWValue, c chan<- string) { // of all the file systems under godoc's observation. // func fsDirnames() <-chan string { - c := make(chan string, 256) // asynchronous for fewer context switches + c := make(chan string, 256) // buffered for fewer context switches go func() { feedDirnames(&fsTree, c) - fsMap.Iterate(func(_ string, root *RWValue) bool { - feedDirnames(root, c) - return true - }) close(c) }() return c diff --git a/src/cmd/godoc/httpzip.go b/src/cmd/godoc/httpzip.go deleted file mode 100644 index 12e99646d..000000000 --- a/src/cmd/godoc/httpzip.go +++ /dev/null @@ -1,190 +0,0 @@ -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// This file provides an implementation of the http.FileSystem -// interface based on the contents of a .zip file. -// -// Assumptions: -// -// - The file paths stored in the zip file must use a slash ('/') as path -// separator; and they must be relative (i.e., they must not start with -// a '/' - this is usually the case if the file was created w/o special -// options). -// - The zip file system treats the file paths found in the zip internally -// like absolute paths w/o a leading '/'; i.e., the paths are considered -// relative to the root of the file system. -// - All path arguments to file system methods are considered relative to -// the root specified with NewHttpZipFS (even if the paths start with a '/'). - -// TODO(gri) Should define a commonly used FileSystem API that is the same -// for http and godoc. Then we only need one zip-file based file -// system implementation. - -package main - -import ( - "archive/zip" - "fmt" - "io" - "net/http" - "os" - "path" - "sort" - "strings" - "time" -) - -type fileInfo struct { - name string - mode os.FileMode - size int64 - mtime time.Time -} - -func (fi *fileInfo) Name() string { return fi.name } -func (fi *fileInfo) Mode() os.FileMode { return fi.mode } -func (fi *fileInfo) Size() int64 { return fi.size } -func (fi *fileInfo) ModTime() time.Time { return fi.mtime } -func (fi *fileInfo) IsDir() bool { return fi.mode.IsDir() } -func (fi *fileInfo) Sys() interface{} { return nil } - -// httpZipFile is the zip-file based implementation of http.File -type httpZipFile struct { - path string // absolute path within zip FS without leading '/' - info os.FileInfo - io.ReadCloser // nil for directory - list zipList -} - -func (f *httpZipFile) Close() error { - if !f.info.IsDir() { - return f.ReadCloser.Close() - } - f.list = nil - return nil -} - -func (f *httpZipFile) Stat() (os.FileInfo, error) { - return f.info, nil -} - -func (f *httpZipFile) Readdir(count int) ([]os.FileInfo, error) { - var list []os.FileInfo - dirname := f.path + "/" - prevname := "" - for i, e := range f.list { - if count == 0 { - f.list = f.list[i:] - break - } - if !strings.HasPrefix(e.Name, dirname) { - f.list = nil - break // not in the same directory anymore - } - name := e.Name[len(dirname):] // local name - var mode os.FileMode - var size int64 - var mtime time.Time - if i := strings.IndexRune(name, '/'); i >= 0 { - // We infer directories from files in subdirectories. - // If we have x/y, return a directory entry for x. - name = name[0:i] // keep local directory name only - mode = os.ModeDir - // no size or mtime for directories - } else { - mode = 0 - size = int64(e.UncompressedSize) - mtime = e.ModTime() - } - // If we have x/y and x/z, don't return two directory entries for x. - // TODO(gri): It should be possible to do this more efficiently - // by determining the (fs.list) range of local directory entries - // (via two binary searches). - if name != prevname { - list = append(list, &fileInfo{ - name, - mode, - size, - mtime, - }) - prevname = name - count-- - } - } - - if count >= 0 && len(list) == 0 { - return nil, io.EOF - } - - return list, nil -} - -func (f *httpZipFile) Seek(offset int64, whence int) (int64, error) { - return 0, fmt.Errorf("Seek not implemented for zip file entry: %s", f.info.Name()) -} - -// httpZipFS is the zip-file based implementation of http.FileSystem -type httpZipFS struct { - *zip.ReadCloser - list zipList - root string -} - -func (fs *httpZipFS) Open(name string) (http.File, error) { - // fs.root does not start with '/'. - path := path.Join(fs.root, name) // path is clean - index, exact := fs.list.lookup(path) - if index < 0 || !strings.HasPrefix(path, fs.root) { - // file not found or not under root - return nil, fmt.Errorf("file not found: %s", name) - } - - if exact { - // exact match found - must be a file - f := fs.list[index] - rc, err := f.Open() - if err != nil { - return nil, err - } - return &httpZipFile{ - path, - &fileInfo{ - name, - 0, - int64(f.UncompressedSize), - f.ModTime(), - }, - rc, - nil, - }, nil - } - - // not an exact match - must be a directory - return &httpZipFile{ - path, - &fileInfo{ - name, - os.ModeDir, - 0, // no size for directory - time.Time{}, // no mtime for directory - }, - nil, - fs.list[index:], - }, nil -} - -func (fs *httpZipFS) Close() error { - fs.list = nil - return fs.ReadCloser.Close() -} - -// NewHttpZipFS creates a new http.FileSystem based on the contents of -// the zip file rc restricted to the directory tree specified by root; -// root must be an absolute path. -func NewHttpZipFS(rc *zip.ReadCloser, root string) http.FileSystem { - list := make(zipList, len(rc.File)) - copy(list, rc.File) // sort a copy of rc.File - sort.Sort(list) - return &httpZipFS{rc, list, zipPath(root)} -} diff --git a/src/cmd/godoc/index.go b/src/cmd/godoc/index.go index 6c36e6f4f..1bef79693 100644 --- a/src/cmd/godoc/index.go +++ b/src/cmd/godoc/index.go @@ -48,7 +48,7 @@ import ( "index/suffixarray" "io" "os" - "path/filepath" + pathpkg "path" "regexp" "sort" "strings" @@ -248,7 +248,7 @@ type File struct { // Path returns the file path of f. func (f *File) Path() string { - return filepath.Join(f.Pak.Path, f.Name) + return pathpkg.Join(f.Pak.Path, f.Name) } // A Spot describes a single occurrence of a word. @@ -695,7 +695,7 @@ var whitelisted = map[string]bool{ // of "permitted" files for indexing. The filename must // be the directory-local name of the file. func isWhitelisted(filename string) bool { - key := filepath.Ext(filename) + key := pathpkg.Ext(filename) if key == "" { // file has no extension - use entire filename key = filename @@ -708,7 +708,7 @@ func (x *Indexer) visitFile(dirname string, f os.FileInfo, fulltextIndex bool) { return } - filename := filepath.Join(dirname, f.Name()) + filename := pathpkg.Join(dirname, f.Name()) goFile := false switch { diff --git a/src/cmd/godoc/main.go b/src/cmd/godoc/main.go index 5f4210539..23f712ab3 100644 --- a/src/cmd/godoc/main.go +++ b/src/cmd/godoc/main.go @@ -38,13 +38,13 @@ import ( "log" "net/http" _ "net/http/pprof" // to serve /debug/pprof/* + "net/url" "os" - "path" + pathpkg "path" "path/filepath" "regexp" "runtime" "strings" - "time" ) const defaultAddr = ":6060" // default webserver address @@ -57,11 +57,6 @@ var ( // file-based index writeIndex = flag.Bool("write_index", false, "write index to a file; the file name must be specified with -index_files") - // periodic sync - syncCmd = flag.String("sync", "", "sync command; disabled if empty") - syncMin = flag.Int("sync_minutes", 0, "sync interval in minutes; disabled if <= 0") - syncDelay delayTime // actual sync interval in minutes; usually syncDelay == syncMin, but syncDelay may back off exponentially - // network httpAddr = flag.String("http", "", "HTTP service address (e.g., '"+defaultAddr+"')") serverAddr = flag.String("server", "", "webserver address for command line searches") @@ -69,6 +64,7 @@ var ( // layout control html = flag.Bool("html", false, "print HTML in command-line mode") srcMode = flag.Bool("src", false, "print (exported) source in command-line mode") + urlFlag = flag.String("url", "", "print HTML for named URL") // command-line searches query = flag.Bool("q", false, "arguments are considered search queries") @@ -77,76 +73,7 @@ var ( func serveError(w http.ResponseWriter, r *http.Request, relpath string, err error) { contents := applyTemplate(errorHTML, "errorHTML", err) // err may contain an absolute path! w.WriteHeader(http.StatusNotFound) - servePage(w, "File "+relpath, "", "", contents) -} - -func exec(rw http.ResponseWriter, args []string) (status int) { - r, w, err := os.Pipe() - if err != nil { - log.Printf("os.Pipe(): %v", err) - return 2 - } - - bin := args[0] - fds := []*os.File{nil, w, w} - if *verbose { - log.Printf("executing %v", args) - } - p, err := os.StartProcess(bin, args, &os.ProcAttr{Files: fds, Dir: *goroot}) - defer r.Close() - w.Close() - if err != nil { - log.Printf("os.StartProcess(%q): %v", bin, err) - return 2 - } - - var buf bytes.Buffer - io.Copy(&buf, r) - wait, err := p.Wait() - if err != nil { - os.Stderr.Write(buf.Bytes()) - log.Printf("os.Wait(%d, 0): %v", p.Pid, err) - return 2 - } - if !wait.Success() { - os.Stderr.Write(buf.Bytes()) - log.Printf("executing %v failed", args) - status = 1 // See comment in default case in dosync. - return - } - - if *verbose { - os.Stderr.Write(buf.Bytes()) - } - if rw != nil { - rw.Header().Set("Content-Type", "text/plain; charset=utf-8") - rw.Write(buf.Bytes()) - } - - return -} - -func dosync(w http.ResponseWriter, r *http.Request) { - args := []string{"/bin/sh", "-c", *syncCmd} - switch exec(w, args) { - case 0: - // sync succeeded and some files have changed; - // update package tree. - // TODO(gri): The directory tree may be temporarily out-of-sync. - // Consider keeping separate time stamps so the web- - // page can indicate this discrepancy. - initFSTree() - fallthrough - case 1: - // sync failed because no files changed; - // don't change the package tree - syncDelay.set(time.Duration(*syncMin) * time.Minute) // revert to regular sync schedule - default: - // TODO(r): this cannot happen now, since Wait has a boolean exit condition, - // not an integer. - // sync failed because of an error - back off exponentially, but try at least once a day - syncDelay.backoff(24 * time.Hour) - } + servePage(w, relpath, "File "+relpath, "", "", contents) } func usage() { @@ -225,7 +152,7 @@ func main() { flag.Parse() // Check usage: either server and no args, command line and args, or index creation mode - if (*httpAddr != "") != (flag.NArg() == 0) && !*writeIndex { + if (*httpAddr != "" || *urlFlag != "") != (flag.NArg() == 0) && !*writeIndex { usage() } @@ -239,19 +166,20 @@ func main() { // same is true for the http handlers in initHandlers. if *zipfile == "" { // use file system of underlying OS - *goroot = filepath.Clean(*goroot) // normalize path separator - fs = OS - fsHttp = http.Dir(*goroot) + fs.Bind("/", OS(*goroot), "/", bindReplace) } else { // use file system specified via .zip file (path separator must be '/') rc, err := zip.OpenReader(*zipfile) if err != nil { log.Fatalf("%s: %s\n", *zipfile, err) } - defer rc.Close() // be nice (e.g., -writeIndex mode) - *goroot = path.Join("/", *goroot) // fsHttp paths are relative to '/' - fs = NewZipFS(rc) - fsHttp = NewHttpZipFS(rc, *goroot) + defer rc.Close() // be nice (e.g., -writeIndex mode) + fs.Bind("/", NewZipFS(rc, *zipfile), *goroot, bindReplace) + } + + // Bind $GOPATH trees into Go root. + for _, p := range filepath.SplitList(build.Default.GOPATH) { + fs.Bind("/src/pkg", OS(p), "/src", bindAfter) } readTemplates() @@ -266,7 +194,6 @@ func main() { log.Println("initialize file systems") *verbose = true // want to see what happens initFSTree() - initDirTrees() *indexThrottle = 1 updateIndex() @@ -286,6 +213,44 @@ func main() { return } + // Print content that would be served at the URL *urlFlag. + if *urlFlag != "" { + registerPublicHandlers(http.DefaultServeMux) + // Try up to 10 fetches, following redirects. + urlstr := *urlFlag + for i := 0; i < 10; i++ { + // Prepare request. + u, err := url.Parse(urlstr) + if err != nil { + log.Fatal(err) + } + req := &http.Request{ + URL: u, + } + + // Invoke default HTTP handler to serve request + // to our buffering httpWriter. + w := &httpWriter{h: http.Header{}, code: 200} + http.DefaultServeMux.ServeHTTP(w, req) + + // Return data, error, or follow redirect. + switch w.code { + case 200: // ok + os.Stdout.Write(w.Bytes()) + return + case 301, 302, 303, 307: // redirect + redirect := w.h.Get("Location") + if redirect == "" { + log.Fatalf("HTTP %d without Location header", w.code) + } + urlstr = redirect + default: + log.Fatalf("HTTP error %d", w.code) + } + } + log.Fatalf("too many redirects") + } + if *httpAddr != "" { // HTTP server mode. var handler http.Handler = http.DefaultServeMux @@ -303,41 +268,20 @@ func main() { default: log.Print("identifier search index enabled") } - if !fsMap.IsEmpty() { - log.Print("user-defined mapping:") - fsMap.Fprint(os.Stderr) - } + fs.Fprint(os.Stderr) handler = loggingHandler(handler) } registerPublicHandlers(http.DefaultServeMux) - if *syncCmd != "" { - http.Handle("/debug/sync", http.HandlerFunc(dosync)) - } + + // Playground handlers are not available in local godoc. + http.HandleFunc("/compile", disabledHandler) + http.HandleFunc("/share", disabledHandler) // Initialize default directory tree with corresponding timestamp. // (Do it in a goroutine so that launch is quick.) go initFSTree() - // Initialize directory trees for user-defined file systems (-path flag). - initDirTrees() - - // Start sync goroutine, if enabled. - if *syncCmd != "" && *syncMin > 0 { - syncDelay.set(*syncMin) // initial sync delay - go func() { - for { - dosync(nil, nil) - delay, _ := syncDelay.get() - dt := delay.(time.Duration) - if *verbose { - log.Printf("next sync in %s", dt) - } - time.Sleep(dt) - } - }() - } - // Immediately update metadata. updateMetadata() // Periodically refresh metadata. @@ -374,27 +318,38 @@ func main() { return } - // determine paths + // Determine paths. + // + // If we are passed an operating system path like . or ./foo or /foo/bar or c:\mysrc, + // we need to map that path somewhere in the fs name space so that routines + // like getPageInfo will see it. We use the arbitrarily-chosen virtual path "/target" + // for this. That is, if we get passed a directory like the above, we map that + // directory so that getPageInfo sees it as /target. + const target = "/target" const cmdPrefix = "cmd/" path := flag.Arg(0) var forceCmd bool - if strings.HasPrefix(path, ".") { - // assume cwd; don't assume -goroot + var abspath, relpath string + if filepath.IsAbs(path) { + fs.Bind(target, OS(path), "/", bindReplace) + abspath = target + } else if build.IsLocalImport(path) { cwd, _ := os.Getwd() // ignore errors path = filepath.Join(cwd, path) + fs.Bind(target, OS(path), "/", bindReplace) + abspath = target } else if strings.HasPrefix(path, cmdPrefix) { path = path[len(cmdPrefix):] forceCmd = true - } - relpath := path - abspath := path - if bp, _ := build.Import(path, "", build.FindOnly); bp.Dir != "" && bp.ImportPath != "" { + } else if bp, _ := build.Import(path, "", build.FindOnly); bp.Dir != "" && bp.ImportPath != "" { + fs.Bind(target, OS(bp.Dir), "/", bindReplace) + abspath = target relpath = bp.ImportPath - abspath = bp.Dir - } else if !filepath.IsAbs(path) { - abspath = absolutePath(path, pkgHandler.fsRoot) } else { - relpath = relativeURL(path) + abspath = pathpkg.Join(pkgHandler.fsRoot, path) + } + if relpath == "" { + relpath = abspath } var mode PageInfoMode @@ -422,7 +377,7 @@ func main() { // (the go command invokes godoc w/ absolute paths; don't override) var cinfo PageInfo if !filepath.IsAbs(path) { - abspath = absolutePath(path, cmdHandler.fsRoot) + abspath = pathpkg.Join(cmdHandler.fsRoot, path) cinfo = cmdHandler.getPageInfo(abspath, relpath, "", mode) } @@ -445,6 +400,10 @@ func main() { if info.Err != nil { log.Fatalf("%v", info.Err) } + if info.PDoc != nil && info.PDoc.ImportPath == target { + // Replace virtual /target with actual argument from command line. + info.PDoc.ImportPath = flag.Arg(0) + } // If we have more than one argument, use the remaining arguments for filtering if flag.NArg() > 1 { @@ -485,3 +444,19 @@ func main() { log.Printf("packageText.Execute: %s", err) } } + +// An httpWriter is an http.ResponseWriter writing to a bytes.Buffer. +type httpWriter struct { + bytes.Buffer + h http.Header + code int +} + +func (w *httpWriter) Header() http.Header { return w.h } +func (w *httpWriter) WriteHeader(code int) { w.code = code } + +// disabledHandler serves a 501 "Not Implemented" response. +func disabledHandler(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotImplemented) + fmt.Fprint(w, "This functionality is not available via local godoc.") +} diff --git a/src/cmd/godoc/mapping.go b/src/cmd/godoc/mapping.go deleted file mode 100644 index 544dd6f66..000000000 --- a/src/cmd/godoc/mapping.go +++ /dev/null @@ -1,202 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// This file implements the Mapping data structure. - -package main - -import ( - "fmt" - "io" - "path" - "path/filepath" - "sort" - "strings" -) - -// A Mapping object maps relative paths (e.g. from URLs) -// to absolute paths (of the file system) and vice versa. -// -// A Mapping object consists of a list of individual mappings -// of the form: prefix -> path which are interpreted as follows: -// A relative path of the form prefix/tail is to be mapped to -// the absolute path/tail, if that absolute path exists in the file -// system. Given a Mapping object, a relative path is mapped to an -// absolute path by trying each of the individual mappings in order, -// until a valid mapping is found. For instance, for the mapping: -// -// user -> /home/user -// public -> /home/user/public -// public -> /home/build/public -// -// the relative paths below are mapped to absolute paths as follows: -// -// user/foo -> /home/user/foo -// public/net/rpc/file1.go -> /home/user/public/net/rpc/file1.go -// -// If there is no /home/user/public/net/rpc/file2.go, the next public -// mapping entry is used to map the relative path to: -// -// public/net/rpc/file2.go -> /home/build/public/net/rpc/file2.go -// -// (assuming that file exists). -// -// Each individual mapping also has a RWValue associated with it that -// may be used to store mapping-specific information. See the Iterate -// method. -// -type Mapping struct { - list []mapping - prefixes []string // lazily computed from list -} - -type mapping struct { - prefix, path string - value *RWValue -} - -// Init initializes the Mapping from a list of paths. -// Empty paths are ignored; relative paths are assumed to be relative to -// the current working directory and converted to absolute paths. -// For each path of the form: -// -// dirname/localname -// -// a mapping -// -// localname -> path -// -// is added to the Mapping object, in the order of occurrence. -// For instance, under Unix, the argument: -// -// /home/user:/home/build/public -// -// leads to the following mapping: -// -// user -> /home/user -// public -> /home/build/public -// -func (m *Mapping) Init(paths []string) { - pathlist := canonicalizePaths(paths, nil) - list := make([]mapping, len(pathlist)) - - // create mapping list - for i, path := range pathlist { - _, prefix := filepath.Split(path) - list[i] = mapping{prefix, path, new(RWValue)} - } - - m.list = list -} - -// IsEmpty returns true if there are no mappings specified. -func (m *Mapping) IsEmpty() bool { return len(m.list) == 0 } - -// PrefixList returns a list of all prefixes, with duplicates removed. -// For instance, for the mapping: -// -// user -> /home/user -// public -> /home/user/public -// public -> /home/build/public -// -// the prefix list is: -// -// user, public -// -func (m *Mapping) PrefixList() []string { - // compute the list lazily - if m.prefixes == nil { - list := make([]string, len(m.list)) - - // populate list - for i, e := range m.list { - list[i] = e.prefix - } - - // sort the list and remove duplicate entries - sort.Strings(list) - i := 0 - prev := "" - for _, path := range list { - if path != prev { - list[i] = path - i++ - prev = path - } - } - - m.prefixes = list[0:i] - } - - return m.prefixes -} - -// Fprint prints the mapping. -func (m *Mapping) Fprint(w io.Writer) { - for _, e := range m.list { - fmt.Fprintf(w, "\t%s -> %s\n", e.prefix, e.path) - } -} - -const sep = string(filepath.Separator) - -// splitFirst splits a path at the first path separator and returns -// the path's head (the top-most directory specified by the path) and -// its tail (the rest of the path). If there is no path separator, -// splitFirst returns path as head, and the empty string as tail. -// Specifically, splitFirst("foo") == splitFirst("foo/"). -// -func splitFirst(path string) (head, tail string) { - if i := strings.Index(path, sep); i > 0 { - // 0 < i < len(path) - return path[0:i], path[i+1:] - } - return path, "" -} - -// ToAbsolute maps a slash-separated relative path to an absolute filesystem -// path using the Mapping specified by the receiver. If the path cannot -// be mapped, the empty string is returned. -// -func (m *Mapping) ToAbsolute(spath string) string { - fpath := filepath.FromSlash(spath) - prefix, tail := splitFirst(fpath) - for _, e := range m.list { - if e.prefix == prefix { - // found potential mapping - abspath := filepath.Join(e.path, tail) - if _, err := fs.Stat(abspath); err == nil { - return abspath - } - } - } - return "" // no match -} - -// ToRelative maps an absolute filesystem path to a relative slash-separated -// path using the Mapping specified by the receiver. If the path cannot -// be mapped, the empty string is returned. -// -func (m *Mapping) ToRelative(fpath string) string { - for _, e := range m.list { - // if fpath has prefix e.path, the next character must be a separator (was issue 3096) - if strings.HasPrefix(fpath, e.path+sep) { - spath := filepath.ToSlash(fpath) - // /absolute/prefix/foo -> prefix/foo - return path.Join(e.prefix, spath[len(e.path):]) // Join will remove a trailing '/' - } - } - return "" // no match -} - -// Iterate calls f for each path and RWValue in the mapping (in uspecified order) -// until f returns false. -// -func (m *Mapping) Iterate(f func(path string, value *RWValue) bool) { - for _, e := range m.list { - if !f(e.path, e.value) { - return - } - } -} diff --git a/src/cmd/godoc/parser.go b/src/cmd/godoc/parser.go index d6cc67cb5..c6b7c2dc8 100644 --- a/src/cmd/godoc/parser.go +++ b/src/cmd/godoc/parser.go @@ -14,7 +14,7 @@ import ( "go/parser" "go/token" "os" - "path/filepath" + pathpkg "path" ) func parseFile(fset *token.FileSet, filename string, mode parser.Mode) (*ast.File, error) { @@ -58,7 +58,7 @@ func parseDir(fset *token.FileSet, path string, filter func(os.FileInfo) bool) ( i := 0 for _, d := range list { if filter == nil || filter(d) { - filenames[i] = filepath.Join(path, d.Name()) + filenames[i] = pathpkg.Join(path, d.Name()) i++ } } diff --git a/src/cmd/godoc/template.go b/src/cmd/godoc/template.go new file mode 100644 index 000000000..d709baef4 --- /dev/null +++ b/src/cmd/godoc/template.go @@ -0,0 +1,182 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Template support for writing HTML documents. +// Documents that include Template: true in their +// metadata are executed as input to text/template. +// +// This file defines functions for those templates to invoke. + +// The template uses the function "code" to inject program +// source into the output by extracting code from files and +// injecting them as HTML-escaped <pre> blocks. +// +// The syntax is simple: 1, 2, or 3 space-separated arguments: +// +// Whole file: +// {{code "foo.go"}} +// One line (here the signature of main): +// {{code "foo.go" `/^func.main/`}} +// Block of text, determined by start and end (here the body of main): +// {{code "foo.go" `/^func.main/` `/^}/` +// +// Patterns can be `/regular expression/`, a decimal number, or "$" +// to signify the end of the file. In multi-line matches, +// lines that end with the four characters +// OMIT +// are omitted from the output, making it easy to provide marker +// lines in the input that will not appear in the output but are easy +// to identify by pattern. + +package main + +import ( + "bytes" + "fmt" + "log" + "regexp" + "strings" + "text/template" +) + +// Functions in this file panic on error, but the panic is recovered +// to an error by 'code'. + +var templateFuncs = template.FuncMap{ + "code": code, +} + +// contents reads and returns the content of the named file +// (from the virtual file system, so for example /doc refers to $GOROOT/doc). +func contents(name string) string { + file, err := ReadFile(fs, name) + if err != nil { + log.Panic(err) + } + return string(file) +} + +// format returns a textual representation of the arg, formatted according to its nature. +func format(arg interface{}) string { + switch arg := arg.(type) { + case int: + return fmt.Sprintf("%d", arg) + case string: + if len(arg) > 2 && arg[0] == '/' && arg[len(arg)-1] == '/' { + return fmt.Sprintf("%#q", arg) + } + return fmt.Sprintf("%q", arg) + default: + log.Panicf("unrecognized argument: %v type %T", arg, arg) + } + return "" +} + +func code(file string, arg ...interface{}) (s string, err error) { + defer func() { + if r := recover(); r != nil { + err = fmt.Errorf("%v", r) + } + }() + + text := contents(file) + var command string + switch len(arg) { + case 0: + // text is already whole file. + command = fmt.Sprintf("code %q", file) + case 1: + command = fmt.Sprintf("code %q %s", file, format(arg[0])) + text = oneLine(file, text, arg[0]) + case 2: + command = fmt.Sprintf("code %q %s %s", file, format(arg[0]), format(arg[1])) + text = multipleLines(file, text, arg[0], arg[1]) + default: + return "", fmt.Errorf("incorrect code invocation: code %q %q", file, arg) + } + // Trim spaces from output. + text = strings.Trim(text, "\n") + // Replace tabs by spaces, which work better in HTML. + text = strings.Replace(text, "\t", " ", -1) + var buf bytes.Buffer + // HTML-escape text and syntax-color comments like elsewhere. + FormatText(&buf, []byte(text), -1, true, "", nil) + // Include the command as a comment. + text = fmt.Sprintf("<pre><!--{{%s}}\n-->%s</pre>", command, buf.Bytes()) + return text, nil +} + +// parseArg returns the integer or string value of the argument and tells which it is. +func parseArg(arg interface{}, file string, max int) (ival int, sval string, isInt bool) { + switch n := arg.(type) { + case int: + if n <= 0 || n > max { + log.Panicf("%q:%d is out of range", file, n) + } + return n, "", true + case string: + return 0, n, false + } + log.Panicf("unrecognized argument %v type %T", arg, arg) + return +} + +// oneLine returns the single line generated by a two-argument code invocation. +func oneLine(file, text string, arg interface{}) string { + lines := strings.SplitAfter(contents(file), "\n") + line, pattern, isInt := parseArg(arg, file, len(lines)) + if isInt { + return lines[line-1] + } + return lines[match(file, 0, lines, pattern)-1] +} + +// multipleLines returns the text generated by a three-argument code invocation. +func multipleLines(file, text string, arg1, arg2 interface{}) string { + lines := strings.SplitAfter(contents(file), "\n") + line1, pattern1, isInt1 := parseArg(arg1, file, len(lines)) + line2, pattern2, isInt2 := parseArg(arg2, file, len(lines)) + if !isInt1 { + line1 = match(file, 0, lines, pattern1) + } + if !isInt2 { + line2 = match(file, line1, lines, pattern2) + } else if line2 < line1 { + log.Panicf("lines out of order for %q: %d %d", text, line1, line2) + } + for k := line1 - 1; k < line2; k++ { + if strings.HasSuffix(lines[k], "OMIT\n") { + lines[k] = "" + } + } + return strings.Join(lines[line1-1:line2], "") +} + +// match identifies the input line that matches the pattern in a code invocation. +// If start>0, match lines starting there rather than at the beginning. +// The return value is 1-indexed. +func match(file string, start int, lines []string, pattern string) int { + // $ matches the end of the file. + if pattern == "$" { + if len(lines) == 0 { + log.Panicf("%q: empty file", file) + } + return len(lines) + } + // /regexp/ matches the line that matches the regexp. + if len(pattern) > 2 && pattern[0] == '/' && pattern[len(pattern)-1] == '/' { + re, err := regexp.Compile(pattern[1 : len(pattern)-1]) + if err != nil { + log.Panic(err) + } + for i := start; i < len(lines); i++ { + if re.MatchString(lines[i]) { + return i + 1 + } + } + log.Panicf("%s: no match for %#q", file, pattern) + } + log.Panicf("unrecognized pattern: %q", pattern) + return 0 +} diff --git a/src/cmd/godoc/utils.go b/src/cmd/godoc/utils.go index be0bdc306..7def015c8 100644 --- a/src/cmd/godoc/utils.go +++ b/src/cmd/godoc/utils.go @@ -7,12 +7,7 @@ package main import ( - "io" - "io/ioutil" - "os" - "path/filepath" - "sort" - "strings" + pathpkg "path" "sync" "time" "unicode/utf8" @@ -40,76 +35,6 @@ func (v *RWValue) get() (interface{}, time.Time) { return v.value, v.timestamp } -// TODO(gri) For now, using os.Getwd() is ok here since the functionality -// based on this code is not invoked for the appengine version, -// but this is fragile. Determine what the right thing to do is, -// here (possibly have some Getwd-equivalent in FileSystem). -var cwd, _ = os.Getwd() // ignore errors - -// canonicalizePaths takes a list of (directory/file) paths and returns -// the list of corresponding absolute paths in sorted (increasing) order. -// Relative paths are assumed to be relative to the current directory, -// empty and duplicate paths as well as paths for which filter(path) is -// false are discarded. filter may be nil in which case it is not used. -// -func canonicalizePaths(list []string, filter func(path string) bool) []string { - i := 0 - for _, path := range list { - path = strings.TrimSpace(path) - if len(path) == 0 { - continue // ignore empty paths (don't assume ".") - } - // len(path) > 0: normalize path - if filepath.IsAbs(path) { - path = filepath.Clean(path) - } else { - path = filepath.Join(cwd, path) - } - // we have a non-empty absolute path - if filter != nil && !filter(path) { - continue - } - // keep the path - list[i] = path - i++ - } - list = list[0:i] - - // sort the list and remove duplicate entries - sort.Strings(list) - i = 0 - prev := "" - for _, path := range list { - if path != prev { - list[i] = path - i++ - prev = path - } - } - - return list[0:i] -} - -// writeFileAtomically writes data to a temporary file and then -// atomically renames that file to the file named by filename. -// -func writeFileAtomically(filename string, data []byte) error { - // TODO(gri) this won't work on appengine - f, err := ioutil.TempFile(filepath.Split(filename)) - if err != nil { - return err - } - n, err := f.Write(data) - f.Close() - if err != nil { - return err - } - if n < len(data) { - return io.ErrShortWrite - } - return os.Rename(f.Name(), filename) -} - // isText returns true if a significant prefix of s looks like correct UTF-8; // that is, if it is likely that s is human-readable text. // @@ -146,7 +71,7 @@ var textExt = map[string]bool{ // func isTextFile(filename string) bool { // if the extension is known, use it for decision making - if isText, found := textExt[filepath.Ext(filename)]; found { + if isText, found := textExt[pathpkg.Ext(filename)]; found { return isText } diff --git a/src/cmd/godoc/zip.go b/src/cmd/godoc/zip.go index 8c4b1101b..620eb4f3c 100644 --- a/src/cmd/godoc/zip.go +++ b/src/cmd/godoc/zip.go @@ -73,6 +73,11 @@ func (fi zipFI) Sys() interface{} { type zipFS struct { *zip.ReadCloser list zipList + name string +} + +func (fs *zipFS) String() string { + return "zip(" + fs.name + ")" } func (fs *zipFS) Close() error { @@ -102,7 +107,7 @@ func (fs *zipFS) stat(abspath string) (int, zipFI, error) { return i, zipFI{name, file}, nil } -func (fs *zipFS) Open(abspath string) (io.ReadCloser, error) { +func (fs *zipFS) Open(abspath string) (readSeekCloser, error) { _, fi, err := fs.stat(zipPath(abspath)) if err != nil { return nil, err @@ -110,7 +115,29 @@ func (fs *zipFS) Open(abspath string) (io.ReadCloser, error) { if fi.IsDir() { return nil, fmt.Errorf("Open: %s is a directory", abspath) } - return fi.file.Open() + r, err := fi.file.Open() + if err != nil { + return nil, err + } + return &zipSeek{fi.file, r}, nil +} + +type zipSeek struct { + file *zip.File + io.ReadCloser +} + +func (f *zipSeek) Seek(offset int64, whence int) (int64, error) { + if whence == 0 && offset == 0 { + r, err := f.file.Open() + if err != nil { + return 0, err + } + f.Close() + f.ReadCloser = r + return 0, nil + } + return 0, fmt.Errorf("unsupported Seek in %s", f.file.Name) } func (fs *zipFS) Lstat(abspath string) (os.FileInfo, error) { @@ -161,11 +188,11 @@ func (fs *zipFS) ReadDir(abspath string) ([]os.FileInfo, error) { return list, nil } -func NewZipFS(rc *zip.ReadCloser) FileSystem { +func NewZipFS(rc *zip.ReadCloser, name string) FileSystem { list := make(zipList, len(rc.File)) copy(list, rc.File) // sort a copy of rc.File sort.Sort(list) - return &zipFS{rc, list} + return &zipFS{rc, list, name} } type zipList []*zip.File diff --git a/src/cmd/gofmt/gofmt.go b/src/cmd/gofmt/gofmt.go index 8e565563e..0bc385b5b 100644 --- a/src/cmd/gofmt/gofmt.go +++ b/src/cmd/gofmt/gofmt.go @@ -26,7 +26,7 @@ var ( // main operation modes list = flag.Bool("l", false, "list files whose formatting differs from gofmt's") write = flag.Bool("w", false, "write result to (source) file instead of stdout") - rewriteRule = flag.String("r", "", "rewrite rule (e.g., 'α[β:len(α)] -> α[β:]')") + rewriteRule = flag.String("r", "", "rewrite rule (e.g., 'a[b:len(a)] -> a[b:]')") simplifyAST = flag.Bool("s", false, "simplify code") doDiff = flag.Bool("d", false, "display diffs instead of rewriting files") allErrors = flag.Bool("e", false, "print all (including spurious) errors") @@ -41,7 +41,7 @@ var ( ) var ( - fset = token.NewFileSet() + fileSet = token.NewFileSet() // per process FileSet exitCode = 0 rewrite func(*ast.File) *ast.File parserMode parser.Mode @@ -98,7 +98,7 @@ func processFile(filename string, in io.Reader, out io.Writer, stdin bool) error return err } - file, adjust, err := parse(filename, src, stdin) + file, adjust, err := parse(fileSet, filename, src, stdin) if err != nil { return err } @@ -111,14 +111,14 @@ func processFile(filename string, in io.Reader, out io.Writer, stdin bool) error } } - ast.SortImports(fset, file) + ast.SortImports(fileSet, file) if *simplifyAST { simplify(file) } var buf bytes.Buffer - err = (&printer.Config{Mode: printerMode, Tabwidth: *tabWidth}).Fprint(&buf, fset, file) + err = (&printer.Config{Mode: printerMode, Tabwidth: *tabWidth}).Fprint(&buf, fileSet, file) if err != nil { return err } @@ -254,7 +254,7 @@ func diff(b1, b2 []byte) (data []byte, err error) { // parse parses src, which was read from filename, // as a Go source file or statement list. -func parse(filename string, src []byte, stdin bool) (*ast.File, func(orig, src []byte) []byte, error) { +func parse(fset *token.FileSet, filename string, src []byte, stdin bool) (*ast.File, func(orig, src []byte) []byte, error) { // Try as whole source file. file, err := parser.ParseFile(fset, filename, src, parserMode) if err == nil { diff --git a/src/cmd/gofmt/long_test.go b/src/cmd/gofmt/long_test.go index 9a589b1ba..edbce606a 100644 --- a/src/cmd/gofmt/long_test.go +++ b/src/cmd/gofmt/long_test.go @@ -14,6 +14,7 @@ import ( "fmt" "go/ast" "go/printer" + "go/token" "io" "os" "path/filepath" @@ -30,8 +31,8 @@ var ( nfiles int // number of files processed ) -func gofmt(filename string, src *bytes.Buffer) error { - f, _, err := parse(filename, src.Bytes(), false) +func gofmt(fset *token.FileSet, filename string, src *bytes.Buffer) error { + f, _, err := parse(fset, filename, src.Bytes(), false) if err != nil { return err } @@ -58,7 +59,8 @@ func testFile(t *testing.T, b1, b2 *bytes.Buffer, filename string) { } // exclude files w/ syntax errors (typically test cases) - if _, _, err = parse(filename, b1.Bytes(), false); err != nil { + fset := token.NewFileSet() + if _, _, err = parse(fset, filename, b1.Bytes(), false); err != nil { if *verbose { fmt.Fprintf(os.Stderr, "ignoring %s\n", err) } @@ -66,7 +68,7 @@ func testFile(t *testing.T, b1, b2 *bytes.Buffer, filename string) { } // gofmt file - if err = gofmt(filename, b1); err != nil { + if err = gofmt(fset, filename, b1); err != nil { t.Errorf("1st gofmt failed: %v", err) return } @@ -76,7 +78,7 @@ func testFile(t *testing.T, b1, b2 *bytes.Buffer, filename string) { b2.Write(b1.Bytes()) // gofmt result again - if err = gofmt(filename, b2); err != nil { + if err = gofmt(fset, filename, b2); err != nil { t.Errorf("2nd gofmt failed: %v", err) return } diff --git a/src/cmd/ld/doc.go b/src/cmd/ld/doc.go index 4728fccb8..e99e50466 100644 --- a/src/cmd/ld/doc.go +++ b/src/cmd/ld/doc.go @@ -9,45 +9,52 @@ Ld is the portable code for a modified version of the Plan 9 linker. The origin http://plan9.bell-labs.com/magic/man2html/1/2l It reads object files (.5, .6, or .8 files) and writes a binary named for the -architecture (5.out, 6.out, 8.out) by default. +architecture (5.out, 6.out, 8.out) by default (if $GOOS is windows, a .exe suffix +will be appended). Major changes include: - - support for ELF and Mach-O binary files + - support for ELF, Mach-O and PE binary files - support for segmented stacks (this feature is implemented here, not in the compilers). Original options are listed on the manual page linked above. -Options new in this version: +Usage: + go tool 6l [flags] mainObj +Substitute 6l with 8l or 5l as appropriate. --d - Elide the dynamic linking header. With this option, the binary - is statically linked and does not refer to dynld. Without this option - (the default), the binary's contents are identical but it is loaded with dynld. --Hdarwin - Write Apple Mach-O binaries (default when $GOOS is darwin) --Hlinux - Write Linux ELF binaries (default when $GOOS is linux) --Hfreebsd - Write FreeBSD ELF binaries (default when $GOOS is freebsd) --Hnetbsd - Write NetBSD ELF binaries (default when $GOOS is netbsd) --Hopenbsd - Write OpenBSD ELF binaries (default when $GOOS is openbsd) --Hwindows - Write Windows PE32+ binaries (default when $GOOS is windows) --I interpreter - Set the ELF dynamic linker to use. --L dir1 -L dir2 - Search for libraries (package files) in dir1, dir2, etc. - The default is the single location $GOROOT/pkg/$GOOS_amd64. --r dir1:dir2:... - Set the dynamic linker search path when using ELF. --V - Print the linker version. --X symbol value - Set the value of an otherwise uninitialized string variable. - The symbol name should be of the form importpath.name, - as displayed in the symbol table printed by "go tool nm". +Options new in this version: + -d + Elide the dynamic linking header. With this option, the binary + is statically linked and does not refer to a dynamic linker. Without this option + (the default), the binary's contents are identical but it is loaded with a dynamic + linker. This flag cannot be used when $GOOS is windows. + -Hdarwin (only in 6l/8l) + Write Apple Mach-O binaries (default when $GOOS is darwin) + -Hlinux + Write Linux ELF binaries (default when $GOOS is linux) + -Hfreebsd (only in 6l/8l) + Write FreeBSD ELF binaries (default when $GOOS is freebsd) + -Hnetbsd (only in 6l/8l) + Write NetBSD ELF binaries (default when $GOOS is netbsd) + -Hopenbsd (only in 6l/8l) + Write OpenBSD ELF binaries (default when $GOOS is openbsd) + -Hwindows (only in 6l/8l) + Write Windows PE32+ Console binaries (default when $GOOS is windows) + -Hwindowsgui (only in 6l/8l) + Write Windows PE32+ GUI binaries + -I interpreter + Set the ELF dynamic linker to use. + -L dir1 -L dir2 + Search for libraries (package files) in dir1, dir2, etc. + The default is the single location $GOROOT/pkg/$GOOS_$GOARCH. + -r dir1:dir2:... + Set the dynamic linker search path when using ELF. + -V + Print the linker version. + -X symbol value + Set the value of an otherwise uninitialized string variable. + The symbol name should be of the form importpath.name, + as displayed in the symbol table printed by "go tool nm". */ package documentation diff --git a/src/cmd/nm/doc.go b/src/cmd/nm/doc.go index 5e216b922..c84369a5f 100644 --- a/src/cmd/nm/doc.go +++ b/src/cmd/nm/doc.go @@ -14,7 +14,8 @@ Plan 9 C compiler. This implementation adds the flag -S, which prints each symbol's size in decimal after its address. -It is installed as go tool nm and is architecture-independent. +Usage: + go tool nm [-aghnsTu] file */ package documentation diff --git a/src/cmd/pack/ar.c b/src/cmd/pack/ar.c index 8d881f876..7e07fbc89 100644 --- a/src/cmd/pack/ar.c +++ b/src/cmd/pack/ar.c @@ -1382,11 +1382,14 @@ mesg(int c, char *file) void trim(char *s, char *buf, int n) { - char *p; + char *p, *q; for(;;) { p = strrchr(s, '/'); - if (!p) { /* no slash in name */ + q = strrchr(s, '\\'); + if (q > p) + p = q; + if (!p) { /* no (back)slash in name */ strncpy(buf, s, n); return; } @@ -1394,7 +1397,7 @@ trim(char *s, char *buf, int n) strncpy(buf, p+1, n); return; } - *p = 0; /* strip trailing slash */ + *p = 0; /* strip trailing (back)slash */ } } diff --git a/src/cmd/vet/doc.go b/src/cmd/vet/doc.go index e51fe3768..620964aaf 100644 --- a/src/cmd/vet/doc.go +++ b/src/cmd/vet/doc.go @@ -13,7 +13,7 @@ Available checks: 1. Printf family -Suspicious calls to functions in the Printf familiy, including any functions +Suspicious calls to functions in the Printf family, including any functions with these names: Print Printf Println Fprint Fprintf Fprintln diff --git a/src/cmd/vet/taglit.go b/src/cmd/vet/taglit.go index 864e7bc60..c3c4f3234 100644 --- a/src/cmd/vet/taglit.go +++ b/src/cmd/vet/taglit.go @@ -81,7 +81,8 @@ var untaggedLiteralWhitelist = map[string]bool{ find $GOROOT/src/pkg -type f | grep -v _test.go | xargs grep '^type.*\[\]' | \ grep -v ' map\[' | sed 's,/[^/]*go.type,,' | sed 's,.*src/pkg/,,' | \ - sed 's, ,.,' | sed 's, .*,,' | grep -v '\.[a-z]' | sort + sed 's, ,.,' | sed 's, .*,,' | grep -v '\.[a-z]' | \ + sort | awk '{ print "\"" $0 "\": true," }' */ "crypto/x509/pkix.RDNSequence": true, "crypto/x509/pkix.RelativeDistinguishedNameSET": true, |