diff options
author | Ondřej Surý <ondrej@sury.org> | 2012-01-30 15:38:19 +0100 |
---|---|---|
committer | Ondřej Surý <ondrej@sury.org> | 2012-01-30 15:38:19 +0100 |
commit | 4cecda6c347bd6902b960c6a35a967add7070b0d (patch) | |
tree | a462e224ff41ec9f3eb1a0b6e815806f9e8804ad /src/pkg/go | |
parent | 6c7ca6e4d4e26e4c8cbe0d183966011b3b088a0a (diff) | |
download | golang-4cecda6c347bd6902b960c6a35a967add7070b0d.tar.gz |
Imported Upstream version 2012.01.27upstream-weekly/2012.01.27
Diffstat (limited to 'src/pkg/go')
94 files changed, 5558 insertions, 5642 deletions
diff --git a/src/pkg/go/ast/Makefile b/src/pkg/go/ast/Makefile index 40be10208..30c386cd8 100644 --- a/src/pkg/go/ast/Makefile +++ b/src/pkg/go/ast/Makefile @@ -8,6 +8,7 @@ TARG=go/ast GOFILES=\ ast.go\ filter.go\ + import.go\ print.go\ resolve.go\ scope.go\ diff --git a/src/pkg/go/ast/ast.go b/src/pkg/go/ast/ast.go index 22bd5ee22..7123fe58f 100644 --- a/src/pkg/go/ast/ast.go +++ b/src/pkg/go/ast/ast.go @@ -9,8 +9,9 @@ package ast import ( "go/token" + "strings" "unicode" - "utf8" + "unicode/utf8" ) // ---------------------------------------------------------------------------- @@ -76,6 +77,74 @@ type CommentGroup struct { func (g *CommentGroup) Pos() token.Pos { return g.List[0].Pos() } func (g *CommentGroup) End() token.Pos { return g.List[len(g.List)-1].End() } +func isWhitespace(ch byte) bool { return ch == ' ' || ch == '\t' || ch == '\n' || ch == '\r' } + +func stripTrailingWhitespace(s string) string { + i := len(s) + for i > 0 && isWhitespace(s[i-1]) { + i-- + } + return s[0:i] +} + +// Text returns the text of the comment, +// with the comment markers - //, /*, and */ - removed. +func (g *CommentGroup) Text() string { + if g == nil { + return "" + } + comments := make([]string, len(g.List)) + for i, c := range g.List { + comments[i] = string(c.Text) + } + + lines := make([]string, 0, 10) // most comments are less than 10 lines + for _, c := range comments { + // Remove comment markers. + // The parser has given us exactly the comment text. + switch c[1] { + case '/': + //-style comment + c = c[2:] + // Remove leading space after //, if there is one. + // TODO(gri) This appears to be necessary in isolated + // cases (bignum.RatFromString) - why? + if len(c) > 0 && c[0] == ' ' { + c = c[1:] + } + case '*': + /*-style comment */ + c = c[2 : len(c)-2] + } + + // Split on newlines. + cl := strings.Split(c, "\n") + + // Walk lines, stripping trailing white space and adding to list. + for _, l := range cl { + lines = append(lines, stripTrailingWhitespace(l)) + } + } + + // Remove leading blank lines; convert runs of + // interior blank lines to a single blank line. + n := 0 + for _, line := range lines { + if line != "" || n > 0 && lines[n-1] != "" { + lines[n] = line + n++ + } + } + lines = lines[0:n] + + // Add final "" entry to get trailing newline from Join. + if n > 0 && lines[n-1] != "" { + lines = append(lines, "") + } + + return strings.Join(lines, "\n") +} + // ---------------------------------------------------------------------------- // Expressions and types @@ -412,29 +481,29 @@ func (x *ChanType) End() token.Pos { return x.Value.End() } // exprNode() ensures that only expression/type nodes can be // assigned to an ExprNode. // -func (x *BadExpr) exprNode() {} -func (x *Ident) exprNode() {} -func (x *Ellipsis) exprNode() {} -func (x *BasicLit) exprNode() {} -func (x *FuncLit) exprNode() {} -func (x *CompositeLit) exprNode() {} -func (x *ParenExpr) exprNode() {} -func (x *SelectorExpr) exprNode() {} -func (x *IndexExpr) exprNode() {} -func (x *SliceExpr) exprNode() {} -func (x *TypeAssertExpr) exprNode() {} -func (x *CallExpr) exprNode() {} -func (x *StarExpr) exprNode() {} -func (x *UnaryExpr) exprNode() {} -func (x *BinaryExpr) exprNode() {} -func (x *KeyValueExpr) exprNode() {} - -func (x *ArrayType) exprNode() {} -func (x *StructType) exprNode() {} -func (x *FuncType) exprNode() {} -func (x *InterfaceType) exprNode() {} -func (x *MapType) exprNode() {} -func (x *ChanType) exprNode() {} +func (*BadExpr) exprNode() {} +func (*Ident) exprNode() {} +func (*Ellipsis) exprNode() {} +func (*BasicLit) exprNode() {} +func (*FuncLit) exprNode() {} +func (*CompositeLit) exprNode() {} +func (*ParenExpr) exprNode() {} +func (*SelectorExpr) exprNode() {} +func (*IndexExpr) exprNode() {} +func (*SliceExpr) exprNode() {} +func (*TypeAssertExpr) exprNode() {} +func (*CallExpr) exprNode() {} +func (*StarExpr) exprNode() {} +func (*UnaryExpr) exprNode() {} +func (*BinaryExpr) exprNode() {} +func (*KeyValueExpr) exprNode() {} + +func (*ArrayType) exprNode() {} +func (*StructType) exprNode() {} +func (*FuncType) exprNode() {} +func (*InterfaceType) exprNode() {} +func (*MapType) exprNode() {} +func (*ChanType) exprNode() {} // ---------------------------------------------------------------------------- // Convenience functions for Idents @@ -711,27 +780,27 @@ func (s *RangeStmt) End() token.Pos { return s.Body.End() } // stmtNode() ensures that only statement nodes can be // assigned to a StmtNode. // -func (s *BadStmt) stmtNode() {} -func (s *DeclStmt) stmtNode() {} -func (s *EmptyStmt) stmtNode() {} -func (s *LabeledStmt) stmtNode() {} -func (s *ExprStmt) stmtNode() {} -func (s *SendStmt) stmtNode() {} -func (s *IncDecStmt) stmtNode() {} -func (s *AssignStmt) stmtNode() {} -func (s *GoStmt) stmtNode() {} -func (s *DeferStmt) stmtNode() {} -func (s *ReturnStmt) stmtNode() {} -func (s *BranchStmt) stmtNode() {} -func (s *BlockStmt) stmtNode() {} -func (s *IfStmt) stmtNode() {} -func (s *CaseClause) stmtNode() {} -func (s *SwitchStmt) stmtNode() {} -func (s *TypeSwitchStmt) stmtNode() {} -func (s *CommClause) stmtNode() {} -func (s *SelectStmt) stmtNode() {} -func (s *ForStmt) stmtNode() {} -func (s *RangeStmt) stmtNode() {} +func (*BadStmt) stmtNode() {} +func (*DeclStmt) stmtNode() {} +func (*EmptyStmt) stmtNode() {} +func (*LabeledStmt) stmtNode() {} +func (*ExprStmt) stmtNode() {} +func (*SendStmt) stmtNode() {} +func (*IncDecStmt) stmtNode() {} +func (*AssignStmt) stmtNode() {} +func (*GoStmt) stmtNode() {} +func (*DeferStmt) stmtNode() {} +func (*ReturnStmt) stmtNode() {} +func (*BranchStmt) stmtNode() {} +func (*BlockStmt) stmtNode() {} +func (*IfStmt) stmtNode() {} +func (*CaseClause) stmtNode() {} +func (*SwitchStmt) stmtNode() {} +func (*TypeSwitchStmt) stmtNode() {} +func (*CommClause) stmtNode() {} +func (*SelectStmt) stmtNode() {} +func (*ForStmt) stmtNode() {} +func (*RangeStmt) stmtNode() {} // ---------------------------------------------------------------------------- // Declarations @@ -752,6 +821,7 @@ type ( Name *Ident // local package name (including "."); or nil Path *BasicLit // import path Comment *CommentGroup // line comments; or nil + EndPos token.Pos // end of spec (overrides Path.Pos if nonzero) } // A ValueSpec node represents a constant or variable declaration @@ -785,7 +855,13 @@ func (s *ImportSpec) Pos() token.Pos { func (s *ValueSpec) Pos() token.Pos { return s.Names[0].Pos() } func (s *TypeSpec) Pos() token.Pos { return s.Name.Pos() } -func (s *ImportSpec) End() token.Pos { return s.Path.End() } +func (s *ImportSpec) End() token.Pos { + if s.EndPos != 0 { + return s.EndPos + } + return s.Path.End() +} + func (s *ValueSpec) End() token.Pos { if n := len(s.Values); n > 0 { return s.Values[n-1].End() @@ -800,9 +876,9 @@ func (s *TypeSpec) End() token.Pos { return s.Type.End() } // specNode() ensures that only spec nodes can be // assigned to a Spec. // -func (s *ImportSpec) specNode() {} -func (s *ValueSpec) specNode() {} -func (s *TypeSpec) specNode() {} +func (*ImportSpec) specNode() {} +func (*ValueSpec) specNode() {} +func (*TypeSpec) specNode() {} // A declaration is represented by one of the following declaration nodes. // @@ -868,9 +944,9 @@ func (d *FuncDecl) End() token.Pos { // declNode() ensures that only declaration nodes can be // assigned to a DeclNode. // -func (d *BadDecl) declNode() {} -func (d *GenDecl) declNode() {} -func (d *FuncDecl) declNode() {} +func (*BadDecl) declNode() {} +func (*GenDecl) declNode() {} +func (*FuncDecl) declNode() {} // ---------------------------------------------------------------------------- // Files and packages diff --git a/src/pkg/go/ast/filter.go b/src/pkg/go/ast/filter.go index 1bd8990f8..4a89b8909 100644 --- a/src/pkg/go/ast/filter.go +++ b/src/pkg/go/ast/filter.go @@ -4,7 +4,10 @@ package ast -import "go/token" +import ( + "go/token" + "sort" +) // ---------------------------------------------------------------------------- // Export filtering @@ -18,14 +21,13 @@ func exportFilter(name string) bool { // only exported nodes remain: all top-level identifiers which are not exported // and their associated information (such as type, initial value, or function // body) are removed. Non-exported fields and methods of exported types are -// stripped, and the function bodies of exported functions are set to nil. -// The File.Comments list is not changed. +// stripped. The File.Comments list is not changed. // -// FileExports returns true if there are exported declarationa; +// FileExports returns true if there are exported declarations; // it returns false otherwise. // func FileExports(src *File) bool { - return FilterFile(src, exportFilter) + return filterFile(src, exportFilter, true) } // PackageExports trims the AST for a Go package in place such that @@ -36,7 +38,7 @@ func FileExports(src *File) bool { // it returns false otherwise. // func PackageExports(pkg *Package) bool { - return FilterPackage(pkg, exportFilter) + return filterPackage(pkg, exportFilter, true) } // ---------------------------------------------------------------------------- @@ -73,7 +75,7 @@ func fieldName(x Expr) *Ident { return nil } -func filterFieldList(fields *FieldList, filter Filter) (removedFields bool) { +func filterFieldList(fields *FieldList, filter Filter, export bool) (removedFields bool) { if fields == nil { return false } @@ -94,8 +96,8 @@ func filterFieldList(fields *FieldList, filter Filter) (removedFields bool) { keepField = len(f.Names) > 0 } if keepField { - if filter == exportFilter { - filterType(f.Type, filter) + if export { + filterType(f.Type, filter, export) } list[j] = f j++ @@ -108,84 +110,84 @@ func filterFieldList(fields *FieldList, filter Filter) (removedFields bool) { return } -func filterParamList(fields *FieldList, filter Filter) bool { +func filterParamList(fields *FieldList, filter Filter, export bool) bool { if fields == nil { return false } var b bool for _, f := range fields.List { - if filterType(f.Type, filter) { + if filterType(f.Type, filter, export) { b = true } } return b } -func filterType(typ Expr, f Filter) bool { +func filterType(typ Expr, f Filter, export bool) bool { switch t := typ.(type) { case *Ident: return f(t.Name) case *ParenExpr: - return filterType(t.X, f) + return filterType(t.X, f, export) case *ArrayType: - return filterType(t.Elt, f) + return filterType(t.Elt, f, export) case *StructType: - if filterFieldList(t.Fields, f) { + if filterFieldList(t.Fields, f, export) { t.Incomplete = true } return len(t.Fields.List) > 0 case *FuncType: - b1 := filterParamList(t.Params, f) - b2 := filterParamList(t.Results, f) + b1 := filterParamList(t.Params, f, export) + b2 := filterParamList(t.Results, f, export) return b1 || b2 case *InterfaceType: - if filterFieldList(t.Methods, f) { + if filterFieldList(t.Methods, f, export) { t.Incomplete = true } return len(t.Methods.List) > 0 case *MapType: - b1 := filterType(t.Key, f) - b2 := filterType(t.Value, f) + b1 := filterType(t.Key, f, export) + b2 := filterType(t.Value, f, export) return b1 || b2 case *ChanType: - return filterType(t.Value, f) + return filterType(t.Value, f, export) } return false } -func filterSpec(spec Spec, f Filter) bool { +func filterSpec(spec Spec, f Filter, export bool) bool { switch s := spec.(type) { case *ValueSpec: s.Names = filterIdentList(s.Names, f) if len(s.Names) > 0 { - if f == exportFilter { - filterType(s.Type, f) + if export { + filterType(s.Type, f, export) } return true } case *TypeSpec: if f(s.Name.Name) { - if f == exportFilter { - filterType(s.Type, f) + if export { + filterType(s.Type, f, export) } return true } - if f != exportFilter { + if !export { // For general filtering (not just exports), // filter type even if name is not filtered // out. // If the type contains filtered elements, // keep the declaration. - return filterType(s.Type, f) + return filterType(s.Type, f, export) } } return false } -func filterSpecList(list []Spec, f Filter) []Spec { +func filterSpecList(list []Spec, f Filter, export bool) []Spec { j := 0 for _, s := range list { - if filterSpec(s, f) { + if filterSpec(s, f, export) { list[j] = s j++ } @@ -201,14 +203,15 @@ func filterSpecList(list []Spec, f Filter) []Spec { // filtering; it returns false otherwise. // func FilterDecl(decl Decl, f Filter) bool { + return filterDecl(decl, f, false) +} + +func filterDecl(decl Decl, f Filter, export bool) bool { switch d := decl.(type) { case *GenDecl: - d.Specs = filterSpecList(d.Specs, f) + d.Specs = filterSpecList(d.Specs, f, export) return len(d.Specs) > 0 case *FuncDecl: - if f == exportFilter { - d.Body = nil // strip body - } return f(d.Name.Name) } return false @@ -225,9 +228,13 @@ func FilterDecl(decl Decl, f Filter) bool { // left after filtering; it returns false otherwise. // func FilterFile(src *File, f Filter) bool { + return filterFile(src, f, false) +} + +func filterFile(src *File, f Filter, export bool) bool { j := 0 for _, d := range src.Decls { - if FilterDecl(d, f) { + if filterDecl(d, f, export) { src.Decls[j] = d j++ } @@ -248,9 +255,13 @@ func FilterFile(src *File, f Filter) bool { // left after filtering; it returns false otherwise. // func FilterPackage(pkg *Package, f Filter) bool { + return filterPackage(pkg, f, false) +} + +func filterPackage(pkg *Package, f Filter, export bool) bool { hasDecls := false for _, src := range pkg.Files { - if FilterFile(src, f) { + if filterFile(src, f, export) { hasDecls = true } } @@ -283,29 +294,35 @@ var separator = &Comment{noPos, "//"} // func MergePackageFiles(pkg *Package, mode MergeMode) *File { // Count the number of package docs, comments and declarations across - // all package files. + // all package files. Also, compute sorted list of filenames, so that + // subsequent iterations can always iterate in the same order. ndocs := 0 ncomments := 0 ndecls := 0 - for _, f := range pkg.Files { + filenames := make([]string, len(pkg.Files)) + i := 0 + for filename, f := range pkg.Files { + filenames[i] = filename + i++ if f.Doc != nil { ndocs += len(f.Doc.List) + 1 // +1 for separator } ncomments += len(f.Comments) ndecls += len(f.Decls) } + sort.Strings(filenames) // Collect package comments from all package files into a single - // CommentGroup - the collected package documentation. The order - // is unspecified. In general there should be only one file with - // a package comment; but it's better to collect extra comments - // than drop them on the floor. + // CommentGroup - the collected package documentation. In general + // there should be only one file with a package comment; but it's + // better to collect extra comments than drop them on the floor. var doc *CommentGroup var pos token.Pos if ndocs > 0 { list := make([]*Comment, ndocs-1) // -1: no separator before first group i := 0 - for _, f := range pkg.Files { + for _, filename := range filenames { + f := pkg.Files[filename] if f.Doc != nil { if i > 0 { // not the first group - add separator @@ -334,7 +351,8 @@ func MergePackageFiles(pkg *Package, mode MergeMode) *File { funcs := make(map[string]int) // map of global function name -> decls index i := 0 // current index n := 0 // number of filtered entries - for _, f := range pkg.Files { + for _, filename := range filenames { + f := pkg.Files[filename] for _, d := range f.Decls { if mode&FilterFuncDuplicates != 0 { // A language entity may be declared multiple @@ -390,7 +408,8 @@ func MergePackageFiles(pkg *Package, mode MergeMode) *File { var imports []*ImportSpec if mode&FilterImportDuplicates != 0 { seen := make(map[string]bool) - for _, f := range pkg.Files { + for _, filename := range filenames { + f := pkg.Files[filename] for _, imp := range f.Imports { if path := imp.Path.Value; !seen[path] { // TODO: consider handling cases where: diff --git a/src/pkg/go/ast/import.go b/src/pkg/go/ast/import.go new file mode 100644 index 000000000..2d4f69aae --- /dev/null +++ b/src/pkg/go/ast/import.go @@ -0,0 +1,134 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ast + +import ( + "go/token" + "sort" + "strconv" +) + +// SortImports sorts runs of consecutive import lines in import blocks in f. +func SortImports(fset *token.FileSet, f *File) { + for _, d := range f.Decls { + d, ok := d.(*GenDecl) + if !ok || d.Tok != token.IMPORT { + // Not an import declaration, so we're done. + // Imports are always first. + break + } + + if d.Lparen == token.NoPos { + // Not a block: sorted by default. + continue + } + + // Identify and sort runs of specs on successive lines. + i := 0 + for j, s := range d.Specs { + if j > i && fset.Position(s.Pos()).Line > 1+fset.Position(d.Specs[j-1].End()).Line { + // j begins a new run. End this one. + sortSpecs(fset, f, d.Specs[i:j]) + i = j + } + } + sortSpecs(fset, f, d.Specs[i:]) + } +} + +func importPath(s Spec) string { + t, err := strconv.Unquote(s.(*ImportSpec).Path.Value) + if err == nil { + return t + } + return "" +} + +type posSpan struct { + Start token.Pos + End token.Pos +} + +func sortSpecs(fset *token.FileSet, f *File, specs []Spec) { + // Avoid work if already sorted (also catches < 2 entries). + sorted := true + for i, s := range specs { + if i > 0 && importPath(specs[i-1]) > importPath(s) { + sorted = false + break + } + } + if sorted { + return + } + + // Record positions for specs. + pos := make([]posSpan, len(specs)) + for i, s := range specs { + pos[i] = posSpan{s.Pos(), s.End()} + } + + // Identify comments in this range. + // Any comment from pos[0].Start to the final line counts. + lastLine := fset.Position(pos[len(pos)-1].End).Line + cstart := len(f.Comments) + cend := len(f.Comments) + for i, g := range f.Comments { + if g.Pos() < pos[0].Start { + continue + } + if i < cstart { + cstart = i + } + if fset.Position(g.End()).Line > lastLine { + cend = i + break + } + } + comments := f.Comments[cstart:cend] + + // Assign each comment to the import spec preceding it. + importComment := map[*ImportSpec][]*CommentGroup{} + specIndex := 0 + for _, g := range comments { + for specIndex+1 < len(specs) && pos[specIndex+1].Start <= g.Pos() { + specIndex++ + } + s := specs[specIndex].(*ImportSpec) + importComment[s] = append(importComment[s], g) + } + + // Sort the import specs by import path. + // Reassign the import paths to have the same position sequence. + // Reassign each comment to abut the end of its spec. + // Sort the comments by new position. + sort.Sort(byImportPath(specs)) + for i, s := range specs { + s := s.(*ImportSpec) + if s.Name != nil { + s.Name.NamePos = pos[i].Start + } + s.Path.ValuePos = pos[i].Start + s.EndPos = pos[i].End + for _, g := range importComment[s] { + for _, c := range g.List { + c.Slash = pos[i].End + } + } + } + sort.Sort(byCommentPos(comments)) +} + +type byImportPath []Spec // slice of *ImportSpec + +func (x byImportPath) Len() int { return len(x) } +func (x byImportPath) Swap(i, j int) { x[i], x[j] = x[j], x[i] } +func (x byImportPath) Less(i, j int) bool { return importPath(x[i]) < importPath(x[j]) } + +type byCommentPos []*CommentGroup + +func (x byCommentPos) Len() int { return len(x) } +func (x byCommentPos) Swap(i, j int) { x[i], x[j] = x[j], x[i] } +func (x byCommentPos) Less(i, j int) bool { return x[i].Pos() < x[j].Pos() } diff --git a/src/pkg/go/ast/print.go b/src/pkg/go/ast/print.go index 62a30481d..f6c63c0d8 100644 --- a/src/pkg/go/ast/print.go +++ b/src/pkg/go/ast/print.go @@ -36,7 +36,7 @@ func NotNilFilter(_ string, v reflect.Value) bool { // struct fields for which f(fieldname, fieldvalue) is true are // are printed; all others are filtered from the output. // -func Fprint(w io.Writer, fset *token.FileSet, x interface{}, f FieldFilter) (n int, err os.Error) { +func Fprint(w io.Writer, fset *token.FileSet, x interface{}, f FieldFilter) (err error) { // setup printer p := printer{ output: w, @@ -48,7 +48,6 @@ func Fprint(w io.Writer, fset *token.FileSet, x interface{}, f FieldFilter) (n i // install error handler defer func() { - n = p.written if e := recover(); e != nil { err = e.(localError).err // re-panics if it's not a localError } @@ -67,24 +66,23 @@ func Fprint(w io.Writer, fset *token.FileSet, x interface{}, f FieldFilter) (n i // Print prints x to standard output, skipping nil fields. // Print(fset, x) is the same as Fprint(os.Stdout, fset, x, NotNilFilter). -func Print(fset *token.FileSet, x interface{}) (int, os.Error) { +func Print(fset *token.FileSet, x interface{}) error { return Fprint(os.Stdout, fset, x, NotNilFilter) } type printer struct { - output io.Writer - fset *token.FileSet - filter FieldFilter - ptrmap map[interface{}]int // *T -> line number - written int // number of bytes written to output - indent int // current indentation level - last byte // the last byte processed by Write - line int // current line number + output io.Writer + fset *token.FileSet + filter FieldFilter + ptrmap map[interface{}]int // *T -> line number + indent int // current indentation level + last byte // the last byte processed by Write + line int // current line number } var indent = []byte(". ") -func (p *printer) Write(data []byte) (n int, err os.Error) { +func (p *printer) Write(data []byte) (n int, err error) { var m int for i, b := range data { // invariant: data[0:n] has been written @@ -114,17 +112,15 @@ func (p *printer) Write(data []byte) (n int, err os.Error) { return } -// localError wraps locally caught os.Errors so we can distinguish +// localError wraps locally caught errors so we can distinguish // them from genuine panics which we don't want to return as errors. type localError struct { - err os.Error + err error } // printf is a convenience wrapper that takes care of print errors. func (p *printer) printf(format string, args ...interface{}) { - n, err := fmt.Fprintf(p, format, args...) - p.written += n - if err != nil { + if _, err := fmt.Fprintf(p, format, args...); err != nil { panic(localError{err}) } } @@ -149,7 +145,7 @@ func (p *printer) print(x reflect.Value) { p.print(x.Elem()) case reflect.Map: - p.printf("%s (len = %d) {\n", x.Type().String(), x.Len()) + p.printf("%s (len = %d) {\n", x.Type(), x.Len()) p.indent++ for _, key := range x.MapKeys() { p.print(key) @@ -178,7 +174,7 @@ func (p *printer) print(x reflect.Value) { p.printf("%#q", s) return } - p.printf("%s (len = %d) {\n", x.Type().String(), x.Len()) + p.printf("%s (len = %d) {\n", x.Type(), x.Len()) p.indent++ for i, n := 0, x.Len(); i < n; i++ { p.printf("%d: ", i) @@ -189,7 +185,7 @@ func (p *printer) print(x reflect.Value) { p.printf("}") case reflect.Struct: - p.printf("%s {\n", x.Type().String()) + p.printf("%s {\n", x.Type()) p.indent++ t := x.Type() for i, n := 0, t.NumField(); i < n; i++ { diff --git a/src/pkg/go/ast/print_test.go b/src/pkg/go/ast/print_test.go index f4e8f7a78..71c028e75 100644 --- a/src/pkg/go/ast/print_test.go +++ b/src/pkg/go/ast/print_test.go @@ -23,11 +23,10 @@ var tests = []struct { {"foobar", "0 \"foobar\""}, // maps - {map[string]int{"a": 1, "b": 2}, - `0 map[string] int (len = 2) { + {map[string]int{"a": 1}, + `0 map[string]int (len = 1) { 1 . "a": 1 - 2 . "b": 2 - 3 }`}, + 2 }`}, // pointers {new(int), "0 *0"}, @@ -41,10 +40,10 @@ var tests = []struct { 4 }`}, // structs - {struct{ x, y int }{42, 991}, - `0 struct { x int; y int } { - 1 . x: 42 - 2 . y: 991 + {struct{ X, Y int }{42, 991}, + `0 struct { X int; Y int } { + 1 . X: 42 + 2 . Y: 991 3 }`}, } @@ -67,7 +66,7 @@ func TestPrint(t *testing.T) { var buf bytes.Buffer for _, test := range tests { buf.Reset() - if _, err := Fprint(&buf, nil, test.x, nil); err != nil { + if err := Fprint(&buf, nil, test.x, nil); err != nil { t.Errorf("Fprint failed: %s", err) } if s, ts := trim(buf.String()), trim(test.s); s != ts { diff --git a/src/pkg/go/ast/resolve.go b/src/pkg/go/ast/resolve.go index 3927a799e..c7c8e7c10 100644 --- a/src/pkg/go/ast/resolve.go +++ b/src/pkg/go/ast/resolve.go @@ -10,7 +10,6 @@ import ( "fmt" "go/scanner" "go/token" - "os" "strconv" ) @@ -61,7 +60,7 @@ func resolve(scope *Scope, ident *Ident) bool { // Importer should load the package data for the given path into // a new *Object (pkg), record pkg in the imports map, and then // return pkg. -type Importer func(imports map[string]*Object, path string) (pkg *Object, err os.Error) +type Importer func(imports map[string]*Object, path string) (pkg *Object, err error) // NewPackage creates a new Package node from a set of File nodes. It resolves // unresolved identifiers across files and updates each file's Unresolved list @@ -72,7 +71,7 @@ type Importer func(imports map[string]*Object, path string) (pkg *Object, err os // different package names are reported and then ignored. // The result is a package node and a scanner.ErrorList if there were errors. // -func NewPackage(fset *token.FileSet, files map[string]*File, importer Importer, universe *Scope) (*Package, os.Error) { +func NewPackage(fset *token.FileSet, files map[string]*File, importer Importer, universe *Scope) (*Package, error) { var p pkgBuilder p.fset = fset @@ -114,7 +113,7 @@ func NewPackage(fset *token.FileSet, files map[string]*File, importer Importer, importErrors = true continue } - path, _ := strconv.Unquote(string(spec.Path.Value)) + path, _ := strconv.Unquote(spec.Path.Value) pkg, err := importer(imports, path) if err != nil { p.errorf(spec.Path.Pos(), "could not import %s (%s)", path, err) diff --git a/src/pkg/go/ast/scope.go b/src/pkg/go/ast/scope.go index 92e366980..11e6b13f1 100644 --- a/src/pkg/go/ast/scope.go +++ b/src/pkg/go/ast/scope.go @@ -80,7 +80,7 @@ func (s *Scope) String() string { type Object struct { Kind ObjKind Name string // declared name - Decl interface{} // corresponding Field, XxxSpec, FuncDecl, or LabeledStmt; or nil + Decl interface{} // corresponding Field, XxxSpec, FuncDecl, LabeledStmt, AssignStmt, Scope; or nil Data interface{} // object-specific data; or nil Type interface{} // place holder for type information; may be nil } @@ -125,6 +125,14 @@ func (obj *Object) Pos() token.Pos { if d.Label.Name == name { return d.Label.Pos() } + case *AssignStmt: + for _, x := range d.Lhs { + if ident, isIdent := x.(*Ident); isIdent && ident.Name == name { + return ident.Pos() + } + } + case *Scope: + // predeclared object - nothing to do for now } return token.NoPos } diff --git a/src/pkg/go/build/Makefile b/src/pkg/go/build/Makefile index 349e00e80..68c731749 100644 --- a/src/pkg/go/build/Makefile +++ b/src/pkg/go/build/Makefile @@ -11,12 +11,13 @@ GOFILES=\ path.go\ syslist.go\ -CLEANFILES+=syslist.go pkgtest/_obj cmdtest/_obj cgotest/_obj +CLEANFILES+=pkgtest/_obj cmdtest/_obj cgotest/_obj include ../../../Make.pkg syslist.go: ../../../Make.inc Makefile echo '// Generated automatically by make.' >$@ echo 'package build' >>$@ + echo >>$@ echo 'const goosList = "$(GOOS_LIST)"' >>$@ echo 'const goarchList = "$(GOARCH_LIST)"' >>$@ diff --git a/src/pkg/go/build/build.go b/src/pkg/go/build/build.go index 97f92bfb6..68e8d3420 100644 --- a/src/pkg/go/build/build.go +++ b/src/pkg/go/build/build.go @@ -5,247 +5,11 @@ // Package build provides tools for building Go packages. package build -import ( - "bytes" - "exec" - "fmt" - "os" - "path/filepath" - "regexp" - "runtime" - "strings" -) - -// Build produces a build Script for the given package. -func Build(tree *Tree, pkg string, info *DirInfo) (*Script, os.Error) { - s := &Script{} - b := &build{ - script: s, - path: filepath.Join(tree.SrcDir(), pkg), - } - b.obj = b.abs("_obj") + string(filepath.Separator) - - b.goarch = runtime.GOARCH - if g := os.Getenv("GOARCH"); g != "" { - b.goarch = g - } - var err os.Error - b.arch, err = ArchChar(b.goarch) - if err != nil { - return nil, err - } - - // add import object files to list of Inputs - for _, pkg := range info.Imports { - t, p, err := FindTree(pkg) - if err != nil && err != ErrNotFound { - // FindTree should always be able to suggest an import - // path and tree. The path must be malformed - // (for example, an absolute or relative path). - return nil, os.NewError("build: invalid import: " + pkg) - } - s.addInput(filepath.Join(t.PkgDir(), p+".a")) - } - - // .go files to be built with gc - gofiles := b.abss(info.GoFiles...) - s.addInput(gofiles...) - - var ofiles []string // object files to be linked or packed - - // make build directory - b.mkdir(b.obj) - s.addIntermediate(b.obj) - - // cgo - if len(info.CgoFiles) > 0 { - cgoFiles := b.abss(info.CgoFiles...) - s.addInput(cgoFiles...) - cgoCFiles := b.abss(info.CFiles...) - s.addInput(cgoCFiles...) - outGo, outObj := b.cgo(cgoFiles, cgoCFiles) - gofiles = append(gofiles, outGo...) - ofiles = append(ofiles, outObj...) - s.addIntermediate(outGo...) - s.addIntermediate(outObj...) - } - - // compile - if len(gofiles) > 0 { - ofile := b.obj + "_go_." + b.arch - b.gc(ofile, gofiles...) - ofiles = append(ofiles, ofile) - s.addIntermediate(ofile) - } - - // assemble - for _, sfile := range info.SFiles { - ofile := b.obj + sfile[:len(sfile)-1] + b.arch - sfile = b.abs(sfile) - s.addInput(sfile) - b.asm(ofile, sfile) - ofiles = append(ofiles, ofile) - s.addIntermediate(ofile) - } - - if len(ofiles) == 0 { - return nil, os.NewError("make: no object files to build") - } - - // choose target file - var targ string - if info.IsCommand() { - // use the last part of the import path as binary name - _, bin := filepath.Split(pkg) - if runtime.GOOS == "windows" { - bin += ".exe" - } - targ = filepath.Join(tree.BinDir(), bin) - } else { - targ = filepath.Join(tree.PkgDir(), pkg+".a") - } - - // make target directory - targDir, _ := filepath.Split(targ) - b.mkdir(targDir) - - // link binary or pack object - if info.IsCommand() { - b.ld(targ, ofiles...) - } else { - b.gopack(targ, ofiles...) - } - s.Output = append(s.Output, targ) - - return b.script, nil -} - -// A Script describes the build process for a Go package. -// The Input, Intermediate, and Output fields are lists of absolute paths. -type Script struct { - Cmd []*Cmd - Input []string - Intermediate []string - Output []string -} - -func (s *Script) addInput(file ...string) { - s.Input = append(s.Input, file...) -} - -func (s *Script) addIntermediate(file ...string) { - s.Intermediate = append(s.Intermediate, file...) -} - -// Run runs the Script's Cmds in order. -func (s *Script) Run() os.Error { - for _, c := range s.Cmd { - if err := c.Run(); err != nil { - return err - } - } - return nil -} - -// Stale returns true if the build's inputs are newer than its outputs. -func (s *Script) Stale() bool { - var latest int64 - // get latest mtime of outputs - for _, file := range s.Output { - fi, err := os.Stat(file) - if err != nil { - // any error reading output files means stale - return true - } - if m := fi.Mtime_ns; m > latest { - latest = m - } - } - for _, file := range s.Input { - fi, err := os.Stat(file) - if err != nil || fi.Mtime_ns > latest { - // any error reading input files means stale - // (attempt to rebuild to figure out why) - return true - } - } - return false -} - -// Clean removes the Script's Intermediate files. -// It tries to remove every file and returns the first error it encounters. -func (s *Script) Clean() (err os.Error) { - // Reverse order so that directories get removed after the files they contain. - for i := len(s.Intermediate) - 1; i >= 0; i-- { - if e := os.Remove(s.Intermediate[i]); err == nil { - err = e - } - } - return -} - -// Nuke removes the Script's Intermediate and Output files. -// It tries to remove every file and returns the first error it encounters. -func (s *Script) Nuke() (err os.Error) { - // Reverse order so that directories get removed after the files they contain. - for i := len(s.Output) - 1; i >= 0; i-- { - if e := os.Remove(s.Output[i]); err == nil { - err = e - } - } - if e := s.Clean(); err == nil { - err = e - } - return -} - -// A Cmd describes an individual build command. -type Cmd struct { - Args []string // command-line - Stdout string // write standard output to this file, "" is passthrough - Dir string // working directory - Env []string // environment - Input []string // file paths (dependencies) - Output []string // file paths -} - -func (c *Cmd) String() string { - return strings.Join(c.Args, " ") -} - -// Run executes the Cmd. -func (c *Cmd) Run() os.Error { - if c.Args[0] == "mkdir" { - for _, p := range c.Output { - if err := os.MkdirAll(p, 0777); err != nil { - return fmt.Errorf("command %q: %v", c, err) - } - } - return nil - } - out := new(bytes.Buffer) - cmd := exec.Command(c.Args[0], c.Args[1:]...) - cmd.Dir = c.Dir - cmd.Env = c.Env - cmd.Stdout = out - cmd.Stderr = out - if c.Stdout != "" { - f, err := os.Create(c.Stdout) - if err != nil { - return err - } - defer f.Close() - cmd.Stdout = f - } - if err := cmd.Run(); err != nil { - return fmt.Errorf("command %q: %v\n%v", c, err, out) - } - return nil -} +import "errors" // ArchChar returns the architecture character for the given goarch. // For example, ArchChar("amd64") returns "6". -func ArchChar(goarch string) (string, os.Error) { +func ArchChar(goarch string) (string, error) { switch goarch { case "386": return "8", nil @@ -254,191 +18,5 @@ func ArchChar(goarch string) (string, os.Error) { case "arm": return "5", nil } - return "", os.NewError("unsupported GOARCH " + goarch) -} - -type build struct { - script *Script - path string - obj string - goarch string - arch string -} - -func (b *build) abs(file string) string { - if filepath.IsAbs(file) { - return file - } - return filepath.Join(b.path, file) -} - -func (b *build) abss(file ...string) []string { - s := make([]string, len(file)) - for i, f := range file { - s[i] = b.abs(f) - } - return s -} - -func (b *build) add(c Cmd) { - b.script.Cmd = append(b.script.Cmd, &c) -} - -func (b *build) mkdir(name string) { - b.add(Cmd{ - Args: []string{"mkdir", "-p", name}, - Output: []string{name}, - }) -} - -func (b *build) gc(ofile string, gofiles ...string) { - gc := b.arch + "g" - args := append([]string{gc, "-o", ofile}, gcImportArgs...) - args = append(args, gofiles...) - b.add(Cmd{ - Args: args, - Input: gofiles, - Output: []string{ofile}, - }) -} - -func (b *build) asm(ofile string, sfile string) { - asm := b.arch + "a" - b.add(Cmd{ - Args: []string{asm, "-o", ofile, sfile}, - Input: []string{sfile}, - Output: []string{ofile}, - }) -} - -func (b *build) ld(targ string, ofiles ...string) { - ld := b.arch + "l" - args := append([]string{ld, "-o", targ}, ldImportArgs...) - args = append(args, ofiles...) - b.add(Cmd{ - Args: args, - Input: ofiles, - Output: []string{targ}, - }) -} - -func (b *build) gopack(targ string, ofiles ...string) { - b.add(Cmd{ - Args: append([]string{"gopack", "grc", targ}, ofiles...), - Input: ofiles, - Output: []string{targ}, - }) -} - -func (b *build) cc(ofile string, cfiles ...string) { - cc := b.arch + "c" - dir := fmt.Sprintf("%s_%s", runtime.GOOS, runtime.GOARCH) - inc := filepath.Join(runtime.GOROOT(), "pkg", dir) - args := []string{cc, "-FVw", "-I", inc, "-o", ofile} - b.add(Cmd{ - Args: append(args, cfiles...), - Input: cfiles, - Output: []string{ofile}, - }) -} - -func (b *build) gccCompile(ofile, cfile string) { - b.add(Cmd{ - Args: b.gccArgs("-o", ofile, "-c", cfile), - Input: []string{cfile}, - Output: []string{ofile}, - }) -} - -func (b *build) gccLink(ofile string, ofiles ...string) { - b.add(Cmd{ - Args: append(b.gccArgs("-o", ofile), ofiles...), - Input: ofiles, - Output: []string{ofile}, - }) -} - -func (b *build) gccArgs(args ...string) []string { - // TODO(adg): HOST_CC - a := []string{"gcc", "-I", b.path, "-g", "-fPIC", "-O2"} - switch b.arch { - case "8": - a = append(a, "-m32") - case "6": - a = append(a, "-m64") - } - return append(a, args...) -} - -var cgoRe = regexp.MustCompile(`[/\\:]`) - -func (b *build) cgo(cgofiles, cgocfiles []string) (outGo, outObj []string) { - // cgo - // TODO(adg): CGOPKGPATH - // TODO(adg): CGO_FLAGS - gofiles := []string{b.obj + "_cgo_gotypes.go"} - cfiles := []string{b.obj + "_cgo_main.c", b.obj + "_cgo_export.c"} - for _, fn := range cgofiles { - f := b.obj + cgoRe.ReplaceAllString(fn[:len(fn)-2], "_") - gofiles = append(gofiles, f+"cgo1.go") - cfiles = append(cfiles, f+"cgo2.c") - } - defunC := b.obj + "_cgo_defun.c" - output := append([]string{defunC}, cfiles...) - output = append(output, gofiles...) - b.add(Cmd{ - Args: append([]string{"cgo", "--"}, cgofiles...), - Dir: b.path, - Env: append(os.Environ(), "GOARCH="+b.goarch), - Input: cgofiles, - Output: output, - }) - outGo = append(outGo, gofiles...) - exportH := filepath.Join(b.path, "_cgo_export.h") - b.script.addIntermediate(defunC, exportH, b.obj+"_cgo_flags") - b.script.addIntermediate(cfiles...) - - // cc _cgo_defun.c - defunObj := b.obj + "_cgo_defun." + b.arch - b.cc(defunObj, defunC) - outObj = append(outObj, defunObj) - - // gcc - linkobj := make([]string, 0, len(cfiles)) - for _, cfile := range cfiles { - ofile := cfile[:len(cfile)-1] + "o" - b.gccCompile(ofile, cfile) - linkobj = append(linkobj, ofile) - if !strings.HasSuffix(ofile, "_cgo_main.o") { - outObj = append(outObj, ofile) - } else { - b.script.addIntermediate(ofile) - } - } - for _, cfile := range cgocfiles { - ofile := b.obj + cgoRe.ReplaceAllString(cfile[:len(cfile)-1], "_") + "o" - b.gccCompile(ofile, cfile) - linkobj = append(linkobj, ofile) - outObj = append(outObj, ofile) - } - dynObj := b.obj + "_cgo_.o" - b.gccLink(dynObj, linkobj...) - b.script.addIntermediate(dynObj) - - // cgo -dynimport - importC := b.obj + "_cgo_import.c" - b.add(Cmd{ - Args: []string{"cgo", "-dynimport", dynObj}, - Stdout: importC, - Input: []string{dynObj}, - Output: []string{importC}, - }) - b.script.addIntermediate(importC) - - // cc _cgo_import.ARCH - importObj := b.obj + "_cgo_import." + b.arch - b.cc(importObj, importC) - outObj = append(outObj, importObj) - - return + return "", errors.New("unsupported GOARCH " + goarch) } diff --git a/src/pkg/go/build/build_test.go b/src/pkg/go/build/build_test.go index 68a4180c9..381e2b2d9 100644 --- a/src/pkg/go/build/build_test.go +++ b/src/pkg/go/build/build_test.go @@ -5,7 +5,6 @@ package build import ( - "exec" "path/filepath" "reflect" "runtime" @@ -28,7 +27,7 @@ var buildPkgs = []struct { GoFiles: []string{"pkgtest.go"}, SFiles: []string{"sqrt_" + runtime.GOARCH + ".s"}, Package: "pkgtest", - Imports: []string{"os"}, + Imports: []string{"bytes"}, TestImports: []string{"fmt", "pkgtest"}, TestGoFiles: sortstr([]string{"sqrt_test.go", "sqrt_" + runtime.GOARCH + "_test.go"}), XTestGoFiles: []string{"xsqrt_test.go"}, @@ -37,23 +36,31 @@ var buildPkgs = []struct { { "go/build/cmdtest", &DirInfo{ - GoFiles: []string{"main.go"}, - Package: "main", - Imports: []string{"go/build/pkgtest"}, + GoFiles: []string{"main.go"}, + Package: "main", + Imports: []string{"go/build/pkgtest"}, + TestImports: []string{}, }, }, { "go/build/cgotest", &DirInfo{ - CgoFiles: []string{"cgotest.go"}, - CFiles: []string{"cgotest.c"}, - Imports: []string{"C", "unsafe"}, - Package: "cgotest", + CgoFiles: ifCgo([]string{"cgotest.go"}), + CFiles: []string{"cgotest.c"}, + HFiles: []string{"cgotest.h"}, + Imports: []string{"C", "unsafe"}, + TestImports: []string{}, + Package: "cgotest", }, }, } -const cmdtestOutput = "3" +func ifCgo(x []string) []string { + if DefaultContext.CgoEnabled { + return x + } + return nil +} func TestBuild(t *testing.T) { for _, tt := range buildPkgs { @@ -68,35 +75,32 @@ func TestBuild(t *testing.T) { t.Errorf("ScanDir(%#q) = %#v, want %#v\n", tt.dir, info, tt.info) continue } + } +} - s, err := Build(tree, tt.dir, info) - if err != nil { - t.Errorf("Build(%#q): %v", tt.dir, err) - continue +func TestMatch(t *testing.T) { + ctxt := DefaultContext + what := "default" + match := func(tag string) { + if !ctxt.match(tag) { + t.Errorf("%s context should match %s, does not", what, tag) } - - if err := s.Run(); err != nil { - t.Errorf("Run(%#q): %v", tt.dir, err) - continue + } + nomatch := func(tag string) { + if ctxt.match(tag) { + t.Errorf("%s context should NOT match %s, does", what, tag) } + } - if tt.dir == "go/build/cmdtest" { - bin := s.Output[0] - b, err := exec.Command(bin).CombinedOutput() - if err != nil { - t.Errorf("exec %s: %v", bin, err) - continue - } - if string(b) != cmdtestOutput { - t.Errorf("cmdtest output: %s want: %s", b, cmdtestOutput) - } - } + match(runtime.GOOS + "," + runtime.GOARCH) + match(runtime.GOOS + "," + runtime.GOARCH + ",!foo") + nomatch(runtime.GOOS + "," + runtime.GOARCH + ",foo") - // Deferred because cmdtest depends on pkgtest. - defer func(s *Script) { - if err := s.Nuke(); err != nil { - t.Errorf("nuking: %v", err) - } - }(s) - } + what = "modified" + ctxt.BuildTags = []string{"foo"} + match(runtime.GOOS + "," + runtime.GOARCH) + match(runtime.GOOS + "," + runtime.GOARCH + ",foo") + nomatch(runtime.GOOS + "," + runtime.GOARCH + ",!foo") + match(runtime.GOOS + "," + runtime.GOARCH + ",!bar") + nomatch(runtime.GOOS + "," + runtime.GOARCH + ",bar") } diff --git a/src/pkg/go/build/dir.go b/src/pkg/go/build/dir.go index 3ee10ab34..d49846f7d 100644 --- a/src/pkg/go/build/dir.go +++ b/src/pkg/go/build/dir.go @@ -6,9 +6,9 @@ package build import ( "bytes" + "errors" "fmt" "go/ast" - "go/doc" "go/parser" "go/token" "io/ioutil" @@ -25,9 +25,10 @@ import ( // A Context specifies the supporting context for a build. type Context struct { - GOARCH string // target architecture - GOOS string // target operating system - // TODO(rsc,adg): GOPATH + GOARCH string // target architecture + GOOS string // target operating system + CgoEnabled bool // whether cgo can be used + BuildTags []string // additional tags to recognize in +build lines // By default, ScanDir uses the operating system's // file system calls to read directories and files. @@ -37,32 +38,32 @@ type Context struct { // format of the strings dir and file: they can be // slash-separated, backslash-separated, even URLs. - // ReadDir returns a slice of *os.FileInfo, sorted by Name, + // ReadDir returns a slice of os.FileInfo, sorted by Name, // describing the content of the named directory. // The dir argument is the argument to ScanDir. // If ReadDir is nil, ScanDir uses io.ReadDir. - ReadDir func(dir string) (fi []*os.FileInfo, err os.Error) + ReadDir func(dir string) (fi []os.FileInfo, err error) // ReadFile returns the content of the file named file // in the directory named dir. The dir argument is the // argument to ScanDir, and the file argument is the - // Name field from an *os.FileInfo returned by ReadDir. + // Name field from an os.FileInfo returned by ReadDir. // The returned path is the full name of the file, to be // used in error messages. // // If ReadFile is nil, ScanDir uses filepath.Join(dir, file) // as the path and ioutil.ReadFile to read the data. - ReadFile func(dir, file string) (path string, content []byte, err os.Error) + ReadFile func(dir, file string) (path string, content []byte, err error) } -func (ctxt *Context) readDir(dir string) ([]*os.FileInfo, os.Error) { +func (ctxt *Context) readDir(dir string) ([]os.FileInfo, error) { if f := ctxt.ReadDir; f != nil { return f(dir) } return ioutil.ReadDir(dir) } -func (ctxt *Context) readFile(dir, file string) (string, []byte, os.Error) { +func (ctxt *Context) readFile(dir, file string) (string, []byte, error) { if f := ctxt.ReadFile; f != nil { return f(dir, file) } @@ -74,9 +75,36 @@ func (ctxt *Context) readFile(dir, file string) (string, []byte, os.Error) { // The DefaultContext is the default Context for builds. // It uses the GOARCH and GOOS environment variables // if set, or else the compiled code's GOARCH and GOOS. -var DefaultContext = Context{ - GOARCH: envOr("GOARCH", runtime.GOARCH), - GOOS: envOr("GOOS", runtime.GOOS), +var DefaultContext Context = defaultContext() + +var cgoEnabled = map[string]bool{ + "darwin/386": true, + "darwin/amd64": true, + "linux/386": true, + "linux/amd64": true, + "freebsd/386": true, + "freebsd/amd64": true, + "windows/386": true, + "windows/amd64": true, +} + +func defaultContext() Context { + var c Context + + c.GOARCH = envOr("GOARCH", runtime.GOARCH) + c.GOOS = envOr("GOOS", runtime.GOOS) + + s := os.Getenv("CGO_ENABLED") + switch s { + case "1": + c.CgoEnabled = true + case "0": + c.CgoEnabled = false + default: + c.CgoEnabled = cgoEnabled[c.GOOS+"/"+c.GOARCH] + } + + return c } func envOr(name, def string) string { @@ -94,9 +122,10 @@ type DirInfo struct { Imports []string // All packages imported by GoFiles // Source files - GoFiles []string // .go files in dir (excluding CgoFiles) + GoFiles []string // .go files in dir (excluding CgoFiles, TestGoFiles, XTestGoFiles) + HFiles []string // .h files in dir CFiles []string // .c files in dir - SFiles []string // .s files in dir + SFiles []string // .s (and, when using cgo, .S files in dir) CgoFiles []string // .go files that import "C" // Cgo directives @@ -116,43 +145,103 @@ func (d *DirInfo) IsCommand() bool { } // ScanDir calls DefaultContext.ScanDir. -func ScanDir(dir string) (info *DirInfo, err os.Error) { +func ScanDir(dir string) (info *DirInfo, err error) { return DefaultContext.ScanDir(dir) } -// ScanDir returns a structure with details about the Go content found -// in the given directory. The file lists exclude: +// TODO(rsc): Move this comment to a more appropriate place. + +// ScanDir returns a structure with details about the Go package +// found in the given directory. +// +// Most .go, .c, .h, and .s files in the directory are considered part +// of the package. The exceptions are: // -// - files in package main (unless no other package is found) -// - files in package documentation -// - files ending in _test.go +// - .go files in package main (unless no other package is found) +// - .go files in package documentation // - files starting with _ or . +// - files with build constraints not satisfied by the context +// +// Build Constraints +// +// A build constraint is a line comment beginning with the directive +build +// that lists the conditions under which a file should be included in the package. +// Constraints may appear in any kind of source file (not just Go), but +// they must be appear near the top of the file, preceded +// only by blank lines and other line comments. +// +// A build constraint is evaluated as the OR of space-separated options; +// each option evaluates as the AND of its comma-separated terms; +// and each term is an alphanumeric word or, preceded by !, its negation. +// That is, the build constraint: +// +// // +build linux,386 darwin,!cgo +// +// corresponds to the boolean formula: +// +// (linux AND 386) OR (darwin AND (NOT cgo)) +// +// During a particular build, the following words are satisfied: +// +// - the target operating system, as spelled by runtime.GOOS +// - the target architecture, as spelled by runtime.GOARCH +// - "cgo", if ctxt.CgoEnabled is true +// - any additional words listed in ctxt.BuildTags +// +// If a file's name, after stripping the extension and a possible _test suffix, +// matches *_GOOS, *_GOARCH, or *_GOOS_GOARCH for any known operating +// system and architecture values, then the file is considered to have an implicit +// build constraint requiring those terms. +// +// Examples +// +// To keep a file from being considered for the build: +// +// // +build ignore +// +// (any other unsatisfied word will work as well, but ``ignore'' is conventional.) +// +// To build a file only when using cgo, and only on Linux and OS X: +// +// // +build linux,cgo darwin,cgo +// +// Such a file is usually paired with another file implementing the +// default functionality for other systems, which in this case would +// carry the constraint: +// +// // +build !linux !darwin !cgo // -func (ctxt *Context) ScanDir(dir string) (info *DirInfo, err os.Error) { +// Naming a file dns_windows.go will cause it to be included only when +// building the package for Windows; similarly, math_386.s will be included +// only when building the package for 32-bit x86. +// +func (ctxt *Context) ScanDir(dir string) (info *DirInfo, err error) { dirs, err := ctxt.readDir(dir) if err != nil { return nil, err } + var Sfiles []string // files with ".S" (capital S) var di DirInfo imported := make(map[string]bool) testImported := make(map[string]bool) fset := token.NewFileSet() for _, d := range dirs { - if !d.IsRegular() { + if d.IsDir() { continue } - if strings.HasPrefix(d.Name, "_") || - strings.HasPrefix(d.Name, ".") { + name := d.Name() + if strings.HasPrefix(name, "_") || + strings.HasPrefix(name, ".") { continue } - if !ctxt.goodOSArchFile(d.Name) { + if !ctxt.goodOSArchFile(name) { continue } - ext := path.Ext(d.Name) + ext := path.Ext(name) switch ext { - case ".go", ".c", ".s": + case ".go", ".c", ".s", ".h", ".S": // tentatively okay default: // skip @@ -160,7 +249,7 @@ func (ctxt *Context) ScanDir(dir string) (info *DirInfo, err os.Error) { } // Look for +build comments to accept or reject the file. - filename, data, err := ctxt.readFile(dir, d.Name) + filename, data, err := ctxt.readFile(dir, name) if err != nil { return nil, err } @@ -171,10 +260,16 @@ func (ctxt *Context) ScanDir(dir string) (info *DirInfo, err os.Error) { // Going to save the file. For non-Go files, can stop here. switch ext { case ".c": - di.CFiles = append(di.CFiles, d.Name) + di.CFiles = append(di.CFiles, name) + continue + case ".h": + di.HFiles = append(di.HFiles, name) continue case ".s": - di.SFiles = append(di.SFiles, d.Name) + di.SFiles = append(di.SFiles, name) + continue + case ".S": + Sfiles = append(Sfiles, name) continue } @@ -191,7 +286,7 @@ func (ctxt *Context) ScanDir(dir string) (info *DirInfo, err os.Error) { continue } - isTest := strings.HasSuffix(d.Name, "_test.go") + isTest := strings.HasSuffix(name, "_test.go") if isTest && strings.HasSuffix(pkg, "_test") { pkg = pkg[:len(pkg)-len("_test")] } @@ -254,15 +349,17 @@ func (ctxt *Context) ScanDir(dir string) (info *DirInfo, err os.Error) { } } if isCgo { - di.CgoFiles = append(di.CgoFiles, d.Name) + if ctxt.CgoEnabled { + di.CgoFiles = append(di.CgoFiles, name) + } } else if isTest { if pkg == string(pf.Name.Name) { - di.TestGoFiles = append(di.TestGoFiles, d.Name) + di.TestGoFiles = append(di.TestGoFiles, name) } else { - di.XTestGoFiles = append(di.XTestGoFiles, d.Name) + di.XTestGoFiles = append(di.XTestGoFiles, name) } } else { - di.GoFiles = append(di.GoFiles, d.Name) + di.GoFiles = append(di.GoFiles, name) } } if di.Package == "" { @@ -280,6 +377,15 @@ func (ctxt *Context) ScanDir(dir string) (info *DirInfo, err os.Error) { di.TestImports[i] = p i++ } + + // add the .S files only if we are using cgo + // (which means gcc will compile them). + // The standard assemblers expect .s files. + if len(di.CgoFiles) > 0 { + di.SFiles = append(di.SFiles, Sfiles...) + sort.Strings(di.SFiles) + } + // File name lists are sorted because ReadDir sorts. sort.Strings(di.Imports) sort.Strings(di.TestImports) @@ -287,7 +393,6 @@ func (ctxt *Context) ScanDir(dir string) (info *DirInfo, err os.Error) { } var slashslash = []byte("//") -var plusBuild = []byte("+build") // shouldBuild reports whether it is okay to use this file, // The rule is that in the file's leading run of // comments @@ -343,7 +448,7 @@ func (ctxt *Context) shouldBuild(content []byte) bool { if f[0] == "+build" { ok := false for _, tok := range f[1:] { - if ctxt.matchOSArch(tok) { + if ctxt.match(tok) { ok = true break } @@ -364,8 +469,8 @@ func (ctxt *Context) shouldBuild(content []byte) bool { // // TODO(rsc): This duplicates code in cgo. // Once the dust settles, remove this code from cgo. -func (ctxt *Context) saveCgo(filename string, di *DirInfo, cg *ast.CommentGroup) os.Error { - text := doc.CommentText(cg) +func (ctxt *Context) saveCgo(filename string, di *DirInfo, cg *ast.CommentGroup) error { + text := cg.Text() for _, line := range strings.Split(text, "\n") { orig := line @@ -395,7 +500,7 @@ func (ctxt *Context) saveCgo(filename string, di *DirInfo, cg *ast.CommentGroup) if len(cond) > 0 { ok := false for _, c := range cond { - if ctxt.matchOSArch(c) { + if ctxt.match(c) { ok = true break } @@ -429,7 +534,7 @@ func (ctxt *Context) saveCgo(filename string, di *DirInfo, cg *ast.CommentGroup) return nil } -var safeBytes = []byte("+-.,/0123456789=ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz") +var safeBytes = []byte("+-.,/0123456789=ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz:") func safeName(s string) bool { if s == "" { @@ -459,12 +564,12 @@ func safeName(s string) bool { // // []string{"a", "b:c d", "ef", `g"`} // -func splitQuoted(s string) (r []string, err os.Error) { +func splitQuoted(s string) (r []string, err error) { var args []string - arg := make([]int, len(s)) + arg := make([]rune, len(s)) escaped := false quoted := false - quote := 0 + quote := '\x00' i := 0 for _, rune := range s { switch { @@ -473,9 +578,9 @@ func splitQuoted(s string) (r []string, err os.Error) { case rune == '\\': escaped = true continue - case quote != 0: + case quote != '\x00': if rune == quote { - quote = 0 + quote = '\x00' continue } case rune == '"' || rune == '\'': @@ -497,25 +602,62 @@ func splitQuoted(s string) (r []string, err os.Error) { args = append(args, string(arg[:i])) } if quote != 0 { - err = os.NewError("unclosed quote") + err = errors.New("unclosed quote") } else if escaped { - err = os.NewError("unfinished escaping") + err = errors.New("unfinished escaping") } return args, err } -// matchOSArch returns true if the name is one of: +// match returns true if the name is one of: // // $GOOS // $GOARCH -// $GOOS/$GOARCH +// cgo (if cgo is enabled) +// !cgo (if cgo is disabled) +// tag (if tag is listed in ctxt.BuildTags) +// !tag (if tag is not listed in ctxt.BuildTags) +// a slash-separated list of any of these // -func (ctxt *Context) matchOSArch(name string) bool { +func (ctxt *Context) match(name string) bool { + if name == "" { + return false + } + if i := strings.Index(name, ","); i >= 0 { + // comma-separated list + return ctxt.match(name[:i]) && ctxt.match(name[i+1:]) + } + if strings.HasPrefix(name, "!!") { // bad syntax, reject always + return false + } + if strings.HasPrefix(name, "!") { // negation + return !ctxt.match(name[1:]) + } + + // Tags must be letters, digits, underscores. + // Unlike in Go identifiers, all digits is fine (e.g., "386"). + for _, c := range name { + if !unicode.IsLetter(c) && !unicode.IsDigit(c) && c != '_' { + return false + } + } + + // special tags + if ctxt.CgoEnabled && name == "cgo" { + return true + } if name == ctxt.GOOS || name == ctxt.GOARCH { return true } - i := strings.Index(name, "/") - return i >= 0 && name[:i] == ctxt.GOOS && name[i+1:] == ctxt.GOARCH + + // other tags + for _, tag := range ctxt.BuildTags { + if tag == name { + return true + } + } + + return false } // goodOSArchFile returns false if the name contains a $GOOS or $GOARCH diff --git a/src/pkg/go/build/path.go b/src/pkg/go/build/path.go index e39b5f8fa..7e931faff 100644 --- a/src/pkg/go/build/path.go +++ b/src/pkg/go/build/path.go @@ -5,8 +5,8 @@ package build import ( + "errors" "fmt" - "log" "os" "path/filepath" "runtime" @@ -21,9 +21,9 @@ type Tree struct { Goroot bool } -func newTree(p string) (*Tree, os.Error) { +func newTree(p string) (*Tree, error) { if !filepath.IsAbs(p) { - return nil, os.NewError("must be absolute") + return nil, errors.New("must be absolute") } ep, err := filepath.EvalSymlinks(p) if err != nil { @@ -56,7 +56,7 @@ func (t *Tree) PkgDir() string { func (t *Tree) BinDir() string { if t.Goroot { if gobin := os.Getenv("GOBIN"); gobin != "" { - return gobin + return filepath.Clean(gobin) } } return filepath.Join(t.Path, "bin") @@ -69,7 +69,7 @@ func (t *Tree) HasSrc(pkg string) bool { if err != nil { return false } - return fi.IsDirectory() + return fi.IsDir() } // HasPkg returns whether the given package's @@ -79,18 +79,17 @@ func (t *Tree) HasPkg(pkg string) bool { if err != nil { return false } - return fi.IsRegular() - // TODO(adg): check object version is consistent + return !fi.IsDir() } var ( - ErrNotFound = os.NewError("go/build: package could not be found locally") - ErrTreeNotFound = os.NewError("go/build: no valid GOROOT or GOPATH could be found") + ErrNotFound = errors.New("package could not be found locally") + ErrTreeNotFound = errors.New("no valid GOROOT or GOPATH could be found") ) // FindTree takes an import or filesystem path and returns the // tree where the package source should be and the package import path. -func FindTree(path string) (tree *Tree, pkg string, err os.Error) { +func FindTree(path string) (tree *Tree, pkg string, err error) { if isLocalPath(path) { if path, err = filepath.Abs(path); err != nil { return @@ -104,14 +103,14 @@ func FindTree(path string) (tree *Tree, pkg string, err os.Error) { continue } tree = t - pkg = path[len(tpath):] + pkg = filepath.ToSlash(path[len(tpath):]) return } err = fmt.Errorf("path %q not inside a GOPATH", path) return } tree = defaultTree - pkg = path + pkg = filepath.ToSlash(path) for _, t := range Path { if t.HasSrc(pkg) { tree = t @@ -149,9 +148,7 @@ var ( func init() { root := runtime.GOROOT() t, err := newTree(root) - if err != nil { - log.Printf("go/build: invalid GOROOT %q: %v", root, err) - } else { + if err == nil { t.Goroot = true Path = []*Tree{t} } @@ -162,9 +159,9 @@ func init() { } t, err := newTree(p) if err != nil { - log.Printf("go/build: invalid GOPATH %q: %v", p, err) continue } + Path = append(Path, t) gcImportArgs = append(gcImportArgs, "-I", t.PkgDir()) ldImportArgs = append(ldImportArgs, "-L", t.PkgDir()) diff --git a/src/pkg/go/build/pkgtest/pkgtest.go b/src/pkg/go/build/pkgtest/pkgtest.go index 03ebb9893..08eea1e2b 100644 --- a/src/pkg/go/build/pkgtest/pkgtest.go +++ b/src/pkg/go/build/pkgtest/pkgtest.go @@ -4,9 +4,9 @@ package pkgtest -import "os" +import "bytes" -func Foo() os.Error { +func Foo() *bytes.Buffer { return nil } diff --git a/src/pkg/go/build/pkgtest/sqrt_test.go b/src/pkg/go/build/pkgtest/sqrt_test.go index 95fb62552..ee9fd5de6 100644 --- a/src/pkg/go/build/pkgtest/sqrt_test.go +++ b/src/pkg/go/build/pkgtest/sqrt_test.go @@ -1,3 +1,7 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + package pkgtest import "fmt" diff --git a/src/pkg/go/build/pkgtest/xsqrt_test.go b/src/pkg/go/build/pkgtest/xsqrt_test.go index 77e903d96..3898d1dda 100644 --- a/src/pkg/go/build/pkgtest/xsqrt_test.go +++ b/src/pkg/go/build/pkgtest/xsqrt_test.go @@ -1,3 +1,7 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + package pkgtest_test import "pkgtest" diff --git a/src/pkg/go/build/syslist.go b/src/pkg/go/build/syslist.go new file mode 100644 index 000000000..0ee982132 --- /dev/null +++ b/src/pkg/go/build/syslist.go @@ -0,0 +1,5 @@ +// Generated automatically by make. +package build + +const goosList = "darwin freebsd linux netbsd openbsd plan9 windows " +const goarchList = "386 amd64 arm " diff --git a/src/pkg/go/doc/Makefile b/src/pkg/go/doc/Makefile index a5152c793..ac2eeb962 100644 --- a/src/pkg/go/doc/Makefile +++ b/src/pkg/go/doc/Makefile @@ -8,5 +8,15 @@ TARG=go/doc GOFILES=\ comment.go\ doc.go\ + example.go\ + exports.go\ + filter.go\ + reader.go\ include ../../../Make.pkg + +# Script to test heading detection heuristic +CLEANFILES+=headscan +headscan: headscan.go + $(GC) $(GCFLAGS) $(GCIMPORTS) headscan.go + $(LD) -o headscan headscan.$(O) diff --git a/src/pkg/go/doc/comment.go b/src/pkg/go/doc/comment.go index e1989226b..d73b13159 100644 --- a/src/pkg/go/doc/comment.go +++ b/src/pkg/go/doc/comment.go @@ -7,114 +7,14 @@ package doc import ( - "go/ast" "io" "regexp" "strings" - "template" // for HTMLEscape + "text/template" // for HTMLEscape + "unicode" + "unicode/utf8" ) -func isWhitespace(ch byte) bool { return ch == ' ' || ch == '\t' || ch == '\n' || ch == '\r' } - -func stripTrailingWhitespace(s string) string { - i := len(s) - for i > 0 && isWhitespace(s[i-1]) { - i-- - } - return s[0:i] -} - -// CommentText returns the text of comment, -// with the comment markers - //, /*, and */ - removed. -func CommentText(comment *ast.CommentGroup) string { - if comment == nil { - return "" - } - comments := make([]string, len(comment.List)) - for i, c := range comment.List { - comments[i] = string(c.Text) - } - - lines := make([]string, 0, 10) // most comments are less than 10 lines - for _, c := range comments { - // Remove comment markers. - // The parser has given us exactly the comment text. - switch c[1] { - case '/': - //-style comment - c = c[2:] - // Remove leading space after //, if there is one. - // TODO(gri) This appears to be necessary in isolated - // cases (bignum.RatFromString) - why? - if len(c) > 0 && c[0] == ' ' { - c = c[1:] - } - case '*': - /*-style comment */ - c = c[2 : len(c)-2] - } - - // Split on newlines. - cl := strings.Split(c, "\n") - - // Walk lines, stripping trailing white space and adding to list. - for _, l := range cl { - lines = append(lines, stripTrailingWhitespace(l)) - } - } - - // Remove leading blank lines; convert runs of - // interior blank lines to a single blank line. - n := 0 - for _, line := range lines { - if line != "" || n > 0 && lines[n-1] != "" { - lines[n] = line - n++ - } - } - lines = lines[0:n] - - // Add final "" entry to get trailing newline from Join. - if n > 0 && lines[n-1] != "" { - lines = append(lines, "") - } - - return strings.Join(lines, "\n") -} - -// Split bytes into lines. -func split(text []byte) [][]byte { - // count lines - n := 0 - last := 0 - for i, c := range text { - if c == '\n' { - last = i + 1 - n++ - } - } - if last < len(text) { - n++ - } - - // split - out := make([][]byte, n) - last = 0 - n = 0 - for i, c := range text { - if c == '\n' { - out[n] = text[last : i+1] - last = i + 1 - n++ - } - } - if last < len(text) { - out[n] = text[last:] - } - - return out -} - var ( ldquo = []byte("“") rdquo = []byte("”") @@ -122,13 +22,13 @@ var ( // Escape comment text for HTML. If nice is set, // also turn `` into “ and '' into ”. -func commentEscape(w io.Writer, s []byte, nice bool) { +func commentEscape(w io.Writer, text string, nice bool) { last := 0 if nice { - for i := 0; i < len(s)-1; i++ { - ch := s[i] - if ch == s[i+1] && (ch == '`' || ch == '\'') { - template.HTMLEscape(w, s[last:i]) + for i := 0; i < len(text)-1; i++ { + ch := text[i] + if ch == text[i+1] && (ch == '`' || ch == '\'') { + template.HTMLEscape(w, []byte(text[last:i])) last = i + 2 switch ch { case '`': @@ -140,7 +40,7 @@ func commentEscape(w io.Writer, s []byte, nice bool) { } } } - template.HTMLEscape(w, s[last:]) + template.HTMLEscape(w, []byte(text[last:])) } const ( @@ -168,6 +68,9 @@ var ( html_endp = []byte("</p>\n") html_pre = []byte("<pre>") html_endpre = []byte("</pre>\n") + html_h = []byte(`<h3 id="`) + html_hq = []byte(`">`) + html_endh = []byte("</h3>\n") ) // Emphasize and escape a line of text for HTML. URLs are converted into links; @@ -178,9 +81,9 @@ var ( // and the word is converted into a link. If nice is set, the remaining text's // appearance is improved where it makes sense (e.g., `` is turned into “ // and '' into ”). -func emphasize(w io.Writer, line []byte, words map[string]string, nice bool) { +func emphasize(w io.Writer, line string, words map[string]string, nice bool) { for { - m := matchRx.FindSubmatchIndex(line) + m := matchRx.FindStringSubmatchIndex(line) if m == nil { break } @@ -228,7 +131,7 @@ func emphasize(w io.Writer, line []byte, words map[string]string, nice bool) { commentEscape(w, line, nice) } -func indentLen(s []byte) int { +func indentLen(s string) int { i := 0 for i < len(s) && (s[i] == ' ' || s[i] == '\t') { i++ @@ -236,9 +139,11 @@ func indentLen(s []byte) int { return i } -func isBlank(s []byte) bool { return len(s) == 0 || (len(s) == 1 && s[0] == '\n') } +func isBlank(s string) bool { + return len(s) == 0 || (len(s) == 1 && s[0] == '\n') +} -func commonPrefix(a, b []byte) []byte { +func commonPrefix(a, b string) string { i := 0 for i < len(a) && i < len(b) && a[i] == b[i] { i++ @@ -246,7 +151,7 @@ func commonPrefix(a, b []byte) []byte { return a[0:i] } -func unindent(block [][]byte) { +func unindent(block []string) { if len(block) == 0 { return } @@ -268,7 +173,66 @@ func unindent(block [][]byte) { } } -// Convert comment text to formatted HTML. +// heading returns the trimmed line if it passes as a section heading; +// otherwise it returns the empty string. +func heading(line string) string { + line = strings.TrimSpace(line) + if len(line) == 0 { + return "" + } + + // a heading must start with an uppercase letter + r, _ := utf8.DecodeRuneInString(line) + if !unicode.IsLetter(r) || !unicode.IsUpper(r) { + return "" + } + + // it must end in a letter or digit: + r, _ = utf8.DecodeLastRuneInString(line) + if !unicode.IsLetter(r) && !unicode.IsDigit(r) { + return "" + } + + // exclude lines with illegal characters + if strings.IndexAny(line, ",.;:!?+*/=()[]{}_^°&§~%#@<\">\\") >= 0 { + return "" + } + + // allow "'" for possessive "'s" only + for b := line; ; { + i := strings.IndexRune(b, '\'') + if i < 0 { + break + } + if i+1 >= len(b) || b[i+1] != 's' || (i+2 < len(b) && b[i+2] != ' ') { + return "" // not followed by "s " + } + b = b[i+2:] + } + + return line +} + +type op int + +const ( + opPara op = iota + opHead + opPre +) + +type block struct { + op op + lines []string +} + +var nonAlphaNumRx = regexp.MustCompile(`[^a-zA-Z0-9]`) + +func anchorID(line string) string { + return nonAlphaNumRx.ReplaceAllString(line, "_") +} + +// ToHTML converts comment text to formatted HTML. // The comment was prepared by DocReader, // so it is known not to have leading, trailing blank lines // nor to have trailing spaces at the end of lines. @@ -276,6 +240,7 @@ func unindent(block [][]byte) { // // Turn each run of multiple \n into </p><p>. // Turn each run of indented lines into a <pre> block without indent. +// Enclose headings with header tags. // // URLs in the comment text are converted into links; if the URL also appears // in the words map, the link is taken from the map (if the corresponding map @@ -284,23 +249,57 @@ func unindent(block [][]byte) { // Go identifiers that appear in the words map are italicized; if the corresponding // map value is not the empty string, it is considered a URL and the word is converted // into a link. -func ToHTML(w io.Writer, s []byte, words map[string]string) { - inpara := false - - close := func() { - if inpara { +func ToHTML(w io.Writer, text string, words map[string]string) { + for _, b := range blocks(text) { + switch b.op { + case opPara: + w.Write(html_p) + for _, line := range b.lines { + emphasize(w, line, words, true) + } w.Write(html_endp) - inpara = false + case opHead: + w.Write(html_h) + id := "" + for _, line := range b.lines { + if id == "" { + id = anchorID(line) + w.Write([]byte(id)) + w.Write(html_hq) + } + commentEscape(w, line, true) + } + if id == "" { + w.Write(html_hq) + } + w.Write(html_endh) + case opPre: + w.Write(html_pre) + for _, line := range b.lines { + emphasize(w, line, nil, false) + } + w.Write(html_endpre) } } - open := func() { - if !inpara { - w.Write(html_p) - inpara = true +} + +func blocks(text string) []block { + var ( + out []block + para []string + + lastWasBlank = false + lastWasHeading = false + ) + + close := func() { + if para != nil { + out = append(out, block{opPara, para}) + para = nil } } - lines := split(s) + lines := strings.SplitAfter(text, "\n") unindent(lines) for i := 0; i < len(lines); { line := lines[i] @@ -308,6 +307,7 @@ func ToHTML(w io.Writer, s []byte, words map[string]string) { // close paragraph close() i++ + lastWasBlank = true continue } if indentLen(line) > 0 { @@ -323,23 +323,119 @@ func ToHTML(w io.Writer, s []byte, words map[string]string) { for j > i && isBlank(lines[j-1]) { j-- } - block := lines[i:j] + pre := lines[i:j] i = j - unindent(block) + unindent(pre) // put those lines in a pre block - w.Write(html_pre) - for _, line := range block { - emphasize(w, line, nil, false) // no nice text formatting - } - w.Write(html_endpre) + out = append(out, block{opPre, pre}) + lastWasHeading = false continue } + + if lastWasBlank && !lastWasHeading && i+2 < len(lines) && + isBlank(lines[i+1]) && !isBlank(lines[i+2]) && indentLen(lines[i+2]) == 0 { + // current line is non-blank, sourounded by blank lines + // and the next non-blank line is not indented: this + // might be a heading. + if head := heading(line); head != "" { + close() + out = append(out, block{opHead, []string{head}}) + i += 2 + lastWasHeading = true + continue + } + } + // open paragraph - open() - emphasize(w, lines[i], words, true) // nice text formatting + lastWasBlank = false + lastWasHeading = false + para = append(para, lines[i]) i++ } close() + + return out +} + +// ToText prepares comment text for presentation in textual output. +// It wraps paragraphs of text to width or fewer Unicode code points +// and then prefixes each line with the indent. In preformatted sections +// (such as program text), it prefixes each non-blank line with preIndent. +func ToText(w io.Writer, text string, indent, preIndent string, width int) { + l := lineWrapper{ + out: w, + width: width, + indent: indent, + } + for _, b := range blocks(text) { + switch b.op { + case opPara: + // l.write will add leading newline if required + for _, line := range b.lines { + l.write(line) + } + l.flush() + case opHead: + w.Write(nl) + for _, line := range b.lines { + l.write(line + "\n") + } + l.flush() + case opPre: + w.Write(nl) + for _, line := range b.lines { + if !isBlank(line) { + w.Write([]byte(preIndent)) + w.Write([]byte(line)) + } + } + } + } +} + +type lineWrapper struct { + out io.Writer + printed bool + width int + indent string + n int + pendSpace int +} + +var nl = []byte("\n") +var space = []byte(" ") + +func (l *lineWrapper) write(text string) { + if l.n == 0 && l.printed { + l.out.Write(nl) // blank line before new paragraph + } + l.printed = true + + for _, f := range strings.Fields(text) { + w := utf8.RuneCountInString(f) + // wrap if line is too long + if l.n > 0 && l.n+l.pendSpace+w > l.width { + l.out.Write(nl) + l.n = 0 + l.pendSpace = 0 + } + if l.n == 0 { + l.out.Write([]byte(l.indent)) + } + l.out.Write(space[:l.pendSpace]) + l.out.Write([]byte(f)) + l.n += l.pendSpace + w + l.pendSpace = 1 + } +} + +func (l *lineWrapper) flush() { + if l.n == 0 { + return + } + l.out.Write(nl) + l.pendSpace = 0 + l.n = 0 } diff --git a/src/pkg/go/doc/comment_test.go b/src/pkg/go/doc/comment_test.go new file mode 100644 index 000000000..e8d7f2e4b --- /dev/null +++ b/src/pkg/go/doc/comment_test.go @@ -0,0 +1,83 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package doc + +import ( + "reflect" + "testing" +) + +var headingTests = []struct { + line string + ok bool +}{ + {"Section", true}, + {"A typical usage", true}, + {"ΔΛΞ is Greek", true}, + {"Foo 42", true}, + {"", false}, + {"section", false}, + {"A typical usage:", false}, + {"This code:", false}, + {"δ is Greek", false}, + {"Foo §", false}, + {"Fermat's Last Sentence", true}, + {"Fermat's", true}, + {"'sX", false}, + {"Ted 'Too' Bar", false}, + {"Use n+m", false}, + {"Scanning:", false}, + {"N:M", false}, +} + +func TestIsHeading(t *testing.T) { + for _, tt := range headingTests { + if h := heading(tt.line); (len(h) > 0) != tt.ok { + t.Errorf("isHeading(%q) = %v, want %v", tt.line, h, tt.ok) + } + } +} + +var blocksTests = []struct { + in string + out []block +}{ + { + in: `Para 1. +Para 1 line 2. + +Para 2. + +Section + +Para 3. + + pre + pre1 + +Para 4. + pre + pre2 +`, + out: []block{ + {opPara, []string{"Para 1.\n", "Para 1 line 2.\n"}}, + {opPara, []string{"Para 2.\n"}}, + {opHead, []string{"Section"}}, + {opPara, []string{"Para 3.\n"}}, + {opPre, []string{"pre\n", "pre1\n"}}, + {opPara, []string{"Para 4.\n"}}, + {opPre, []string{"pre\n", "pre2\n"}}, + }, + }, +} + +func TestBlocks(t *testing.T) { + for i, tt := range blocksTests { + b := blocks(tt.in) + if !reflect.DeepEqual(b, tt.out) { + t.Errorf("#%d: mismatch\nhave: %v\nwant: %v", i, b, tt.out) + } + } +} diff --git a/src/pkg/go/doc/doc.go b/src/pkg/go/doc/doc.go index c7fed9784..d4aae8ff0 100644 --- a/src/pkg/go/doc/doc.go +++ b/src/pkg/go/doc/doc.go @@ -8,634 +8,86 @@ package doc import ( "go/ast" "go/token" - "regexp" - "sort" ) -// ---------------------------------------------------------------------------- +// Package is the documentation for an entire package. +type Package struct { + Doc string + Name string + ImportPath string + Imports []string + Filenames []string + Bugs []string -type typeDoc struct { - // len(decl.Specs) == 1, and the element type is *ast.TypeSpec - // if the type declaration hasn't been seen yet, decl is nil - decl *ast.GenDecl - // values, factory functions, and methods associated with the type - values []*ast.GenDecl // consts and vars - factories map[string]*ast.FuncDecl - methods map[string]*ast.FuncDecl + // declarations + Consts []*Value + Types []*Type + Vars []*Value + Funcs []*Func } -// docReader accumulates documentation for a single package. -// It modifies the AST: Comments (declaration documentation) -// that have been collected by the DocReader are set to nil -// in the respective AST nodes so that they are not printed -// twice (once when printing the documentation and once when -// printing the corresponding AST node). -// -type docReader struct { - doc *ast.CommentGroup // package documentation, if any - pkgName string - values []*ast.GenDecl // consts and vars - types map[string]*typeDoc - funcs map[string]*ast.FuncDecl - bugs []*ast.CommentGroup -} - -func (doc *docReader) init(pkgName string) { - doc.pkgName = pkgName - doc.types = make(map[string]*typeDoc) - doc.funcs = make(map[string]*ast.FuncDecl) -} - -func (doc *docReader) addDoc(comments *ast.CommentGroup) { - if doc.doc == nil { - // common case: just one package comment - doc.doc = comments - return - } - - // More than one package comment: Usually there will be only - // one file with a package comment, but it's better to collect - // all comments than drop them on the floor. - // (This code isn't particularly clever - no amortized doubling is - // used - but this situation occurs rarely and is not time-critical.) - n1 := len(doc.doc.List) - n2 := len(comments.List) - list := make([]*ast.Comment, n1+1+n2) // + 1 for separator line - copy(list, doc.doc.List) - list[n1] = &ast.Comment{token.NoPos, "//"} // separator line - copy(list[n1+1:], comments.List) - doc.doc = &ast.CommentGroup{list} -} - -func (doc *docReader) addType(decl *ast.GenDecl) { - spec := decl.Specs[0].(*ast.TypeSpec) - typ := doc.lookupTypeDoc(spec.Name.Name) - // typ should always be != nil since declared types - // are always named - be conservative and check - if typ != nil { - // a type should be added at most once, so typ.decl - // should be nil - if it isn't, simply overwrite it - typ.decl = decl - } -} - -func (doc *docReader) lookupTypeDoc(name string) *typeDoc { - if name == "" { - return nil // no type docs for anonymous types - } - if tdoc, found := doc.types[name]; found { - return tdoc - } - // type wasn't found - add one without declaration - tdoc := &typeDoc{nil, nil, make(map[string]*ast.FuncDecl), make(map[string]*ast.FuncDecl)} - doc.types[name] = tdoc - return tdoc -} - -func baseTypeName(typ ast.Expr) string { - switch t := typ.(type) { - case *ast.Ident: - // if the type is not exported, the effect to - // a client is as if there were no type name - if t.IsExported() { - return t.Name - } - case *ast.StarExpr: - return baseTypeName(t.X) - } - return "" -} - -func (doc *docReader) addValue(decl *ast.GenDecl) { - // determine if decl should be associated with a type - // Heuristic: For each typed entry, determine the type name, if any. - // If there is exactly one type name that is sufficiently - // frequent, associate the decl with the respective type. - domName := "" - domFreq := 0 - prev := "" - for _, s := range decl.Specs { - if v, ok := s.(*ast.ValueSpec); ok { - name := "" - switch { - case v.Type != nil: - // a type is present; determine its name - name = baseTypeName(v.Type) - case decl.Tok == token.CONST: - // no type is present but we have a constant declaration; - // use the previous type name (w/o more type information - // we cannot handle the case of unnamed variables with - // initializer expressions except for some trivial cases) - name = prev - } - if name != "" { - // entry has a named type - if domName != "" && domName != name { - // more than one type name - do not associate - // with any type - domName = "" - break - } - domName = name - domFreq++ - } - prev = name - } - } - - // determine values list - const threshold = 0.75 - values := &doc.values - if domName != "" && domFreq >= int(float64(len(decl.Specs))*threshold) { - // typed entries are sufficiently frequent - typ := doc.lookupTypeDoc(domName) - if typ != nil { - values = &typ.values // associate with that type - } - } - - *values = append(*values, decl) -} - -// Helper function to set the table entry for function f. Makes sure that -// at least one f with associated documentation is stored in table, if there -// are multiple f's with the same name. -func setFunc(table map[string]*ast.FuncDecl, f *ast.FuncDecl) { - name := f.Name.Name - if g, exists := table[name]; exists && g.Doc != nil { - // a function with the same name has already been registered; - // since it has documentation, assume f is simply another - // implementation and ignore it - // TODO(gri) consider collecting all functions, or at least - // all comments - return - } - // function doesn't exist or has no documentation; use f - table[name] = f -} - -func (doc *docReader) addFunc(fun *ast.FuncDecl) { - name := fun.Name.Name - - // determine if it should be associated with a type - if fun.Recv != nil { - // method - typ := doc.lookupTypeDoc(baseTypeName(fun.Recv.List[0].Type)) - if typ != nil { - // exported receiver type - setFunc(typ.methods, fun) - } - // otherwise don't show the method - // TODO(gri): There may be exported methods of non-exported types - // that can be called because of exported values (consts, vars, or - // function results) of that type. Could determine if that is the - // case and then show those methods in an appropriate section. - return - } - - // perhaps a factory function - // determine result type, if any - if fun.Type.Results.NumFields() >= 1 { - res := fun.Type.Results.List[0] - if len(res.Names) <= 1 { - // exactly one (named or anonymous) result associated - // with the first type in result signature (there may - // be more than one result) - tname := baseTypeName(res.Type) - typ := doc.lookupTypeDoc(tname) - if typ != nil { - // named and exported result type - - // Work-around for failure of heuristic: In package os - // too many functions are considered factory functions - // for the Error type. Eliminate manually for now as - // this appears to be the only important case in the - // current library where the heuristic fails. - if doc.pkgName == "os" && tname == "Error" && - name != "NewError" && name != "NewSyscallError" { - // not a factory function for os.Error - setFunc(doc.funcs, fun) // treat as ordinary function - return - } - - setFunc(typ.factories, fun) - return - } - } - } - - // ordinary function - setFunc(doc.funcs, fun) -} - -func (doc *docReader) addDecl(decl ast.Decl) { - switch d := decl.(type) { - case *ast.GenDecl: - if len(d.Specs) > 0 { - switch d.Tok { - case token.CONST, token.VAR: - // constants and variables are always handled as a group - doc.addValue(d) - case token.TYPE: - // types are handled individually - for _, spec := range d.Specs { - // make a (fake) GenDecl node for this TypeSpec - // (we need to do this here - as opposed to just - // for printing - so we don't lose the GenDecl - // documentation) - // - // TODO(gri): Consider just collecting the TypeSpec - // node (and copy in the GenDecl.doc if there is no - // doc in the TypeSpec - this is currently done in - // makeTypeDocs below). Simpler data structures, but - // would lose GenDecl documentation if the TypeSpec - // has documentation as well. - doc.addType(&ast.GenDecl{d.Doc, d.Pos(), token.TYPE, token.NoPos, []ast.Spec{spec}, token.NoPos}) - // A new GenDecl node is created, no need to nil out d.Doc. - } - } - } - case *ast.FuncDecl: - doc.addFunc(d) - } -} - -func copyCommentList(list []*ast.Comment) []*ast.Comment { - return append([]*ast.Comment(nil), list...) -} - -var ( - bug_markers = regexp.MustCompile("^/[/*][ \t]*BUG\\(.*\\):[ \t]*") // BUG(uid): - bug_content = regexp.MustCompile("[^ \n\r\t]+") // at least one non-whitespace char -) - -// addFile adds the AST for a source file to the docReader. -// Adding the same AST multiple times is a no-op. -// -func (doc *docReader) addFile(src *ast.File) { - // add package documentation - if src.Doc != nil { - doc.addDoc(src.Doc) - src.Doc = nil // doc consumed - remove from ast.File node - } - - // add all declarations - for _, decl := range src.Decls { - doc.addDecl(decl) - } - - // collect BUG(...) comments - for _, c := range src.Comments { - text := c.List[0].Text - if m := bug_markers.FindStringIndex(text); m != nil { - // found a BUG comment; maybe empty - if btxt := text[m[1]:]; bug_content.MatchString(btxt) { - // non-empty BUG comment; collect comment without BUG prefix - list := copyCommentList(c.List) - list[0].Text = text[m[1]:] - doc.bugs = append(doc.bugs, &ast.CommentGroup{list}) - } - } - } - src.Comments = nil // consumed unassociated comments - remove from ast.File node -} - -func NewFileDoc(file *ast.File) *PackageDoc { - var r docReader - r.init(file.Name.Name) - r.addFile(file) - return r.newDoc("", nil) -} - -func NewPackageDoc(pkg *ast.Package, importpath string) *PackageDoc { - var r docReader - r.init(pkg.Name) - filenames := make([]string, len(pkg.Files)) - i := 0 - for filename, f := range pkg.Files { - r.addFile(f) - filenames[i] = filename - i++ - } - return r.newDoc(importpath, filenames) -} - -// ---------------------------------------------------------------------------- -// Conversion to external representation - -// ValueDoc is the documentation for a group of declared -// values, either vars or consts. -// -type ValueDoc struct { +// Value is the documentation for a (possibly grouped) var or const declaration. +type Value struct { Doc string + Names []string // var or const names in declaration order Decl *ast.GenDecl - order int -} - -type sortValueDoc []*ValueDoc - -func (p sortValueDoc) Len() int { return len(p) } -func (p sortValueDoc) Swap(i, j int) { p[i], p[j] = p[j], p[i] } -func declName(d *ast.GenDecl) string { - if len(d.Specs) != 1 { - return "" - } - - switch v := d.Specs[0].(type) { - case *ast.ValueSpec: - return v.Names[0].Name - case *ast.TypeSpec: - return v.Name.Name - } - - return "" + order int } -func (p sortValueDoc) Less(i, j int) bool { - // sort by name - // pull blocks (name = "") up to top - // in original order - if ni, nj := declName(p[i].Decl), declName(p[j].Decl); ni != nj { - return ni < nj - } - return p[i].order < p[j].order -} +// Type is the documentation for a type declaration. +type Type struct { + Doc string + Name string + Decl *ast.GenDecl -func makeValueDocs(list []*ast.GenDecl, tok token.Token) []*ValueDoc { - d := make([]*ValueDoc, len(list)) // big enough in any case - n := 0 - for i, decl := range list { - if decl.Tok == tok { - d[n] = &ValueDoc{CommentText(decl.Doc), decl, i} - n++ - decl.Doc = nil // doc consumed - removed from AST - } - } - d = d[0:n] - sort.Sort(sortValueDoc(d)) - return d + // associated declarations + Consts []*Value // sorted list of constants of (mostly) this type + Vars []*Value // sorted list of variables of (mostly) this type + Funcs []*Func // sorted list of functions returning this type + Methods []*Func // sorted list of methods (including embedded ones) of this type } -// FuncDoc is the documentation for a func declaration, -// either a top-level function or a method function. -// -type FuncDoc struct { +// Func is the documentation for a func declaration. +type Func struct { Doc string - Recv ast.Expr // TODO(rsc): Would like string here Name string Decl *ast.FuncDecl -} - -type sortFuncDoc []*FuncDoc - -func (p sortFuncDoc) Len() int { return len(p) } -func (p sortFuncDoc) Swap(i, j int) { p[i], p[j] = p[j], p[i] } -func (p sortFuncDoc) Less(i, j int) bool { return p[i].Name < p[j].Name } -func makeFuncDocs(m map[string]*ast.FuncDecl) []*FuncDoc { - d := make([]*FuncDoc, len(m)) - i := 0 - for _, f := range m { - doc := new(FuncDoc) - doc.Doc = CommentText(f.Doc) - f.Doc = nil // doc consumed - remove from ast.FuncDecl node - if f.Recv != nil { - doc.Recv = f.Recv.List[0].Type - } - doc.Name = f.Name.Name - doc.Decl = f - d[i] = doc - i++ - } - sort.Sort(sortFuncDoc(d)) - return d -} - -// TypeDoc is the documentation for a declared type. -// Consts and Vars are sorted lists of constants and variables of (mostly) that type. -// Factories is a sorted list of factory functions that return that type. -// Methods is a sorted list of method functions on that type. -type TypeDoc struct { - Doc string - Type *ast.TypeSpec - Consts []*ValueDoc - Vars []*ValueDoc - Factories []*FuncDoc - Methods []*FuncDoc - Decl *ast.GenDecl - order int + // methods + // (for functions, these fields have the respective zero value) + Recv string // actual receiver "T" or "*T" + Orig string // original receiver "T" or "*T" + Level int // embedding level; 0 means not embedded } -type sortTypeDoc []*TypeDoc +// Mode values control the operation of New. +type Mode int -func (p sortTypeDoc) Len() int { return len(p) } -func (p sortTypeDoc) Swap(i, j int) { p[i], p[j] = p[j], p[i] } -func (p sortTypeDoc) Less(i, j int) bool { - // sort by name - // pull blocks (name = "") up to top - // in original order - if ni, nj := p[i].Type.Name.Name, p[j].Type.Name.Name; ni != nj { - return ni < nj - } - return p[i].order < p[j].order -} - -// NOTE(rsc): This would appear not to be correct for type ( ) -// blocks, but the doc extractor above has split them into -// individual declarations. -func (doc *docReader) makeTypeDocs(m map[string]*typeDoc) []*TypeDoc { - d := make([]*TypeDoc, len(m)) - i := 0 - for _, old := range m { - // all typeDocs should have a declaration associated with - // them after processing an entire package - be conservative - // and check - if decl := old.decl; decl != nil { - typespec := decl.Specs[0].(*ast.TypeSpec) - t := new(TypeDoc) - doc := typespec.Doc - typespec.Doc = nil // doc consumed - remove from ast.TypeSpec node - if doc == nil { - // no doc associated with the spec, use the declaration doc, if any - doc = decl.Doc - } - decl.Doc = nil // doc consumed - remove from ast.Decl node - t.Doc = CommentText(doc) - t.Type = typespec - t.Consts = makeValueDocs(old.values, token.CONST) - t.Vars = makeValueDocs(old.values, token.VAR) - t.Factories = makeFuncDocs(old.factories) - t.Methods = makeFuncDocs(old.methods) - t.Decl = old.decl - t.order = i - d[i] = t - i++ - } else { - // no corresponding type declaration found - move any associated - // values, factory functions, and methods back to the top-level - // so that they are not lost (this should only happen if a package - // file containing the explicit type declaration is missing or if - // an unqualified type name was used after a "." import) - // 1) move values - doc.values = append(doc.values, old.values...) - // 2) move factory functions - for name, f := range old.factories { - doc.funcs[name] = f - } - // 3) move methods - for name, f := range old.methods { - // don't overwrite functions with the same name - if _, found := doc.funcs[name]; !found { - doc.funcs[name] = f - } - } - } - } - d = d[0:i] // some types may have been ignored - sort.Sort(sortTypeDoc(d)) - return d -} - -func makeBugDocs(list []*ast.CommentGroup) []string { - d := make([]string, len(list)) - for i, g := range list { - d[i] = CommentText(g) - } - return d -} - -// PackageDoc is the documentation for an entire package. -// -type PackageDoc struct { - PackageName string - ImportPath string - Filenames []string - Doc string - Consts []*ValueDoc - Types []*TypeDoc - Vars []*ValueDoc - Funcs []*FuncDoc - Bugs []string -} +const ( + // extract documentation for all package-level declarations, + // not just exported ones + AllDecls Mode = 1 << iota +) -// newDoc returns the accumulated documentation for the package. +// New computes the package documentation for the given package AST. +// New takes ownership of the AST pkg and may edit or overwrite it. // -func (doc *docReader) newDoc(importpath string, filenames []string) *PackageDoc { - p := new(PackageDoc) - p.PackageName = doc.pkgName - p.ImportPath = importpath - sort.Strings(filenames) - p.Filenames = filenames - p.Doc = CommentText(doc.doc) - // makeTypeDocs may extend the list of doc.values and - // doc.funcs and thus must be called before any other - // function consuming those lists - p.Types = doc.makeTypeDocs(doc.types) - p.Consts = makeValueDocs(doc.values, token.CONST) - p.Vars = makeValueDocs(doc.values, token.VAR) - p.Funcs = makeFuncDocs(doc.funcs) - p.Bugs = makeBugDocs(doc.bugs) - return p -} - -// ---------------------------------------------------------------------------- -// Filtering by name - -type Filter func(string) bool - -func matchFields(fields *ast.FieldList, f Filter) bool { - if fields != nil { - for _, field := range fields.List { - for _, name := range field.Names { - if f(name.Name) { - return true - } - } - } +func New(pkg *ast.Package, importPath string, mode Mode) *Package { + var r reader + r.readPackage(pkg, mode) + r.computeMethodSets() + r.cleanupTypes() + return &Package{ + Doc: r.doc, + Name: pkg.Name, + ImportPath: importPath, + Imports: sortedKeys(r.imports), + Filenames: r.filenames, + Bugs: r.bugs, + Consts: sortedValues(r.values, token.CONST), + Types: sortedTypes(r.types), + Vars: sortedValues(r.values, token.VAR), + Funcs: sortedFuncs(r.funcs), } - return false -} - -func matchDecl(d *ast.GenDecl, f Filter) bool { - for _, d := range d.Specs { - switch v := d.(type) { - case *ast.ValueSpec: - for _, name := range v.Names { - if f(name.Name) { - return true - } - } - case *ast.TypeSpec: - if f(v.Name.Name) { - return true - } - switch t := v.Type.(type) { - case *ast.StructType: - if matchFields(t.Fields, f) { - return true - } - case *ast.InterfaceType: - if matchFields(t.Methods, f) { - return true - } - } - } - } - return false -} - -func filterValueDocs(a []*ValueDoc, f Filter) []*ValueDoc { - w := 0 - for _, vd := range a { - if matchDecl(vd.Decl, f) { - a[w] = vd - w++ - } - } - return a[0:w] -} - -func filterFuncDocs(a []*FuncDoc, f Filter) []*FuncDoc { - w := 0 - for _, fd := range a { - if f(fd.Name) { - a[w] = fd - w++ - } - } - return a[0:w] -} - -func filterTypeDocs(a []*TypeDoc, f Filter) []*TypeDoc { - w := 0 - for _, td := range a { - n := 0 // number of matches - if matchDecl(td.Decl, f) { - n = 1 - } else { - // type name doesn't match, but we may have matching consts, vars, factories or methods - td.Consts = filterValueDocs(td.Consts, f) - td.Vars = filterValueDocs(td.Vars, f) - td.Factories = filterFuncDocs(td.Factories, f) - td.Methods = filterFuncDocs(td.Methods, f) - n += len(td.Consts) + len(td.Vars) + len(td.Factories) + len(td.Methods) - } - if n > 0 { - a[w] = td - w++ - } - } - return a[0:w] -} - -// Filter eliminates documentation for names that don't pass through the filter f. -// TODO: Recognize "Type.Method" as a name. -// -func (p *PackageDoc) Filter(f Filter) { - p.Consts = filterValueDocs(p.Consts, f) - p.Vars = filterValueDocs(p.Vars, f) - p.Types = filterTypeDocs(p.Types, f) - p.Funcs = filterFuncDocs(p.Funcs, f) - p.Doc = "" // don't show top-level package doc } diff --git a/src/pkg/go/doc/doc_test.go b/src/pkg/go/doc/doc_test.go new file mode 100644 index 000000000..d9ffe47b6 --- /dev/null +++ b/src/pkg/go/doc/doc_test.go @@ -0,0 +1,121 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package doc + +import ( + "bytes" + "flag" + "fmt" + "go/parser" + "go/printer" + "go/token" + "io/ioutil" + "os" + "path/filepath" + "strings" + "testing" + "text/template" +) + +var update = flag.Bool("update", false, "update golden (.out) files") + +const dataDir = "testdata" + +var templateTxt = readTemplate("template.txt") + +func readTemplate(filename string) *template.Template { + t := template.New(filename) + t.Funcs(template.FuncMap{ + "node": nodeFmt, + "synopsis": synopsisFmt, + }) + return template.Must(t.ParseFiles(filepath.Join(dataDir, filename))) +} + +func nodeFmt(node interface{}, fset *token.FileSet) string { + var buf bytes.Buffer + printer.Fprint(&buf, fset, node) + return strings.Replace(strings.TrimSpace(buf.String()), "\n", "\n\t", -1) +} + +func synopsisFmt(s string) string { + const n = 64 + if len(s) > n { + // cut off excess text and go back to a word boundary + s = s[0:n] + if i := strings.LastIndexAny(s, "\t\n "); i >= 0 { + s = s[0:i] + } + s = strings.TrimSpace(s) + " ..." + } + return "// " + strings.Replace(s, "\n", " ", -1) +} + +func isGoFile(fi os.FileInfo) bool { + name := fi.Name() + return !fi.IsDir() && + len(name) > 0 && name[0] != '.' && // ignore .files + filepath.Ext(name) == ".go" +} + +type bundle struct { + *Package + FSet *token.FileSet +} + +func test(t *testing.T, mode Mode) { + // get all packages + fset := token.NewFileSet() + pkgs, err := parser.ParseDir(fset, dataDir, isGoFile, parser.ParseComments) + if err != nil { + t.Fatal(err) + } + + // test all packages + for _, pkg := range pkgs { + importpath := dataDir + "/" + pkg.Name + doc := New(pkg, importpath, mode) + + // golden files always use / in filenames - canonicalize them + for i, filename := range doc.Filenames { + doc.Filenames[i] = filepath.ToSlash(filename) + } + + // print documentation + var buf bytes.Buffer + if err := templateTxt.Execute(&buf, bundle{doc, fset}); err != nil { + t.Error(err) + continue + } + got := buf.Bytes() + + // update golden file if necessary + golden := filepath.Join(dataDir, fmt.Sprintf("%s.%d.golden", pkg.Name, mode)) + if *update { + err := ioutil.WriteFile(golden, got, 0644) + if err != nil { + t.Error(err) + } + continue + } + + // get golden file + want, err := ioutil.ReadFile(golden) + if err != nil { + t.Error(err) + continue + } + + // compare + if bytes.Compare(got, want) != 0 { + t.Errorf("package %s\n\tgot:\n%s\n\twant:\n%s", pkg.Name, got, want) + } + } +} + +func Test(t *testing.T) { + test(t, 0) + test(t, AllDecls) +} diff --git a/src/pkg/go/doc/example.go b/src/pkg/go/doc/example.go new file mode 100644 index 000000000..7c59bf9bd --- /dev/null +++ b/src/pkg/go/doc/example.go @@ -0,0 +1,57 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Extract example functions from package ASTs. + +package doc + +import ( + "go/ast" + "go/printer" + "strings" + "unicode" + "unicode/utf8" +) + +type Example struct { + Name string // name of the item being demonstrated + Body *printer.CommentedNode // code + Output string // expected output +} + +func Examples(pkg *ast.Package) []*Example { + var examples []*Example + for _, src := range pkg.Files { + for _, decl := range src.Decls { + f, ok := decl.(*ast.FuncDecl) + if !ok { + continue + } + name := f.Name.Name + if !isTest(name, "Example") { + continue + } + examples = append(examples, &Example{ + Name: name[len("Example"):], + Body: &printer.CommentedNode{f.Body, src.Comments}, + Output: f.Doc.Text(), + }) + } + } + return examples +} + +// isTest tells whether name looks like a test, example, or benchmark. +// It is a Test (say) if there is a character after Test that is not a +// lower-case letter. (We don't want Testiness.) +func isTest(name, prefix string) bool { + if !strings.HasPrefix(name, prefix) { + return false + } + if len(name) == len(prefix) { // "Test" is ok + return true + } + rune, _ := utf8.DecodeRuneInString(name[len(prefix):]) + return !unicode.IsLower(rune) +} diff --git a/src/pkg/go/doc/exports.go b/src/pkg/go/doc/exports.go new file mode 100644 index 000000000..e6f58ccb3 --- /dev/null +++ b/src/pkg/go/doc/exports.go @@ -0,0 +1,169 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file implements export filtering of an AST. + +package doc + +import "go/ast" + +// filterIdentList removes unexported names from list in place +// and returns the resulting list. +// +func filterIdentList(list []*ast.Ident) []*ast.Ident { + j := 0 + for _, x := range list { + if ast.IsExported(x.Name) { + list[j] = x + j++ + } + } + return list[0:j] +} + +// filterFieldList removes unexported fields (field names) from the field list +// in place and returns true if fields were removed. Removed fields that are +// anonymous (embedded) fields are added as embedded types to base. filterType +// is called with the types of all remaining fields. +// +func (r *reader) filterFieldList(base *baseType, fields *ast.FieldList) (removedFields bool) { + if fields == nil { + return + } + list := fields.List + j := 0 + for _, field := range list { + keepField := false + if n := len(field.Names); n == 0 { + // anonymous field + name, imp := baseTypeName(field.Type) + if ast.IsExported(name) { + // we keep the field - in this case r.readDecl + // will take care of adding the embedded type + keepField = true + } else if base != nil && !imp { + // we don't keep the field - add it as an embedded + // type so we won't loose its methods, if any + if embedded := r.lookupType(name); embedded != nil { + _, ptr := field.Type.(*ast.StarExpr) + base.addEmbeddedType(embedded, ptr) + } + } + } else { + field.Names = filterIdentList(field.Names) + if len(field.Names) < n { + removedFields = true + } + if len(field.Names) > 0 { + keepField = true + } + } + if keepField { + r.filterType(nil, field.Type) + list[j] = field + j++ + } + } + if j < len(list) { + removedFields = true + } + fields.List = list[0:j] + return +} + +// filterParamList applies filterType to each parameter type in fields. +// +func (r *reader) filterParamList(fields *ast.FieldList) { + if fields != nil { + for _, f := range fields.List { + r.filterType(nil, f.Type) + } + } +} + +// filterType strips any unexported struct fields or method types from typ +// in place. If fields (or methods) have been removed, the corresponding +// struct or interface type has the Incomplete field set to true. +// +func (r *reader) filterType(base *baseType, typ ast.Expr) { + switch t := typ.(type) { + case *ast.Ident: + // nothing to do + case *ast.ParenExpr: + r.filterType(nil, t.X) + case *ast.ArrayType: + r.filterType(nil, t.Elt) + case *ast.StructType: + if r.filterFieldList(base, t.Fields) { + t.Incomplete = true + } + case *ast.FuncType: + r.filterParamList(t.Params) + r.filterParamList(t.Results) + case *ast.InterfaceType: + if r.filterFieldList(base, t.Methods) { + t.Incomplete = true + } + case *ast.MapType: + r.filterType(nil, t.Key) + r.filterType(nil, t.Value) + case *ast.ChanType: + r.filterType(nil, t.Value) + } +} + +func (r *reader) filterSpec(spec ast.Spec) bool { + switch s := spec.(type) { + case *ast.ImportSpec: + // always keep imports so we can collect them + return true + case *ast.ValueSpec: + s.Names = filterIdentList(s.Names) + if len(s.Names) > 0 { + r.filterType(nil, s.Type) + return true + } + case *ast.TypeSpec: + if ast.IsExported(s.Name.Name) { + r.filterType(r.lookupType(s.Name.Name), s.Type) + return true + } + } + return false +} + +func (r *reader) filterSpecList(list []ast.Spec) []ast.Spec { + j := 0 + for _, s := range list { + if r.filterSpec(s) { + list[j] = s + j++ + } + } + return list[0:j] +} + +func (r *reader) filterDecl(decl ast.Decl) bool { + switch d := decl.(type) { + case *ast.GenDecl: + d.Specs = r.filterSpecList(d.Specs) + return len(d.Specs) > 0 + case *ast.FuncDecl: + return ast.IsExported(d.Name.Name) + } + return false +} + +// fileExports removes unexported declarations from src in place. +// +func (r *reader) fileExports(src *ast.File) { + j := 0 + for _, d := range src.Decls { + if r.filterDecl(d) { + src.Decls[j] = d + j++ + } + } + src.Decls = src.Decls[0:j] +} diff --git a/src/pkg/go/doc/filter.go b/src/pkg/go/doc/filter.go new file mode 100644 index 000000000..02b66ccef --- /dev/null +++ b/src/pkg/go/doc/filter.go @@ -0,0 +1,105 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package doc + +import "go/ast" + +type Filter func(string) bool + +func matchFields(fields *ast.FieldList, f Filter) bool { + if fields != nil { + for _, field := range fields.List { + for _, name := range field.Names { + if f(name.Name) { + return true + } + } + } + } + return false +} + +func matchDecl(d *ast.GenDecl, f Filter) bool { + for _, d := range d.Specs { + switch v := d.(type) { + case *ast.ValueSpec: + for _, name := range v.Names { + if f(name.Name) { + return true + } + } + case *ast.TypeSpec: + if f(v.Name.Name) { + return true + } + switch t := v.Type.(type) { + case *ast.StructType: + if matchFields(t.Fields, f) { + return true + } + case *ast.InterfaceType: + if matchFields(t.Methods, f) { + return true + } + } + } + } + return false +} + +func filterValues(a []*Value, f Filter) []*Value { + w := 0 + for _, vd := range a { + if matchDecl(vd.Decl, f) { + a[w] = vd + w++ + } + } + return a[0:w] +} + +func filterFuncs(a []*Func, f Filter) []*Func { + w := 0 + for _, fd := range a { + if f(fd.Name) { + a[w] = fd + w++ + } + } + return a[0:w] +} + +func filterTypes(a []*Type, f Filter) []*Type { + w := 0 + for _, td := range a { + n := 0 // number of matches + if matchDecl(td.Decl, f) { + n = 1 + } else { + // type name doesn't match, but we may have matching consts, vars, factories or methods + td.Consts = filterValues(td.Consts, f) + td.Vars = filterValues(td.Vars, f) + td.Funcs = filterFuncs(td.Funcs, f) + td.Methods = filterFuncs(td.Methods, f) + n += len(td.Consts) + len(td.Vars) + len(td.Funcs) + len(td.Methods) + } + if n > 0 { + a[w] = td + w++ + } + } + return a[0:w] +} + +// Filter eliminates documentation for names that don't pass through the filter f. +// TODO: Recognize "Type.Method" as a name. +// +func (p *Package) Filter(f Filter) { + p.Consts = filterValues(p.Consts, f) + p.Vars = filterValues(p.Vars, f) + p.Types = filterTypes(p.Types, f) + p.Funcs = filterFuncs(p.Funcs, f) + p.Doc = "" // don't show top-level package doc +} diff --git a/src/pkg/go/doc/headscan.go b/src/pkg/go/doc/headscan.go new file mode 100644 index 000000000..37486b126 --- /dev/null +++ b/src/pkg/go/doc/headscan.go @@ -0,0 +1,111 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* + The headscan command extracts comment headings from package files; + it is used to detect false positives which may require an adjustment + to the comment formatting heuristics in comment.go. + + Usage: headscan [-root root_directory] + + By default, the $GOROOT/src directory is scanned. +*/ +package main + +import ( + "bytes" + "flag" + "fmt" + "go/doc" + "go/parser" + "go/token" + "os" + "path/filepath" + "runtime" + "strings" +) + +var ( + root = flag.String("root", filepath.Join(runtime.GOROOT(), "src"), "root of filesystem tree to scan") + verbose = flag.Bool("v", false, "verbose mode") +) + +const ( + html_h = "<h3>" + html_endh = "</h3>\n" +) + +func isGoFile(fi os.FileInfo) bool { + return strings.HasSuffix(fi.Name(), ".go") && + !strings.HasSuffix(fi.Name(), "_test.go") +} + +func appendHeadings(list []string, comment string) []string { + var buf bytes.Buffer + doc.ToHTML(&buf, comment, nil) + for s := buf.String(); ; { + i := strings.Index(s, html_h) + if i < 0 { + break + } + i += len(html_h) + j := strings.Index(s, html_endh) + if j < 0 { + list = append(list, s[i:]) // incorrect HTML + break + } + list = append(list, s[i:j]) + s = s[j+len(html_endh):] + } + return list +} + +func main() { + flag.Parse() + fset := token.NewFileSet() + nheadings := 0 + err := filepath.Walk(*root, func(path string, fi os.FileInfo, err error) error { + if !fi.IsDir() { + return nil + } + pkgs, err := parser.ParseDir(fset, path, isGoFile, parser.ParseComments) + if err != nil { + if *verbose { + fmt.Fprintln(os.Stderr, err) + } + return nil + } + for _, pkg := range pkgs { + d := doc.New(pkg, path, doc.Mode(0)) + list := appendHeadings(nil, d.Doc) + for _, d := range d.Consts { + list = appendHeadings(list, d.Doc) + } + for _, d := range d.Types { + list = appendHeadings(list, d.Doc) + } + for _, d := range d.Vars { + list = appendHeadings(list, d.Doc) + } + for _, d := range d.Funcs { + list = appendHeadings(list, d.Doc) + } + if len(list) > 0 { + // directories may contain multiple packages; + // print path and package name + fmt.Printf("%s (package %s)\n", path, pkg.Name) + for _, h := range list { + fmt.Printf("\t%s\n", h) + } + nheadings += len(list) + } + } + return nil + }) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + fmt.Println(nheadings, "headings found") +} diff --git a/src/pkg/go/doc/reader.go b/src/pkg/go/doc/reader.go new file mode 100644 index 000000000..9c6f0816b --- /dev/null +++ b/src/pkg/go/doc/reader.go @@ -0,0 +1,734 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package doc + +import ( + "go/ast" + "go/token" + "regexp" + "sort" + "strconv" +) + +// ---------------------------------------------------------------------------- +// function/method sets +// +// Internally, we treat functions like methods and collect them in method sets. + +// methodSet describes a set of methods. Entries where Decl == nil are conflict +// entries (more then one method with the same name at the same embedding level). +// +type methodSet map[string]*Func + +// recvString returns a string representation of recv of the +// form "T", "*T", or "BADRECV" (if not a proper receiver type). +// +func recvString(recv ast.Expr) string { + switch t := recv.(type) { + case *ast.Ident: + return t.Name + case *ast.StarExpr: + return "*" + recvString(t.X) + } + return "BADRECV" +} + +// set creates the corresponding Func for f and adds it to mset. +// If there are multiple f's with the same name, set keeps the first +// one with documentation; conflicts are ignored. +// +func (mset methodSet) set(f *ast.FuncDecl) { + name := f.Name.Name + if g := mset[name]; g != nil && g.Doc != "" { + // A function with the same name has already been registered; + // since it has documentation, assume f is simply another + // implementation and ignore it. This does not happen if the + // caller is using go/build.ScanDir to determine the list of + // files implementing a package. + return + } + // function doesn't exist or has no documentation; use f + recv := "" + if f.Recv != nil { + var typ ast.Expr + // be careful in case of incorrect ASTs + if list := f.Recv.List; len(list) == 1 { + typ = list[0].Type + } + recv = recvString(typ) + } + mset[name] = &Func{ + Doc: f.Doc.Text(), + Name: name, + Decl: f, + Recv: recv, + Orig: recv, + } + f.Doc = nil // doc consumed - remove from AST +} + +// add adds method m to the method set; m is ignored if the method set +// already contains a method with the same name at the same or a higher +// level then m. +// +func (mset methodSet) add(m *Func) { + old := mset[m.Name] + if old == nil || m.Level < old.Level { + mset[m.Name] = m + return + } + if old != nil && m.Level == old.Level { + // conflict - mark it using a method with nil Decl + mset[m.Name] = &Func{ + Name: m.Name, + Level: m.Level, + } + } +} + +// ---------------------------------------------------------------------------- +// Base types + +// baseTypeName returns the name of the base type of x (or "") +// and whether the type is imported or not. +// +func baseTypeName(x ast.Expr) (name string, imported bool) { + switch t := x.(type) { + case *ast.Ident: + return t.Name, false + case *ast.SelectorExpr: + if _, ok := t.X.(*ast.Ident); ok { + // only possible for qualified type names; + // assume type is imported + return t.Sel.Name, true + } + case *ast.StarExpr: + return baseTypeName(t.X) + } + return +} + +// embeddedType describes the type of an anonymous field. +// +type embeddedType struct { + typ *baseType // the corresponding base type + ptr bool // if set, the anonymous field type is a pointer +} + +type baseType struct { + doc string // doc comment for type + name string // local type name (excluding package qualifier) + decl *ast.GenDecl // nil if declaration hasn't been seen yet + + // associated declarations + values []*Value // consts and vars + funcs methodSet + methods methodSet + + isEmbedded bool // true if this type is embedded + isStruct bool // true if this type is a struct + embedded []embeddedType // list of embedded types +} + +func (typ *baseType) addEmbeddedType(e *baseType, isPtr bool) { + e.isEmbedded = true + typ.embedded = append(typ.embedded, embeddedType{e, isPtr}) +} + +// ---------------------------------------------------------------------------- +// AST reader + +// reader accumulates documentation for a single package. +// It modifies the AST: Comments (declaration documentation) +// that have been collected by the reader are set to nil +// in the respective AST nodes so that they are not printed +// twice (once when printing the documentation and once when +// printing the corresponding AST node). +// +type reader struct { + mode Mode + + // package properties + doc string // package documentation, if any + filenames []string + bugs []string + + // declarations + imports map[string]int + values []*Value // consts and vars + types map[string]*baseType + funcs methodSet +} + +// isVisible reports whether name is visible in the documentation. +// +func (r *reader) isVisible(name string) bool { + return r.mode&AllDecls != 0 || ast.IsExported(name) +} + +// lookupType returns the base type with the given name. +// If the base type has not been encountered yet, a new +// type with the given name but no associated declaration +// is added to the type map. +// +func (r *reader) lookupType(name string) *baseType { + if name == "" || name == "_" { + return nil // no type docs for anonymous types + } + if typ, found := r.types[name]; found { + return typ + } + // type not found - add one without declaration + typ := &baseType{ + name: name, + funcs: make(methodSet), + methods: make(methodSet), + } + r.types[name] = typ + return typ +} + +func (r *reader) readDoc(comment *ast.CommentGroup) { + // By convention there should be only one package comment + // but collect all of them if there are more then one. + text := comment.Text() + if r.doc == "" { + r.doc = text + return + } + r.doc += "\n" + text +} + +func specNames(specs []ast.Spec) []string { + names := make([]string, 0, len(specs)) // reasonable estimate + for _, s := range specs { + // s guaranteed to be an *ast.ValueSpec by readValue + for _, ident := range s.(*ast.ValueSpec).Names { + names = append(names, ident.Name) + } + } + return names +} + +// readValue processes a const or var declaration. +// +func (r *reader) readValue(decl *ast.GenDecl) { + // determine if decl should be associated with a type + // Heuristic: For each typed entry, determine the type name, if any. + // If there is exactly one type name that is sufficiently + // frequent, associate the decl with the respective type. + domName := "" + domFreq := 0 + prev := "" + n := 0 + for _, spec := range decl.Specs { + s, ok := spec.(*ast.ValueSpec) + if !ok { + continue // should not happen, but be conservative + } + name := "" + switch { + case s.Type != nil: + // a type is present; determine its name + if n, imp := baseTypeName(s.Type); !imp && r.isVisible(n) { + name = n + } + case decl.Tok == token.CONST: + // no type is present but we have a constant declaration; + // use the previous type name (w/o more type information + // we cannot handle the case of unnamed variables with + // initializer expressions except for some trivial cases) + name = prev + } + if name != "" { + // entry has a named type + if domName != "" && domName != name { + // more than one type name - do not associate + // with any type + domName = "" + break + } + domName = name + domFreq++ + } + prev = name + n++ + } + + // nothing to do w/o a legal declaration + if n == 0 { + return + } + + // determine values list with which to associate the Value for this decl + values := &r.values + const threshold = 0.75 + if domName != "" && domFreq >= int(float64(len(decl.Specs))*threshold) { + // typed entries are sufficiently frequent + typ := r.lookupType(domName) + if typ != nil { + values = &typ.values // associate with that type + } + } + + *values = append(*values, &Value{ + Doc: decl.Doc.Text(), + Names: specNames(decl.Specs), + Decl: decl, + order: len(*values), + }) + decl.Doc = nil // doc consumed - remove from AST +} + +// fields returns a struct's fields or an interface's methods. +// +func fields(typ ast.Expr) (list []*ast.Field, isStruct bool) { + var fields *ast.FieldList + switch t := typ.(type) { + case *ast.StructType: + fields = t.Fields + isStruct = true + case *ast.InterfaceType: + fields = t.Methods + } + if fields != nil { + list = fields.List + } + return +} + +// readType processes a type declaration. +// +func (r *reader) readType(decl *ast.GenDecl, spec *ast.TypeSpec) { + typ := r.lookupType(spec.Name.Name) + if typ == nil { + return // no name or blank name - ignore the type + } + + // A type should be added at most once, so info.decl + // should be nil - if it is not, simply overwrite it. + typ.decl = decl + + // compute documentation + doc := spec.Doc + spec.Doc = nil // doc consumed - remove from AST + if doc == nil { + // no doc associated with the spec, use the declaration doc, if any + doc = decl.Doc + } + decl.Doc = nil // doc consumed - remove from AST + typ.doc = doc.Text() + + // look for anonymous fields that might contribute methods + var list []*ast.Field + list, typ.isStruct = fields(spec.Type) + for _, field := range list { + if len(field.Names) == 0 { + // anonymous field - add corresponding field type to typ + n, imp := baseTypeName(field.Type) + if imp { + // imported type - we don't handle this case + // at the moment + return + } + if embedded := r.lookupType(n); embedded != nil { + _, ptr := field.Type.(*ast.StarExpr) + typ.addEmbeddedType(embedded, ptr) + } + } + } +} + +// readFunc processes a func or method declaration. +// +func (r *reader) readFunc(fun *ast.FuncDecl) { + // strip function body + fun.Body = nil + + // determine if it should be associated with a type + if fun.Recv != nil { + // method + recvTypeName, imp := baseTypeName(fun.Recv.List[0].Type) + if imp { + // should not happen (incorrect AST); + // don't show this method + return + } + var typ *baseType + if r.isVisible(recvTypeName) { + // visible recv type: if not found, add it to r.types + typ = r.lookupType(recvTypeName) + } else { + // invisible recv type: if not found, do not add it + // (invisible embedded types are added before this + // phase, so if the type doesn't exist yet, we don't + // care about this method) + typ = r.types[recvTypeName] + } + if typ != nil { + // associate method with the type + // (if the type is not exported, it may be embedded + // somewhere so we need to collect the method anyway) + typ.methods.set(fun) + } + // otherwise don't show the method + // TODO(gri): There may be exported methods of non-exported types + // that can be called because of exported values (consts, vars, or + // function results) of that type. Could determine if that is the + // case and then show those methods in an appropriate section. + return + } + + // perhaps a factory function + // determine result type, if any + if fun.Type.Results.NumFields() >= 1 { + res := fun.Type.Results.List[0] + if len(res.Names) <= 1 { + // exactly one (named or anonymous) result associated + // with the first type in result signature (there may + // be more than one result) + if n, imp := baseTypeName(res.Type); !imp && r.isVisible(n) { + if typ := r.lookupType(n); typ != nil { + // associate Func with typ + typ.funcs.set(fun) + return + } + } + } + } + + // just an ordinary function + r.funcs.set(fun) +} + +var ( + bug_markers = regexp.MustCompile("^/[/*][ \t]*BUG\\(.*\\):[ \t]*") // BUG(uid): + bug_content = regexp.MustCompile("[^ \n\r\t]+") // at least one non-whitespace char +) + +// readFile adds the AST for a source file to the reader. +// +func (r *reader) readFile(src *ast.File) { + // add package documentation + if src.Doc != nil { + r.readDoc(src.Doc) + src.Doc = nil // doc consumed - remove from AST + } + + // add all declarations + for _, decl := range src.Decls { + switch d := decl.(type) { + case *ast.GenDecl: + switch d.Tok { + case token.IMPORT: + // imports are handled individually + for _, spec := range d.Specs { + if s, ok := spec.(*ast.ImportSpec); ok { + if import_, err := strconv.Unquote(s.Path.Value); err == nil { + r.imports[import_] = 1 + } + } + } + case token.CONST, token.VAR: + // constants and variables are always handled as a group + r.readValue(d) + case token.TYPE: + // types are handled individually + for _, spec := range d.Specs { + if s, ok := spec.(*ast.TypeSpec); ok { + // use an individual (possibly fake) declaration + // for each type; this also ensures that each type + // gets to (re-)use the declaration documentation + // if there's none associated with the spec itself + fake := &ast.GenDecl{ + d.Doc, d.Pos(), token.TYPE, token.NoPos, + []ast.Spec{s}, token.NoPos, + } + r.readType(fake, s) + } + } + } + case *ast.FuncDecl: + r.readFunc(d) + } + } + + // collect BUG(...) comments + for _, c := range src.Comments { + text := c.List[0].Text + if m := bug_markers.FindStringIndex(text); m != nil { + // found a BUG comment; maybe empty + if btxt := text[m[1]:]; bug_content.MatchString(btxt) { + // non-empty BUG comment; collect comment without BUG prefix + list := append([]*ast.Comment(nil), c.List...) // make a copy + list[0].Text = text[m[1]:] + r.bugs = append(r.bugs, (&ast.CommentGroup{list}).Text()) + } + } + } + src.Comments = nil // consumed unassociated comments - remove from AST +} + +func (r *reader) readPackage(pkg *ast.Package, mode Mode) { + // initialize reader + r.filenames = make([]string, len(pkg.Files)) + r.imports = make(map[string]int) + r.mode = mode + r.types = make(map[string]*baseType) + r.funcs = make(methodSet) + + // sort package files before reading them so that the + // result result does not depend on map iteration order + i := 0 + for filename := range pkg.Files { + r.filenames[i] = filename + i++ + } + sort.Strings(r.filenames) + + // process files in sorted order + for _, filename := range r.filenames { + f := pkg.Files[filename] + if mode&AllDecls == 0 { + r.fileExports(f) + } + r.readFile(f) + } +} + +// ---------------------------------------------------------------------------- +// Types + +var predeclaredTypes = map[string]bool{ + "bool": true, + "byte": true, + "complex64": true, + "complex128": true, + "float32": true, + "float64": true, + "int": true, + "int8": true, + "int16": true, + "int32": true, + "int64": true, + "string": true, + "uint": true, + "uint8": true, + "uint16": true, + "uint32": true, + "uint64": true, + "uintptr": true, +} + +func customizeRecv(f *Func, recvTypeName string, embeddedIsPtr bool, level int) *Func { + if f == nil || f.Decl == nil || f.Decl.Recv == nil || len(f.Decl.Recv.List) != 1 { + return f // shouldn't happen, but be safe + } + + // copy existing receiver field and set new type + newField := *f.Decl.Recv.List[0] + _, origRecvIsPtr := newField.Type.(*ast.StarExpr) + var typ ast.Expr = ast.NewIdent(recvTypeName) + if !embeddedIsPtr && origRecvIsPtr { + typ = &ast.StarExpr{token.NoPos, typ} + } + newField.Type = typ + + // copy existing receiver field list and set new receiver field + newFieldList := *f.Decl.Recv + newFieldList.List = []*ast.Field{&newField} + + // copy existing function declaration and set new receiver field list + newFuncDecl := *f.Decl + newFuncDecl.Recv = &newFieldList + + // copy existing function documentation and set new declaration + newF := *f + newF.Decl = &newFuncDecl + newF.Recv = recvString(typ) + // the Orig field never changes + newF.Level = level + + return &newF +} + +// collectEmbeddedMethods collects the embedded methods from +// all processed embedded types found in info in mset. +// +func collectEmbeddedMethods(mset methodSet, typ *baseType, recvTypeName string, embeddedIsPtr bool, level int) { + for _, e := range typ.embedded { + // Once an embedded type is embedded as a pointer type + // all embedded types in those types are treated like + // pointer types for the purpose of the receiver type + // computation; i.e., embeddedIsPtr is sticky for this + // embedding hierarchy. + thisEmbeddedIsPtr := embeddedIsPtr || e.ptr + for _, m := range e.typ.methods { + // only top-level methods are embedded + if m.Level == 0 { + mset.add(customizeRecv(m, recvTypeName, thisEmbeddedIsPtr, level)) + } + } + collectEmbeddedMethods(mset, e.typ, recvTypeName, thisEmbeddedIsPtr, level+1) + } +} + +// computeMethodSets determines the actual method sets for each type encountered. +// +func (r *reader) computeMethodSets() { + for _, t := range r.types { + // collect embedded methods for t + if t.isStruct { + // struct + collectEmbeddedMethods(t.methods, t, t.name, false, 1) + } else { + // interface + // TODO(gri) fix this + } + } +} + +// cleanupTypes removes the association of functions and methods with +// types that have no declaration. Instead, these functions and methods +// are shown at the package level. It also removes types with missing +// declarations or which are not visible. +// +func (r *reader) cleanupTypes() { + for _, t := range r.types { + visible := r.isVisible(t.name) + if t.decl == nil && (predeclaredTypes[t.name] || t.isEmbedded && visible) { + // t.name is a predeclared type (and was not redeclared in this package), + // or it was embedded somewhere but its declaration is missing (because + // the AST is incomplete): move any associated values, funcs, and methods + // back to the top-level so that they are not lost. + // 1) move values + r.values = append(r.values, t.values...) + // 2) move factory functions + for name, f := range t.funcs { + r.funcs[name] = f + } + // 3) move methods + for name, m := range t.methods { + // don't overwrite functions with the same name - drop them + if _, found := r.funcs[name]; !found { + r.funcs[name] = m + } + } + } + // remove types w/o declaration or which are not visible + if t.decl == nil || !visible { + delete(r.types, t.name) + } + } +} + +// ---------------------------------------------------------------------------- +// Sorting + +type data struct { + n int + swap func(i, j int) + less func(i, j int) bool +} + +func (d *data) Len() int { return d.n } +func (d *data) Swap(i, j int) { d.swap(i, j) } +func (d *data) Less(i, j int) bool { return d.less(i, j) } + +// sortBy is a helper function for sorting +func sortBy(less func(i, j int) bool, swap func(i, j int), n int) { + sort.Sort(&data{n, swap, less}) +} + +func sortedKeys(m map[string]int) []string { + list := make([]string, len(m)) + i := 0 + for key := range m { + list[i] = key + i++ + } + sort.Strings(list) + return list +} + +// sortingName returns the name to use when sorting d into place. +// +func sortingName(d *ast.GenDecl) string { + if len(d.Specs) == 1 { + if s, ok := d.Specs[0].(*ast.ValueSpec); ok { + return s.Names[0].Name + } + } + return "" +} + +func sortedValues(m []*Value, tok token.Token) []*Value { + list := make([]*Value, len(m)) // big enough in any case + i := 0 + for _, val := range m { + if val.Decl.Tok == tok { + list[i] = val + i++ + } + } + list = list[0:i] + + sortBy( + func(i, j int) bool { + if ni, nj := sortingName(list[i].Decl), sortingName(list[j].Decl); ni != nj { + return ni < nj + } + return list[i].order < list[j].order + }, + func(i, j int) { list[i], list[j] = list[j], list[i] }, + len(list), + ) + + return list +} + +func sortedTypes(m map[string]*baseType) []*Type { + list := make([]*Type, len(m)) + i := 0 + for _, t := range m { + list[i] = &Type{ + Doc: t.doc, + Name: t.name, + Decl: t.decl, + Consts: sortedValues(t.values, token.CONST), + Vars: sortedValues(t.values, token.VAR), + Funcs: sortedFuncs(t.funcs), + Methods: sortedFuncs(t.methods), + } + i++ + } + + sortBy( + func(i, j int) bool { return list[i].Name < list[j].Name }, + func(i, j int) { list[i], list[j] = list[j], list[i] }, + len(list), + ) + + return list +} + +func sortedFuncs(m methodSet) []*Func { + list := make([]*Func, len(m)) + i := 0 + for _, m := range m { + // exclude conflict entries + if m.Decl != nil { + list[i] = m + i++ + } + } + list = list[0:i] + sortBy( + func(i, j int) bool { return list[i].Name < list[j].Name }, + func(i, j int) { list[i], list[j] = list[j], list[i] }, + len(list), + ) + return list +} diff --git a/src/pkg/go/doc/testdata/a.0.golden b/src/pkg/go/doc/testdata/a.0.golden new file mode 100644 index 000000000..24db02d34 --- /dev/null +++ b/src/pkg/go/doc/testdata/a.0.golden @@ -0,0 +1,13 @@ +// comment 0 comment 1 +PACKAGE a + +IMPORTPATH + testdata/a + +FILENAMES + testdata/a0.go + testdata/a1.go + +BUGS + // bug0 + // bug1 diff --git a/src/pkg/go/doc/testdata/a.1.golden b/src/pkg/go/doc/testdata/a.1.golden new file mode 100644 index 000000000..24db02d34 --- /dev/null +++ b/src/pkg/go/doc/testdata/a.1.golden @@ -0,0 +1,13 @@ +// comment 0 comment 1 +PACKAGE a + +IMPORTPATH + testdata/a + +FILENAMES + testdata/a0.go + testdata/a1.go + +BUGS + // bug0 + // bug1 diff --git a/src/pkg/go/doc/testdata/a0.go b/src/pkg/go/doc/testdata/a0.go new file mode 100644 index 000000000..dc552989e --- /dev/null +++ b/src/pkg/go/doc/testdata/a0.go @@ -0,0 +1,8 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// comment 0 +package a + +//BUG(uid): bug0 diff --git a/src/pkg/go/doc/testdata/a1.go b/src/pkg/go/doc/testdata/a1.go new file mode 100644 index 000000000..098776c1b --- /dev/null +++ b/src/pkg/go/doc/testdata/a1.go @@ -0,0 +1,8 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// comment 1 +package a + +//BUG(uid): bug1 diff --git a/src/pkg/go/doc/testdata/b.0.golden b/src/pkg/go/doc/testdata/b.0.golden new file mode 100644 index 000000000..7c3330061 --- /dev/null +++ b/src/pkg/go/doc/testdata/b.0.golden @@ -0,0 +1,43 @@ +// +PACKAGE b + +IMPORTPATH + testdata/b + +IMPORTS + a + +FILENAMES + testdata/b.go + +CONSTANTS + // + const Pi = 3.14 // Pi + + +VARIABLES + // + var MaxInt int // MaxInt + + +FUNCTIONS + // + func F(x int) int + + // Always under the package functions list. + func NotAFactory() int + + // Associated with uint type if AllDecls is set. + func UintFactory() uint + + +TYPES + // + type T struct{} // T + + // + var V T // v + + // + func (x *T) M() + diff --git a/src/pkg/go/doc/testdata/b.1.golden b/src/pkg/go/doc/testdata/b.1.golden new file mode 100644 index 000000000..f30380516 --- /dev/null +++ b/src/pkg/go/doc/testdata/b.1.golden @@ -0,0 +1,49 @@ +// +PACKAGE b + +IMPORTPATH + testdata/b + +IMPORTS + a + +FILENAMES + testdata/b.go + +CONSTANTS + // + const Pi = 3.14 // Pi + + +VARIABLES + // + var MaxInt int // MaxInt + + +FUNCTIONS + // + func F(x int) int + + // Always under the package functions list. + func NotAFactory() int + + +TYPES + // + type T struct{} // T + + // + var V T // v + + // + func (x *T) M() + + // Should only appear if AllDecls is set. + type uint struct{} + + // Associated with uint type if AllDecls is set. + func UintFactory() uint + + // Associated with uint type if AllDecls is set. + func uintFactory() uint + diff --git a/src/pkg/go/doc/testdata/b.go b/src/pkg/go/doc/testdata/b.go new file mode 100644 index 000000000..28660f9be --- /dev/null +++ b/src/pkg/go/doc/testdata/b.go @@ -0,0 +1,30 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package b + +import "a" + +// Basic declarations + +const Pi = 3.14 // Pi +var MaxInt int // MaxInt +type T struct{} // T +var V T // v +func F(x int) int {} // F +func (x *T) M() {} // M + +// Corner cases: association with (presumed) predeclared types + +// Always under the package functions list. +func NotAFactory() int {} + +// Associated with uint type if AllDecls is set. +func UintFactory() uint {} + +// Associated with uint type if AllDecls is set. +func uintFactory() uint {} + +// Should only appear if AllDecls is set. +type uint struct{} // overrides a predeclared type uint diff --git a/src/pkg/go/doc/testdata/benchmark.go b/src/pkg/go/doc/testdata/benchmark.go new file mode 100644 index 000000000..0bf567b7c --- /dev/null +++ b/src/pkg/go/doc/testdata/benchmark.go @@ -0,0 +1,293 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package testing + +import ( + "flag" + "fmt" + "os" + "runtime" + "time" +) + +var matchBenchmarks = flag.String("test.bench", "", "regular expression to select benchmarks to run") +var benchTime = flag.Float64("test.benchtime", 1, "approximate run time for each benchmark, in seconds") + +// An internal type but exported because it is cross-package; part of the implementation +// of gotest. +type InternalBenchmark struct { + Name string + F func(b *B) +} + +// B is a type passed to Benchmark functions to manage benchmark +// timing and to specify the number of iterations to run. +type B struct { + common + N int + benchmark InternalBenchmark + bytes int64 + timerOn bool + result BenchmarkResult +} + +// StartTimer starts timing a test. This function is called automatically +// before a benchmark starts, but it can also used to resume timing after +// a call to StopTimer. +func (b *B) StartTimer() { + if !b.timerOn { + b.start = time.Now() + b.timerOn = true + } +} + +// StopTimer stops timing a test. This can be used to pause the timer +// while performing complex initialization that you don't +// want to measure. +func (b *B) StopTimer() { + if b.timerOn { + b.duration += time.Now().Sub(b.start) + b.timerOn = false + } +} + +// ResetTimer sets the elapsed benchmark time to zero. +// It does not affect whether the timer is running. +func (b *B) ResetTimer() { + if b.timerOn { + b.start = time.Now() + } + b.duration = 0 +} + +// SetBytes records the number of bytes processed in a single operation. +// If this is called, the benchmark will report ns/op and MB/s. +func (b *B) SetBytes(n int64) { b.bytes = n } + +func (b *B) nsPerOp() int64 { + if b.N <= 0 { + return 0 + } + return b.duration.Nanoseconds() / int64(b.N) +} + +// runN runs a single benchmark for the specified number of iterations. +func (b *B) runN(n int) { + // Try to get a comparable environment for each run + // by clearing garbage from previous runs. + runtime.GC() + b.N = n + b.ResetTimer() + b.StartTimer() + b.benchmark.F(b) + b.StopTimer() +} + +func min(x, y int) int { + if x > y { + return y + } + return x +} + +func max(x, y int) int { + if x < y { + return y + } + return x +} + +// roundDown10 rounds a number down to the nearest power of 10. +func roundDown10(n int) int { + var tens = 0 + // tens = floor(log_10(n)) + for n > 10 { + n = n / 10 + tens++ + } + // result = 10^tens + result := 1 + for i := 0; i < tens; i++ { + result *= 10 + } + return result +} + +// roundUp rounds x up to a number of the form [1eX, 2eX, 5eX]. +func roundUp(n int) int { + base := roundDown10(n) + if n < (2 * base) { + return 2 * base + } + if n < (5 * base) { + return 5 * base + } + return 10 * base +} + +// run times the benchmark function in a separate goroutine. +func (b *B) run() BenchmarkResult { + go b.launch() + <-b.signal + return b.result +} + +// launch launches the benchmark function. It gradually increases the number +// of benchmark iterations until the benchmark runs for a second in order +// to get a reasonable measurement. It prints timing information in this form +// testing.BenchmarkHello 100000 19 ns/op +// launch is run by the fun function as a separate goroutine. +func (b *B) launch() { + // Run the benchmark for a single iteration in case it's expensive. + n := 1 + + // Signal that we're done whether we return normally + // or by FailNow's runtime.Goexit. + defer func() { + b.signal <- b + }() + + b.runN(n) + // Run the benchmark for at least the specified amount of time. + d := time.Duration(*benchTime * float64(time.Second)) + for !b.failed && b.duration < d && n < 1e9 { + last := n + // Predict iterations/sec. + if b.nsPerOp() == 0 { + n = 1e9 + } else { + n = int(d.Nanoseconds() / b.nsPerOp()) + } + // Run more iterations than we think we'll need for a second (1.5x). + // Don't grow too fast in case we had timing errors previously. + // Be sure to run at least one more than last time. + n = max(min(n+n/2, 100*last), last+1) + // Round up to something easy to read. + n = roundUp(n) + b.runN(n) + } + b.result = BenchmarkResult{b.N, b.duration, b.bytes} +} + +// The results of a benchmark run. +type BenchmarkResult struct { + N int // The number of iterations. + T time.Duration // The total time taken. + Bytes int64 // Bytes processed in one iteration. +} + +func (r BenchmarkResult) NsPerOp() int64 { + if r.N <= 0 { + return 0 + } + return r.T.Nanoseconds() / int64(r.N) +} + +func (r BenchmarkResult) mbPerSec() float64 { + if r.Bytes <= 0 || r.T <= 0 || r.N <= 0 { + return 0 + } + return (float64(r.Bytes) * float64(r.N) / 1e6) / r.T.Seconds() +} + +func (r BenchmarkResult) String() string { + mbs := r.mbPerSec() + mb := "" + if mbs != 0 { + mb = fmt.Sprintf("\t%7.2f MB/s", mbs) + } + nsop := r.NsPerOp() + ns := fmt.Sprintf("%10d ns/op", nsop) + if r.N > 0 && nsop < 100 { + // The format specifiers here make sure that + // the ones digits line up for all three possible formats. + if nsop < 10 { + ns = fmt.Sprintf("%13.2f ns/op", float64(r.T.Nanoseconds())/float64(r.N)) + } else { + ns = fmt.Sprintf("%12.1f ns/op", float64(r.T.Nanoseconds())/float64(r.N)) + } + } + return fmt.Sprintf("%8d\t%s%s", r.N, ns, mb) +} + +// An internal function but exported because it is cross-package; part of the implementation +// of gotest. +func RunBenchmarks(matchString func(pat, str string) (bool, error), benchmarks []InternalBenchmark) { + // If no flag was specified, don't run benchmarks. + if len(*matchBenchmarks) == 0 { + return + } + for _, Benchmark := range benchmarks { + matched, err := matchString(*matchBenchmarks, Benchmark.Name) + if err != nil { + fmt.Fprintf(os.Stderr, "testing: invalid regexp for -test.bench: %s\n", err) + os.Exit(1) + } + if !matched { + continue + } + for _, procs := range cpuList { + runtime.GOMAXPROCS(procs) + b := &B{ + common: common{ + signal: make(chan interface{}), + }, + benchmark: Benchmark, + } + benchName := Benchmark.Name + if procs != 1 { + benchName = fmt.Sprintf("%s-%d", Benchmark.Name, procs) + } + fmt.Printf("%s\t", benchName) + r := b.run() + if b.failed { + // The output could be very long here, but probably isn't. + // We print it all, regardless, because we don't want to trim the reason + // the benchmark failed. + fmt.Printf("--- FAIL: %s\n%s", benchName, b.output) + continue + } + fmt.Printf("%v\n", r) + // Unlike with tests, we ignore the -chatty flag and always print output for + // benchmarks since the output generation time will skew the results. + if len(b.output) > 0 { + b.trimOutput() + fmt.Printf("--- BENCH: %s\n%s", benchName, b.output) + } + if p := runtime.GOMAXPROCS(-1); p != procs { + fmt.Fprintf(os.Stderr, "testing: %s left GOMAXPROCS set to %d\n", benchName, p) + } + } + } +} + +// trimOutput shortens the output from a benchmark, which can be very long. +func (b *B) trimOutput() { + // The output is likely to appear multiple times because the benchmark + // is run multiple times, but at least it will be seen. This is not a big deal + // because benchmarks rarely print, but just in case, we trim it if it's too long. + const maxNewlines = 10 + for nlCount, j := 0, 0; j < len(b.output); j++ { + if b.output[j] == '\n' { + nlCount++ + if nlCount >= maxNewlines { + b.output = append(b.output[:j], "\n\t... [output truncated]\n"...) + break + } + } + } +} + +// Benchmark benchmarks a single function. Useful for creating +// custom benchmarks that do not use gotest. +func Benchmark(f func(b *B)) BenchmarkResult { + b := &B{ + common: common{ + signal: make(chan interface{}), + }, + benchmark: InternalBenchmark{"", f}, + } + return b.run() +} diff --git a/src/pkg/go/doc/testdata/c.0.golden b/src/pkg/go/doc/testdata/c.0.golden new file mode 100644 index 000000000..e21959b19 --- /dev/null +++ b/src/pkg/go/doc/testdata/c.0.golden @@ -0,0 +1,48 @@ +// +PACKAGE c + +IMPORTPATH + testdata/c + +IMPORTS + a + +FILENAMES + testdata/c.go + +TYPES + // A (should see this) + type A struct{} + + // B (should see this) + type B struct{} + + // C (should see this) + type C struct{} + + // D (should see this) + type D struct{} + + // E1 (should see this) + type E1 struct{} + + // E (should see this for E2 and E3) + type E2 struct{} + + // E (should see this for E2 and E3) + type E3 struct{} + + // E4 (should see this) + type E4 struct{} + + // + type T1 struct{} + + // + func (t1 *T1) M() + + // T2 must not show methods of local T1 + type T2 struct { + a.T1 // not the same as locally declared T1 + } + diff --git a/src/pkg/go/doc/testdata/c.1.golden b/src/pkg/go/doc/testdata/c.1.golden new file mode 100644 index 000000000..e21959b19 --- /dev/null +++ b/src/pkg/go/doc/testdata/c.1.golden @@ -0,0 +1,48 @@ +// +PACKAGE c + +IMPORTPATH + testdata/c + +IMPORTS + a + +FILENAMES + testdata/c.go + +TYPES + // A (should see this) + type A struct{} + + // B (should see this) + type B struct{} + + // C (should see this) + type C struct{} + + // D (should see this) + type D struct{} + + // E1 (should see this) + type E1 struct{} + + // E (should see this for E2 and E3) + type E2 struct{} + + // E (should see this for E2 and E3) + type E3 struct{} + + // E4 (should see this) + type E4 struct{} + + // + type T1 struct{} + + // + func (t1 *T1) M() + + // T2 must not show methods of local T1 + type T2 struct { + a.T1 // not the same as locally declared T1 + } + diff --git a/src/pkg/go/doc/testdata/c.go b/src/pkg/go/doc/testdata/c.go new file mode 100644 index 000000000..e0f39196d --- /dev/null +++ b/src/pkg/go/doc/testdata/c.go @@ -0,0 +1,62 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package c + +import "a" + +// ---------------------------------------------------------------------------- +// Test that empty declarations don't cause problems + +const () + +type () + +var () + +// ---------------------------------------------------------------------------- +// Test that types with documentation on both, the Decl and the Spec node +// are handled correctly. + +// A (should see this) +type A struct{} + +// B (should see this) +type ( + B struct{} +) + +type ( + // C (should see this) + C struct{} +) + +// D (should not see this) +type ( + // D (should see this) + D struct{} +) + +// E (should see this for E2 and E3) +type ( + // E1 (should see this) + E1 struct{} + E2 struct{} + E3 struct{} + // E4 (should see this) + E4 struct{} +) + +// ---------------------------------------------------------------------------- +// Test that local and imported types are different when +// handling anonymous fields. + +type T1 struct{} + +func (t1 *T1) M() {} + +// T2 must not show methods of local T1 +type T2 struct { + a.T1 // not the same as locally declared T1 +} diff --git a/src/pkg/go/doc/testdata/d.0.golden b/src/pkg/go/doc/testdata/d.0.golden new file mode 100644 index 000000000..c00519953 --- /dev/null +++ b/src/pkg/go/doc/testdata/d.0.golden @@ -0,0 +1,104 @@ +// +PACKAGE d + +IMPORTPATH + testdata/d + +FILENAMES + testdata/d1.go + testdata/d2.go + +CONSTANTS + // CBx constants should appear before CAx constants. + const ( + CB2 = iota // before CB1 + CB1 // before CB0 + CB0 // at end + ) + + // CAx constants should appear after CBx constants. + const ( + CA2 = iota // before CA1 + CA1 // before CA0 + CA0 // at end + ) + + // C0 should be first. + const C0 = 0 + + // C1 should be second. + const C1 = 1 + + // C2 should be third. + const C2 = 2 + + // + const ( + // Single const declarations inside ()'s are considered ungrouped + // and show up in sorted order. + Cungrouped = 0 + ) + + +VARIABLES + // VBx variables should appear before VAx variables. + var ( + VB2 int // before VB1 + VB1 int // before VB0 + VB0 int // at end + ) + + // VAx variables should appear after VBx variables. + var ( + VA2 int // before VA1 + VA1 int // before VA0 + VA0 int // at end + ) + + // V0 should be first. + var V0 uintptr + + // V1 should be second. + var V1 uint + + // V2 should be third. + var V2 int + + // + var ( + // Single var declarations inside ()'s are considered ungrouped + // and show up in sorted order. + Vungrouped = 0 + ) + + +FUNCTIONS + // F0 should be first. + func F0() + + // F1 should be second. + func F1() + + // F2 should be third. + func F2() + + +TYPES + // T0 should be first. + type T0 struct{} + + // T1 should be second. + type T1 struct{} + + // T2 should be third. + type T2 struct{} + + // TG0 should be first. + type TG0 struct{} + + // TG1 should be second. + type TG1 struct{} + + // TG2 should be third. + type TG2 struct{} + diff --git a/src/pkg/go/doc/testdata/d.1.golden b/src/pkg/go/doc/testdata/d.1.golden new file mode 100644 index 000000000..c00519953 --- /dev/null +++ b/src/pkg/go/doc/testdata/d.1.golden @@ -0,0 +1,104 @@ +// +PACKAGE d + +IMPORTPATH + testdata/d + +FILENAMES + testdata/d1.go + testdata/d2.go + +CONSTANTS + // CBx constants should appear before CAx constants. + const ( + CB2 = iota // before CB1 + CB1 // before CB0 + CB0 // at end + ) + + // CAx constants should appear after CBx constants. + const ( + CA2 = iota // before CA1 + CA1 // before CA0 + CA0 // at end + ) + + // C0 should be first. + const C0 = 0 + + // C1 should be second. + const C1 = 1 + + // C2 should be third. + const C2 = 2 + + // + const ( + // Single const declarations inside ()'s are considered ungrouped + // and show up in sorted order. + Cungrouped = 0 + ) + + +VARIABLES + // VBx variables should appear before VAx variables. + var ( + VB2 int // before VB1 + VB1 int // before VB0 + VB0 int // at end + ) + + // VAx variables should appear after VBx variables. + var ( + VA2 int // before VA1 + VA1 int // before VA0 + VA0 int // at end + ) + + // V0 should be first. + var V0 uintptr + + // V1 should be second. + var V1 uint + + // V2 should be third. + var V2 int + + // + var ( + // Single var declarations inside ()'s are considered ungrouped + // and show up in sorted order. + Vungrouped = 0 + ) + + +FUNCTIONS + // F0 should be first. + func F0() + + // F1 should be second. + func F1() + + // F2 should be third. + func F2() + + +TYPES + // T0 should be first. + type T0 struct{} + + // T1 should be second. + type T1 struct{} + + // T2 should be third. + type T2 struct{} + + // TG0 should be first. + type TG0 struct{} + + // TG1 should be second. + type TG1 struct{} + + // TG2 should be third. + type TG2 struct{} + diff --git a/src/pkg/go/doc/testdata/d1.go b/src/pkg/go/doc/testdata/d1.go new file mode 100644 index 000000000..ebd694195 --- /dev/null +++ b/src/pkg/go/doc/testdata/d1.go @@ -0,0 +1,57 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Test cases for sort order of declarations. + +package d + +// C2 should be third. +const C2 = 2 + +// V2 should be third. +var V2 int + +// CBx constants should appear before CAx constants. +const ( + CB2 = iota // before CB1 + CB1 // before CB0 + CB0 // at end +) + +// VBx variables should appear before VAx variables. +var ( + VB2 int // before VB1 + VB1 int // before VB0 + VB0 int // at end +) + +const ( + // Single const declarations inside ()'s are considered ungrouped + // and show up in sorted order. + Cungrouped = 0 +) + +var ( + // Single var declarations inside ()'s are considered ungrouped + // and show up in sorted order. + Vungrouped = 0 +) + +// T2 should be third. +type T2 struct{} + +// Grouped types are sorted nevertheless. +type ( + // TG2 should be third. + TG2 struct{} + + // TG1 should be second. + TG1 struct{} + + // TG0 should be first. + TG0 struct{} +) + +// F2 should be third. +func F2() {} diff --git a/src/pkg/go/doc/testdata/d2.go b/src/pkg/go/doc/testdata/d2.go new file mode 100644 index 000000000..2f56f4fa4 --- /dev/null +++ b/src/pkg/go/doc/testdata/d2.go @@ -0,0 +1,45 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Test cases for sort order of declarations. + +package d + +// C1 should be second. +const C1 = 1 + +// C0 should be first. +const C0 = 0 + +// V1 should be second. +var V1 uint + +// V0 should be first. +var V0 uintptr + +// CAx constants should appear after CBx constants. +const ( + CA2 = iota // before CA1 + CA1 // before CA0 + CA0 // at end +) + +// VAx variables should appear after VBx variables. +var ( + VA2 int // before VA1 + VA1 int // before VA0 + VA0 int // at end +) + +// T1 should be second. +type T1 struct{} + +// T0 should be first. +type T0 struct{} + +// F1 should be second. +func F1() {} + +// F0 should be first. +func F0() {} diff --git a/src/pkg/go/doc/testdata/e.0.golden b/src/pkg/go/doc/testdata/e.0.golden new file mode 100644 index 000000000..3b128f7be --- /dev/null +++ b/src/pkg/go/doc/testdata/e.0.golden @@ -0,0 +1,31 @@ +// +PACKAGE e + +IMPORTPATH + testdata/e + +FILENAMES + testdata/e.go + +TYPES + // T1 has no (top-level) M method due to conflict. + type T1 struct { + // contains filtered or unexported fields + } + + // T2 has only M as top-level method. + type T2 struct { + // contains filtered or unexported fields + } + + // T2.M should appear as method of T2. + func (T2) M() + + // T3 has only M as top-level method. + type T3 struct { + // contains filtered or unexported fields + } + + // T3.M should appear as method of T3. + func (T3) M() + diff --git a/src/pkg/go/doc/testdata/e.1.golden b/src/pkg/go/doc/testdata/e.1.golden new file mode 100644 index 000000000..d05602d82 --- /dev/null +++ b/src/pkg/go/doc/testdata/e.1.golden @@ -0,0 +1,61 @@ +// +PACKAGE e + +IMPORTPATH + testdata/e + +FILENAMES + testdata/e.go + +TYPES + // T1 has no (top-level) M method due to conflict. + type T1 struct { + t1 + t2 + } + + // T2 has only M as top-level method. + type T2 struct { + t1 + } + + // T2.M should appear as method of T2. + func (T2) M() + + // T3 has only M as top-level method. + type T3 struct { + t1e + t2e + } + + // T3.M should appear as method of T3. + func (T3) M() + + // + type t1 struct{} + + // t1.M should not appear as method in a Tx type. + func (t1) M() + + // + type t1e struct { + t1 + } + + // t1.M should not appear as method in a Tx type. + func (t1e) M() + + // + type t2 struct{} + + // t2.M should not appear as method in a Tx type. + func (t2) M() + + // + type t2e struct { + t2 + } + + // t2.M should not appear as method in a Tx type. + func (t2e) M() + diff --git a/src/pkg/go/doc/testdata/e.go b/src/pkg/go/doc/testdata/e.go new file mode 100644 index 000000000..8ea6a83b6 --- /dev/null +++ b/src/pkg/go/doc/testdata/e.go @@ -0,0 +1,58 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Embedding tests. +// TODO(gri): This should be comprehensive. + +package e + +// ---------------------------------------------------------------------------- +// Conflicting methods M must not show up. + +type t1 struct{} + +// t1.M should not appear as method in a Tx type. +func (t1) M() {} + +type t2 struct{} + +// t2.M should not appear as method in a Tx type. +func (t2) M() {} + +// T1 has no (top-level) M method due to conflict. +type T1 struct { + t1 + t2 +} + +// ---------------------------------------------------------------------------- +// Higher-level method M wins over lower-level method M. + +// T2 has only M as top-level method. +type T2 struct { + t1 +} + +// T2.M should appear as method of T2. +func (T2) M() {} + +// ---------------------------------------------------------------------------- +// Higher-level method M wins over lower-level conflicting methods M. + +type t1e struct { + t1 +} + +type t2e struct { + t2 +} + +// T3 has only M as top-level method. +type T3 struct { + t1e + t2e +} + +// T3.M should appear as method of T3. +func (T3) M() {} diff --git a/src/pkg/go/doc/testdata/example.go b/src/pkg/go/doc/testdata/example.go new file mode 100644 index 000000000..fdeda137e --- /dev/null +++ b/src/pkg/go/doc/testdata/example.go @@ -0,0 +1,81 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package testing + +import ( + "bytes" + "fmt" + "io" + "os" + "strings" + "time" +) + +type InternalExample struct { + Name string + F func() + Output string +} + +func RunExamples(examples []InternalExample) (ok bool) { + ok = true + + var eg InternalExample + + stdout, stderr := os.Stdout, os.Stderr + defer func() { + os.Stdout, os.Stderr = stdout, stderr + if e := recover(); e != nil { + fmt.Printf("--- FAIL: %s\npanic: %v\n", eg.Name, e) + os.Exit(1) + } + }() + + for _, eg = range examples { + if *chatty { + fmt.Printf("=== RUN: %s\n", eg.Name) + } + + // capture stdout and stderr + r, w, err := os.Pipe() + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + os.Stdout, os.Stderr = w, w + outC := make(chan string) + go func() { + buf := new(bytes.Buffer) + _, err := io.Copy(buf, r) + if err != nil { + fmt.Fprintf(stderr, "testing: copying pipe: %v\n", err) + os.Exit(1) + } + outC <- buf.String() + }() + + // run example + t0 := time.Now() + eg.F() + dt := time.Now().Sub(t0) + + // close pipe, restore stdout/stderr, get output + w.Close() + os.Stdout, os.Stderr = stdout, stderr + out := <-outC + + // report any errors + tstr := fmt.Sprintf("(%.2f seconds)", dt.Seconds()) + if g, e := strings.TrimSpace(out), strings.TrimSpace(eg.Output); g != e { + fmt.Printf("--- FAIL: %s %s\ngot:\n%s\nwant:\n%s\n", + eg.Name, tstr, g, e) + ok = false + } else if *chatty { + fmt.Printf("--- PASS: %s %s\n", eg.Name, tstr) + } + } + + return +} diff --git a/src/pkg/go/doc/testdata/template.txt b/src/pkg/go/doc/testdata/template.txt new file mode 100644 index 000000000..32e331cdd --- /dev/null +++ b/src/pkg/go/doc/testdata/template.txt @@ -0,0 +1,65 @@ +{{synopsis .Doc}} +PACKAGE {{.Name}} + +IMPORTPATH + {{.ImportPath}} + +{{with .Imports}}IMPORTS +{{range .}} {{.}} +{{end}} +{{end}}{{/* + +*/}}FILENAMES +{{range .Filenames}} {{.}} +{{end}}{{/* + +*/}}{{with .Consts}} +CONSTANTS +{{range .}} {{synopsis .Doc}} + {{node .Decl $.FSet}} + +{{end}}{{end}}{{/* + +*/}}{{with .Vars}} +VARIABLES +{{range .}} {{synopsis .Doc}} + {{node .Decl $.FSet}} + +{{end}}{{end}}{{/* + +*/}}{{with .Funcs}} +FUNCTIONS +{{range .}} {{synopsis .Doc}} + {{node .Decl $.FSet}} + +{{end}}{{end}}{{/* + +*/}}{{with .Types}} +TYPES +{{range .}} {{synopsis .Doc}} + {{node .Decl $.FSet}} + +{{range .Consts}} {{synopsis .Doc}} + {{node .Decl $.FSet}} + +{{end}}{{/* + +*/}}{{range .Vars}} {{synopsis .Doc}} + {{node .Decl $.FSet}} + +{{end}}{{/* + +*/}}{{range .Funcs}} {{synopsis .Doc}} + {{node .Decl $.FSet}} + +{{end}}{{/* + +*/}}{{range .Methods}} {{synopsis .Doc}} + {{node .Decl $.FSet}} + +{{end}}{{end}}{{end}}{{/* + +*/}}{{with .Bugs}} +BUGS +{{range .}} {{synopsis .}} +{{end}}{{end}}
\ No newline at end of file diff --git a/src/pkg/go/doc/testdata/testing.0.golden b/src/pkg/go/doc/testdata/testing.0.golden new file mode 100644 index 000000000..15a903986 --- /dev/null +++ b/src/pkg/go/doc/testdata/testing.0.golden @@ -0,0 +1,156 @@ +// Package testing provides support for automated testing of Go ... +PACKAGE testing + +IMPORTPATH + testdata/testing + +IMPORTS + bytes + flag + fmt + io + os + runtime + runtime/pprof + strconv + strings + time + +FILENAMES + testdata/benchmark.go + testdata/example.go + testdata/testing.go + +FUNCTIONS + // An internal function but exported because it is cross-package; ... + func Main(matchString func(pat, str string) (bool, error), tests []InternalTest, benchmarks []InternalBenchmark, examples []InternalExample) + + // An internal function but exported because it is cross-package; ... + func RunBenchmarks(matchString func(pat, str string) (bool, error), benchmarks []InternalBenchmark) + + // + func RunExamples(examples []InternalExample) (ok bool) + + // + func RunTests(matchString func(pat, str string) (bool, error), tests []InternalTest) (ok bool) + + // Short reports whether the -test.short flag is set. + func Short() bool + + +TYPES + // B is a type passed to Benchmark functions to manage benchmark ... + type B struct { + N int + // contains filtered or unexported fields + } + + // Error is equivalent to Log() followed by Fail(). + func (c *B) Error(args ...interface{}) + + // Errorf is equivalent to Logf() followed by Fail(). + func (c *B) Errorf(format string, args ...interface{}) + + // Fail marks the function as having failed but continues ... + func (c *B) Fail() + + // FailNow marks the function as having failed and stops its ... + func (c *B) FailNow() + + // Failed returns whether the function has failed. + func (c *B) Failed() bool + + // Fatal is equivalent to Log() followed by FailNow(). + func (c *B) Fatal(args ...interface{}) + + // Fatalf is equivalent to Logf() followed by FailNow(). + func (c *B) Fatalf(format string, args ...interface{}) + + // Log formats its arguments using default formatting, analogous ... + func (c *B) Log(args ...interface{}) + + // Logf formats its arguments according to the format, analogous ... + func (c *B) Logf(format string, args ...interface{}) + + // ResetTimer sets the elapsed benchmark time to zero. It does not ... + func (b *B) ResetTimer() + + // SetBytes records the number of bytes processed in a single ... + func (b *B) SetBytes(n int64) + + // StartTimer starts timing a test. This function is called ... + func (b *B) StartTimer() + + // StopTimer stops timing a test. This can be used to pause the ... + func (b *B) StopTimer() + + // The results of a benchmark run. + type BenchmarkResult struct { + N int // The number of iterations. + T time.Duration // The total time taken. + Bytes int64 // Bytes processed in one iteration. + } + + // Benchmark benchmarks a single function. Useful for creating ... + func Benchmark(f func(b *B)) BenchmarkResult + + // + func (r BenchmarkResult) NsPerOp() int64 + + // + func (r BenchmarkResult) String() string + + // An internal type but exported because it is cross-package; part ... + type InternalBenchmark struct { + Name string + F func(b *B) + } + + // + type InternalExample struct { + Name string + F func() + Output string + } + + // An internal type but exported because it is cross-package; part ... + type InternalTest struct { + Name string + F func(*T) + } + + // T is a type passed to Test functions to manage test state and ... + type T struct { + // contains filtered or unexported fields + } + + // Error is equivalent to Log() followed by Fail(). + func (c *T) Error(args ...interface{}) + + // Errorf is equivalent to Logf() followed by Fail(). + func (c *T) Errorf(format string, args ...interface{}) + + // Fail marks the function as having failed but continues ... + func (c *T) Fail() + + // FailNow marks the function as having failed and stops its ... + func (c *T) FailNow() + + // Failed returns whether the function has failed. + func (c *T) Failed() bool + + // Fatal is equivalent to Log() followed by FailNow(). + func (c *T) Fatal(args ...interface{}) + + // Fatalf is equivalent to Logf() followed by FailNow(). + func (c *T) Fatalf(format string, args ...interface{}) + + // Log formats its arguments using default formatting, analogous ... + func (c *T) Log(args ...interface{}) + + // Logf formats its arguments according to the format, analogous ... + func (c *T) Logf(format string, args ...interface{}) + + // Parallel signals that this test is to be run in parallel with ... + func (t *T) Parallel() + diff --git a/src/pkg/go/doc/testdata/testing.1.golden b/src/pkg/go/doc/testdata/testing.1.golden new file mode 100644 index 000000000..1f92f8fe3 --- /dev/null +++ b/src/pkg/go/doc/testdata/testing.1.golden @@ -0,0 +1,298 @@ +// Package testing provides support for automated testing of Go ... +PACKAGE testing + +IMPORTPATH + testdata/testing + +IMPORTS + bytes + flag + fmt + io + os + runtime + runtime/pprof + strconv + strings + time + +FILENAMES + testdata/benchmark.go + testdata/example.go + testdata/testing.go + +VARIABLES + // + var ( + // The short flag requests that tests run more quickly, but its functionality + // is provided by test writers themselves. The testing package is just its + // home. The all.bash installation script sets it to make installation more + // efficient, but by default the flag is off so a plain "gotest" will do a + // full test of the package. + short = flag.Bool("test.short", false, "run smaller test suite to save time") + + // Report as tests are run; default is silent for success. + chatty = flag.Bool("test.v", false, "verbose: print additional output") + match = flag.String("test.run", "", "regular expression to select tests to run") + memProfile = flag.String("test.memprofile", "", "write a memory profile to the named file after execution") + memProfileRate = flag.Int("test.memprofilerate", 0, "if >=0, sets runtime.MemProfileRate") + cpuProfile = flag.String("test.cpuprofile", "", "write a cpu profile to the named file during execution") + timeout = flag.Duration("test.timeout", 0, "if positive, sets an aggregate time limit for all tests") + cpuListStr = flag.String("test.cpu", "", "comma-separated list of number of CPUs to use for each test") + parallel = flag.Int("test.parallel", runtime.GOMAXPROCS(0), "maximum test parallelism") + + cpuList []int + ) + + // + var benchTime = flag.Float64("test.benchtime", 1, "approximate run time for each benchmark, in seconds") + + // + var matchBenchmarks = flag.String("test.bench", "", "regular expression to select benchmarks to run") + + // + var timer *time.Timer + + +FUNCTIONS + // An internal function but exported because it is cross-package; ... + func Main(matchString func(pat, str string) (bool, error), tests []InternalTest, benchmarks []InternalBenchmark, examples []InternalExample) + + // An internal function but exported because it is cross-package; ... + func RunBenchmarks(matchString func(pat, str string) (bool, error), benchmarks []InternalBenchmark) + + // + func RunExamples(examples []InternalExample) (ok bool) + + // + func RunTests(matchString func(pat, str string) (bool, error), tests []InternalTest) (ok bool) + + // Short reports whether the -test.short flag is set. + func Short() bool + + // after runs after all testing. + func after() + + // alarm is called if the timeout expires. + func alarm() + + // before runs before all testing. + func before() + + // decorate inserts the final newline if needed and indentation ... + func decorate(s string, addFileLine bool) string + + // + func max(x, y int) int + + // + func min(x, y int) int + + // + func parseCpuList() + + // roundDown10 rounds a number down to the nearest power of 10. + func roundDown10(n int) int + + // roundUp rounds x up to a number of the form [1eX, 2eX, 5eX]. + func roundUp(n int) int + + // startAlarm starts an alarm if requested. + func startAlarm() + + // stopAlarm turns off the alarm. + func stopAlarm() + + // + func tRunner(t *T, test *InternalTest) + + +TYPES + // B is a type passed to Benchmark functions to manage benchmark ... + type B struct { + common + N int + benchmark InternalBenchmark + bytes int64 + timerOn bool + result BenchmarkResult + } + + // Error is equivalent to Log() followed by Fail(). + func (c *B) Error(args ...interface{}) + + // Errorf is equivalent to Logf() followed by Fail(). + func (c *B) Errorf(format string, args ...interface{}) + + // Fail marks the function as having failed but continues ... + func (c *B) Fail() + + // FailNow marks the function as having failed and stops its ... + func (c *B) FailNow() + + // Failed returns whether the function has failed. + func (c *B) Failed() bool + + // Fatal is equivalent to Log() followed by FailNow(). + func (c *B) Fatal(args ...interface{}) + + // Fatalf is equivalent to Logf() followed by FailNow(). + func (c *B) Fatalf(format string, args ...interface{}) + + // Log formats its arguments using default formatting, analogous ... + func (c *B) Log(args ...interface{}) + + // Logf formats its arguments according to the format, analogous ... + func (c *B) Logf(format string, args ...interface{}) + + // ResetTimer sets the elapsed benchmark time to zero. It does not ... + func (b *B) ResetTimer() + + // SetBytes records the number of bytes processed in a single ... + func (b *B) SetBytes(n int64) + + // StartTimer starts timing a test. This function is called ... + func (b *B) StartTimer() + + // StopTimer stops timing a test. This can be used to pause the ... + func (b *B) StopTimer() + + // launch launches the benchmark function. It gradually increases ... + func (b *B) launch() + + // log generates the output. It's always at the same stack depth. + func (c *B) log(s string) + + // + func (b *B) nsPerOp() int64 + + // run times the benchmark function in a separate goroutine. + func (b *B) run() BenchmarkResult + + // runN runs a single benchmark for the specified number of ... + func (b *B) runN(n int) + + // trimOutput shortens the output from a benchmark, which can be ... + func (b *B) trimOutput() + + // The results of a benchmark run. + type BenchmarkResult struct { + N int // The number of iterations. + T time.Duration // The total time taken. + Bytes int64 // Bytes processed in one iteration. + } + + // Benchmark benchmarks a single function. Useful for creating ... + func Benchmark(f func(b *B)) BenchmarkResult + + // + func (r BenchmarkResult) NsPerOp() int64 + + // + func (r BenchmarkResult) String() string + + // + func (r BenchmarkResult) mbPerSec() float64 + + // An internal type but exported because it is cross-package; part ... + type InternalBenchmark struct { + Name string + F func(b *B) + } + + // + type InternalExample struct { + Name string + F func() + Output string + } + + // An internal type but exported because it is cross-package; part ... + type InternalTest struct { + Name string + F func(*T) + } + + // T is a type passed to Test functions to manage test state and ... + type T struct { + common + name string // Name of test. + startParallel chan bool // Parallel tests will wait on this. + } + + // Error is equivalent to Log() followed by Fail(). + func (c *T) Error(args ...interface{}) + + // Errorf is equivalent to Logf() followed by Fail(). + func (c *T) Errorf(format string, args ...interface{}) + + // Fail marks the function as having failed but continues ... + func (c *T) Fail() + + // FailNow marks the function as having failed and stops its ... + func (c *T) FailNow() + + // Failed returns whether the function has failed. + func (c *T) Failed() bool + + // Fatal is equivalent to Log() followed by FailNow(). + func (c *T) Fatal(args ...interface{}) + + // Fatalf is equivalent to Logf() followed by FailNow(). + func (c *T) Fatalf(format string, args ...interface{}) + + // Log formats its arguments using default formatting, analogous ... + func (c *T) Log(args ...interface{}) + + // Logf formats its arguments according to the format, analogous ... + func (c *T) Logf(format string, args ...interface{}) + + // Parallel signals that this test is to be run in parallel with ... + func (t *T) Parallel() + + // log generates the output. It's always at the same stack depth. + func (c *T) log(s string) + + // + func (t *T) report() + + // common holds the elements common between T and B and captures ... + type common struct { + output []byte // Output generated by test or benchmark. + failed bool // Test or benchmark has failed. + start time.Time // Time test or benchmark started + duration time.Duration + self interface{} // To be sent on signal channel when done. + signal chan interface{} // Output for serial tests. + } + + // Error is equivalent to Log() followed by Fail(). + func (c *common) Error(args ...interface{}) + + // Errorf is equivalent to Logf() followed by Fail(). + func (c *common) Errorf(format string, args ...interface{}) + + // Fail marks the function as having failed but continues ... + func (c *common) Fail() + + // FailNow marks the function as having failed and stops its ... + func (c *common) FailNow() + + // Failed returns whether the function has failed. + func (c *common) Failed() bool + + // Fatal is equivalent to Log() followed by FailNow(). + func (c *common) Fatal(args ...interface{}) + + // Fatalf is equivalent to Logf() followed by FailNow(). + func (c *common) Fatalf(format string, args ...interface{}) + + // Log formats its arguments using default formatting, analogous ... + func (c *common) Log(args ...interface{}) + + // Logf formats its arguments according to the format, analogous ... + func (c *common) Logf(format string, args ...interface{}) + + // log generates the output. It's always at the same stack depth. + func (c *common) log(s string) + diff --git a/src/pkg/go/doc/testdata/testing.go b/src/pkg/go/doc/testdata/testing.go new file mode 100644 index 000000000..cfe212dc1 --- /dev/null +++ b/src/pkg/go/doc/testdata/testing.go @@ -0,0 +1,404 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package testing provides support for automated testing of Go packages. +// It is intended to be used in concert with the ``gotest'' utility, which automates +// execution of any function of the form +// func TestXxx(*testing.T) +// where Xxx can be any alphanumeric string (but the first letter must not be in +// [a-z]) and serves to identify the test routine. +// These TestXxx routines should be declared within the package they are testing. +// +// Functions of the form +// func BenchmarkXxx(*testing.B) +// are considered benchmarks, and are executed by gotest when the -test.bench +// flag is provided. +// +// A sample benchmark function looks like this: +// func BenchmarkHello(b *testing.B) { +// for i := 0; i < b.N; i++ { +// fmt.Sprintf("hello") +// } +// } +// The benchmark package will vary b.N until the benchmark function lasts +// long enough to be timed reliably. The output +// testing.BenchmarkHello 10000000 282 ns/op +// means that the loop ran 10000000 times at a speed of 282 ns per loop. +// +// If a benchmark needs some expensive setup before running, the timer +// may be stopped: +// func BenchmarkBigLen(b *testing.B) { +// b.StopTimer() +// big := NewBig() +// b.StartTimer() +// for i := 0; i < b.N; i++ { +// big.Len() +// } +// } +package testing + +import ( + "flag" + "fmt" + "os" + "runtime" + "runtime/pprof" + "strconv" + "strings" + "time" +) + +var ( + // The short flag requests that tests run more quickly, but its functionality + // is provided by test writers themselves. The testing package is just its + // home. The all.bash installation script sets it to make installation more + // efficient, but by default the flag is off so a plain "gotest" will do a + // full test of the package. + short = flag.Bool("test.short", false, "run smaller test suite to save time") + + // Report as tests are run; default is silent for success. + chatty = flag.Bool("test.v", false, "verbose: print additional output") + match = flag.String("test.run", "", "regular expression to select tests to run") + memProfile = flag.String("test.memprofile", "", "write a memory profile to the named file after execution") + memProfileRate = flag.Int("test.memprofilerate", 0, "if >=0, sets runtime.MemProfileRate") + cpuProfile = flag.String("test.cpuprofile", "", "write a cpu profile to the named file during execution") + timeout = flag.Duration("test.timeout", 0, "if positive, sets an aggregate time limit for all tests") + cpuListStr = flag.String("test.cpu", "", "comma-separated list of number of CPUs to use for each test") + parallel = flag.Int("test.parallel", runtime.GOMAXPROCS(0), "maximum test parallelism") + + cpuList []int +) + +// common holds the elements common between T and B and +// captures common methods such as Errorf. +type common struct { + output []byte // Output generated by test or benchmark. + failed bool // Test or benchmark has failed. + start time.Time // Time test or benchmark started + duration time.Duration + self interface{} // To be sent on signal channel when done. + signal chan interface{} // Output for serial tests. +} + +// Short reports whether the -test.short flag is set. +func Short() bool { + return *short +} + +// decorate inserts the final newline if needed and indentation tabs for formatting. +// If addFileLine is true, it also prefixes the string with the file and line of the call site. +func decorate(s string, addFileLine bool) string { + if addFileLine { + _, file, line, ok := runtime.Caller(3) // decorate + log + public function. + if ok { + // Truncate file name at last file name separator. + if index := strings.LastIndex(file, "/"); index >= 0 { + file = file[index+1:] + } else if index = strings.LastIndex(file, "\\"); index >= 0 { + file = file[index+1:] + } + } else { + file = "???" + line = 1 + } + s = fmt.Sprintf("%s:%d: %s", file, line, s) + } + s = "\t" + s // Every line is indented at least one tab. + n := len(s) + if n > 0 && s[n-1] != '\n' { + s += "\n" + n++ + } + for i := 0; i < n-1; i++ { // -1 to avoid final newline + if s[i] == '\n' { + // Second and subsequent lines are indented an extra tab. + return s[0:i+1] + "\t" + decorate(s[i+1:n], false) + } + } + return s +} + +// T is a type passed to Test functions to manage test state and support formatted test logs. +// Logs are accumulated during execution and dumped to standard error when done. +type T struct { + common + name string // Name of test. + startParallel chan bool // Parallel tests will wait on this. +} + +// Fail marks the function as having failed but continues execution. +func (c *common) Fail() { c.failed = true } + +// Failed returns whether the function has failed. +func (c *common) Failed() bool { return c.failed } + +// FailNow marks the function as having failed and stops its execution. +// Execution will continue at the next Test. +func (c *common) FailNow() { + c.Fail() + + // Calling runtime.Goexit will exit the goroutine, which + // will run the deferred functions in this goroutine, + // which will eventually run the deferred lines in tRunner, + // which will signal to the test loop that this test is done. + // + // A previous version of this code said: + // + // c.duration = ... + // c.signal <- c.self + // runtime.Goexit() + // + // This previous version duplicated code (those lines are in + // tRunner no matter what), but worse the goroutine teardown + // implicit in runtime.Goexit was not guaranteed to complete + // before the test exited. If a test deferred an important cleanup + // function (like removing temporary files), there was no guarantee + // it would run on a test failure. Because we send on c.signal during + // a top-of-stack deferred function now, we know that the send + // only happens after any other stacked defers have completed. + runtime.Goexit() +} + +// log generates the output. It's always at the same stack depth. +func (c *common) log(s string) { + c.output = append(c.output, decorate(s, true)...) +} + +// Log formats its arguments using default formatting, analogous to Println(), +// and records the text in the error log. +func (c *common) Log(args ...interface{}) { c.log(fmt.Sprintln(args...)) } + +// Logf formats its arguments according to the format, analogous to Printf(), +// and records the text in the error log. +func (c *common) Logf(format string, args ...interface{}) { c.log(fmt.Sprintf(format, args...)) } + +// Error is equivalent to Log() followed by Fail(). +func (c *common) Error(args ...interface{}) { + c.log(fmt.Sprintln(args...)) + c.Fail() +} + +// Errorf is equivalent to Logf() followed by Fail(). +func (c *common) Errorf(format string, args ...interface{}) { + c.log(fmt.Sprintf(format, args...)) + c.Fail() +} + +// Fatal is equivalent to Log() followed by FailNow(). +func (c *common) Fatal(args ...interface{}) { + c.log(fmt.Sprintln(args...)) + c.FailNow() +} + +// Fatalf is equivalent to Logf() followed by FailNow(). +func (c *common) Fatalf(format string, args ...interface{}) { + c.log(fmt.Sprintf(format, args...)) + c.FailNow() +} + +// Parallel signals that this test is to be run in parallel with (and only with) +// other parallel tests in this CPU group. +func (t *T) Parallel() { + t.signal <- (*T)(nil) // Release main testing loop + <-t.startParallel // Wait for serial tests to finish +} + +// An internal type but exported because it is cross-package; part of the implementation +// of gotest. +type InternalTest struct { + Name string + F func(*T) +} + +func tRunner(t *T, test *InternalTest) { + t.start = time.Now() + + // When this goroutine is done, either because test.F(t) + // returned normally or because a test failure triggered + // a call to runtime.Goexit, record the duration and send + // a signal saying that the test is done. + defer func() { + t.duration = time.Now().Sub(t.start) + t.signal <- t + }() + + test.F(t) +} + +// An internal function but exported because it is cross-package; part of the implementation +// of gotest. +func Main(matchString func(pat, str string) (bool, error), tests []InternalTest, benchmarks []InternalBenchmark, examples []InternalExample) { + flag.Parse() + parseCpuList() + + before() + startAlarm() + testOk := RunTests(matchString, tests) + exampleOk := RunExamples(examples) + if !testOk || !exampleOk { + fmt.Println("FAIL") + os.Exit(1) + } + fmt.Println("PASS") + stopAlarm() + RunBenchmarks(matchString, benchmarks) + after() +} + +func (t *T) report() { + tstr := fmt.Sprintf("(%.2f seconds)", t.duration.Seconds()) + format := "--- %s: %s %s\n%s" + if t.failed { + fmt.Printf(format, "FAIL", t.name, tstr, t.output) + } else if *chatty { + fmt.Printf(format, "PASS", t.name, tstr, t.output) + } +} + +func RunTests(matchString func(pat, str string) (bool, error), tests []InternalTest) (ok bool) { + ok = true + if len(tests) == 0 { + fmt.Fprintln(os.Stderr, "testing: warning: no tests to run") + return + } + for _, procs := range cpuList { + runtime.GOMAXPROCS(procs) + // We build a new channel tree for each run of the loop. + // collector merges in one channel all the upstream signals from parallel tests. + // If all tests pump to the same channel, a bug can occur where a test + // kicks off a goroutine that Fails, yet the test still delivers a completion signal, + // which skews the counting. + var collector = make(chan interface{}) + + numParallel := 0 + startParallel := make(chan bool) + + for i := 0; i < len(tests); i++ { + matched, err := matchString(*match, tests[i].Name) + if err != nil { + fmt.Fprintf(os.Stderr, "testing: invalid regexp for -test.run: %s\n", err) + os.Exit(1) + } + if !matched { + continue + } + testName := tests[i].Name + if procs != 1 { + testName = fmt.Sprintf("%s-%d", tests[i].Name, procs) + } + t := &T{ + common: common{ + signal: make(chan interface{}), + }, + name: testName, + startParallel: startParallel, + } + t.self = t + if *chatty { + fmt.Printf("=== RUN %s\n", t.name) + } + go tRunner(t, &tests[i]) + out := (<-t.signal).(*T) + if out == nil { // Parallel run. + go func() { + collector <- <-t.signal + }() + numParallel++ + continue + } + t.report() + ok = ok && !out.failed + } + + running := 0 + for numParallel+running > 0 { + if running < *parallel && numParallel > 0 { + startParallel <- true + running++ + numParallel-- + continue + } + t := (<-collector).(*T) + t.report() + ok = ok && !t.failed + running-- + } + } + return +} + +// before runs before all testing. +func before() { + if *memProfileRate > 0 { + runtime.MemProfileRate = *memProfileRate + } + if *cpuProfile != "" { + f, err := os.Create(*cpuProfile) + if err != nil { + fmt.Fprintf(os.Stderr, "testing: %s", err) + return + } + if err := pprof.StartCPUProfile(f); err != nil { + fmt.Fprintf(os.Stderr, "testing: can't start cpu profile: %s", err) + f.Close() + return + } + // Could save f so after can call f.Close; not worth the effort. + } + +} + +// after runs after all testing. +func after() { + if *cpuProfile != "" { + pprof.StopCPUProfile() // flushes profile to disk + } + if *memProfile != "" { + f, err := os.Create(*memProfile) + if err != nil { + fmt.Fprintf(os.Stderr, "testing: %s", err) + return + } + if err = pprof.WriteHeapProfile(f); err != nil { + fmt.Fprintf(os.Stderr, "testing: can't write %s: %s", *memProfile, err) + } + f.Close() + } +} + +var timer *time.Timer + +// startAlarm starts an alarm if requested. +func startAlarm() { + if *timeout > 0 { + timer = time.AfterFunc(*timeout, alarm) + } +} + +// stopAlarm turns off the alarm. +func stopAlarm() { + if *timeout > 0 { + timer.Stop() + } +} + +// alarm is called if the timeout expires. +func alarm() { + panic("test timed out") +} + +func parseCpuList() { + if len(*cpuListStr) == 0 { + cpuList = append(cpuList, runtime.GOMAXPROCS(-1)) + } else { + for _, val := range strings.Split(*cpuListStr, ",") { + cpu, err := strconv.Atoi(val) + if err != nil || cpu <= 0 { + fmt.Fprintf(os.Stderr, "testing: invalid value %q for -test.cpu", val) + os.Exit(1) + } + cpuList = append(cpuList, cpu) + } + } +} diff --git a/src/pkg/go/parser/interface.go b/src/pkg/go/parser/interface.go index 4f980fc65..f1b4ce34d 100644 --- a/src/pkg/go/parser/interface.go +++ b/src/pkg/go/parser/interface.go @@ -8,8 +8,8 @@ package parser import ( "bytes" + "errors" "go/ast" - "go/scanner" "go/token" "io" "io/ioutil" @@ -21,7 +21,7 @@ import ( // otherwise it returns an error. If src == nil, readSource returns // the result of reading the file specified by filename. // -func readSource(filename string, src interface{}) ([]byte, os.Error) { +func readSource(filename string, src interface{}) ([]byte, error) { if src != nil { switch s := src.(type) { case string: @@ -35,86 +35,30 @@ func readSource(filename string, src interface{}) ([]byte, os.Error) { } case io.Reader: var buf bytes.Buffer - _, err := io.Copy(&buf, s) - if err != nil { + if _, err := io.Copy(&buf, s); err != nil { return nil, err } return buf.Bytes(), nil - default: - return nil, os.NewError("invalid source") } + return nil, errors.New("invalid source") } - return ioutil.ReadFile(filename) } -func (p *parser) errors() os.Error { - mode := scanner.Sorted - if p.mode&SpuriousErrors == 0 { - mode = scanner.NoMultiples - } - return p.GetError(mode) -} - -// ParseExpr parses a Go expression and returns the corresponding -// AST node. The fset, filename, and src arguments have the same interpretation -// as for ParseFile. If there is an error, the result expression -// may be nil or contain a partial AST. -// -func ParseExpr(fset *token.FileSet, filename string, src interface{}) (ast.Expr, os.Error) { - data, err := readSource(filename, src) - if err != nil { - return nil, err - } - - var p parser - p.init(fset, filename, data, 0) - x := p.parseRhs() - if p.tok == token.SEMICOLON { - p.next() // consume automatically inserted semicolon, if any - } - p.expect(token.EOF) - - return x, p.errors() -} - -// ParseStmtList parses a list of Go statements and returns the list -// of corresponding AST nodes. The fset, filename, and src arguments have the same -// interpretation as for ParseFile. If there is an error, the node -// list may be nil or contain partial ASTs. +// A Mode value is a set of flags (or 0). +// They control the amount of source code parsed and other optional +// parser functionality. // -func ParseStmtList(fset *token.FileSet, filename string, src interface{}) ([]ast.Stmt, os.Error) { - data, err := readSource(filename, src) - if err != nil { - return nil, err - } - - var p parser - p.init(fset, filename, data, 0) - list := p.parseStmtList() - p.expect(token.EOF) - - return list, p.errors() -} - -// ParseDeclList parses a list of Go declarations and returns the list -// of corresponding AST nodes. The fset, filename, and src arguments have the same -// interpretation as for ParseFile. If there is an error, the node -// list may be nil or contain partial ASTs. -// -func ParseDeclList(fset *token.FileSet, filename string, src interface{}) ([]ast.Decl, os.Error) { - data, err := readSource(filename, src) - if err != nil { - return nil, err - } - - var p parser - p.init(fset, filename, data, 0) - list := p.parseDeclList() - p.expect(token.EOF) - - return list, p.errors() -} +type Mode uint + +const ( + PackageClauseOnly Mode = 1 << iota // parsing stops after package clause + ImportsOnly // parsing stops after import declarations + ParseComments // parse comments and add them to AST + Trace // print a trace of parsed productions + DeclarationErrors // report declaration errors + SpuriousErrors // report all (not just the first) errors per line +) // ParseFile parses the source code of a single Go source file and returns // the corresponding ast.File node. The source code may be provided via @@ -123,7 +67,6 @@ func ParseDeclList(fset *token.FileSet, filename string, src interface{}) ([]ast // If src != nil, ParseFile parses the source from src and the filename is // only used when recording position information. The type of the argument // for the src parameter must be string, []byte, or io.Reader. -// // If src == nil, ParseFile parses the file specified by filename. // // The mode parameter controls the amount of source text parsed and other @@ -132,49 +75,18 @@ func ParseDeclList(fset *token.FileSet, filename string, src interface{}) ([]ast // // If the source couldn't be read, the returned AST is nil and the error // indicates the specific failure. If the source was read but syntax -// errors were found, the result is a partial AST (with ast.BadX nodes +// errors were found, the result is a partial AST (with ast.Bad* nodes // representing the fragments of erroneous source code). Multiple errors // are returned via a scanner.ErrorList which is sorted by file position. // -func ParseFile(fset *token.FileSet, filename string, src interface{}, mode uint) (*ast.File, os.Error) { - data, err := readSource(filename, src) +func ParseFile(fset *token.FileSet, filename string, src interface{}, mode Mode) (*ast.File, error) { + text, err := readSource(filename, src) if err != nil { return nil, err } - var p parser - p.init(fset, filename, data, mode) - file := p.parseFile() // parseFile reads to EOF - - return file, p.errors() -} - -// ParseFiles calls ParseFile for each file in the filenames list and returns -// a map of package name -> package AST with all the packages found. The mode -// bits are passed to ParseFile unchanged. Position information is recorded -// in the file set fset. -// -// Files with parse errors are ignored. In this case the map of packages may -// be incomplete (missing packages and/or incomplete packages) and the first -// error encountered is returned. -// -func ParseFiles(fset *token.FileSet, filenames []string, mode uint) (pkgs map[string]*ast.Package, first os.Error) { - pkgs = make(map[string]*ast.Package) - for _, filename := range filenames { - if src, err := ParseFile(fset, filename, nil, mode); err == nil { - name := src.Name.Name - pkg, found := pkgs[name] - if !found { - // TODO(gri) Use NewPackage here; reconsider ParseFiles API. - pkg = &ast.Package{name, nil, nil, make(map[string]*ast.File)} - pkgs[name] = pkg - } - pkg.Files[filename] = src - } else if first == nil { - first = err - } - } - return + p.init(fset, filename, text, mode) + return p.parseFile(), p.errors() } // ParseDir calls ParseFile for the files in the directory specified by path and @@ -185,9 +97,9 @@ func ParseFiles(fset *token.FileSet, filenames []string, mode uint) (pkgs map[st // // If the directory couldn't be read, a nil map and the respective error are // returned. If a parse error occurred, a non-nil but incomplete map and the -// error are returned. +// first error encountered are returned. // -func ParseDir(fset *token.FileSet, path string, filter func(*os.FileInfo) bool, mode uint) (map[string]*ast.Package, os.Error) { +func ParseDir(fset *token.FileSet, path string, filter func(os.FileInfo) bool, mode Mode) (pkgs map[string]*ast.Package, first error) { fd, err := os.Open(path) if err != nil { return nil, err @@ -199,16 +111,41 @@ func ParseDir(fset *token.FileSet, path string, filter func(*os.FileInfo) bool, return nil, err } - filenames := make([]string, len(list)) - n := 0 - for i := 0; i < len(list); i++ { - d := &list[i] + pkgs = make(map[string]*ast.Package) + for _, d := range list { if filter == nil || filter(d) { - filenames[n] = filepath.Join(path, d.Name) - n++ + filename := filepath.Join(path, d.Name()) + if src, err := ParseFile(fset, filename, nil, mode); err == nil { + name := src.Name.Name + pkg, found := pkgs[name] + if !found { + pkg = &ast.Package{ + Name: name, + Files: make(map[string]*ast.File), + } + pkgs[name] = pkg + } + pkg.Files[filename] = src + } else if first == nil { + first = err + } } } - filenames = filenames[0:n] - return ParseFiles(fset, filenames, mode) + return +} + +// ParseExpr is a convenience function for obtaining the AST of an expression x. +// The position information recorded in the AST is undefined. +// +func ParseExpr(x string) (ast.Expr, error) { + // parse x within the context of a complete package for correct scopes; + // use //line directive for correct positions in error messages and put + // x alone on a separate line (handles line comments), followed by a ';' + // to force an error if the expression is incomplete + file, err := ParseFile(token.NewFileSet(), "", "package p;func _(){_=\n//line :1\n"+x+"\n;}", 0) + if err != nil { + return nil, err + } + return file.Decls[0].(*ast.FuncDecl).Body.List[0].(*ast.AssignStmt).Rhs[0], nil } diff --git a/src/pkg/go/parser/parser.go b/src/pkg/go/parser/parser.go index be82b2f80..6bee8de9f 100644 --- a/src/pkg/go/parser/parser.go +++ b/src/pkg/go/parser/parser.go @@ -16,19 +16,6 @@ import ( "go/token" ) -// The mode parameter to the Parse* functions is a set of flags (or 0). -// They control the amount of source code parsed and other optional -// parser functionality. -// -const ( - PackageClauseOnly uint = 1 << iota // parsing stops after package clause - ImportsOnly // parsing stops after import declarations - ParseComments // parse comments and add them to AST - Trace // print a trace of parsed productions - DeclarationErrors // report declaration errors - SpuriousErrors // report all (not just the first) errors per line -) - // The parser structure holds the parser's internal state. type parser struct { file *token.File @@ -36,7 +23,7 @@ type parser struct { scanner scanner.Scanner // Tracing/debugging - mode uint // parsing mode + mode Mode // parsing mode trace bool // == (mode & Trace != 0) indent uint // indentation used for tracing output @@ -65,18 +52,13 @@ type parser struct { targetStack [][]*ast.Ident // stack of unresolved labels } -// scannerMode returns the scanner mode bits given the parser's mode bits. -func scannerMode(mode uint) uint { - var m uint = scanner.InsertSemis +func (p *parser) init(fset *token.FileSet, filename string, src []byte, mode Mode) { + p.file = fset.AddFile(filename, fset.Base(), len(src)) + var m scanner.Mode if mode&ParseComments != 0 { - m |= scanner.ScanComments + m = scanner.ScanComments } - return m -} - -func (p *parser) init(fset *token.FileSet, filename string, src []byte, mode uint) { - p.file = fset.AddFile(filename, fset.Base(), len(src)) - p.scanner.Init(p.file, src, p, scannerMode(mode)) + p.scanner.Init(p.file, src, p, m) p.mode = mode p.trace = mode&Trace != 0 // for convenience (p.trace is used frequently) @@ -92,6 +74,14 @@ func (p *parser) init(fset *token.FileSet, filename string, src []byte, mode uin p.openLabelScope() } +func (p *parser) errors() error { + m := scanner.Sorted + if p.mode&SpuriousErrors == 0 { + m = scanner.NoMultiples + } + return p.GetError(m) +} + // ---------------------------------------------------------------------------- // Scoping support @@ -144,28 +134,31 @@ func (p *parser) declare(decl, data interface{}, scope *ast.Scope, kind ast.ObjK } } -func (p *parser) shortVarDecl(idents []*ast.Ident) { +func (p *parser) shortVarDecl(decl *ast.AssignStmt, list []ast.Expr) { // Go spec: A short variable declaration may redeclare variables // provided they were originally declared in the same block with // the same type, and at least one of the non-blank variables is new. n := 0 // number of new variables - for _, ident := range idents { - assert(ident.Obj == nil, "identifier already declared or resolved") - obj := ast.NewObj(ast.Var, ident.Name) - // short var declarations cannot have redeclaration errors - // and are not global => no need to remember the respective - // declaration - ident.Obj = obj - if ident.Name != "_" { - if alt := p.topScope.Insert(obj); alt != nil { - ident.Obj = alt // redeclaration - } else { - n++ // new declaration + for _, x := range list { + if ident, isIdent := x.(*ast.Ident); isIdent { + assert(ident.Obj == nil, "identifier already declared or resolved") + obj := ast.NewObj(ast.Var, ident.Name) + // remember corresponding assignment for other tools + obj.Decl = decl + ident.Obj = obj + if ident.Name != "_" { + if alt := p.topScope.Insert(obj); alt != nil { + ident.Obj = alt // redeclaration + } else { + n++ // new declaration + } } + } else { + p.errorExpected(x.Pos(), "identifier") } } if n == 0 && p.mode&DeclarationErrors != 0 { - p.error(idents[0].Pos(), "no new variables on left side of :=") + p.error(list[0].Pos(), "no new variables on left side of :=") } } @@ -434,7 +427,9 @@ func (p *parser) parseLhsList() []ast.Expr { switch p.tok { case token.DEFINE: // lhs of a short variable declaration - p.shortVarDecl(p.makeIdentList(list)) + // but doesn't enter scope until later: + // caller must call p.shortVarDecl(p.makeIdentList(list)) + // at appropriate time. case token.COLON: // lhs of a label declaration or a communication clause of a select // statement (parseLhsList is not called when parsing the case clause @@ -520,7 +515,7 @@ func (p *parser) makeIdentList(list []ast.Expr) []*ast.Ident { for i, x := range list { ident, isIdent := x.(*ast.Ident) if !isIdent { - pos := x.(ast.Expr).Pos() + pos := x.Pos() p.errorExpected(pos, "identifier") ident = &ast.Ident{pos, "_", nil} } @@ -1129,7 +1124,7 @@ func (p *parser) parseLiteralValue(typ ast.Expr) ast.Expr { // checkExpr checks that x is an expression (and not a type). func (p *parser) checkExpr(x ast.Expr) ast.Expr { - switch t := unparen(x).(type) { + switch unparen(x).(type) { case *ast.BadExpr: case *ast.Ident: case *ast.BasicLit: @@ -1398,7 +1393,11 @@ func (p *parser) parseSimpleStmt(mode int) (ast.Stmt, bool) { } else { y = p.parseRhsList() } - return &ast.AssignStmt{x, pos, tok, y}, isRange + as := &ast.AssignStmt{x, pos, tok, y} + if tok == token.DEFINE { + p.shortVarDecl(as, x) + } + return as, isRange } if len(x) > 1 { @@ -1710,31 +1709,28 @@ func (p *parser) parseCommClause() *ast.CommClause { comm = &ast.SendStmt{lhs[0], arrow, rhs} } else { // RecvStmt - pos := p.pos - tok := p.tok - var rhs ast.Expr - if tok == token.ASSIGN || tok == token.DEFINE { + if tok := p.tok; tok == token.ASSIGN || tok == token.DEFINE { // RecvStmt with assignment if len(lhs) > 2 { p.errorExpected(lhs[0].Pos(), "1 or 2 expressions") // continue with first two expressions lhs = lhs[0:2] } + pos := p.pos p.next() - rhs = p.parseRhs() + rhs := p.parseRhs() + as := &ast.AssignStmt{lhs, pos, tok, []ast.Expr{rhs}} + if tok == token.DEFINE { + p.shortVarDecl(as, lhs) + } + comm = as } else { - // rhs must be single receive operation + // lhs must be single receive operation if len(lhs) > 1 { p.errorExpected(lhs[0].Pos(), "1 expression") // continue with first expression } - rhs = lhs[0] - lhs = nil // there is no lhs - } - if lhs != nil { - comm = &ast.AssignStmt{lhs, pos, tok, []ast.Expr{rhs}} - } else { - comm = &ast.ExprStmt{rhs} + comm = &ast.ExprStmt{lhs[0]} } } } else { @@ -1909,7 +1905,7 @@ func parseImportSpec(p *parser, doc *ast.CommentGroup, _ int) ast.Spec { p.expectSemi() // call before accessing p.linecomment // collect imports - spec := &ast.ImportSpec{doc, ident, path, p.lineComment} + spec := &ast.ImportSpec{doc, ident, path, p.lineComment, token.NoPos} p.imports = append(p.imports, spec) return spec @@ -2018,7 +2014,7 @@ func (p *parser) parseReceiver(scope *ast.Scope) *ast.FieldList { // must have exactly one receiver if par.NumFields() != 1 { p.errorExpected(par.Opening, "exactly one receiver") - par.List = []*ast.Field{&ast.Field{Type: &ast.BadExpr{par.Opening, par.Closing + 1}}} + par.List = []*ast.Field{{Type: &ast.BadExpr{par.Opening, par.Closing + 1}}} return par } @@ -2027,7 +2023,7 @@ func (p *parser) parseReceiver(scope *ast.Scope) *ast.FieldList { base := deref(recv.Type) if _, isIdent := base.(*ast.Ident); !isIdent { p.errorExpected(base.Pos(), "(unqualified) identifier") - par.List = []*ast.Field{&ast.Field{Type: &ast.BadExpr{recv.Pos(), recv.End()}}} + par.List = []*ast.Field{{Type: &ast.BadExpr{recv.Pos(), recv.End()}}} } return par @@ -2103,18 +2099,6 @@ func (p *parser) parseDecl() ast.Decl { return p.parseGenDecl(p.tok, f) } -func (p *parser) parseDeclList() (list []ast.Decl) { - if p.trace { - defer un(trace(p, "DeclList")) - } - - for p.tok != token.EOF { - list = append(list, p.parseDecl()) - } - - return -} - // ---------------------------------------------------------------------------- // Source files diff --git a/src/pkg/go/parser/parser_test.go b/src/pkg/go/parser/parser_test.go index 9705dcff2..a3ee8525d 100644 --- a/src/pkg/go/parser/parser_test.go +++ b/src/pkg/go/parser/parser_test.go @@ -5,6 +5,7 @@ package parser import ( + "go/ast" "go/token" "os" "testing" @@ -53,7 +54,7 @@ func TestParseIllegalInputs(t *testing.T) { } } -var validPrograms = []interface{}{ +var validPrograms = []string{ "package p\n", `package p;`, `package p; import "fmt"; func f() { fmt.Println("Hello, World!") };`, @@ -112,7 +113,7 @@ func nameFilter(filename string) bool { return true } -func dirFilter(f *os.FileInfo) bool { return nameFilter(f.Name) } +func dirFilter(f os.FileInfo) bool { return nameFilter(f.Name()) } func TestParse4(t *testing.T) { path := "." @@ -134,3 +135,72 @@ func TestParse4(t *testing.T) { } } } + +func TestParseExpr(t *testing.T) { + // just kicking the tires: + // a valid expression + src := "a + b" + x, err := ParseExpr(src) + if err != nil { + t.Errorf("ParseExpr(%s): %v", src, err) + } + // sanity check + if _, ok := x.(*ast.BinaryExpr); !ok { + t.Errorf("ParseExpr(%s): got %T, expected *ast.BinaryExpr", src, x) + } + + // an invalid expression + src = "a + *" + _, err = ParseExpr(src) + if err == nil { + t.Errorf("ParseExpr(%s): %v", src, err) + } + + // it must not crash + for _, src := range validPrograms { + ParseExpr(src) + } +} + +func TestColonEqualsScope(t *testing.T) { + f, err := ParseFile(fset, "", `package p; func f() { x, y, z := x, y, z }`, 0) + if err != nil { + t.Errorf("parse: %s", err) + } + + // RHS refers to undefined globals; LHS does not. + as := f.Decls[0].(*ast.FuncDecl).Body.List[0].(*ast.AssignStmt) + for _, v := range as.Rhs { + id := v.(*ast.Ident) + if id.Obj != nil { + t.Errorf("rhs %s has Obj, should not", id.Name) + } + } + for _, v := range as.Lhs { + id := v.(*ast.Ident) + if id.Obj == nil { + t.Errorf("lhs %s does not have Obj, should", id.Name) + } + } +} + +func TestVarScope(t *testing.T) { + f, err := ParseFile(fset, "", `package p; func f() { var x, y, z = x, y, z }`, 0) + if err != nil { + t.Errorf("parse: %s", err) + } + + // RHS refers to undefined globals; LHS does not. + as := f.Decls[0].(*ast.FuncDecl).Body.List[0].(*ast.DeclStmt).Decl.(*ast.GenDecl).Specs[0].(*ast.ValueSpec) + for _, v := range as.Values { + id := v.(*ast.Ident) + if id.Obj != nil { + t.Errorf("rhs %s has Obj, should not", id.Name) + } + } + for _, id := range as.Names { + if id.Obj == nil { + t.Errorf("lhs %s does not have Obj, should", id.Name) + } + } +} diff --git a/src/pkg/go/printer/nodes.go b/src/pkg/go/printer/nodes.go index 364530634..5f3b4d4a7 100644 --- a/src/pkg/go/printer/nodes.go +++ b/src/pkg/go/printer/nodes.go @@ -39,7 +39,10 @@ import ( // future (not yet interspersed) comments in this function. // func (p *printer) linebreak(line, min int, ws whiteSpace, newSection bool) (printedBreak bool) { - n := p.nlines(line-p.pos.Line, min) + n := nlimit(line - p.pos.Line) + if n < min { + n = min + } if n > 0 { p.print(ws) if newSection { @@ -69,7 +72,7 @@ func (p *printer) setComment(g *ast.CommentGroup) { // for some reason there are pending comments; this // should never happen - handle gracefully and flush // all comments up to g, ignore anything after that - p.flush(p.fset.Position(g.List[0].Pos()), token.ILLEGAL) + p.flush(p.posFor(g.List[0].Pos()), token.ILLEGAL) } p.comments[0] = g p.cindex = 0 @@ -119,10 +122,10 @@ func (p *printer) exprList(prev0 token.Pos, list []ast.Expr, depth int, mode exp p.print(blank) } - prev := p.fset.Position(prev0) - next := p.fset.Position(next0) - line := p.fset.Position(list[0].Pos()).Line - endLine := p.fset.Position(list[len(list)-1].End()).Line + prev := p.posFor(prev0) + next := p.posFor(next0) + line := p.lineFor(list[0].Pos()) + endLine := p.lineFor(list[len(list)-1].End()) if prev.IsValid() && prev.Line == line && line == endLine { // all list entries on a single line @@ -166,7 +169,7 @@ func (p *printer) exprList(prev0 token.Pos, list []ast.Expr, depth int, mode exp // print all list elements for i, x := range list { prevLine := line - line = p.fset.Position(x.Pos()).Line + line = p.lineFor(x.Pos()) // determine if the next linebreak, if any, needs to use formfeed: // in general, use the entire node size to make the decision; for @@ -269,23 +272,32 @@ func (p *printer) exprList(prev0 token.Pos, list []ast.Expr, depth int, mode exp func (p *printer) parameters(fields *ast.FieldList, multiLine *bool) { p.print(fields.Opening, token.LPAREN) if len(fields.List) > 0 { + prevLine := p.lineFor(fields.Opening) ws := indent - var prevLine, line int for i, par := range fields.List { + // determine par begin and end line (may be different + // if there are multiple parameter names for this par + // or the type is on a separate line) + var parLineBeg int + var parLineEnd = p.lineFor(par.Type.Pos()) + if len(par.Names) > 0 { + parLineBeg = p.lineFor(par.Names[0].Pos()) + } else { + parLineBeg = parLineEnd + } + // separating "," if needed if i > 0 { p.print(token.COMMA) - if len(par.Names) > 0 { - line = p.fset.Position(par.Names[0].Pos()).Line - } else { - line = p.fset.Position(par.Type.Pos()).Line - } - if 0 < prevLine && prevLine < line && p.linebreak(line, 0, ws, true) { - ws = ignore - *multiLine = true - } else { - p.print(blank) - } } + // separator if needed (linebreak or blank) + if 0 < prevLine && prevLine < parLineBeg && p.linebreak(parLineBeg, 0, ws, true) { + // break line if the opening "(" or previous parameter ended on a different line + ws = ignore + *multiLine = true + } else if i > 0 { + p.print(blank) + } + // parameter names if len(par.Names) > 0 { // Very subtle: If we indented before (ws == ignore), identList // won't indent again. If we didn't (ws == indent), identList will @@ -296,11 +308,18 @@ func (p *printer) parameters(fields *ast.FieldList, multiLine *bool) { p.identList(par.Names, ws == indent, multiLine) p.print(blank) } + // parameter type p.expr(par.Type, multiLine) - prevLine = p.fset.Position(par.Type.Pos()).Line + prevLine = parLineEnd } + // if the closing ")" is on a separate line from the last parameter, + // print an additional "," and line break + if closing := p.lineFor(fields.Closing); 0 < prevLine && prevLine < closing { + p.print(",") + p.linebreak(closing, 0, ignore, true) + } + // unindent if we indented if ws == ignore { - // unindent if we indented p.print(unindent) } } @@ -354,16 +373,17 @@ func (p *printer) isOneLineFieldList(list []*ast.Field) bool { } func (p *printer) setLineComment(text string) { - p.setComment(&ast.CommentGroup{[]*ast.Comment{&ast.Comment{token.NoPos, text}}}) + p.setComment(&ast.CommentGroup{[]*ast.Comment{{token.NoPos, text}}}) } func (p *printer) fieldList(fields *ast.FieldList, isStruct, isIncomplete bool) { lbrace := fields.Opening list := fields.List rbrace := fields.Closing - srcIsOneLine := lbrace.IsValid() && rbrace.IsValid() && p.fset.Position(lbrace).Line == p.fset.Position(rbrace).Line + hasComments := isIncomplete || p.commentBefore(p.posFor(rbrace)) + srcIsOneLine := lbrace.IsValid() && rbrace.IsValid() && p.lineFor(lbrace) == p.lineFor(rbrace) - if !isIncomplete && !p.commentBefore(p.fset.Position(rbrace)) && srcIsOneLine { + if !hasComments && srcIsOneLine { // possibly a one-line struct/interface if len(list) == 0 { // no blank between keyword and {} in this case @@ -388,9 +408,13 @@ func (p *printer) fieldList(fields *ast.FieldList, isStruct, isIncomplete bool) return } } + // hasComments || !srcIsOneLine + + p.print(blank, lbrace, token.LBRACE, indent) + if hasComments || len(list) > 0 { + p.print(formfeed) + } - // at least one entry or incomplete - p.print(blank, lbrace, token.LBRACE, indent, formfeed) if isStruct { sep := vtab @@ -400,7 +424,7 @@ func (p *printer) fieldList(fields *ast.FieldList, isStruct, isIncomplete bool) var ml bool for i, f := range list { if i > 0 { - p.linebreak(p.fset.Position(f.Pos()).Line, 1, ignore, ml) + p.linebreak(p.lineFor(f.Pos()), 1, ignore, ml) } ml = false extraTabs := 0 @@ -435,7 +459,7 @@ func (p *printer) fieldList(fields *ast.FieldList, isStruct, isIncomplete bool) if len(list) > 0 { p.print(formfeed) } - p.flush(p.fset.Position(rbrace), token.RBRACE) // make sure we don't lose the last line comment + p.flush(p.posFor(rbrace), token.RBRACE) // make sure we don't lose the last line comment p.setLineComment("// contains filtered or unexported fields") } @@ -444,7 +468,7 @@ func (p *printer) fieldList(fields *ast.FieldList, isStruct, isIncomplete bool) var ml bool for i, f := range list { if i > 0 { - p.linebreak(p.fset.Position(f.Pos()).Line, 1, ignore, ml) + p.linebreak(p.lineFor(f.Pos()), 1, ignore, ml) } ml = false p.setComment(f.Doc) @@ -462,7 +486,7 @@ func (p *printer) fieldList(fields *ast.FieldList, isStruct, isIncomplete bool) if len(list) > 0 { p.print(formfeed) } - p.flush(p.fset.Position(rbrace), token.RBRACE) // make sure we don't lose the last line comment + p.flush(p.posFor(rbrace), token.RBRACE) // make sure we don't lose the last line comment p.setLineComment("// contains filtered or unexported methods") } @@ -618,7 +642,7 @@ func (p *printer) binaryExpr(x *ast.BinaryExpr, prec1, cutoff, depth int, multiL p.print(blank) } xline := p.pos.Line // before the operator (it may be on the next line!) - yline := p.fset.Position(x.Y.Pos()).Line + yline := p.lineFor(x.Y.Pos()) p.print(x.OpPos, x.Op) if xline != yline && xline > 0 && yline > 0 { // at least one line break, but respect an extra empty line @@ -911,7 +935,7 @@ func (p *printer) stmtList(list []ast.Stmt, _indent int, nextIsRBrace bool) { for i, s := range list { // _indent == 0 only for lists of switch/select case clauses; // in those cases each clause is a new section - p.linebreak(p.fset.Position(s.Pos()).Line, 1, ignore, i == 0 || _indent == 0 || multiLine) + p.linebreak(p.lineFor(s.Pos()), 1, ignore, i == 0 || _indent == 0 || multiLine) multiLine = false p.stmt(s, nextIsRBrace && i == len(list)-1, &multiLine) } @@ -924,7 +948,7 @@ func (p *printer) stmtList(list []ast.Stmt, _indent int, nextIsRBrace bool) { func (p *printer) block(s *ast.BlockStmt, indent int) { p.print(s.Pos(), token.LBRACE) p.stmtList(s.List, indent, true) - p.linebreak(p.fset.Position(s.Rbrace).Line, 1, ignore, true) + p.linebreak(p.lineFor(s.Rbrace), 1, ignore, true) p.print(s.Rbrace, token.RBRACE) } @@ -1025,7 +1049,7 @@ func (p *printer) stmt(stmt ast.Stmt, nextIsRBrace bool, multiLine *bool) { break } } else { - p.linebreak(p.fset.Position(s.Stmt.Pos()).Line, 1, ignore, true) + p.linebreak(p.lineFor(s.Stmt.Pos()), 1, ignore, true) } p.stmt(s.Stmt, nextIsRBrace, multiLine) @@ -1137,7 +1161,7 @@ func (p *printer) stmt(stmt ast.Stmt, nextIsRBrace bool, multiLine *bool) { case *ast.SelectStmt: p.print(token.SELECT, blank) body := s.Body - if len(body.List) == 0 && !p.commentBefore(p.fset.Position(body.Rbrace)) { + if len(body.List) == 0 && !p.commentBefore(p.posFor(body.Rbrace)) { // print empty select statement w/o comments on one line p.print(body.Lbrace, token.LBRACE, body.Rbrace, token.RBRACE) } else { @@ -1278,6 +1302,7 @@ func (p *printer) spec(spec ast.Spec, n int, doIndent bool, multiLine *bool) { } p.expr(s.Path, multiLine) p.setComment(s.Comment) + p.print(s.EndPos) case *ast.ValueSpec: if n != 1 { @@ -1328,7 +1353,7 @@ func (p *printer) genDecl(d *ast.GenDecl, multiLine *bool) { var ml bool for i, s := range d.Specs { if i > 0 { - p.linebreak(p.fset.Position(s.Pos()).Line, 1, ignore, ml) + p.linebreak(p.lineFor(s.Pos()), 1, ignore, ml) } ml = false p.valueSpec(s.(*ast.ValueSpec), keepType[i], false, &ml) @@ -1337,7 +1362,7 @@ func (p *printer) genDecl(d *ast.GenDecl, multiLine *bool) { var ml bool for i, s := range d.Specs { if i > 0 { - p.linebreak(p.fset.Position(s.Pos()).Line, 1, ignore, ml) + p.linebreak(p.lineFor(s.Pos()), 1, ignore, ml) } ml = false p.spec(s, n, false, &ml) @@ -1376,7 +1401,7 @@ func (p *printer) nodeSize(n ast.Node, maxSize int) (size int) { // in RawFormat cfg := Config{Mode: RawFormat} var buf bytes.Buffer - if _, err := cfg.fprint(&buf, p.fset, n, p.nodeSizes); err != nil { + if err := cfg.fprint(&buf, p.fset, n, p.nodeSizes); err != nil { return } if buf.Len() <= maxSize { @@ -1394,11 +1419,11 @@ func (p *printer) nodeSize(n ast.Node, maxSize int) (size int) { func (p *printer) isOneLineFunc(b *ast.BlockStmt, headerSize int) bool { pos1 := b.Pos() pos2 := b.Rbrace - if pos1.IsValid() && pos2.IsValid() && p.fset.Position(pos1).Line != p.fset.Position(pos2).Line { + if pos1.IsValid() && pos2.IsValid() && p.lineFor(pos1) != p.lineFor(pos2) { // opening and closing brace are on different lines - don't make it a one-liner return false } - if len(b.List) > 5 || p.commentBefore(p.fset.Position(pos2)) { + if len(b.List) > 5 || p.commentBefore(p.posFor(pos2)) { // too many statements or there is a comment inside - don't make it a one-liner return false } @@ -1449,7 +1474,7 @@ func (p *printer) funcBody(b *ast.BlockStmt, headerSize int, isLit bool, multiLi // are on the same line; if they are on different lines (or unknown) // the result is infinity. func (p *printer) distance(from0 token.Pos, to token.Position) int { - from := p.fset.Position(from0) + from := p.posFor(from0) if from.IsValid() && to.IsValid() && from.Line == to.Line { return to.Column - from.Column } @@ -1508,12 +1533,17 @@ func (p *printer) file(src *ast.File) { prev := tok tok = declToken(d) // if the declaration token changed (e.g., from CONST to TYPE) + // or the next declaration has documentation associated with it, // print an empty line between top-level declarations + // (because p.linebreak is called with the position of d, which + // is past any documentation, the minimum requirement is satisfied + // even w/o the extra getDoc(d) nil-check - leave it in case the + // linebreak logic improves - there's already a TODO). min := 1 - if prev != tok { + if prev != tok || getDoc(d) != nil { min = 2 } - p.linebreak(p.fset.Position(d.Pos()).Line, min, ignore, false) + p.linebreak(p.lineFor(d.Pos()), min, ignore, false) p.decl(d, ignoreMultiLine) } } diff --git a/src/pkg/go/printer/performance_test.go b/src/pkg/go/printer/performance_test.go index 84fb2808e..dbd942292 100644 --- a/src/pkg/go/printer/performance_test.go +++ b/src/pkg/go/printer/performance_test.go @@ -20,7 +20,7 @@ import ( var testfile *ast.File func testprint(out io.Writer, file *ast.File) { - if _, err := (&Config{TabIndent | UseSpaces, 8}).Fprint(out, fset, file); err != nil { + if err := (&Config{TabIndent | UseSpaces, 8}).Fprint(out, fset, file); err != nil { log.Fatalf("print error: %s", err) } } diff --git a/src/pkg/go/printer/printer.go b/src/pkg/go/printer/printer.go index bfabd749a..52dfff6f4 100644 --- a/src/pkg/go/printer/printer.go +++ b/src/pkg/go/printer/printer.go @@ -13,13 +13,18 @@ import ( "io" "os" "path/filepath" - "tabwriter" + "strconv" + "strings" + "text/tabwriter" ) -const debug = false // enable for debugging - +const ( + maxNewlines = 2 // max. number of newlines between source text + debug = false // enable for debugging + infinity = 1 << 30 +) -type whiteSpace int +type whiteSpace byte const ( ignore = whiteSpace(0) @@ -31,18 +36,6 @@ const ( unindent = whiteSpace('<') ) -var ( - esc = []byte{tabwriter.Escape} - htab = []byte{'\t'} - htabs = []byte("\t\t\t\t\t\t\t\t") - newlines = []byte("\n\n\n\n\n\n\n\n") // more than the max determined by nlines - formfeeds = []byte("\f\f\f\f\f\f\f\f") // more than the max determined by nlines -) - -// Special positions -var noPos token.Position // use noPos when a position is needed but not known -var infinity = 1 << 30 - // Use ignoreMultiLine if the multiLine information is not important. var ignoreMultiLine = new(bool) @@ -50,31 +43,20 @@ var ignoreMultiLine = new(bool) type pmode int const ( - inLiteral pmode = 1 << iota - noExtraLinebreak + noExtraLinebreak pmode = 1 << iota ) -// local error wrapper so we can distinguish os.Errors we want to return -// as errors from genuine panics (which we don't want to return as errors) -type osError struct { - err os.Error -} - type printer struct { // Configuration (does not change after initialization) - output io.Writer Config fset *token.FileSet // Current state - written int // number of bytes written - indent int // current indentation - mode pmode // current printer mode - lastTok token.Token // the last token printed (token.ILLEGAL if it's whitespace) - - // Reused buffers - wsbuf []whiteSpace // delayed white space - litbuf bytes.Buffer // for creation of escaped literals and comments + output bytes.Buffer // raw printer result + indent int // current indentation + mode pmode // current printer mode + lastTok token.Token // the last token printed (token.ILLEGAL if it's whitespace) + wsbuf []whiteSpace // delayed white space // The (possibly estimated) position in the generated output; // in AST space (i.e., pos is set whenever a token position is @@ -93,14 +75,18 @@ type printer struct { // Cache of already computed node sizes. nodeSizes map[ast.Node]int + + // Cache of most recently computed line position. + cachedPos token.Pos + cachedLine int // line corresponding to cachedPos } -func (p *printer) init(output io.Writer, cfg *Config, fset *token.FileSet, nodeSizes map[ast.Node]int) { - p.output = output +func (p *printer) init(cfg *Config, fset *token.FileSet, nodeSizes map[ast.Node]int) { p.Config = *cfg p.fset = fset p.wsbuf = make([]whiteSpace, 0, 16) // whitespace sequences are short p.nodeSizes = nodeSizes + p.cachedPos = -1 } func (p *printer) internalError(msg ...interface{}) { @@ -111,109 +97,90 @@ func (p *printer) internalError(msg ...interface{}) { } } -// escape escapes string s by bracketing it with tabwriter.Escape. -// Escaped strings pass through tabwriter unchanged. (Note that -// valid Go programs cannot contain tabwriter.Escape bytes since -// they do not appear in legal UTF-8 sequences). -// -func (p *printer) escape(s string) string { - p.litbuf.Reset() - p.litbuf.WriteByte(tabwriter.Escape) - p.litbuf.WriteString(s) - p.litbuf.WriteByte(tabwriter.Escape) - return p.litbuf.String() +func (p *printer) posFor(pos token.Pos) token.Position { + // not used frequently enough to cache entire token.Position + return p.fset.Position(pos) } -// nlines returns the adjusted number of linebreaks given the desired number -// of breaks n such that min <= result <= max. -// -func (p *printer) nlines(n, min int) int { - const max = 2 // max. number of newlines - switch { - case n < min: - return min - case n > max: - return max +func (p *printer) lineFor(pos token.Pos) int { + if pos != p.cachedPos { + p.cachedPos = pos + p.cachedLine = p.fset.Position(pos).Line } - return n + return p.cachedLine } -// write0 writes raw (uninterpreted) data to p.output and handles errors. -// write0 does not indent after newlines, and does not HTML-escape or update p.pos. -// -func (p *printer) write0(data []byte) { - if len(data) > 0 { - n, err := p.output.Write(data) - p.written += n - if err != nil { - panic(osError{err}) +// writeByte writes ch to p.output and updates p.pos. +func (p *printer) writeByte(ch byte) { + p.output.WriteByte(ch) + p.pos.Offset++ + p.pos.Column++ + + if ch == '\n' || ch == '\f' { + // write indentation + // use "hard" htabs - indentation columns + // must not be discarded by the tabwriter + const htabs = "\t\t\t\t\t\t\t\t" + j := p.indent + for j > len(htabs) { + p.output.WriteString(htabs) + j -= len(htabs) } + p.output.WriteString(htabs[0:j]) + + // update p.pos + p.pos.Line++ + p.pos.Offset += p.indent + p.pos.Column = 1 + p.indent } } -// write interprets data and writes it to p.output. It inserts indentation -// after a line break unless in a tabwriter escape sequence. -// It updates p.pos as a side-effect. -// -func (p *printer) write(data []byte) { - i0 := 0 - for i, b := range data { - switch b { - case '\n', '\f': - // write segment ending in b - p.write0(data[i0 : i+1]) - - // update p.pos - p.pos.Offset += i + 1 - i0 - p.pos.Line++ - p.pos.Column = 1 - - if p.mode&inLiteral == 0 { - // write indentation - // use "hard" htabs - indentation columns - // must not be discarded by the tabwriter - j := p.indent - for ; j > len(htabs); j -= len(htabs) { - p.write0(htabs) - } - p.write0(htabs[0:j]) - - // update p.pos - p.pos.Offset += p.indent - p.pos.Column += p.indent - } - - // next segment start - i0 = i + 1 - - case tabwriter.Escape: - p.mode ^= inLiteral +// writeByteN writes ch n times to p.output and updates p.pos. +func (p *printer) writeByteN(ch byte, n int) { + for n > 0 { + p.writeByte(ch) + n-- + } +} - // ignore escape chars introduced by printer - they are - // invisible and must not affect p.pos (was issue #1089) - p.pos.Offset-- - p.pos.Column-- - } +// writeString writes the string s to p.output and updates p.pos. +// If isLit is set, s is escaped w/ tabwriter.Escape characters +// to protect s from being interpreted by the tabwriter. +// +// Note: writeString is only used to write Go tokens, literals, and +// comments, all of which must be written literally. Thus, it is correct +// to always set isLit = true. However, setting it explicitly only when +// needed (i.e., when we don't know that s contains no tabs or line breaks) +// avoids processing extra escape characters and reduces run time of the +// printer benchmark by up to 10%. +// +func (p *printer) writeString(s string, isLit bool) { + if isLit { + // Protect s such that is passes through the tabwriter + // unchanged. Note that valid Go programs cannot contain + // tabwriter.Escape bytes since they do not appear in legal + // UTF-8 sequences. + p.output.WriteByte(tabwriter.Escape) } - // write remaining segment - p.write0(data[i0:]) + p.output.WriteString(s) // update p.pos - d := len(data) - i0 - p.pos.Offset += d - p.pos.Column += d -} - -func (p *printer) writeNewlines(n int, useFF bool) { - if n > 0 { - n = p.nlines(n, 0) - if useFF { - p.write(formfeeds[0:n]) - } else { - p.write(newlines[0:n]) + nlines := 0 + column := p.pos.Column + len(s) + for i := 0; i < len(s); i++ { + if s[i] == '\n' { + nlines++ + column = len(s) - i } } + p.pos.Offset += len(s) + p.pos.Line += nlines + p.pos.Column = column + + if isLit { + p.output.WriteByte(tabwriter.Escape) + } } // writeItem writes data at position pos. data is the text corresponding to @@ -222,7 +189,7 @@ func (p *printer) writeNewlines(n int, useFF bool) { // source text. writeItem updates p.last to the position immediately following // the data. // -func (p *printer) writeItem(pos token.Position, data string) { +func (p *printer) writeItem(pos token.Position, data string, isLit bool) { if pos.IsValid() { // continue with previous position if we don't have a valid pos if p.last.IsValid() && p.last.Filename != pos.Filename { @@ -238,12 +205,14 @@ func (p *printer) writeItem(pos token.Position, data string) { if debug { // do not update p.pos - use write0 _, filename := filepath.Split(pos.Filename) - p.write0([]byte(fmt.Sprintf("[%s:%d:%d]", filename, pos.Line, pos.Column))) + fmt.Fprintf(&p.output, "[%s:%d:%d]", filename, pos.Line, pos.Column) } - p.write([]byte(data)) + p.writeString(data, isLit) p.last = p.pos } +const linePrefix = "//line " + // writeCommentPrefix writes the whitespace before a comment. // If there is any pending whitespace, it consumes as much of // it as is likely to help position the comment nicely. @@ -252,15 +221,15 @@ func (p *printer) writeItem(pos token.Position, data string) { // a group of comments (or nil), and isKeyword indicates if the // next item is a keyword. // -func (p *printer) writeCommentPrefix(pos, next token.Position, prev *ast.Comment, isKeyword bool) { - if p.written == 0 { +func (p *printer) writeCommentPrefix(pos, next token.Position, prev, comment *ast.Comment, isKeyword bool) { + if p.output.Len() == 0 { // the comment is the first item to be printed - don't write any whitespace return } if pos.IsValid() && pos.Filename != p.last.Filename { - // comment in a different file - separate with newlines (writeNewlines will limit the number) - p.writeNewlines(10, true) + // comment in a different file - separate with newlines + p.writeByteN('\f', maxNewlines) return } @@ -293,19 +262,20 @@ func (p *printer) writeCommentPrefix(pos, next token.Position, prev *ast.Comment } // make sure there is at least one separator if !hasSep { + sep := byte('\t') if pos.Line == next.Line { // next item is on the same line as the comment // (which must be a /*-style comment): separate // with a blank instead of a tab - p.write([]byte{' '}) - } else { - p.write(htab) + sep = ' ' } + p.writeByte(sep) } } else { // comment on a different line: // separate with at least one line break + droppedLinebreak := false if prev == nil { // first comment of a comment group j := 0 @@ -331,44 +301,70 @@ func (p *printer) writeCommentPrefix(pos, next token.Position, prev *ast.Comment case newline, formfeed: // TODO(gri): may want to keep formfeed info in some cases p.wsbuf[i] = ignore + droppedLinebreak = true } j = i break } p.writeWhitespace(j) } - // use formfeeds to break columns before a comment; - // this is analogous to using formfeeds to separate - // individual lines of /*-style comments - but make - // sure there is at least one line break if the previous - // comment was a line comment - n := pos.Line - p.last.Line // if !pos.IsValid(), pos.Line == 0, and n will be 0 - if n <= 0 && prev != nil && prev.Text[1] == '/' { + + // determine number of linebreaks before the comment + n := 0 + if pos.IsValid() && p.last.IsValid() { + n = pos.Line - p.last.Line + if n < 0 { // should never happen + n = 0 + } + } + + // at the package scope level only (p.indent == 0), + // add an extra newline if we dropped one before: + // this preserves a blank line before documentation + // comments at the package scope level (issue 2570) + if p.indent == 0 && droppedLinebreak { + n++ + } + + // make sure there is at least one line break + // if the previous comment was a line comment + if n == 0 && prev != nil && prev.Text[1] == '/' { n = 1 } - p.writeNewlines(n, true) + + if n > 0 { + // turn off indent if we're about to print a line directive + indent := p.indent + if strings.HasPrefix(comment.Text, linePrefix) { + p.indent = 0 + } + // use formfeeds to break columns before a comment; + // this is analogous to using formfeeds to separate + // individual lines of /*-style comments + p.writeByteN('\f', nlimit(n)) + p.indent = indent // restore indent + } } } -// TODO(gri): It should be possible to convert the code below from using -// []byte to string and in the process eliminate some conversions. - // Split comment text into lines -func split(text []byte) [][]byte { +// (using strings.Split(text, "\n") is significantly slower for +// this specific purpose, as measured with: gotest -bench=Print) +func split(text string) []string { // count lines (comment text never ends in a newline) n := 1 - for _, c := range text { - if c == '\n' { + for i := 0; i < len(text); i++ { + if text[i] == '\n' { n++ } } // split - lines := make([][]byte, n) + lines := make([]string, n) n = 0 i := 0 - for j, c := range text { - if c == '\n' { + for j := 0; j < len(text); j++ { + if text[j] == '\n' { lines[n] = text[i:j] // exclude newline i = j + 1 // discard newline n++ @@ -379,16 +375,18 @@ func split(text []byte) [][]byte { return lines } -func isBlank(s []byte) bool { - for _, b := range s { - if b > ' ' { +// Returns true if s contains only white space +// (only tabs and blanks can appear in the printer's context). +func isBlank(s string) bool { + for i := 0; i < len(s); i++ { + if s[i] > ' ' { return false } } return true } -func commonPrefix(a, b []byte) []byte { +func commonPrefix(a, b string) string { i := 0 for i < len(a) && i < len(b) && a[i] == b[i] && (a[i] <= ' ' || a[i] == '*') { i++ @@ -396,7 +394,7 @@ func commonPrefix(a, b []byte) []byte { return a[0:i] } -func stripCommonPrefix(lines [][]byte) { +func stripCommonPrefix(lines []string) { if len(lines) < 2 { return // at most one line - nothing to do } @@ -420,19 +418,21 @@ func stripCommonPrefix(lines [][]byte) { // Note that the first and last line are never empty (they // contain the opening /* and closing */ respectively) and // thus they can be ignored by the blank line check. - var prefix []byte + var prefix string if len(lines) > 2 { + first := true for i, line := range lines[1 : len(lines)-1] { switch { case isBlank(line): - lines[1+i] = nil // range starts at line 1 - case prefix == nil: + lines[1+i] = "" // range starts at line 1 + case first: prefix = commonPrefix(line, line) + first = false default: prefix = commonPrefix(prefix, line) } } - } else { // len(lines) == 2 + } else { // len(lines) == 2, lines cannot be blank (contain /* and */) line := lines[1] prefix = commonPrefix(line, line) } @@ -441,7 +441,7 @@ func stripCommonPrefix(lines [][]byte) { * Check for vertical "line of stars" and correct prefix accordingly. */ lineOfStars := false - if i := bytes.Index(prefix, []byte{'*'}); i >= 0 { + if i := strings.Index(prefix, "*"); i >= 0 { // Line of stars present. if i > 0 && prefix[i-1] == ' ' { i-- // remove trailing blank from prefix so stars remain aligned @@ -489,7 +489,7 @@ func stripCommonPrefix(lines [][]byte) { } // Shorten the computed common prefix by the length of // suffix, if it is found as suffix of the prefix. - if bytes.HasSuffix(prefix, suffix) { + if strings.HasSuffix(prefix, string(suffix)) { prefix = prefix[0 : len(prefix)-len(suffix)] } } @@ -499,19 +499,18 @@ func stripCommonPrefix(lines [][]byte) { // with the opening /*, otherwise align the text with the other // lines. last := lines[len(lines)-1] - closing := []byte("*/") - i := bytes.Index(last, closing) + closing := "*/" + i := strings.Index(last, closing) // i >= 0 (closing is always present) if isBlank(last[0:i]) { // last line only contains closing */ - var sep []byte if lineOfStars { - // insert an aligning blank - sep = []byte{' '} + closing = " */" // add blank to align final star } - lines[len(lines)-1] = bytes.Join([][]byte{prefix, closing}, sep) + lines[len(lines)-1] = prefix + closing } else { // last line contains more comment text - assume - // it is aligned like the other lines + // it is aligned like the other lines and include + // in prefix computation prefix = commonPrefix(prefix, last) } @@ -526,28 +525,47 @@ func stripCommonPrefix(lines [][]byte) { func (p *printer) writeComment(comment *ast.Comment) { text := comment.Text + if strings.HasPrefix(text, linePrefix) { + pos := strings.TrimSpace(text[len(linePrefix):]) + i := strings.LastIndex(pos, ":") + if i >= 0 { + // The line directive we are about to print changed + // the Filename and Line number used by go/token + // as it was reading the input originally. + // In order to match the original input, we have to + // update our own idea of the file and line number + // accordingly, after printing the directive. + file := pos[:i] + line, _ := strconv.Atoi(pos[i+1:]) + defer func() { + p.pos.Filename = file + p.pos.Line = line + p.pos.Column = 1 + }() + } + } + // shortcut common case of //-style comments if text[1] == '/' { - p.writeItem(p.fset.Position(comment.Pos()), p.escape(text)) + p.writeItem(p.posFor(comment.Pos()), text, true) return } // for /*-style comments, print line by line and let the // write function take care of the proper indentation - lines := split([]byte(text)) + lines := split(text) stripCommonPrefix(lines) // write comment lines, separated by formfeed, // without a line break after the last line - linebreak := formfeeds[0:1] - pos := p.fset.Position(comment.Pos()) + pos := p.posFor(comment.Pos()) for i, line := range lines { if i > 0 { - p.write(linebreak) + p.writeByte('\f') pos = p.pos } if len(line) > 0 { - p.writeItem(pos, p.escape(string(line))) + p.writeItem(pos, line, true) } } } @@ -555,10 +573,11 @@ func (p *printer) writeComment(comment *ast.Comment) { // writeCommentSuffix writes a line break after a comment if indicated // and processes any leftover indentation information. If a line break // is needed, the kind of break (newline vs formfeed) depends on the -// pending whitespace. writeCommentSuffix returns true if a pending -// formfeed was dropped from the whitespace buffer. +// pending whitespace. The writeCommentSuffix result indicates if a +// newline was written or if a formfeed was dropped from the whitespace +// buffer. // -func (p *printer) writeCommentSuffix(needsLinebreak bool) (droppedFF bool) { +func (p *printer) writeCommentSuffix(needsLinebreak bool) (wroteNewline, droppedFF bool) { for i, ch := range p.wsbuf { switch ch { case blank, vtab: @@ -571,6 +590,7 @@ func (p *printer) writeCommentSuffix(needsLinebreak bool) (droppedFF bool) { // but remember if we dropped any formfeeds if needsLinebreak { needsLinebreak = false + wroteNewline = true } else { if ch == formfeed { droppedFF = true @@ -583,7 +603,8 @@ func (p *printer) writeCommentSuffix(needsLinebreak bool) (droppedFF bool) { // make sure we have a line break if needsLinebreak { - p.write([]byte{'\n'}) + p.writeByte('\n') + wroteNewline = true } return @@ -592,24 +613,24 @@ func (p *printer) writeCommentSuffix(needsLinebreak bool) (droppedFF bool) { // intersperseComments consumes all comments that appear before the next token // tok and prints it together with the buffered whitespace (i.e., the whitespace // that needs to be written before the next token). A heuristic is used to mix -// the comments and whitespace. intersperseComments returns true if a pending -// formfeed was dropped from the whitespace buffer. +// the comments and whitespace. The intersperseComments result indicates if a +// newline was written or if a formfeed was dropped from the whitespace buffer. // -func (p *printer) intersperseComments(next token.Position, tok token.Token) (droppedFF bool) { +func (p *printer) intersperseComments(next token.Position, tok token.Token) (wroteNewline, droppedFF bool) { var last *ast.Comment for ; p.commentBefore(next); p.cindex++ { for _, c := range p.comments[p.cindex].List { - p.writeCommentPrefix(p.fset.Position(c.Pos()), next, last, tok.IsKeyword()) + p.writeCommentPrefix(p.posFor(c.Pos()), next, last, c, tok.IsKeyword()) p.writeComment(c) last = c } } if last != nil { - if last.Text[1] == '*' && p.fset.Position(last.Pos()).Line == next.Line { + if last.Text[1] == '*' && p.lineFor(last.Pos()) == next.Line { // the last comment is a /*-style comment and the next item // follows on the same line: separate with an extra blank - p.write([]byte{' '}) + p.writeByte(' ') } // ensure that there is a line break after a //-style comment, // before a closing '}' unless explicitly disabled, or at eof @@ -623,13 +644,12 @@ func (p *printer) intersperseComments(next token.Position, tok token.Token) (dro // no comment was written - we should never reach here since // intersperseComments should not be called in that case p.internalError("intersperseComments called without pending comments") - return false + return } // whiteWhitespace writes the first n whitespace entries. func (p *printer) writeWhitespace(n int) { // write entries - var data [1]byte for i := 0; i < n; i++ { switch ch := p.wsbuf[i]; ch { case ignore: @@ -661,8 +681,7 @@ func (p *printer) writeWhitespace(n int) { } fallthrough default: - data[0] = byte(ch) - p.write(data[0:]) + p.writeByte(byte(ch)) } } @@ -678,6 +697,13 @@ func (p *printer) writeWhitespace(n int) { // ---------------------------------------------------------------------------- // Printing interface +// nlines limits n to maxNewlines. +func nlimit(n int) int { + if n > maxNewlines { + n = maxNewlines + } + return n +} func mayCombine(prev token.Token, next byte) (b bool) { switch prev { @@ -711,7 +737,8 @@ func mayCombine(prev token.Token, next byte) (b bool) { func (p *printer) print(args ...interface{}) { for _, f := range args { next := p.pos // estimated position of next item - var data string + data := "" + isLit := false var tok token.Token switch x := f.(type) { @@ -739,7 +766,8 @@ func (p *printer) print(args ...interface{}) { data = x.Name tok = token.IDENT case *ast.BasicLit: - data = p.escape(x.Value) + data = x.Value + isLit = true tok = x.Kind case token.Token: s := x.String() @@ -760,26 +788,41 @@ func (p *printer) print(args ...interface{}) { tok = x case token.Pos: if x.IsValid() { - next = p.fset.Position(x) // accurate position of next item + next = p.posFor(x) // accurate position of next item } tok = p.lastTok + case string: + // incorrect AST - print error message + data = x + isLit = true + tok = token.STRING default: - fmt.Fprintf(os.Stderr, "print: unsupported argument type %T\n", f) + fmt.Fprintf(os.Stderr, "print: unsupported argument %v (%T)\n", f, f) panic("go/printer type") } p.lastTok = tok p.pos = next if data != "" { - droppedFF := p.flush(next, tok) + wroteNewline, droppedFF := p.flush(next, tok) // intersperse extra newlines if present in the source // (don't do this in flush as it will cause extra newlines - // at the end of a file) - use formfeeds if we dropped one - // before - p.writeNewlines(next.Line-p.pos.Line, droppedFF) + // at the end of a file) + n := nlimit(next.Line - p.pos.Line) + // don't exceed maxNewlines if we already wrote one + if wroteNewline && n == maxNewlines { + n = maxNewlines - 1 + } + if n > 0 { + ch := byte('\n') + if droppedFF { + ch = '\f' // use formfeed since we dropped one before + } + p.writeByteN(ch, n) + } - p.writeItem(next, data) + p.writeItem(next, data, isLit) } } } @@ -788,19 +831,18 @@ func (p *printer) print(args ...interface{}) { // before the next position in the source code. // func (p *printer) commentBefore(next token.Position) bool { - return p.cindex < len(p.comments) && p.fset.Position(p.comments[p.cindex].List[0].Pos()).Offset < next.Offset + return p.cindex < len(p.comments) && p.posFor(p.comments[p.cindex].List[0].Pos()).Offset < next.Offset } -// Flush prints any pending comments and whitespace occurring -// textually before the position of the next token tok. Flush -// returns true if a pending formfeed character was dropped -// from the whitespace buffer as a result of interspersing -// comments. +// Flush prints any pending comments and whitespace occurring textually +// before the position of the next token tok. The Flush result indicates +// if a newline was written or if a formfeed was dropped from the whitespace +// buffer. // -func (p *printer) flush(next token.Position, tok token.Token) (droppedFF bool) { +func (p *printer) flush(next token.Position, tok token.Token) (wroteNewline, droppedFF bool) { if p.commentBefore(next) { // if there are comments before the next item, intersperse them - droppedFF = p.intersperseComments(next, tok) + wroteNewline, droppedFF = p.intersperseComments(next, tok) } else { // otherwise, write any leftover whitespace p.writeWhitespace(len(p.wsbuf)) @@ -808,6 +850,98 @@ func (p *printer) flush(next token.Position, tok token.Token) (droppedFF bool) { return } +// getNode returns the ast.CommentGroup associated with n, if any. +func getDoc(n ast.Node) *ast.CommentGroup { + switch n := n.(type) { + case *ast.Field: + return n.Doc + case *ast.ImportSpec: + return n.Doc + case *ast.ValueSpec: + return n.Doc + case *ast.TypeSpec: + return n.Doc + case *ast.GenDecl: + return n.Doc + case *ast.FuncDecl: + return n.Doc + case *ast.File: + return n.Doc + } + return nil +} + +func (p *printer) printNode(node interface{}) error { + // unpack *CommentedNode, if any + var comments []*ast.CommentGroup + if cnode, ok := node.(*CommentedNode); ok { + node = cnode.Node + comments = cnode.Comments + } + + if comments != nil { + // commented node - restrict comment list to relevant range + n, ok := node.(ast.Node) + if !ok { + goto unsupported + } + beg := n.Pos() + end := n.End() + // if the node has associated documentation, + // include that commentgroup in the range + // (the comment list is sorted in the order + // of the comment appearance in the source code) + if doc := getDoc(n); doc != nil { + beg = doc.Pos() + } + // token.Pos values are global offsets, we can + // compare them directly + i := 0 + for i < len(comments) && comments[i].End() < beg { + i++ + } + j := i + for j < len(comments) && comments[j].Pos() < end { + j++ + } + if i < j { + p.comments = comments[i:j] + } + } else if n, ok := node.(*ast.File); ok { + // use ast.File comments, if any + p.comments = n.Comments + } + + // if there are no comments, use node comments + p.useNodeComments = p.comments == nil + + // format node + switch n := node.(type) { + case ast.Expr: + p.expr(n, ignoreMultiLine) + case ast.Stmt: + // A labeled statement will un-indent to position the + // label. Set indent to 1 so we don't get indent "underflow". + if _, labeledStmt := n.(*ast.LabeledStmt); labeledStmt { + p.indent = 1 + } + p.stmt(n, false, ignoreMultiLine) + case ast.Decl: + p.decl(n, ignoreMultiLine) + case ast.Spec: + p.spec(n, 1, false, ignoreMultiLine) + case *ast.File: + p.file(n) + default: + goto unsupported + } + + return nil + +unsupported: + return fmt.Errorf("go/printer: unsupported node type %T", node) +} + // ---------------------------------------------------------------------------- // Trimmer @@ -837,7 +971,9 @@ const ( // However, this would mess up any formatting done by // the tabwriter. -func (p *trimmer) Write(data []byte) (n int, err os.Error) { +var aNewline = []byte("\n") + +func (p *trimmer) Write(data []byte) (n int, err error) { // invariants: // p.state == inSpace: // p.space is unwritten @@ -855,8 +991,8 @@ func (p *trimmer) Write(data []byte) (n int, err os.Error) { case '\t', ' ': p.space.WriteByte(b) // WriteByte returns no errors case '\n', '\f': - p.space.Reset() // discard trailing space - _, err = p.output.Write(newlines[0:1]) // write newline + p.space.Reset() // discard trailing space + _, err = p.output.Write(aNewline) case tabwriter.Escape: _, err = p.output.Write(p.space.Bytes()) p.state = inEscape @@ -883,7 +1019,7 @@ func (p *trimmer) Write(data []byte) (n int, err os.Error) { _, err = p.output.Write(data[m:n]) p.state = inSpace p.space.Reset() - _, err = p.output.Write(newlines[0:1]) // write newline + _, err = p.output.Write(aNewline) case tabwriter.Escape: _, err = p.output.Write(data[m:n]) p.state = inEscape @@ -925,15 +1061,22 @@ type Config struct { } // fprint implements Fprint and takes a nodesSizes map for setting up the printer state. -func (cfg *Config) fprint(output io.Writer, fset *token.FileSet, node interface{}, nodeSizes map[ast.Node]int) (written int, err os.Error) { +func (cfg *Config) fprint(output io.Writer, fset *token.FileSet, node interface{}, nodeSizes map[ast.Node]int) (err error) { + // print node + var p printer + p.init(cfg, fset, nodeSizes) + if err = p.printNode(node); err != nil { + return + } + p.flush(token.Position{Offset: infinity, Line: infinity}, token.EOF) + // redirect output through a trimmer to eliminate trailing whitespace // (Input to a tabwriter must be untrimmed since trailing tabs provide // formatting information. The tabwriter could provide trimming // functionality but no tabwriter is used when RawFormat is set.) output = &trimmer{output: output} - // setup tabwriter if needed and redirect output - var tw *tabwriter.Writer + // redirect output through a tabwriter if necessary if cfg.Mode&RawFormat == 0 { minwidth := cfg.Tabwidth @@ -948,70 +1091,42 @@ func (cfg *Config) fprint(output io.Writer, fset *token.FileSet, node interface{ twmode |= tabwriter.TabIndent } - tw = tabwriter.NewWriter(output, minwidth, cfg.Tabwidth, 1, padchar, twmode) - output = tw + output = tabwriter.NewWriter(output, minwidth, cfg.Tabwidth, 1, padchar, twmode) } - // setup printer - var p printer - p.init(output, cfg, fset, nodeSizes) - defer func() { - written = p.written - if e := recover(); e != nil { - err = e.(osError).err // re-panics if it's not a local osError - } - }() - - // print node - switch n := node.(type) { - case ast.Expr: - p.useNodeComments = true - p.expr(n, ignoreMultiLine) - case ast.Stmt: - p.useNodeComments = true - // A labeled statement will un-indent to position the - // label. Set indent to 1 so we don't get indent "underflow". - if _, labeledStmt := n.(*ast.LabeledStmt); labeledStmt { - p.indent = 1 - } - p.stmt(n, false, ignoreMultiLine) - case ast.Decl: - p.useNodeComments = true - p.decl(n, ignoreMultiLine) - case ast.Spec: - p.useNodeComments = true - p.spec(n, 1, false, ignoreMultiLine) - case *ast.File: - p.comments = n.Comments - p.useNodeComments = n.Comments == nil - p.file(n) - default: - panic(osError{fmt.Errorf("printer.Fprint: unsupported node type %T", n)}) + // write printer result via tabwriter/trimmer to output + if _, err = output.Write(p.output.Bytes()); err != nil { + return } - p.flush(token.Position{Offset: infinity, Line: infinity}, token.EOF) // flush tabwriter, if any - if tw != nil { - tw.Flush() // ignore errors + if tw, _ := (output).(*tabwriter.Writer); tw != nil { + err = tw.Flush() } return } -// Fprint "pretty-prints" an AST node to output and returns the number -// of bytes written and an error (if any) for a given configuration cfg. +// A CommentedNode bundles an AST node and corresponding comments. +// It may be provided as argument to any of the FPrint functions. +// +type CommentedNode struct { + Node interface{} // *ast.File, or ast.Expr, ast.Decl, ast.Spec, or ast.Stmt + Comments []*ast.CommentGroup +} + +// Fprint "pretty-prints" an AST node to output for a given configuration cfg. // Position information is interpreted relative to the file set fset. -// The node type must be *ast.File, or assignment-compatible to ast.Expr, -// ast.Decl, ast.Spec, or ast.Stmt. +// The node type must be *ast.File, *CommentedNode, or assignment-compatible +// to ast.Expr, ast.Decl, ast.Spec, or ast.Stmt. // -func (cfg *Config) Fprint(output io.Writer, fset *token.FileSet, node interface{}) (int, os.Error) { +func (cfg *Config) Fprint(output io.Writer, fset *token.FileSet, node interface{}) error { return cfg.fprint(output, fset, node, make(map[ast.Node]int)) } // Fprint "pretty-prints" an AST node to output. // It calls Config.Fprint with default settings. // -func Fprint(output io.Writer, fset *token.FileSet, node interface{}) os.Error { - _, err := (&Config{Tabwidth: 8}).Fprint(output, fset, node) // don't care about number of bytes written - return err +func Fprint(output io.Writer, fset *token.FileSet, node interface{}) error { + return (&Config{Tabwidth: 8}).Fprint(output, fset, node) } diff --git a/src/pkg/go/printer/printer_test.go b/src/pkg/go/printer/printer_test.go index a644aa383..525fcc159 100644 --- a/src/pkg/go/printer/printer_test.go +++ b/src/pkg/go/printer/printer_test.go @@ -62,7 +62,7 @@ func runcheck(t *testing.T, source, golden string, mode checkMode) { // format source var buf bytes.Buffer - if _, err := cfg.Fprint(&buf, fset, prog); err != nil { + if err := cfg.Fprint(&buf, fset, prog); err != nil { t.Error(err) } res := buf.Bytes() @@ -107,7 +107,7 @@ func check(t *testing.T, source, golden string, mode checkMode) { // start a timer to produce a time-out signal tc := make(chan int) go func() { - time.Sleep(10e9) // plenty of a safety margin, even for very slow machines + time.Sleep(10 * time.Second) // plenty of a safety margin, even for very slow machines tc <- 0 }() @@ -204,3 +204,18 @@ func init() { panic("got " + s + ", want " + name) } } + +// Verify that the printer doesn't crash if the AST contains BadXXX nodes. +func TestBadNodes(t *testing.T) { + const src = "package p\n(" + const res = "package p\nBadDecl\n" + f, err := parser.ParseFile(fset, "", src, parser.ParseComments) + if err == nil { + t.Errorf("expected illegal program") + } + var buf bytes.Buffer + Fprint(&buf, fset, f) + if buf.String() != res { + t.Errorf("got %q, expected %q", buf.String(), res) + } +} diff --git a/src/pkg/go/printer/testdata/comments.golden b/src/pkg/go/printer/testdata/comments.golden index 7b332252c..d2ad9e3a2 100644 --- a/src/pkg/go/printer/testdata/comments.golden +++ b/src/pkg/go/printer/testdata/comments.golden @@ -106,7 +106,7 @@ type S3 struct { var x int // x var () -// This comment SHOULD be associated with the next declaration. +// This comment SHOULD be associated with f0. func f0() { const pi = 3.14 // pi var s1 struct{} /* an empty struct */ /* foo */ @@ -115,8 +115,9 @@ func f0() { var s2 struct{} = struct{}{} x := pi } + // -// NO SPACE HERE +// This comment should be associated with f1, with one blank line before the comment. // func f1() { f0() diff --git a/src/pkg/go/printer/testdata/comments.input b/src/pkg/go/printer/testdata/comments.input index 2a9a86b68..222e0a713 100644 --- a/src/pkg/go/printer/testdata/comments.input +++ b/src/pkg/go/printer/testdata/comments.input @@ -107,7 +107,7 @@ var x int // x var () -// This comment SHOULD be associated with the next declaration. +// This comment SHOULD be associated with f0. func f0() { const pi = 3.14 // pi var s1 struct {} /* an empty struct */ /* foo */ @@ -117,7 +117,7 @@ func f0() { x := pi } // -// NO SPACE HERE +// This comment should be associated with f1, with one blank line before the comment. // func f1() { f0() @@ -130,7 +130,7 @@ func f1() { func _() { - // this comment should be properly indented +// this comment should be properly indented } diff --git a/src/pkg/go/printer/testdata/declarations.golden b/src/pkg/go/printer/testdata/declarations.golden index bfa2568c2..928b8ce0a 100644 --- a/src/pkg/go/printer/testdata/declarations.golden +++ b/src/pkg/go/printer/testdata/declarations.golden @@ -115,6 +115,18 @@ import _ "io" var _ int +// at least one empty line between declarations of the same kind +// if there is associated documentation (was issue 2570) +type T1 struct{} + +// T2 comment +type T2 struct { +} // should be a two-line struct + +// T3 comment +type T2 struct { +} // should be a two-line struct + // printing of constant literals const ( _ = "foobar" @@ -286,6 +298,15 @@ type _ struct { } } +// no blank lines in empty structs and interfaces, but leave 1- or 2-line layout alone +type _ struct{} +type _ struct { +} + +type _ interface{} +type _ interface { +} + // no tabs for single or ungrouped decls func _() { const xxxxxx = 0 @@ -752,30 +773,39 @@ func ManageStatus(in <-chan *Status, req <-chan Request, TargetHistorySize int) { } -func MultiLineSignature0(a, b, c int) { +func MultiLineSignature0( + a, b, c int, +) { } -func MultiLineSignature1(a, b, c int, - u, v, w float) { +func MultiLineSignature1( + a, b, c int, + u, v, w float, +) { } -func MultiLineSignature2(a, b, - c int) { +func MultiLineSignature2( + a, b, + c int, +) { } -func MultiLineSignature3(a, b, +func MultiLineSignature3( + a, b, c int, u, v, w float, x ...int) { } -func MultiLineSignature4(a, b, c int, +func MultiLineSignature4( + a, b, c int, u, v, w float, x ...int) { } -func MultiLineSignature5(a, b, c int, +func MultiLineSignature5( + a, b, c int, u, v, w float, p, q, r string, @@ -784,25 +814,34 @@ func MultiLineSignature5(a, b, c int, // make sure it also works for methods in interfaces type _ interface { - MultiLineSignature0(a, b, c int) + MultiLineSignature0( + a, b, c int, + ) - MultiLineSignature1(a, b, c int, - u, v, w float) + MultiLineSignature1( + a, b, c int, + u, v, w float, + ) - MultiLineSignature2(a, b, - c int) + MultiLineSignature2( + a, b, + c int, + ) - MultiLineSignature3(a, b, + MultiLineSignature3( + a, b, c int, u, v, w float, x ...int) - MultiLineSignature4(a, b, c int, + MultiLineSignature4( + a, b, c int, u, v, w float, x ...int) - MultiLineSignature5(a, b, c int, + MultiLineSignature5( + a, b, c int, u, v, w float, p, q, r string, diff --git a/src/pkg/go/printer/testdata/declarations.input b/src/pkg/go/printer/testdata/declarations.input index 1d69c57b5..68f90308a 100644 --- a/src/pkg/go/printer/testdata/declarations.input +++ b/src/pkg/go/printer/testdata/declarations.input @@ -115,6 +115,20 @@ import ( import _ "io" var _ int +// at least one empty line between declarations of the same kind +// if there is associated documentation (was issue 2570) +type T1 struct{} +// T2 comment +type T2 struct { +} // should be a two-line struct + + +// T3 comment +type T2 struct { + + +} // should be a two-line struct + // printing of constant literals const ( @@ -293,6 +307,18 @@ type _ struct { } +// no blank lines in empty structs and interfaces, but leave 1- or 2-line layout alone +type _ struct{ } +type _ struct { + +} + +type _ interface{ } +type _ interface { + +} + + // no tabs for single or ungrouped decls func _() { const xxxxxx = 0 diff --git a/src/pkg/go/printer/testdata/linebreaks.golden b/src/pkg/go/printer/testdata/linebreaks.golden index be780da67..006cf1718 100644 --- a/src/pkg/go/printer/testdata/linebreaks.golden +++ b/src/pkg/go/printer/testdata/linebreaks.golden @@ -220,4 +220,56 @@ testLoop: } } +// Respect line breaks in function calls. +func _() { + f(x) + f(x, + x) + f(x, + x, + ) + f( + x, + x) + f( + x, + x, + ) +} + +// Respect line breaks in function declarations. +func _(x T) {} +func _(x T, + y T) { +} +func _(x T, + y T, +) { +} +func _( + x T, + y T) { +} +func _( + x T, + y T, +) { +} + +// Example from issue 2597. +func ManageStatus0( + in <-chan *Status, + req <-chan Request, + stat chan<- *TargetInfo, + TargetHistorySize int) { +} + +func ManageStatus1( + in <-chan *Status, + req <-chan Request, + stat chan<- *TargetInfo, + TargetHistorySize int, +) { +} + // There should be exactly one linebreak after this comment. diff --git a/src/pkg/go/printer/testdata/linebreaks.input b/src/pkg/go/printer/testdata/linebreaks.input index 457b491e6..e782bb044 100644 --- a/src/pkg/go/printer/testdata/linebreaks.input +++ b/src/pkg/go/printer/testdata/linebreaks.input @@ -220,4 +220,52 @@ testLoop: } } +// Respect line breaks in function calls. +func _() { + f(x) + f(x, + x) + f(x, + x, + ) + f( + x, + x) + f( + x, + x, + ) +} + +// Respect line breaks in function declarations. +func _(x T) {} +func _(x T, + y T) {} +func _(x T, + y T, +) {} +func _( + x T, + y T) {} +func _( + x T, + y T, +) {} + +// Example from issue 2597. +func ManageStatus0( + in <-chan *Status, + req <-chan Request, + stat chan<- *TargetInfo, + TargetHistorySize int) { +} + +func ManageStatus1( + in <-chan *Status, + req <-chan Request, + stat chan<- *TargetInfo, + TargetHistorySize int, +) { +} + // There should be exactly one linebreak after this comment. diff --git a/src/pkg/go/printer/testdata/parser.go b/src/pkg/go/printer/testdata/parser.go index 2d27af499..89e341c5f 100644 --- a/src/pkg/go/printer/testdata/parser.go +++ b/src/pkg/go/printer/testdata/parser.go @@ -1999,7 +1999,7 @@ func (p *parser) parseReceiver(scope *ast.Scope) *ast.FieldList { if par.NumFields() != 1 { p.errorExpected(pos, "exactly one receiver") // TODO determine a better range for BadExpr below - par.List = []*ast.Field{&ast.Field{Type: &ast.BadExpr{pos, pos}}} + par.List = []*ast.Field{{Type: &ast.BadExpr{pos, pos}}} return par } @@ -2008,7 +2008,7 @@ func (p *parser) parseReceiver(scope *ast.Scope) *ast.FieldList { base := deref(recv.Type) if _, isIdent := base.(*ast.Ident); !isIdent { p.errorExpected(base.Pos(), "(unqualified) identifier") - par.List = []*ast.Field{&ast.Field{Type: &ast.BadExpr{recv.Pos(), recv.End()}}} + par.List = []*ast.Field{{Type: &ast.BadExpr{recv.Pos(), recv.End()}}} } return par diff --git a/src/pkg/go/printer/testdata/statements.golden b/src/pkg/go/printer/testdata/statements.golden index a6d85107f..90e174355 100644 --- a/src/pkg/go/printer/testdata/statements.golden +++ b/src/pkg/go/printer/testdata/statements.golden @@ -271,7 +271,6 @@ func _() { // Known bug: The first use call may have more than one empty line before // (see go/printer/nodes.go, func linebreak). - use(x) if x < x { @@ -386,7 +385,6 @@ L: // A comment on the same line as the label, followed by a single empty line. // Known bug: There may be more than one empty line before MoreCode() // (see go/printer/nodes.go, func linebreak). - MoreCode() } diff --git a/src/pkg/go/scanner/errors.go b/src/pkg/go/scanner/errors.go index a0927e416..cd9620b87 100644 --- a/src/pkg/go/scanner/errors.go +++ b/src/pkg/go/scanner/errors.go @@ -8,7 +8,6 @@ import ( "fmt" "go/token" "io" - "os" "sort" ) @@ -49,7 +48,7 @@ type Error struct { Msg string } -func (e *Error) String() string { +func (e *Error) Error() string { if e.Pos.Filename != "" || e.Pos.IsValid() { // don't print "<unknown position>" // TODO(gri) reconsider the semantics of Position.IsValid @@ -85,14 +84,14 @@ func (p ErrorList) Less(i, j int) bool { return false } -func (p ErrorList) String() string { +func (p ErrorList) Error() string { switch len(p) { case 0: return "unspecified error" case 1: - return p[0].String() + return p[0].Error() } - return fmt.Sprintf("%s (and %d more errors)", p[0].String(), len(p)-1) + return fmt.Sprintf("%s (and %d more errors)", p[0], len(p)-1) } // These constants control the construction of the ErrorList @@ -136,11 +135,11 @@ func (h *ErrorVector) GetErrorList(mode int) ErrorList { return list } -// GetError is like GetErrorList, but it returns an os.Error instead -// so that a nil result can be assigned to an os.Error variable and +// GetError is like GetErrorList, but it returns an error instead +// so that a nil result can be assigned to an error variable and // remains nil. // -func (h *ErrorVector) GetError(mode int) os.Error { +func (h *ErrorVector) GetError(mode int) error { if len(h.errors) == 0 { return nil } @@ -157,7 +156,7 @@ func (h *ErrorVector) Error(pos token.Position, msg string) { // one error per line, if the err parameter is an ErrorList. Otherwise // it prints the err string. // -func PrintError(w io.Writer, err os.Error) { +func PrintError(w io.Writer, err error) { if list, ok := err.(ErrorList); ok { for _, e := range list { fmt.Fprintf(w, "%s\n", e) diff --git a/src/pkg/go/scanner/scanner.go b/src/pkg/go/scanner/scanner.go index 7f3dd2373..7c72c0a46 100644 --- a/src/pkg/go/scanner/scanner.go +++ b/src/pkg/go/scanner/scanner.go @@ -6,7 +6,7 @@ // source which can then be tokenized through repeated calls to the Scan // function. Typical use: // -// var s Scanner +// var s scanner.Scanner // fset := token.NewFileSet() // position information is relative to fset // file := fset.AddFile(filename, fset.Base(), len(src)) // register file // s.Init(file, src, nil /* no error handler */, 0) @@ -27,7 +27,7 @@ import ( "path/filepath" "strconv" "unicode" - "utf8" + "unicode/utf8" ) // A Scanner holds the scanner's internal state while processing @@ -40,10 +40,10 @@ type Scanner struct { dir string // directory portion of file.Name() src []byte // source err ErrorHandler // error reporting; or nil - mode uint // scanning mode + mode Mode // scanning mode // scanning state - ch int // current character + ch rune // current character offset int // character offset rdOffset int // reading offset (position after current character) lineOffset int // current line offset @@ -63,7 +63,7 @@ func (S *Scanner) next() { S.lineOffset = S.offset S.file.AddLine(S.offset) } - r, w := int(S.src[S.rdOffset]), 1 + r, w := rune(S.src[S.rdOffset]), 1 switch { case r == 0: S.error(S.offset, "illegal character NUL") @@ -86,13 +86,14 @@ func (S *Scanner) next() { } } -// The mode parameter to the Init function is a set of flags (or 0). +// A mode value is set of flags (or 0). // They control scanner behavior. // +type Mode uint + const ( - ScanComments = 1 << iota // return comments as COMMENT tokens - AllowIllegalChars // do not report an error for illegal chars - InsertSemis // automatically insert semicolons + ScanComments Mode = 1 << iota // return comments as COMMENT tokens + dontInsertSemis // do not automatically insert semicolons - for testing only ) // Init prepares the scanner S to tokenize the text src by setting the @@ -105,12 +106,12 @@ const ( // Calls to Scan will use the error handler err if they encounter a // syntax error and err is not nil. Also, for each error encountered, // the Scanner field ErrorCount is incremented by one. The mode parameter -// determines how comments, illegal characters, and semicolons are handled. +// determines how comments are handled. // // Note that Init may call err if there is an error in the first character // of the file. // -func (S *Scanner) Init(file *token.File, src []byte, err ErrorHandler, mode uint) { +func (S *Scanner) Init(file *token.File, src []byte, err ErrorHandler, mode Mode) { // Explicitly initialize all fields since a scanner may be reused. if file.Size() != len(src) { panic("file size does not match src len") @@ -152,13 +153,13 @@ func (S *Scanner) interpretLineComment(text []byte) { filename = filepath.Join(S.dir, filename) } // update scanner position - S.file.AddLineInfo(S.lineOffset, filename, line-1) // -1 since comment applies to next line + S.file.AddLineInfo(S.lineOffset+len(text)+1, filename, line) // +len(text)+1 since comment applies to next line } } } } -func (S *Scanner) scanComment() { +func (S *Scanner) scanComment() string { // initial '/' already consumed; S.ch == '/' || S.ch == '*' offs := S.offset - 1 // position of initial '/' @@ -172,7 +173,7 @@ func (S *Scanner) scanComment() { // comment starts at the beginning of the current line S.interpretLineComment(S.src[offs:S.offset]) } - return + goto exit } /*-style comment */ @@ -182,11 +183,14 @@ func (S *Scanner) scanComment() { S.next() if ch == '*' && S.ch == '/' { S.next() - return + goto exit } } S.error(offs, "comment not terminated") + +exit: + return string(S.src[offs:S.offset]) } func (S *Scanner) findLineEnd() bool { @@ -233,30 +237,30 @@ func (S *Scanner) findLineEnd() bool { return false } -func isLetter(ch int) bool { +func isLetter(ch rune) bool { return 'a' <= ch && ch <= 'z' || 'A' <= ch && ch <= 'Z' || ch == '_' || ch >= 0x80 && unicode.IsLetter(ch) } -func isDigit(ch int) bool { +func isDigit(ch rune) bool { return '0' <= ch && ch <= '9' || ch >= 0x80 && unicode.IsDigit(ch) } -func (S *Scanner) scanIdentifier() token.Token { +func (S *Scanner) scanIdentifier() string { offs := S.offset for isLetter(S.ch) || isDigit(S.ch) { S.next() } - return token.Lookup(S.src[offs:S.offset]) + return string(S.src[offs:S.offset]) } -func digitVal(ch int) int { +func digitVal(ch rune) int { switch { case '0' <= ch && ch <= '9': - return ch - '0' + return int(ch - '0') case 'a' <= ch && ch <= 'f': - return ch - 'a' + 10 + return int(ch - 'a' + 10) case 'A' <= ch && ch <= 'F': - return ch - 'A' + 10 + return int(ch - 'A' + 10) } return 16 // larger than any legal digit val } @@ -267,11 +271,13 @@ func (S *Scanner) scanMantissa(base int) { } } -func (S *Scanner) scanNumber(seenDecimalPoint bool) token.Token { +func (S *Scanner) scanNumber(seenDecimalPoint bool) (token.Token, string) { // digitVal(S.ch) < 10 + offs := S.offset tok := token.INT if seenDecimalPoint { + offs-- tok = token.FLOAT S.scanMantissa(10) goto exponent @@ -335,10 +341,10 @@ exponent: } exit: - return tok + return tok, string(S.src[offs:S.offset]) } -func (S *Scanner) scanEscape(quote int) { +func (S *Scanner) scanEscape(quote rune) { offs := S.offset var i, base, max uint32 @@ -382,7 +388,7 @@ func (S *Scanner) scanEscape(quote int) { } } -func (S *Scanner) scanChar() { +func (S *Scanner) scanChar() string { // '\'' opening already consumed offs := S.offset - 1 @@ -406,9 +412,11 @@ func (S *Scanner) scanChar() { if n != 1 { S.error(offs, "illegal character literal") } + + return string(S.src[offs:S.offset]) } -func (S *Scanner) scanString() { +func (S *Scanner) scanString() string { // '"' opening already consumed offs := S.offset - 1 @@ -425,15 +433,33 @@ func (S *Scanner) scanString() { } S.next() + + return string(S.src[offs:S.offset]) +} + +func stripCR(b []byte) []byte { + c := make([]byte, len(b)) + i := 0 + for _, ch := range b { + if ch != '\r' { + c[i] = ch + i++ + } + } + return c[:i] } -func (S *Scanner) scanRawString() { +func (S *Scanner) scanRawString() string { // '`' opening already consumed offs := S.offset - 1 + hasCR := false for S.ch != '`' { ch := S.ch S.next() + if ch == '\r' { + hasCR = true + } if ch < 0 { S.error(offs, "string not terminated") break @@ -441,6 +467,13 @@ func (S *Scanner) scanRawString() { } S.next() + + lit := S.src[offs:S.offset] + if hasCR { + lit = stripCR(lit) + } + + return string(lit) } func (S *Scanner) skipWhitespace() { @@ -463,7 +496,7 @@ func (S *Scanner) switch2(tok0, tok1 token.Token) token.Token { return tok0 } -func (S *Scanner) switch3(tok0, tok1 token.Token, ch2 int, tok2 token.Token) token.Token { +func (S *Scanner) switch3(tok0, tok1 token.Token, ch2 rune, tok2 token.Token) token.Token { if S.ch == '=' { S.next() return tok1 @@ -475,7 +508,7 @@ func (S *Scanner) switch3(tok0, tok1 token.Token, ch2 int, tok2 token.Token) tok return tok0 } -func (S *Scanner) switch4(tok0, tok1 token.Token, ch2 int, tok2, tok3 token.Token) token.Token { +func (S *Scanner) switch4(tok0, tok1 token.Token, ch2 rune, tok2, tok3 token.Token) token.Token { if S.ch == '=' { S.next() return tok1 @@ -491,15 +524,24 @@ func (S *Scanner) switch4(tok0, tok1 token.Token, ch2 int, tok2, tok3 token.Toke return tok0 } -// Scan scans the next token and returns the token position, -// the token, and the literal string corresponding to the -// token. The source end is indicated by token.EOF. +// Scan scans the next token and returns the token position, the token, +// and its literal string if applicable. The source end is indicated by +// token.EOF. +// +// If the returned token is a literal (token.IDENT, token.INT, token.FLOAT, +// token.IMAG, token.CHAR, token.STRING) or token.COMMENT, the literal string +// has the corresponding value. // // If the returned token is token.SEMICOLON, the corresponding // literal string is ";" if the semicolon was present in the source, // and "\n" if the semicolon was inserted because of a newline or // at EOF. // +// If the returned token is token.ILLEGAL, the literal string is the +// offending character. +// +// In all other cases, Scan returns an empty literal string. +// // For more tolerant parsing, Scan will return a valid token if // possible even if a syntax error was encountered. Thus, even // if the resulting token sequence contains no illegal tokens, @@ -511,33 +553,33 @@ func (S *Scanner) switch4(tok0, tok1 token.Token, ch2 int, tok2, tok3 token.Toke // set with Init. Token positions are relative to that file // and thus relative to the file set. // -func (S *Scanner) Scan() (token.Pos, token.Token, string) { +func (S *Scanner) Scan() (pos token.Pos, tok token.Token, lit string) { scanAgain: S.skipWhitespace() // current token start - insertSemi := false - offs := S.offset - tok := token.ILLEGAL + pos = S.file.Pos(S.offset) // determine token value + insertSemi := false switch ch := S.ch; { case isLetter(ch): - tok = S.scanIdentifier() + lit = S.scanIdentifier() + tok = token.Lookup(lit) switch tok { case token.IDENT, token.BREAK, token.CONTINUE, token.FALLTHROUGH, token.RETURN: insertSemi = true } case digitVal(ch) < 10: insertSemi = true - tok = S.scanNumber(false) + tok, lit = S.scanNumber(false) default: S.next() // always make progress switch ch { case -1: if S.insertSemi { S.insertSemi = false // EOF consumed - return S.file.Pos(offs), token.SEMICOLON, "\n" + return pos, token.SEMICOLON, "\n" } tok = token.EOF case '\n': @@ -545,25 +587,25 @@ scanAgain: // set in the first place and exited early // from S.skipWhitespace() S.insertSemi = false // newline consumed - return S.file.Pos(offs), token.SEMICOLON, "\n" + return pos, token.SEMICOLON, "\n" case '"': insertSemi = true tok = token.STRING - S.scanString() + lit = S.scanString() case '\'': insertSemi = true tok = token.CHAR - S.scanChar() + lit = S.scanChar() case '`': insertSemi = true tok = token.STRING - S.scanRawString() + lit = S.scanRawString() case ':': tok = S.switch2(token.COLON, token.DEFINE) case '.': if digitVal(S.ch) < 10 { insertSemi = true - tok = S.scanNumber(true) + tok, lit = S.scanNumber(true) } else if S.ch == '.' { S.next() if S.ch == '.' { @@ -577,6 +619,7 @@ scanAgain: tok = token.COMMA case ';': tok = token.SEMICOLON + lit = ";" case '(': tok = token.LPAREN case ')': @@ -610,12 +653,12 @@ scanAgain: if S.insertSemi && S.findLineEnd() { // reset position to the beginning of the comment S.ch = '/' - S.offset = offs - S.rdOffset = offs + 1 + S.offset = S.file.Offset(pos) + S.rdOffset = S.offset + 1 S.insertSemi = false // newline consumed - return S.file.Pos(offs), token.SEMICOLON, "\n" + return pos, token.SEMICOLON, "\n" } - S.scanComment() + lit = S.scanComment() if S.mode&ScanComments == 0 { // skip comment S.insertSemi = false // newline consumed @@ -652,19 +695,15 @@ scanAgain: case '|': tok = S.switch3(token.OR, token.OR_ASSIGN, '|', token.LOR) default: - if S.mode&AllowIllegalChars == 0 { - S.error(offs, fmt.Sprintf("illegal character %#U", ch)) - } + S.error(S.file.Offset(pos), fmt.Sprintf("illegal character %#U", ch)) insertSemi = S.insertSemi // preserve insertSemi info + tok = token.ILLEGAL + lit = string(ch) } } - - if S.mode&InsertSemis != 0 { + if S.mode&dontInsertSemis == 0 { S.insertSemi = insertSemi } - // TODO(gri): The scanner API should change such that the literal string - // is only valid if an actual literal was scanned. This will - // permit a more efficient implementation. - return S.file.Pos(offs), tok, string(S.src[offs:S.offset]) + return } diff --git a/src/pkg/go/scanner/scanner_test.go b/src/pkg/go/scanner/scanner_test.go index eb9e1cb81..af45bc5b1 100644 --- a/src/pkg/go/scanner/scanner_test.go +++ b/src/pkg/go/scanner/scanner_test.go @@ -83,6 +83,8 @@ var tokens = [...]elt{ "`", literal, }, + {token.STRING, "`\r`", literal}, + {token.STRING, "`foo\r\nbar`", literal}, // Operators and delimiters {token.ADD, "+", operator}, @@ -175,6 +177,15 @@ var tokens = [...]elt{ const whitespace = " \t \n\n\n" // to separate tokens +var source = func() []byte { + var src []byte + for _, t := range tokens { + src = append(src, t.lit...) + src = append(src, whitespace...) + } + return src +}() + type testErrorHandler struct { t *testing.T } @@ -212,20 +223,20 @@ func checkPos(t *testing.T, lit string, p token.Pos, expected token.Position) { // Verify that calling Scan() provides the correct results. func TestScan(t *testing.T) { // make source - var src string - for _, e := range tokens { - src += e.lit + whitespace - } - src_linecount := newlineCount(src) + src_linecount := newlineCount(string(source)) whitespace_linecount := newlineCount(whitespace) // verify scan var s Scanner - s.Init(fset.AddFile("", fset.Base(), len(src)), []byte(src), &testErrorHandler{t}, ScanComments) + s.Init(fset.AddFile("", fset.Base(), len(source)), source, &testErrorHandler{t}, ScanComments|dontInsertSemis) index := 0 epos := token.Position{"", 0, 1, 1} // expected position for { pos, tok, lit := s.Scan() + if lit == "" { + // no literal value for non-literal tokens + lit = tok.String() + } e := elt{token.EOF, "", special} if index < len(tokens) { e = tokens[index] @@ -237,10 +248,18 @@ func TestScan(t *testing.T) { } checkPos(t, lit, pos, epos) if tok != e.tok { - t.Errorf("bad token for %q: got %s, expected %s", lit, tok.String(), e.tok.String()) + t.Errorf("bad token for %q: got %s, expected %s", lit, tok, e.tok) } - if e.tok.IsLiteral() && lit != e.lit { - t.Errorf("bad literal for %q: got %q, expected %q", lit, lit, e.lit) + if e.tok.IsLiteral() { + // no CRs in raw string literals + elit := e.lit + if elit[0] == '`' { + elit = string(stripCR([]byte(elit))) + epos.Offset += len(e.lit) - len(lit) // correct position + } + if lit != elit { + t.Errorf("bad literal for %q: got %q, expected %q", lit, lit, elit) + } } if tokenclass(tok) != e.class { t.Errorf("bad class for %q: got %d, expected %d", lit, tokenclass(tok), e.class) @@ -262,7 +281,7 @@ func TestScan(t *testing.T) { } } -func checkSemi(t *testing.T, line string, mode uint) { +func checkSemi(t *testing.T, line string, mode Mode) { var S Scanner file := fset.AddFile("TestSemis", fset.Base(), len(line)) S.Init(file, []byte(line), nil, mode) @@ -286,7 +305,7 @@ func checkSemi(t *testing.T, line string, mode uint) { } checkPos(t, line, pos, semiPos) } else { - t.Errorf("bad token for %q: got %s, expected ;", line, tok.String()) + t.Errorf("bad token for %q: got %s, expected ;", line, tok) } } else if tok == token.SEMICOLON { t.Errorf("bad token for %q: got ;, expected no ;", line) @@ -420,14 +439,14 @@ var lines = []string{ func TestSemis(t *testing.T) { for _, line := range lines { - checkSemi(t, line, AllowIllegalChars|InsertSemis) - checkSemi(t, line, AllowIllegalChars|InsertSemis|ScanComments) + checkSemi(t, line, 0) + checkSemi(t, line, ScanComments) // if the input ended in newlines, the input must tokenize the // same with or without those newlines for i := len(line) - 1; i >= 0 && line[i] == '\n'; i-- { - checkSemi(t, line[0:i], AllowIllegalChars|InsertSemis) - checkSemi(t, line[0:i], AllowIllegalChars|InsertSemis|ScanComments) + checkSemi(t, line[0:i], 0) + checkSemi(t, line[0:i], ScanComments) } } } @@ -482,7 +501,7 @@ func TestLineComments(t *testing.T) { // verify scan var S Scanner file := fset.AddFile(filepath.Join("dir", "TestLineComments"), fset.Base(), len(src)) - S.Init(file, []byte(src), nil, 0) + S.Init(file, []byte(src), nil, dontInsertSemis) for _, s := range segs { p, _, lit := S.Scan() pos := file.Position(p) @@ -501,7 +520,7 @@ func TestInit(t *testing.T) { // 1st init src1 := "if true { }" f1 := fset.AddFile("src1", fset.Base(), len(src1)) - s.Init(f1, []byte(src1), nil, 0) + s.Init(f1, []byte(src1), nil, dontInsertSemis) if f1.Size() != len(src1) { t.Errorf("bad file size: got %d, expected %d", f1.Size(), len(src1)) } @@ -509,40 +528,19 @@ func TestInit(t *testing.T) { s.Scan() // true _, tok, _ := s.Scan() // { if tok != token.LBRACE { - t.Errorf("bad token: got %s, expected %s", tok.String(), token.LBRACE) + t.Errorf("bad token: got %s, expected %s", tok, token.LBRACE) } // 2nd init src2 := "go true { ]" f2 := fset.AddFile("src2", fset.Base(), len(src2)) - s.Init(f2, []byte(src2), nil, 0) + s.Init(f2, []byte(src2), nil, dontInsertSemis) if f2.Size() != len(src2) { t.Errorf("bad file size: got %d, expected %d", f2.Size(), len(src2)) } _, tok, _ = s.Scan() // go if tok != token.GO { - t.Errorf("bad token: got %s, expected %s", tok.String(), token.GO) - } - - if s.ErrorCount != 0 { - t.Errorf("found %d errors", s.ErrorCount) - } -} - -func TestIllegalChars(t *testing.T) { - var s Scanner - - const src = "*?*$*@*" - file := fset.AddFile("", fset.Base(), len(src)) - s.Init(file, []byte(src), &testErrorHandler{t}, AllowIllegalChars) - for offs, ch := range src { - pos, tok, lit := s.Scan() - if poffs := file.Offset(pos); poffs != offs { - t.Errorf("bad position for %s: got %d, expected %d", lit, poffs, offs) - } - if tok == token.ILLEGAL && lit != string(ch) { - t.Errorf("bad token: got %s, expected %s", lit, string(ch)) - } + t.Errorf("bad token: got %s, expected %s", tok, token.GO) } if s.ErrorCount != 0 { @@ -562,7 +560,7 @@ func TestStdErrorHander(t *testing.T) { v := new(ErrorVector) var s Scanner - s.Init(fset.AddFile("File1", fset.Base(), len(src)), []byte(src), v, 0) + s.Init(fset.AddFile("File1", fset.Base(), len(src)), []byte(src), v, dontInsertSemis) for { if _, tok, _ := s.Scan(); tok == token.EOF { break @@ -607,7 +605,7 @@ func (h *errorCollector) Error(pos token.Position, msg string) { func checkError(t *testing.T, src string, tok token.Token, pos int, err string) { var s Scanner var h errorCollector - s.Init(fset.AddFile("", fset.Base(), len(src)), []byte(src), &h, ScanComments) + s.Init(fset.AddFile("", fset.Base(), len(src)), []byte(src), &h, ScanComments|dontInsertSemis) _, tok0, _ := s.Scan() _, tok1, _ := s.Scan() if tok0 != tok { @@ -670,3 +668,20 @@ func TestScanErrors(t *testing.T) { checkError(t, e.src, e.tok, e.pos, e.err) } } + +func BenchmarkScan(b *testing.B) { + b.StopTimer() + fset := token.NewFileSet() + file := fset.AddFile("", fset.Base(), len(source)) + var s Scanner + b.StartTimer() + for i := b.N - 1; i >= 0; i-- { + s.Init(file, source, nil, ScanComments) + for { + _, tok, _ := s.Scan() + if tok == token.EOF { + break + } + } + } +} diff --git a/src/pkg/go/token/position.go b/src/pkg/go/token/position.go index 9155b501d..647d1b770 100644 --- a/src/pkg/go/token/position.go +++ b/src/pkg/go/token/position.go @@ -12,6 +12,9 @@ import ( "sync" ) +// ----------------------------------------------------------------------------- +// Positions + // Position describes an arbitrary source position // including the file, line, and column location. // A Position is valid if the line number is > 0. @@ -81,86 +84,8 @@ func (p Pos) IsValid() bool { return p != NoPos } -func searchFiles(a []*File, x int) int { - return sort.Search(len(a), func(i int) bool { return a[i].base > x }) - 1 -} - -func (s *FileSet) file(p Pos) *File { - if f := s.last; f != nil && f.base <= int(p) && int(p) <= f.base+f.size { - return f - } - if i := searchFiles(s.files, int(p)); i >= 0 { - f := s.files[i] - // f.base <= int(p) by definition of searchFiles - if int(p) <= f.base+f.size { - s.last = f - return f - } - } - return nil -} - -// File returns the file which contains the position p. -// If no such file is found (for instance for p == NoPos), -// the result is nil. -// -func (s *FileSet) File(p Pos) (f *File) { - if p != NoPos { - s.mutex.RLock() - f = s.file(p) - s.mutex.RUnlock() - } - return -} - -func (f *File) position(p Pos) (pos Position) { - offset := int(p) - f.base - pos.Offset = offset - pos.Filename, pos.Line, pos.Column = f.info(offset) - return -} - -// Position converts a Pos in the fileset into a general Position. -func (s *FileSet) Position(p Pos) (pos Position) { - if p != NoPos { - // TODO(gri) consider optimizing the case where p - // is in the last file added, or perhaps - // looked at - will eliminate one level - // of search - s.mutex.RLock() - if f := s.file(p); f != nil { - pos = f.position(p) - } - s.mutex.RUnlock() - } - return -} - -// A lineInfo object describes alternative file and line number -// information (such as provided via a //line comment in a .go -// file) for a given file offset. -type lineInfo struct { - // fields are exported to make them accessible to gob - Offset int - Filename string - Line int -} - -// AddLineInfo adds alternative file and line number information for -// a given file offset. The offset must be larger than the offset for -// the previously added alternative line info and smaller than the -// file size; otherwise the information is ignored. -// -// AddLineInfo is typically used to register alternative position -// information for //line filename:line comments in source files. -// -func (f *File) AddLineInfo(offset int, filename string, line int) { - f.set.mutex.Lock() - if i := len(f.infos); i == 0 || f.infos[i-1].Offset < offset && offset < f.size { - f.infos = append(f.infos, lineInfo{offset, filename, line}) - } - f.set.mutex.Unlock() -} +// ----------------------------------------------------------------------------- +// File // A File is a handle for a file belonging to a FileSet. // A File has a name, size, and line offset table. @@ -255,6 +180,32 @@ func (f *File) SetLinesForContent(content []byte) { f.set.mutex.Unlock() } +// A lineInfo object describes alternative file and line number +// information (such as provided via a //line comment in a .go +// file) for a given file offset. +type lineInfo struct { + // fields are exported to make them accessible to gob + Offset int + Filename string + Line int +} + +// AddLineInfo adds alternative file and line number information for +// a given file offset. The offset must be larger than the offset for +// the previously added alternative line info and smaller than the +// file size; otherwise the information is ignored. +// +// AddLineInfo is typically used to register alternative position +// information for //line filename:line comments in source files. +// +func (f *File) AddLineInfo(offset int, filename string, line int) { + f.set.mutex.Lock() + if i := len(f.infos); i == 0 || f.infos[i-1].Offset < offset && offset < f.size { + f.infos = append(f.infos, lineInfo{offset, filename, line}) + } + f.set.mutex.Unlock() +} + // Pos returns the Pos value for the given file offset; // the offset must be <= f.Size(). // f.Pos(f.Offset(p)) == p. @@ -285,41 +236,6 @@ func (f *File) Line(p Pos) int { return f.Position(p).Line } -// Position returns the Position value for the given file position p; -// p must be a Pos value in that file or NoPos. -// -func (f *File) Position(p Pos) (pos Position) { - if p != NoPos { - if int(p) < f.base || int(p) > f.base+f.size { - panic("illegal Pos value") - } - pos = f.position(p) - } - return -} - -func searchInts(a []int, x int) int { - // This function body is a manually inlined version of: - // - // return sort.Search(len(a), func(i int) bool { return a[i] > x }) - 1 - // - // With better compiler optimizations, this may not be needed in the - // future, but at the moment this change improves the go/printer - // benchmark performance by ~30%. This has a direct impact on the - // speed of gofmt and thus seems worthwhile (2011-04-29). - i, j := 0, len(a) - for i < j { - h := i + (j-i)/2 // avoid overflow when computing h - // i ≤ h < j - if a[h] <= x { - i = h + 1 - } else { - j = h - } - } - return i - 1 -} - func searchLineInfos(a []lineInfo, x int) int { return sort.Search(len(a), func(i int) bool { return a[i].Offset > x }) - 1 } @@ -343,6 +259,29 @@ func (f *File) info(offset int) (filename string, line, column int) { return } +func (f *File) position(p Pos) (pos Position) { + offset := int(p) - f.base + pos.Offset = offset + pos.Filename, pos.Line, pos.Column = f.info(offset) + return +} + +// Position returns the Position value for the given file position p; +// p must be a Pos value in that file or NoPos. +// +func (f *File) Position(p Pos) (pos Position) { + if p != NoPos { + if int(p) < f.base || int(p) > f.base+f.size { + panic("illegal Pos value") + } + pos = f.position(p) + } + return +} + +// ----------------------------------------------------------------------------- +// FileSet + // A FileSet represents a set of source files. // Methods of file sets are synchronized; multiple goroutines // may invoke them concurrently. @@ -406,23 +345,91 @@ func (s *FileSet) AddFile(filename string, base, size int) *File { return f } -// Files returns the files added to the file set. -func (s *FileSet) Files() <-chan *File { - ch := make(chan *File) - go func() { - for i := 0; ; i++ { - var f *File - s.mutex.RLock() - if i < len(s.files) { - f = s.files[i] - } - s.mutex.RUnlock() - if f == nil { - break - } - ch <- f +// Iterate calls f for the files in the file set in the order they were added +// until f returns false. +// +func (s *FileSet) Iterate(f func(*File) bool) { + for i := 0; ; i++ { + var file *File + s.mutex.RLock() + if i < len(s.files) { + file = s.files[i] + } + s.mutex.RUnlock() + if file == nil || !f(file) { + break } - close(ch) - }() - return ch + } +} + +func searchFiles(a []*File, x int) int { + return sort.Search(len(a), func(i int) bool { return a[i].base > x }) - 1 +} + +func (s *FileSet) file(p Pos) *File { + // common case: p is in last file + if f := s.last; f != nil && f.base <= int(p) && int(p) <= f.base+f.size { + return f + } + // p is not in last file - search all files + if i := searchFiles(s.files, int(p)); i >= 0 { + f := s.files[i] + // f.base <= int(p) by definition of searchFiles + if int(p) <= f.base+f.size { + s.last = f + return f + } + } + return nil +} + +// File returns the file that contains the position p. +// If no such file is found (for instance for p == NoPos), +// the result is nil. +// +func (s *FileSet) File(p Pos) (f *File) { + if p != NoPos { + s.mutex.RLock() + f = s.file(p) + s.mutex.RUnlock() + } + return +} + +// Position converts a Pos in the fileset into a general Position. +func (s *FileSet) Position(p Pos) (pos Position) { + if p != NoPos { + s.mutex.RLock() + if f := s.file(p); f != nil { + pos = f.position(p) + } + s.mutex.RUnlock() + } + return +} + +// ----------------------------------------------------------------------------- +// Helper functions + +func searchInts(a []int, x int) int { + // This function body is a manually inlined version of: + // + // return sort.Search(len(a), func(i int) bool { return a[i] > x }) - 1 + // + // With better compiler optimizations, this may not be needed in the + // future, but at the moment this change improves the go/printer + // benchmark performance by ~30%. This has a direct impact on the + // speed of gofmt and thus seems worthwhile (2011-04-29). + // TODO(gri): Remove this when compilers have caught up. + i, j := 0, len(a) + for i < j { + h := i + (j-i)/2 // avoid overflow when computing h + // i ≤ h < j + if a[h] <= x { + i = h + 1 + } else { + j = h + } + } + return i - 1 } diff --git a/src/pkg/go/token/position_test.go b/src/pkg/go/token/position_test.go index 30bec5991..160107df4 100644 --- a/src/pkg/go/token/position_test.go +++ b/src/pkg/go/token/position_test.go @@ -167,12 +167,13 @@ func TestFiles(t *testing.T) { for i, test := range tests { fset.AddFile(test.filename, fset.Base(), test.size) j := 0 - for g := range fset.Files() { - if g.Name() != tests[j].filename { - t.Errorf("expected filename = %s; got %s", tests[j].filename, g.Name()) + fset.Iterate(func(f *File) bool { + if f.Name() != tests[j].filename { + t.Errorf("expected filename = %s; got %s", tests[j].filename, f.Name()) } j++ - } + return true + }) if j != i+1 { t.Errorf("expected %d files; got %d", i+1, j) } diff --git a/src/pkg/go/token/serialize.go b/src/pkg/go/token/serialize.go index 80a3323f9..042d6abdf 100644 --- a/src/pkg/go/token/serialize.go +++ b/src/pkg/go/token/serialize.go @@ -5,9 +5,8 @@ package token import ( - "gob" + "encoding/gob" "io" - "os" ) type serializedFile struct { @@ -24,10 +23,19 @@ type serializedFileSet struct { Files []serializedFile } +func (s *serializedFileSet) Read(r io.Reader) error { + return gob.NewDecoder(r).Decode(s) +} + +func (s *serializedFileSet) Write(w io.Writer) error { + return gob.NewEncoder(w).Encode(s) +} + // Read reads the fileset from r into s; s must not be nil. -func (s *FileSet) Read(r io.Reader) os.Error { +// If r does not also implement io.ByteReader, it will be wrapped in a bufio.Reader. +func (s *FileSet) Read(r io.Reader) error { var ss serializedFileSet - if err := gob.NewDecoder(r).Decode(&ss); err != nil { + if err := ss.Read(r); err != nil { return err } @@ -46,7 +54,7 @@ func (s *FileSet) Read(r io.Reader) os.Error { } // Write writes the fileset s to w. -func (s *FileSet) Write(w io.Writer) os.Error { +func (s *FileSet) Write(w io.Writer) error { var ss serializedFileSet s.mutex.Lock() @@ -58,5 +66,5 @@ func (s *FileSet) Write(w io.Writer) os.Error { ss.Files = files s.mutex.Unlock() - return gob.NewEncoder(w).Encode(ss) + return ss.Write(w) } diff --git a/src/pkg/go/token/serialize_test.go b/src/pkg/go/token/serialize_test.go index 24e419abf..a8ce30ab2 100644 --- a/src/pkg/go/token/serialize_test.go +++ b/src/pkg/go/token/serialize_test.go @@ -7,13 +7,12 @@ package token import ( "bytes" "fmt" - "os" "testing" ) // equal returns nil if p and q describe the same file set; // otherwise it returns an error describing the discrepancy. -func equal(p, q *FileSet) os.Error { +func equal(p, q *FileSet) error { if p == q { // avoid deadlock if p == q return nil diff --git a/src/pkg/go/token/token.go b/src/pkg/go/token/token.go index 557374052..84b6314d5 100644 --- a/src/pkg/go/token/token.go +++ b/src/pkg/go/token/token.go @@ -283,10 +283,8 @@ func init() { // Lookup maps an identifier to its keyword token or IDENT (if not a keyword). // -func Lookup(ident []byte) Token { - // TODO Maps with []byte key are illegal because []byte does not - // support == . Should find a more efficient solution eventually. - if tok, is_keyword := keywords[string(ident)]; is_keyword { +func Lookup(ident string) Token { + if tok, is_keyword := keywords[ident]; is_keyword { return tok } return IDENT @@ -295,16 +293,16 @@ func Lookup(ident []byte) Token { // Predicates // IsLiteral returns true for tokens corresponding to identifiers -// and basic type literals; returns false otherwise. +// and basic type literals; it returns false otherwise. // func (tok Token) IsLiteral() bool { return literal_beg < tok && tok < literal_end } // IsOperator returns true for tokens corresponding to operators and -// delimiters; returns false otherwise. +// delimiters; it returns false otherwise. // func (tok Token) IsOperator() bool { return operator_beg < tok && tok < operator_end } // IsKeyword returns true for tokens corresponding to keywords; -// returns false otherwise. +// it returns false otherwise. // func (tok Token) IsKeyword() bool { return keyword_beg < tok && tok < keyword_end } diff --git a/src/pkg/go/typechecker/Makefile b/src/pkg/go/typechecker/Makefile deleted file mode 100644 index 83af3ef4e..000000000 --- a/src/pkg/go/typechecker/Makefile +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright 2010 The Go Authors. All rights reserved. -# Use of this source code is governed by a BSD-style -# license that can be found in the LICENSE file. - -include ../../../Make.inc - -TARG=go/typechecker -GOFILES=\ - scope.go\ - type.go\ - typechecker.go\ - universe.go\ - -include ../../../Make.pkg diff --git a/src/pkg/go/typechecker/scope.go b/src/pkg/go/typechecker/scope.go deleted file mode 100644 index d73d1a450..000000000 --- a/src/pkg/go/typechecker/scope.go +++ /dev/null @@ -1,69 +0,0 @@ -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// DEPRECATED FILE - WILL GO AWAY EVENTUALLY. -// -// Scope handling is now done in go/parser. -// The functionality here is only present to -// keep the typechecker running for now. - -package typechecker - -import "go/ast" - -func (tc *typechecker) openScope() *ast.Scope { - tc.topScope = ast.NewScope(tc.topScope) - return tc.topScope -} - -func (tc *typechecker) closeScope() { - tc.topScope = tc.topScope.Outer -} - -// declInScope declares an object of a given kind and name in scope and sets the object's Decl and N fields. -// It returns the newly allocated object. If an object with the same name already exists in scope, an error -// is reported and the object is not inserted. -func (tc *typechecker) declInScope(scope *ast.Scope, kind ast.ObjKind, name *ast.Ident, decl interface{}, n int) *ast.Object { - obj := ast.NewObj(kind, name.Name) - obj.Decl = decl - //obj.N = n - name.Obj = obj - if name.Name != "_" { - if alt := scope.Insert(obj); alt != nil { - tc.Errorf(name.Pos(), "%s already declared at %s", name.Name, tc.fset.Position(alt.Pos()).String()) - } - } - return obj -} - -// decl is the same as declInScope(tc.topScope, ...) -func (tc *typechecker) decl(kind ast.ObjKind, name *ast.Ident, decl interface{}, n int) *ast.Object { - return tc.declInScope(tc.topScope, kind, name, decl, n) -} - -// find returns the object with the given name if visible in the current scope hierarchy. -// If no such object is found, an error is reported and a bad object is returned instead. -func (tc *typechecker) find(name *ast.Ident) (obj *ast.Object) { - for s := tc.topScope; s != nil && obj == nil; s = s.Outer { - obj = s.Lookup(name.Name) - } - if obj == nil { - tc.Errorf(name.Pos(), "%s not declared", name.Name) - obj = ast.NewObj(ast.Bad, name.Name) - } - name.Obj = obj - return -} - -// findField returns the object with the given name if visible in the type's scope. -// If no such object is found, an error is reported and a bad object is returned instead. -func (tc *typechecker) findField(typ *Type, name *ast.Ident) (obj *ast.Object) { - // TODO(gri) This is simplistic at the moment and ignores anonymous fields. - obj = typ.Scope.Lookup(name.Name) - if obj == nil { - tc.Errorf(name.Pos(), "%s not declared", name.Name) - obj = ast.NewObj(ast.Bad, name.Name) - } - return -} diff --git a/src/pkg/go/typechecker/testdata/test0.src b/src/pkg/go/typechecker/testdata/test0.src deleted file mode 100644 index 4e317f214..000000000 --- a/src/pkg/go/typechecker/testdata/test0.src +++ /dev/null @@ -1,94 +0,0 @@ -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// type declarations - -package P0 - -type ( - B bool - I int32 - A [10]P - T struct { - x, y P - } - P *T - R *R - F func(A) I - Y interface { - f(A) I - } - S []P - M map[I]F - C chan<- I -) - -type ( - a/* ERROR "illegal cycle" */ a - a/* ERROR "already declared" */ int - - b/* ERROR "illegal cycle" */ c - c d - d e - e b /* ERROR "not a type" */ - - t *t - - U V - V W - W *U - - P1 *S2 - P2 P1 - - S1 struct { - a, b, c int - u, v, a/* ERROR "already declared" */ float - } - S2/* ERROR "illegal cycle" */ struct { - x S2 - } - - L1 []L1 - L2 []int - - A1 [10]int - A2/* ERROR "illegal cycle" */ [10]A2 - A3/* ERROR "illegal cycle" */ [10]struct { - x A4 - } - A4 [10]A3 - - F1 func() - F2 func(x, y, z float) - F3 func(x, y, x /* ERROR "already declared" */ float) - F4 func() (x, y, x /* ERROR "already declared" */ float) - F5 func(x int) (x /* ERROR "already declared" */ float) - - I1 interface{} - I2 interface { - m1() - } - I3 interface { - m1() - m1 /* ERROR "already declared" */ () - } - I4 interface { - m1(x, y, x /* ERROR "already declared" */ float) - m2() (x, y, x /* ERROR "already declared" */ float) - m3(x int) (x /* ERROR "already declared" */ float) - } - I5 interface { - m1(I5) - } - - C1 chan int - C2 <-chan int - C3 chan<- C3 - - M1 map[Last]string - M2 map[string]M2 - - Last int -) diff --git a/src/pkg/go/typechecker/testdata/test1.src b/src/pkg/go/typechecker/testdata/test1.src deleted file mode 100644 index b5531fb9f..000000000 --- a/src/pkg/go/typechecker/testdata/test1.src +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// const and var declarations - -package P1 - -const ( - c1 = 0 - c2 int = 0 - c3, c4 = 0 -) diff --git a/src/pkg/go/typechecker/testdata/test3.src b/src/pkg/go/typechecker/testdata/test3.src deleted file mode 100644 index 2e1a9fa8f..000000000 --- a/src/pkg/go/typechecker/testdata/test3.src +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package P3 - -// function and method signatures - -func _() {} -func _() {} -func _(x, x /* ERROR "already declared" */ int) {} - -func f() {} -func f /* ERROR "already declared" */ () {} - -func (*foo /* ERROR "invalid receiver" */ ) m() {} -func (bar /* ERROR "not a type" */ ) m() {} - -func f1(x, _, _ int) (_, _ float) {} -func f2(x, y, x /* ERROR "already declared" */ int) {} -func f3(x, y int) (a, b, x /* ERROR "already declared" */ int) {} - -func (x *T) m1() {} -func (x *T) m1 /* ERROR "already declared" */ () {} -func (x T) m1 /* ERROR "already declared" */ () {} -func (T) m1 /* ERROR "already declared" */ () {} - -func (x *T) m2(u, x /* ERROR "already declared" */ int) {} -func (x *T) m3(a, b, c int) (u, x /* ERROR "already declared" */ int) {} -// The following are disabled for now because the typechecker -// in in the process of being rewritten and cannot handle them -// at the moment -//func (T) _(x, x /* "already declared" */ int) {} -//func (T) _() (x, x /* "already declared" */ int) {} - -//func (PT) _() {} - -var bar int - -type T struct{} -type PT (T) diff --git a/src/pkg/go/typechecker/testdata/test4.src b/src/pkg/go/typechecker/testdata/test4.src deleted file mode 100644 index 94d3558f9..000000000 --- a/src/pkg/go/typechecker/testdata/test4.src +++ /dev/null @@ -1,11 +0,0 @@ -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Constant declarations - -package P4 - -const ( - c0 = 0 -) diff --git a/src/pkg/go/typechecker/type.go b/src/pkg/go/typechecker/type.go deleted file mode 100644 index 1b88eb54b..000000000 --- a/src/pkg/go/typechecker/type.go +++ /dev/null @@ -1,118 +0,0 @@ -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package typechecker - -import "go/ast" - -// A Type represents a Go type. -type Type struct { - Form Form - Obj *ast.Object // corresponding type name, or nil - Scope *ast.Scope // fields and methods, always present - N uint // basic type id, array length, number of function results, or channel direction - Key, Elt *Type // map key and array, pointer, slice, map or channel element - Params *ast.Scope // function (receiver, input and result) parameters, tuple expressions (results of function calls), or nil - Expr ast.Expr // corresponding AST expression -} - -// NewType creates a new type of a given form. -func NewType(form Form) *Type { - return &Type{Form: form, Scope: ast.NewScope(nil)} -} - -// Form describes the form of a type. -type Form int - -// The list of possible type forms. -const ( - BadType Form = iota // for error handling - Unresolved // type not fully setup - Basic - Array - Struct - Pointer - Function - Method - Interface - Slice - Map - Channel - Tuple -) - -var formStrings = [...]string{ - BadType: "badType", - Unresolved: "unresolved", - Basic: "basic", - Array: "array", - Struct: "struct", - Pointer: "pointer", - Function: "function", - Method: "method", - Interface: "interface", - Slice: "slice", - Map: "map", - Channel: "channel", - Tuple: "tuple", -} - -func (form Form) String() string { return formStrings[form] } - -// The list of basic type id's. -const ( - Bool = iota - Byte - Uint - Int - Float - Complex - Uintptr - String - - Uint8 - Uint16 - Uint32 - Uint64 - - Int8 - Int16 - Int32 - Int64 - - Float32 - Float64 - - Complex64 - Complex128 - - // TODO(gri) ideal types are missing -) - -var BasicTypes = map[uint]string{ - Bool: "bool", - Byte: "byte", - Uint: "uint", - Int: "int", - Float: "float", - Complex: "complex", - Uintptr: "uintptr", - String: "string", - - Uint8: "uint8", - Uint16: "uint16", - Uint32: "uint32", - Uint64: "uint64", - - Int8: "int8", - Int16: "int16", - Int32: "int32", - Int64: "int64", - - Float32: "float32", - Float64: "float64", - - Complex64: "complex64", - Complex128: "complex128", -} diff --git a/src/pkg/go/typechecker/typechecker.go b/src/pkg/go/typechecker/typechecker.go deleted file mode 100644 index 24480165b..000000000 --- a/src/pkg/go/typechecker/typechecker.go +++ /dev/null @@ -1,468 +0,0 @@ -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// DEPRECATED PACKAGE - SEE go/types INSTEAD. -// This package implements typechecking of a Go AST. -// The result of the typecheck is an augmented AST -// with object and type information for each identifier. -// -package typechecker - -import ( - "fmt" - "go/ast" - "go/token" - "go/scanner" - "os" -) - -// TODO(gri) don't report errors for objects/types that are marked as bad. - - -const debug = true // set for debugging output - -// An importer takes an import path and returns the data describing the -// respective package's exported interface. The data format is TBD. -// -type Importer func(path string) ([]byte, os.Error) - -// CheckPackage typechecks a package and augments the AST by setting -// *ast.Object, *ast.Type, and *ast.Scope fields accordingly. If an -// importer is provided, it is used to handle imports, otherwise they -// are ignored (likely leading to typechecking errors). -// -// If errors are reported, the AST may be incompletely augmented (fields -// may be nil) or contain incomplete object, type, or scope information. -// -func CheckPackage(fset *token.FileSet, pkg *ast.Package, importer Importer) os.Error { - var tc typechecker - tc.fset = fset - tc.importer = importer - tc.checkPackage(pkg) - return tc.GetError(scanner.Sorted) -} - -// CheckFile typechecks a single file, but otherwise behaves like -// CheckPackage. If the complete package consists of more than just -// one file, the file may not typecheck without errors. -// -func CheckFile(fset *token.FileSet, file *ast.File, importer Importer) os.Error { - // create a single-file dummy package - pkg := &ast.Package{file.Name.Name, nil, nil, map[string]*ast.File{fset.Position(file.Name.NamePos).Filename: file}} - return CheckPackage(fset, pkg, importer) -} - -// ---------------------------------------------------------------------------- -// Typechecker state - -type typechecker struct { - fset *token.FileSet - scanner.ErrorVector - importer Importer - globals []*ast.Object // list of global objects - topScope *ast.Scope // current top-most scope - cyclemap map[*ast.Object]bool // for cycle detection - iota int // current value of iota -} - -func (tc *typechecker) Errorf(pos token.Pos, format string, args ...interface{}) { - tc.Error(tc.fset.Position(pos), fmt.Sprintf(format, args...)) -} - -func assert(pred bool) { - if !pred { - panic("internal error") - } -} - -/* -Typechecking is done in several phases: - -phase 1: declare all global objects; also collect all function and method declarations - - all objects have kind, name, decl fields; the decl field permits - quick lookup of an object's declaration - - constant objects have an iota value - - type objects have unresolved types with empty scopes, all others have nil types - - report global double declarations - -phase 2: bind methods to their receiver base types - - receiver base types must be declared in the package, thus for - each method a corresponding (unresolved) type must exist - - report method double declarations and errors with base types - -phase 3: resolve all global objects - - sequentially iterate through all objects in the global scope - - resolve types for all unresolved types and assign types to - all attached methods - - assign types to all other objects, possibly by evaluating - constant and initializer expressions - - resolution may recurse; a cyclemap is used to detect cycles - - report global typing errors - -phase 4: sequentially typecheck function and method bodies - - all global objects are declared and have types and values; - all methods have types - - sequentially process statements in each body; any object - referred to must be fully defined at this point - - report local typing errors -*/ - -func (tc *typechecker) checkPackage(pkg *ast.Package) { - // setup package scope - tc.topScope = Universe - tc.openScope() - defer tc.closeScope() - - // TODO(gri) there's no file scope at the moment since we ignore imports - - // phase 1: declare all global objects; also collect all function and method declarations - var funcs []*ast.FuncDecl - for _, file := range pkg.Files { - for _, decl := range file.Decls { - tc.declGlobal(decl) - if f, isFunc := decl.(*ast.FuncDecl); isFunc { - funcs = append(funcs, f) - } - } - } - - // phase 2: bind methods to their receiver base types - for _, m := range funcs { - if m.Recv != nil { - tc.bindMethod(m) - } - } - - // phase 3: resolve all global objects - tc.cyclemap = make(map[*ast.Object]bool) - for _, obj := range tc.globals { - tc.resolve(obj) - } - assert(len(tc.cyclemap) == 0) - - // 4: sequentially typecheck function and method bodies - for _, f := range funcs { - ftype, _ := f.Name.Obj.Type.(*Type) - tc.checkBlock(f.Body.List, ftype) - } - - pkg.Scope = tc.topScope -} - -func (tc *typechecker) declGlobal(global ast.Decl) { - switch d := global.(type) { - case *ast.BadDecl: - // ignore - - case *ast.GenDecl: - iota := 0 - var prev *ast.ValueSpec - for _, spec := range d.Specs { - switch s := spec.(type) { - case *ast.ImportSpec: - // TODO(gri) imports go into file scope - case *ast.ValueSpec: - switch d.Tok { - case token.CONST: - if s.Values == nil { - // create a new spec with type and values from the previous one - if prev != nil { - s = &ast.ValueSpec{s.Doc, s.Names, prev.Type, prev.Values, s.Comment} - } else { - // TODO(gri) this should probably go into the const decl code - tc.Errorf(s.Pos(), "missing initializer for const %s", s.Names[0].Name) - } - } - for _, name := range s.Names { - tc.globals = append(tc.globals, tc.decl(ast.Con, name, s, iota)) - } - case token.VAR: - for _, name := range s.Names { - tc.globals = append(tc.globals, tc.decl(ast.Var, name, s, 0)) - } - default: - panic("unreachable") - } - prev = s - iota++ - case *ast.TypeSpec: - obj := tc.decl(ast.Typ, s.Name, s, 0) - tc.globals = append(tc.globals, obj) - // give all type objects an unresolved type so - // that we can collect methods in the type scope - typ := NewType(Unresolved) - obj.Type = typ - typ.Obj = obj - default: - panic("unreachable") - } - } - - case *ast.FuncDecl: - if d.Recv == nil { - tc.globals = append(tc.globals, tc.decl(ast.Fun, d.Name, d, 0)) - } - - default: - panic("unreachable") - } -} - -// If x is of the form *T, deref returns T, otherwise it returns x. -func deref(x ast.Expr) ast.Expr { - if p, isPtr := x.(*ast.StarExpr); isPtr { - x = p.X - } - return x -} - -func (tc *typechecker) bindMethod(method *ast.FuncDecl) { - // a method is declared in the receiver base type's scope - var scope *ast.Scope - base := deref(method.Recv.List[0].Type) - if name, isIdent := base.(*ast.Ident); isIdent { - // if base is not an *ast.Ident, we had a syntax - // error and the parser reported an error already - obj := tc.topScope.Lookup(name.Name) - if obj == nil { - tc.Errorf(name.Pos(), "invalid receiver: %s is not declared in this package", name.Name) - } else if obj.Kind != ast.Typ { - tc.Errorf(name.Pos(), "invalid receiver: %s is not a type", name.Name) - } else { - typ := obj.Type.(*Type) - assert(typ.Form == Unresolved) - scope = typ.Scope - } - } - if scope == nil { - // no receiver type found; use a dummy scope - // (we still want to type-check the method - // body, so make sure there is a name object - // and type) - // TODO(gri) should we record the scope so - // that we don't lose the receiver for type- - // checking of the method body? - scope = ast.NewScope(nil) - } - tc.declInScope(scope, ast.Fun, method.Name, method, 0) -} - -func (tc *typechecker) resolve(obj *ast.Object) { - // check for declaration cycles - if tc.cyclemap[obj] { - tc.Errorf(obj.Pos(), "illegal cycle in declaration of %s", obj.Name) - obj.Kind = ast.Bad - return - } - tc.cyclemap[obj] = true - defer func() { - tc.cyclemap[obj] = false, false - }() - - // resolve non-type objects - typ, _ := obj.Type.(*Type) - if typ == nil { - switch obj.Kind { - case ast.Bad: - // ignore - - case ast.Con: - tc.declConst(obj) - - case ast.Var: - tc.declVar(obj) - obj.Type = tc.typeFor(nil, obj.Decl.(*ast.ValueSpec).Type, false) - - case ast.Fun: - obj.Type = NewType(Function) - t := obj.Decl.(*ast.FuncDecl).Type - tc.declSignature(obj.Type.(*Type), nil, t.Params, t.Results) - - default: - // type objects have non-nil types when resolve is called - if debug { - fmt.Printf("kind = %s\n", obj.Kind) - } - panic("unreachable") - } - return - } - - // resolve type objects - if typ.Form == Unresolved { - tc.typeFor(typ, typ.Obj.Decl.(*ast.TypeSpec).Type, false) - - // provide types for all methods - for _, obj := range typ.Scope.Objects { - if obj.Kind == ast.Fun { - assert(obj.Type == nil) - obj.Type = NewType(Method) - f := obj.Decl.(*ast.FuncDecl) - t := f.Type - tc.declSignature(obj.Type.(*Type), f.Recv, t.Params, t.Results) - } - } - } -} - -func (tc *typechecker) checkBlock(body []ast.Stmt, ftype *Type) { - tc.openScope() - defer tc.closeScope() - - // inject function/method parameters into block scope, if any - if ftype != nil { - for _, par := range ftype.Params.Objects { - if par.Name != "_" { - alt := tc.topScope.Insert(par) - assert(alt == nil) // ftype has no double declarations - } - } - } - - for _, stmt := range body { - tc.checkStmt(stmt) - } -} - -// ---------------------------------------------------------------------------- -// Types - -// unparen removes parentheses around x, if any. -func unparen(x ast.Expr) ast.Expr { - if ux, hasParens := x.(*ast.ParenExpr); hasParens { - return unparen(ux.X) - } - return x -} - -func (tc *typechecker) declFields(scope *ast.Scope, fields *ast.FieldList, ref bool) (n uint) { - if fields != nil { - for _, f := range fields.List { - typ := tc.typeFor(nil, f.Type, ref) - for _, name := range f.Names { - fld := tc.declInScope(scope, ast.Var, name, f, 0) - fld.Type = typ - n++ - } - } - } - return n -} - -func (tc *typechecker) declSignature(typ *Type, recv, params, results *ast.FieldList) { - assert((typ.Form == Method) == (recv != nil)) - typ.Params = ast.NewScope(nil) - tc.declFields(typ.Params, recv, true) - tc.declFields(typ.Params, params, true) - typ.N = tc.declFields(typ.Params, results, true) -} - -func (tc *typechecker) typeFor(def *Type, x ast.Expr, ref bool) (typ *Type) { - x = unparen(x) - - // type name - if t, isIdent := x.(*ast.Ident); isIdent { - obj := tc.find(t) - - if obj.Kind != ast.Typ { - tc.Errorf(t.Pos(), "%s is not a type", t.Name) - if def == nil { - typ = NewType(BadType) - } else { - typ = def - typ.Form = BadType - } - typ.Expr = x - return - } - - if !ref { - tc.resolve(obj) // check for cycles even if type resolved - } - typ = obj.Type.(*Type) - - if def != nil { - // new type declaration: copy type structure - def.Form = typ.Form - def.N = typ.N - def.Key, def.Elt = typ.Key, typ.Elt - def.Params = typ.Params - def.Expr = x - typ = def - } - return - } - - // type literal - typ = def - if typ == nil { - typ = NewType(BadType) - } - typ.Expr = x - - switch t := x.(type) { - case *ast.SelectorExpr: - if debug { - fmt.Println("qualified identifier unimplemented") - } - typ.Form = BadType - - case *ast.StarExpr: - typ.Form = Pointer - typ.Elt = tc.typeFor(nil, t.X, true) - - case *ast.ArrayType: - if t.Len != nil { - typ.Form = Array - // TODO(gri) compute the real length - // (this may call resolve recursively) - (*typ).N = 42 - } else { - typ.Form = Slice - } - typ.Elt = tc.typeFor(nil, t.Elt, t.Len == nil) - - case *ast.StructType: - typ.Form = Struct - tc.declFields(typ.Scope, t.Fields, false) - - case *ast.FuncType: - typ.Form = Function - tc.declSignature(typ, nil, t.Params, t.Results) - - case *ast.InterfaceType: - typ.Form = Interface - tc.declFields(typ.Scope, t.Methods, true) - - case *ast.MapType: - typ.Form = Map - typ.Key = tc.typeFor(nil, t.Key, true) - typ.Elt = tc.typeFor(nil, t.Value, true) - - case *ast.ChanType: - typ.Form = Channel - typ.N = uint(t.Dir) - typ.Elt = tc.typeFor(nil, t.Value, true) - - default: - if debug { - fmt.Printf("x is %T\n", x) - } - panic("unreachable") - } - - return -} - -// ---------------------------------------------------------------------------- -// TODO(gri) implement these place holders - -func (tc *typechecker) declConst(*ast.Object) { -} - -func (tc *typechecker) declVar(*ast.Object) { -} - -func (tc *typechecker) checkStmt(ast.Stmt) { -} diff --git a/src/pkg/go/typechecker/typechecker_test.go b/src/pkg/go/typechecker/typechecker_test.go deleted file mode 100644 index 4bad4499a..000000000 --- a/src/pkg/go/typechecker/typechecker_test.go +++ /dev/null @@ -1,163 +0,0 @@ -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// This file implements a simple typechecker test harness. Packages found -// in the testDir directory are typechecked. Error messages reported by -// the typechecker are compared against the error messages expected for -// the test files. -// -// Expected errors are indicated in the test files by putting a comment -// of the form /* ERROR "rx" */ immediately following an offending token. -// The harness will verify that an error matching the regular expression -// rx is reported at that source position. Consecutive comments may be -// used to indicate multiple errors for the same token position. -// -// For instance, the following test file indicates that a "not declared" -// error should be reported for the undeclared variable x: -// -// package P0 -// func f() { -// _ = x /* ERROR "not declared" */ + 1 -// } -// -// If the -pkg flag is set, only packages with package names matching -// the regular expression provided via the flag value are tested. - -package typechecker - -import ( - "flag" - "fmt" - "go/ast" - "go/parser" - "go/scanner" - "go/token" - "io/ioutil" - "os" - "regexp" - "sort" - "strings" - "testing" -) - -const testDir = "./testdata" // location of test packages - -var fset = token.NewFileSet() - -var ( - pkgPat = flag.String("pkg", ".*", "regular expression to select test packages by package name") - trace = flag.Bool("trace", false, "print package names") -) - -// ERROR comments must be of the form /* ERROR "rx" */ and rx is -// a regular expression that matches the expected error message. -var errRx = regexp.MustCompile(`^/\* *ERROR *"([^"]*)" *\*/$`) - -// expectedErrors collects the regular expressions of ERROR comments -// found in the package files of pkg and returns them in sorted order -// (by filename and position). -func expectedErrors(t *testing.T, pkg *ast.Package) (list scanner.ErrorList) { - // scan all package files - for filename := range pkg.Files { - src, err := ioutil.ReadFile(filename) - if err != nil { - t.Fatalf("expectedErrors(%s): %v", pkg.Name, err) - } - - var s scanner.Scanner - file := fset.AddFile(filename, fset.Base(), len(src)) - s.Init(file, src, nil, scanner.ScanComments) - var prev token.Pos // position of last non-comment token - loop: - for { - pos, tok, lit := s.Scan() - switch tok { - case token.EOF: - break loop - case token.COMMENT: - s := errRx.FindStringSubmatch(lit) - if len(s) == 2 { - list = append(list, &scanner.Error{fset.Position(prev), string(s[1])}) - } - default: - prev = pos - } - } - } - sort.Sort(list) // multiple files may not be sorted - return -} - -func testFilter(f *os.FileInfo) bool { - return strings.HasSuffix(f.Name, ".src") && f.Name[0] != '.' -} - -func checkError(t *testing.T, expected, found *scanner.Error) { - rx, err := regexp.Compile(expected.Msg) - if err != nil { - t.Errorf("%s: %v", expected.Pos, err) - return - } - - match := rx.MatchString(found.Msg) - - if expected.Pos.Offset != found.Pos.Offset { - if match { - t.Errorf("%s: expected error should have been at %s", expected.Pos, found.Pos) - } else { - t.Errorf("%s: error matching %q expected", expected.Pos, expected.Msg) - return - } - } - - if !match { - t.Errorf("%s: %q does not match %q", expected.Pos, expected.Msg, found.Msg) - } -} - -func TestTypeCheck(t *testing.T) { - flag.Parse() - pkgRx, err := regexp.Compile(*pkgPat) - if err != nil { - t.Fatalf("illegal flag value %q: %s", *pkgPat, err) - } - - pkgs, err := parser.ParseDir(fset, testDir, testFilter, 0) - if err != nil { - scanner.PrintError(os.Stderr, err) - t.Fatalf("packages in %s contain syntax errors", testDir) - } - - for _, pkg := range pkgs { - if !pkgRx.MatchString(pkg.Name) { - continue // only test selected packages - } - - if *trace { - fmt.Println(pkg.Name) - } - - xlist := expectedErrors(t, pkg) - err := CheckPackage(fset, pkg, nil) - if err != nil { - if elist, ok := err.(scanner.ErrorList); ok { - // verify that errors match - for i := 0; i < len(xlist) && i < len(elist); i++ { - checkError(t, xlist[i], elist[i]) - } - // the correct number or errors must have been found - if len(xlist) != len(elist) { - fmt.Fprintf(os.Stderr, "%s\n", pkg.Name) - scanner.PrintError(os.Stderr, elist) - fmt.Fprintln(os.Stderr) - t.Errorf("TypeCheck(%s): %d errors expected but %d reported", pkg.Name, len(xlist), len(elist)) - } - } else { - t.Errorf("TypeCheck(%s): %v", pkg.Name, err) - } - } else if len(xlist) > 0 { - t.Errorf("TypeCheck(%s): %d errors expected but 0 reported", pkg.Name, len(xlist)) - } - } -} diff --git a/src/pkg/go/typechecker/universe.go b/src/pkg/go/typechecker/universe.go deleted file mode 100644 index 81c14a05e..000000000 --- a/src/pkg/go/typechecker/universe.go +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package typechecker - -import "go/ast" - -// TODO(gri) should this be in package ast? - -// The Universe scope contains all predeclared identifiers. -var Universe *ast.Scope - -func def(obj *ast.Object) { - alt := Universe.Insert(obj) - if alt != nil { - panic("object declared twice") - } -} - -func init() { - Universe = ast.NewScope(nil) - - // basic types - for n, name := range BasicTypes { - typ := NewType(Basic) - typ.N = n - obj := ast.NewObj(ast.Typ, name) - obj.Type = typ - typ.Obj = obj - def(obj) - } - - // built-in functions - // TODO(gri) implement this -} diff --git a/src/pkg/go/types/Makefile b/src/pkg/go/types/Makefile deleted file mode 100644 index 4ca707c73..000000000 --- a/src/pkg/go/types/Makefile +++ /dev/null @@ -1,16 +0,0 @@ -# Copyright 2010 The Go Authors. All rights reserved. -# Use of this source code is governed by a BSD-style -# license that can be found in the LICENSE file. - -include ../../../Make.inc - -TARG=go/types -GOFILES=\ - check.go\ - const.go\ - exportdata.go\ - gcimporter.go\ - types.go\ - universe.go\ - -include ../../../Make.pkg diff --git a/src/pkg/go/types/check.go b/src/pkg/go/types/check.go deleted file mode 100644 index 87e3e93da..000000000 --- a/src/pkg/go/types/check.go +++ /dev/null @@ -1,226 +0,0 @@ -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// This file implements the Check function, which typechecks a package. - -package types - -import ( - "fmt" - "go/ast" - "go/scanner" - "go/token" - "os" - "strconv" -) - -const debug = false - -type checker struct { - fset *token.FileSet - scanner.ErrorVector - types map[ast.Expr]Type -} - -func (c *checker) errorf(pos token.Pos, format string, args ...interface{}) string { - msg := fmt.Sprintf(format, args...) - c.Error(c.fset.Position(pos), msg) - return msg -} - -// collectFields collects struct fields tok = token.STRUCT), interface methods -// (tok = token.INTERFACE), and function arguments/results (tok = token.FUNC). -func (c *checker) collectFields(tok token.Token, list *ast.FieldList, cycleOk bool) (fields ObjList, tags []string, isVariadic bool) { - if list != nil { - for _, field := range list.List { - ftype := field.Type - if t, ok := ftype.(*ast.Ellipsis); ok { - ftype = t.Elt - isVariadic = true - } - typ := c.makeType(ftype, cycleOk) - tag := "" - if field.Tag != nil { - assert(field.Tag.Kind == token.STRING) - tag, _ = strconv.Unquote(field.Tag.Value) - } - if len(field.Names) > 0 { - // named fields - for _, name := range field.Names { - obj := name.Obj - obj.Type = typ - fields = append(fields, obj) - if tok == token.STRUCT { - tags = append(tags, tag) - } - } - } else { - // anonymous field - switch tok { - case token.STRUCT: - tags = append(tags, tag) - fallthrough - case token.FUNC: - obj := ast.NewObj(ast.Var, "") - obj.Type = typ - fields = append(fields, obj) - case token.INTERFACE: - utyp := Underlying(typ) - if typ, ok := utyp.(*Interface); ok { - // TODO(gri) This is not good enough. Check for double declarations! - fields = append(fields, typ.Methods...) - } else if _, ok := utyp.(*Bad); !ok { - // if utyp is Bad, don't complain (the root cause was reported before) - c.errorf(ftype.Pos(), "interface contains embedded non-interface type") - } - default: - panic("unreachable") - } - } - } - } - return -} - -// makeType makes a new type for an AST type specification x or returns -// the type referred to by a type name x. If cycleOk is set, a type may -// refer to itself directly or indirectly; otherwise cycles are errors. -// -func (c *checker) makeType(x ast.Expr, cycleOk bool) (typ Type) { - if debug { - fmt.Printf("makeType (cycleOk = %v)\n", cycleOk) - ast.Print(c.fset, x) - defer func() { - fmt.Printf("-> %T %v\n\n", typ, typ) - }() - } - - switch t := x.(type) { - case *ast.BadExpr: - return &Bad{} - - case *ast.Ident: - // type name - obj := t.Obj - if obj == nil { - // unresolved identifier (error has been reported before) - return &Bad{Msg: "unresolved identifier"} - } - if obj.Kind != ast.Typ { - msg := c.errorf(t.Pos(), "%s is not a type", t.Name) - return &Bad{Msg: msg} - } - c.checkObj(obj, cycleOk) - if !cycleOk && obj.Type.(*Name).Underlying == nil { - // TODO(gri) Enable this message again once its position - // is independent of the underlying map implementation. - // msg := c.errorf(obj.Pos(), "illegal cycle in declaration of %s", obj.Name) - msg := "illegal cycle" - return &Bad{Msg: msg} - } - return obj.Type.(Type) - - case *ast.ParenExpr: - return c.makeType(t.X, cycleOk) - - case *ast.SelectorExpr: - // qualified identifier - // TODO (gri) eventually, this code belongs to expression - // type checking - here for the time being - if ident, ok := t.X.(*ast.Ident); ok { - if obj := ident.Obj; obj != nil { - if obj.Kind != ast.Pkg { - msg := c.errorf(ident.Pos(), "%s is not a package", obj.Name) - return &Bad{Msg: msg} - } - // TODO(gri) we have a package name but don't - // have the mapping from package name to package - // scope anymore (created in ast.NewPackage). - return &Bad{} // for now - } - } - // TODO(gri) can this really happen (the parser should have excluded this)? - msg := c.errorf(t.Pos(), "expected qualified identifier") - return &Bad{Msg: msg} - - case *ast.StarExpr: - return &Pointer{Base: c.makeType(t.X, true)} - - case *ast.ArrayType: - if t.Len != nil { - // TODO(gri) compute length - return &Array{Elt: c.makeType(t.Elt, cycleOk)} - } - return &Slice{Elt: c.makeType(t.Elt, true)} - - case *ast.StructType: - fields, tags, _ := c.collectFields(token.STRUCT, t.Fields, cycleOk) - return &Struct{Fields: fields, Tags: tags} - - case *ast.FuncType: - params, _, _ := c.collectFields(token.FUNC, t.Params, true) - results, _, isVariadic := c.collectFields(token.FUNC, t.Results, true) - return &Func{Recv: nil, Params: params, Results: results, IsVariadic: isVariadic} - - case *ast.InterfaceType: - methods, _, _ := c.collectFields(token.INTERFACE, t.Methods, cycleOk) - methods.Sort() - return &Interface{Methods: methods} - - case *ast.MapType: - return &Map{Key: c.makeType(t.Key, true), Elt: c.makeType(t.Key, true)} - - case *ast.ChanType: - return &Chan{Dir: t.Dir, Elt: c.makeType(t.Value, true)} - } - - panic(fmt.Sprintf("unreachable (%T)", x)) -} - -// checkObj type checks an object. -func (c *checker) checkObj(obj *ast.Object, ref bool) { - if obj.Type != nil { - // object has already been type checked - return - } - - switch obj.Kind { - case ast.Bad: - // ignore - - case ast.Con: - // TODO(gri) complete this - - case ast.Typ: - typ := &Name{Obj: obj} - obj.Type = typ // "mark" object so recursion terminates - typ.Underlying = Underlying(c.makeType(obj.Decl.(*ast.TypeSpec).Type, ref)) - - case ast.Var: - // TODO(gri) complete this - - case ast.Fun: - // TODO(gri) complete this - - default: - panic("unreachable") - } -} - -// Check typechecks a package. -// It augments the AST by assigning types to all ast.Objects and returns a map -// of types for all expression nodes in statements, and a scanner.ErrorList if -// there are errors. -// -func Check(fset *token.FileSet, pkg *ast.Package) (types map[ast.Expr]Type, err os.Error) { - var c checker - c.fset = fset - c.types = make(map[ast.Expr]Type) - - for _, obj := range pkg.Scope.Objects { - c.checkObj(obj, false) - } - - return c.types, c.GetError(scanner.NoMultiples) -} diff --git a/src/pkg/go/types/check_test.go b/src/pkg/go/types/check_test.go deleted file mode 100644 index 8be653fcb..000000000 --- a/src/pkg/go/types/check_test.go +++ /dev/null @@ -1,215 +0,0 @@ -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// This file implements a typechecker test harness. The packages specified -// in tests are typechecked. Error messages reported by the typechecker are -// compared against the error messages expected in the test files. -// -// Expected errors are indicated in the test files by putting a comment -// of the form /* ERROR "rx" */ immediately following an offending token. -// The harness will verify that an error matching the regular expression -// rx is reported at that source position. Consecutive comments may be -// used to indicate multiple errors for the same token position. -// -// For instance, the following test file indicates that a "not declared" -// error should be reported for the undeclared variable x: -// -// package p -// func f() { -// _ = x /* ERROR "not declared" */ + 1 -// } - -package types - -import ( - "fmt" - "go/ast" - "go/parser" - "go/scanner" - "go/token" - "io/ioutil" - "os" - "regexp" - "testing" -) - -// The test filenames do not end in .go so that they are invisible -// to gofmt since they contain comments that must not change their -// positions relative to surrounding tokens. - -var tests = []struct { - name string - files []string -}{ - {"test0", []string{"testdata/test0.src"}}, -} - -var fset = token.NewFileSet() - -// TODO(gri) This functionality should be in token.Fileset. -func getFile(filename string) *token.File { - for f := range fset.Files() { - if f.Name() == filename { - return f - } - } - return nil -} - -// TODO(gri) This functionality should be in token.Fileset. -func getPos(filename string, offset int) token.Pos { - if f := getFile(filename); f != nil { - return f.Pos(offset) - } - return token.NoPos -} - -// TODO(gri) Need to revisit parser interface. We should be able to use parser.ParseFiles -// or a similar function instead. -func parseFiles(t *testing.T, testname string, filenames []string) (map[string]*ast.File, os.Error) { - files := make(map[string]*ast.File) - var errors scanner.ErrorList - for _, filename := range filenames { - if _, exists := files[filename]; exists { - t.Fatalf("%s: duplicate file %s", testname, filename) - } - file, err := parser.ParseFile(fset, filename, nil, parser.DeclarationErrors) - if file == nil { - t.Fatalf("%s: could not parse file %s", testname, filename) - } - files[filename] = file - if err != nil { - // if the parser returns a non-scanner.ErrorList error - // the file couldn't be read in the first place and - // file == nil; in that case we shouldn't reach here - errors = append(errors, err.(scanner.ErrorList)...) - } - - } - return files, errors -} - -// ERROR comments must be of the form /* ERROR "rx" */ and rx is -// a regular expression that matches the expected error message. -// -var errRx = regexp.MustCompile(`^/\* *ERROR *"([^"]*)" *\*/$`) - -// expectedErrors collects the regular expressions of ERROR comments found -// in files and returns them as a map of error positions to error messages. -// -func expectedErrors(t *testing.T, testname string, files map[string]*ast.File) map[token.Pos]string { - errors := make(map[token.Pos]string) - for filename := range files { - src, err := ioutil.ReadFile(filename) - if err != nil { - t.Fatalf("%s: could not read %s", testname, filename) - } - - var s scanner.Scanner - // file was parsed already - do not add it again to the file - // set otherwise the position information returned here will - // not match the position information collected by the parser - s.Init(getFile(filename), src, nil, scanner.ScanComments) - var prev token.Pos // position of last non-comment token - - scanFile: - for { - pos, tok, lit := s.Scan() - switch tok { - case token.EOF: - break scanFile - case token.COMMENT: - s := errRx.FindStringSubmatch(lit) - if len(s) == 2 { - errors[prev] = string(s[1]) - } - default: - prev = pos - } - } - } - return errors -} - -func eliminate(t *testing.T, expected map[token.Pos]string, errors os.Error) { - if errors == nil { - return - } - for _, error := range errors.(scanner.ErrorList) { - // error.Pos is a token.Position, but we want - // a token.Pos so we can do a map lookup - // TODO(gri) Need to move scanner.Errors over - // to use token.Pos and file set info. - pos := getPos(error.Pos.Filename, error.Pos.Offset) - if msg, found := expected[pos]; found { - // we expect a message at pos; check if it matches - rx, err := regexp.Compile(msg) - if err != nil { - t.Errorf("%s: %v", error.Pos, err) - continue - } - if match := rx.MatchString(error.Msg); !match { - t.Errorf("%s: %q does not match %q", error.Pos, error.Msg, msg) - continue - } - // we have a match - eliminate this error - expected[pos] = "", false - } else { - // To keep in mind when analyzing failed test output: - // If the same error position occurs multiple times in errors, - // this message will be triggered (because the first error at - // the position removes this position from the expected errors). - t.Errorf("%s: no (multiple?) error expected, but found: %s", error.Pos, error.Msg) - } - } -} - -func check(t *testing.T, testname string, testfiles []string) { - // TODO(gri) Eventually all these different phases should be - // subsumed into a single function call that takes - // a set of files and creates a fully resolved and - // type-checked AST. - - files, err := parseFiles(t, testname, testfiles) - - // we are expecting the following errors - // (collect these after parsing the files so that - // they are found in the file set) - errors := expectedErrors(t, testname, files) - - // verify errors returned by the parser - eliminate(t, errors, err) - - // verify errors returned after resolving identifiers - pkg, err := ast.NewPackage(fset, files, GcImporter, Universe) - eliminate(t, errors, err) - - // verify errors returned by the typechecker - _, err = Check(fset, pkg) - eliminate(t, errors, err) - - // there should be no expected errors left - if len(errors) > 0 { - t.Errorf("%s: %d errors not reported:", testname, len(errors)) - for pos, msg := range errors { - t.Errorf("%s: %s\n", fset.Position(pos), msg) - } - } -} - -func TestCheck(t *testing.T) { - // For easy debugging w/o changing the testing code, - // if there is a local test file, only test that file. - const testfile = "test.go" - if fi, err := os.Stat(testfile); err == nil && fi.IsRegular() { - fmt.Printf("WARNING: Testing only %s (remove it to run all tests)\n", testfile) - check(t, testfile, []string{testfile}) - return - } - - // Otherwise, run all the tests. - for _, test := range tests { - check(t, test.name, test.files) - } -} diff --git a/src/pkg/go/types/const.go b/src/pkg/go/types/const.go deleted file mode 100644 index 1ef95d9f9..000000000 --- a/src/pkg/go/types/const.go +++ /dev/null @@ -1,332 +0,0 @@ -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// This file implements operations on ideal constants. - -package types - -import ( - "big" - "go/token" - "strconv" -) - -// TODO(gri) Consider changing the API so Const is an interface -// and operations on consts don't have to type switch. - -// A Const implements an ideal constant Value. -// The zero value z for a Const is not a valid constant value. -type Const struct { - // representation of constant values: - // ideal bool -> bool - // ideal int -> *big.Int - // ideal float -> *big.Rat - // ideal complex -> cmplx - // ideal string -> string - val interface{} -} - -// Representation of complex values. -type cmplx struct { - re, im *big.Rat -} - -func assert(cond bool) { - if !cond { - panic("go/types internal error: assertion failed") - } -} - -// MakeConst makes an ideal constant from a literal -// token and the corresponding literal string. -func MakeConst(tok token.Token, lit string) Const { - switch tok { - case token.INT: - var x big.Int - _, ok := x.SetString(lit, 0) - assert(ok) - return Const{&x} - case token.FLOAT: - var y big.Rat - _, ok := y.SetString(lit) - assert(ok) - return Const{&y} - case token.IMAG: - assert(lit[len(lit)-1] == 'i') - var im big.Rat - _, ok := im.SetString(lit[0 : len(lit)-1]) - assert(ok) - return Const{cmplx{big.NewRat(0, 1), &im}} - case token.CHAR: - assert(lit[0] == '\'' && lit[len(lit)-1] == '\'') - code, _, _, err := strconv.UnquoteChar(lit[1:len(lit)-1], '\'') - assert(err == nil) - return Const{big.NewInt(int64(code))} - case token.STRING: - s, err := strconv.Unquote(lit) - assert(err == nil) - return Const{s} - } - panic("unreachable") -} - -// MakeZero returns the zero constant for the given type. -func MakeZero(typ *Type) Const { - // TODO(gri) fix this - return Const{0} -} - -// Match attempts to match the internal constant representations of x and y. -// If the attempt is successful, the result is the values of x and y, -// if necessary converted to have the same internal representation; otherwise -// the results are invalid. -func (x Const) Match(y Const) (u, v Const) { - switch a := x.val.(type) { - case bool: - if _, ok := y.val.(bool); ok { - u, v = x, y - } - case *big.Int: - switch y.val.(type) { - case *big.Int: - u, v = x, y - case *big.Rat: - var z big.Rat - z.SetInt(a) - u, v = Const{&z}, y - case cmplx: - var z big.Rat - z.SetInt(a) - u, v = Const{cmplx{&z, big.NewRat(0, 1)}}, y - } - case *big.Rat: - switch y.val.(type) { - case *big.Int: - v, u = y.Match(x) - case *big.Rat: - u, v = x, y - case cmplx: - u, v = Const{cmplx{a, big.NewRat(0, 0)}}, y - } - case cmplx: - switch y.val.(type) { - case *big.Int, *big.Rat: - v, u = y.Match(x) - case cmplx: - u, v = x, y - } - case string: - if _, ok := y.val.(string); ok { - u, v = x, y - } - default: - panic("unreachable") - } - return -} - -// Convert attempts to convert the constant x to a given type. -// If the attempt is successful, the result is the new constant; -// otherwise the result is invalid. -func (x Const) Convert(typ *Type) Const { - // TODO(gri) implement this - switch x := x.val.(type) { - case bool: - case *big.Int: - case *big.Rat: - case cmplx: - case string: - } - return x -} - -func (x Const) String() string { - switch x := x.val.(type) { - case bool: - if x { - return "true" - } - return "false" - case *big.Int: - return x.String() - case *big.Rat: - return x.FloatString(10) // 10 digits of precision after decimal point seems fine - case cmplx: - // TODO(gri) don't print 0 components - return x.re.FloatString(10) + " + " + x.im.FloatString(10) + "i" - case string: - return x - } - panic("unreachable") -} - -func (x Const) UnaryOp(op token.Token) Const { - panic("unimplemented") -} - -func (x Const) BinaryOp(op token.Token, y Const) Const { - var z interface{} - switch x := x.val.(type) { - case bool: - z = binaryBoolOp(x, op, y.val.(bool)) - case *big.Int: - z = binaryIntOp(x, op, y.val.(*big.Int)) - case *big.Rat: - z = binaryFloatOp(x, op, y.val.(*big.Rat)) - case cmplx: - z = binaryCmplxOp(x, op, y.val.(cmplx)) - case string: - z = binaryStringOp(x, op, y.val.(string)) - default: - panic("unreachable") - } - return Const{z} -} - -func binaryBoolOp(x bool, op token.Token, y bool) interface{} { - switch op { - case token.EQL: - return x == y - case token.NEQ: - return x != y - } - panic("unreachable") -} - -func binaryIntOp(x *big.Int, op token.Token, y *big.Int) interface{} { - var z big.Int - switch op { - case token.ADD: - return z.Add(x, y) - case token.SUB: - return z.Sub(x, y) - case token.MUL: - return z.Mul(x, y) - case token.QUO: - return z.Quo(x, y) - case token.REM: - return z.Rem(x, y) - case token.AND: - return z.And(x, y) - case token.OR: - return z.Or(x, y) - case token.XOR: - return z.Xor(x, y) - case token.AND_NOT: - return z.AndNot(x, y) - case token.SHL: - panic("unimplemented") - case token.SHR: - panic("unimplemented") - case token.EQL: - return x.Cmp(y) == 0 - case token.NEQ: - return x.Cmp(y) != 0 - case token.LSS: - return x.Cmp(y) < 0 - case token.LEQ: - return x.Cmp(y) <= 0 - case token.GTR: - return x.Cmp(y) > 0 - case token.GEQ: - return x.Cmp(y) >= 0 - } - panic("unreachable") -} - -func binaryFloatOp(x *big.Rat, op token.Token, y *big.Rat) interface{} { - var z big.Rat - switch op { - case token.ADD: - return z.Add(x, y) - case token.SUB: - return z.Sub(x, y) - case token.MUL: - return z.Mul(x, y) - case token.QUO: - return z.Quo(x, y) - case token.EQL: - return x.Cmp(y) == 0 - case token.NEQ: - return x.Cmp(y) != 0 - case token.LSS: - return x.Cmp(y) < 0 - case token.LEQ: - return x.Cmp(y) <= 0 - case token.GTR: - return x.Cmp(y) > 0 - case token.GEQ: - return x.Cmp(y) >= 0 - } - panic("unreachable") -} - -func binaryCmplxOp(x cmplx, op token.Token, y cmplx) interface{} { - a, b := x.re, x.im - c, d := y.re, y.im - switch op { - case token.ADD: - // (a+c) + i(b+d) - var re, im big.Rat - re.Add(a, c) - im.Add(b, d) - return cmplx{&re, &im} - case token.SUB: - // (a-c) + i(b-d) - var re, im big.Rat - re.Sub(a, c) - im.Sub(b, d) - return cmplx{&re, &im} - case token.MUL: - // (ac-bd) + i(bc+ad) - var ac, bd, bc, ad big.Rat - ac.Mul(a, c) - bd.Mul(b, d) - bc.Mul(b, c) - ad.Mul(a, d) - var re, im big.Rat - re.Sub(&ac, &bd) - im.Add(&bc, &ad) - return cmplx{&re, &im} - case token.QUO: - // (ac+bd)/s + i(bc-ad)/s, with s = cc + dd - var ac, bd, bc, ad, s big.Rat - ac.Mul(a, c) - bd.Mul(b, d) - bc.Mul(b, c) - ad.Mul(a, d) - s.Add(c.Mul(c, c), d.Mul(d, d)) - var re, im big.Rat - re.Add(&ac, &bd) - re.Quo(&re, &s) - im.Sub(&bc, &ad) - im.Quo(&im, &s) - return cmplx{&re, &im} - case token.EQL: - return a.Cmp(c) == 0 && b.Cmp(d) == 0 - case token.NEQ: - return a.Cmp(c) != 0 || b.Cmp(d) != 0 - } - panic("unreachable") -} - -func binaryStringOp(x string, op token.Token, y string) interface{} { - switch op { - case token.ADD: - return x + y - case token.EQL: - return x == y - case token.NEQ: - return x != y - case token.LSS: - return x < y - case token.LEQ: - return x <= y - case token.GTR: - return x > y - case token.GEQ: - return x >= y - } - panic("unreachable") -} diff --git a/src/pkg/go/types/exportdata.go b/src/pkg/go/types/exportdata.go deleted file mode 100644 index 383520320..000000000 --- a/src/pkg/go/types/exportdata.go +++ /dev/null @@ -1,132 +0,0 @@ -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// This file implements ExportData. - -package types - -import ( - "bufio" - "fmt" - "io" - "os" - "strconv" - "strings" -) - -func readGopackHeader(buf *bufio.Reader) (name string, size int, err os.Error) { - // See $GOROOT/include/ar.h. - hdr := make([]byte, 64+12+6+6+8+10+2) - _, err = io.ReadFull(buf, hdr) - if err != nil { - return - } - if trace { - fmt.Printf("header: %s", hdr) - } - s := strings.TrimSpace(string(hdr[64+12+6+6+8:][:10])) - size, err = strconv.Atoi(s) - if err != nil || hdr[len(hdr)-2] != '`' || hdr[len(hdr)-1] != '\n' { - err = os.NewError("invalid archive header") - return - } - name = strings.TrimSpace(string(hdr[:64])) - return -} - -type dataReader struct { - *bufio.Reader - io.Closer -} - -// ExportData returns a readCloser positioned at the beginning of the -// export data section of the given object/archive file, or an error. -// It is the caller's responsibility to close the readCloser. -// -func ExportData(filename string) (rc io.ReadCloser, err os.Error) { - file, err := os.Open(filename) - if err != nil { - return - } - - defer func() { - if err != nil { - file.Close() - // Add file name to error. - err = fmt.Errorf("reading export data: %s: %v", filename, err) - } - }() - - buf := bufio.NewReader(file) - - // Read first line to make sure this is an object file. - line, err := buf.ReadSlice('\n') - if err != nil { - return - } - if string(line) == "!<arch>\n" { - // Archive file. Scan to __.PKGDEF, which should - // be second archive entry. - var name string - var size int - - // First entry should be __.SYMDEF. - // Read and discard. - if name, size, err = readGopackHeader(buf); err != nil { - return - } - if name != "__.SYMDEF" { - err = os.NewError("go archive does not begin with __.SYMDEF") - return - } - const block = 4096 - tmp := make([]byte, block) - for size > 0 { - n := size - if n > block { - n = block - } - _, err = io.ReadFull(buf, tmp[:n]) - if err != nil { - return - } - size -= n - } - - // Second entry should be __.PKGDEF. - if name, size, err = readGopackHeader(buf); err != nil { - return - } - if name != "__.PKGDEF" { - err = os.NewError("go archive is missing __.PKGDEF") - return - } - - // Read first line of __.PKGDEF data, so that line - // is once again the first line of the input. - line, err = buf.ReadSlice('\n') - if err != nil { - return - } - } - - // Now at __.PKGDEF in archive or still at beginning of file. - // Either way, line should begin with "go object ". - if !strings.HasPrefix(string(line), "go object ") { - err = os.NewError("not a go object file") - return - } - - // Skip over object header to export data. - // Begins after first line with $$. - for line[0] != '$' { - line, err = buf.ReadSlice('\n') - if err != nil { - return - } - } - - rc = &dataReader{buf, file} - return -} diff --git a/src/pkg/go/types/gcimporter.go b/src/pkg/go/types/gcimporter.go deleted file mode 100644 index 6ab1806b6..000000000 --- a/src/pkg/go/types/gcimporter.go +++ /dev/null @@ -1,799 +0,0 @@ -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// This file implements an ast.Importer for gc generated object files. -// TODO(gri) Eventually move this into a separate package outside types. - -package types - -import ( - "big" - "fmt" - "go/ast" - "go/token" - "io" - "os" - "path/filepath" - "runtime" - "scanner" - "strconv" -) - -const trace = false // set to true for debugging - -var ( - pkgRoot = filepath.Join(runtime.GOROOT(), "pkg", runtime.GOOS+"_"+runtime.GOARCH) - pkgExts = [...]string{".a", ".5", ".6", ".8"} -) - -// findPkg returns the filename and package id for an import path. -// If no file was found, an empty filename is returned. -func findPkg(path string) (filename, id string) { - if len(path) == 0 { - return - } - - id = path - var noext string - switch path[0] { - default: - // "x" -> "$GOROOT/pkg/$GOOS_$GOARCH/x.ext", "x" - noext = filepath.Join(pkgRoot, path) - - case '.': - // "./x" -> "/this/directory/x.ext", "/this/directory/x" - cwd, err := os.Getwd() - if err != nil { - return - } - noext = filepath.Join(cwd, path) - id = noext - - case '/': - // "/x" -> "/x.ext", "/x" - noext = path - } - - // try extensions - for _, ext := range pkgExts { - filename = noext + ext - if f, err := os.Stat(filename); err == nil && f.IsRegular() { - return - } - } - - filename = "" // not found - return -} - -// gcParser parses the exports inside a gc compiler-produced -// object/archive file and populates its scope with the results. -type gcParser struct { - scanner scanner.Scanner - tok int // current token - lit string // literal string; only valid for Ident, Int, String tokens - id string // package id of imported package - imports map[string]*ast.Object // package id -> package object -} - -func (p *gcParser) init(filename, id string, src io.Reader, imports map[string]*ast.Object) { - p.scanner.Init(src) - p.scanner.Error = func(_ *scanner.Scanner, msg string) { p.error(msg) } - p.scanner.Mode = scanner.ScanIdents | scanner.ScanInts | scanner.ScanStrings | scanner.ScanComments | scanner.SkipComments - p.scanner.Whitespace = 1<<'\t' | 1<<' ' - p.scanner.Filename = filename // for good error messages - p.next() - p.id = id - p.imports = imports -} - -func (p *gcParser) next() { - p.tok = p.scanner.Scan() - switch p.tok { - case scanner.Ident, scanner.Int, scanner.String: - p.lit = p.scanner.TokenText() - default: - p.lit = "" - } - if trace { - fmt.Printf("%s: %q -> %q\n", scanner.TokenString(p.tok), p.scanner.TokenText(), p.lit) - } -} - -// GcImporter implements the ast.Importer signature. -func GcImporter(imports map[string]*ast.Object, path string) (pkg *ast.Object, err os.Error) { - if path == "unsafe" { - return Unsafe, nil - } - - defer func() { - if r := recover(); r != nil { - err = r.(importError) // will re-panic if r is not an importError - if trace { - panic(err) // force a stack trace - } - } - }() - - filename, id := findPkg(path) - if filename == "" { - err = os.NewError("can't find import: " + id) - return - } - - if pkg = imports[id]; pkg != nil { - return // package was imported before - } - - buf, err := ExportData(filename) - if err != nil { - return - } - defer buf.Close() - - if trace { - fmt.Printf("importing %s (%s)\n", id, filename) - } - - var p gcParser - p.init(filename, id, buf, imports) - pkg = p.parseExport() - return -} - -// ---------------------------------------------------------------------------- -// Error handling - -// Internal errors are boxed as importErrors. -type importError struct { - pos scanner.Position - err os.Error -} - -func (e importError) String() string { - return fmt.Sprintf("import error %s (byte offset = %d): %s", e.pos, e.pos.Offset, e.err) -} - -func (p *gcParser) error(err interface{}) { - if s, ok := err.(string); ok { - err = os.NewError(s) - } - // panic with a runtime.Error if err is not an os.Error - panic(importError{p.scanner.Pos(), err.(os.Error)}) -} - -func (p *gcParser) errorf(format string, args ...interface{}) { - p.error(fmt.Sprintf(format, args...)) -} - -func (p *gcParser) expect(tok int) string { - lit := p.lit - if p.tok != tok { - p.errorf("expected %q, got %q (%q)", scanner.TokenString(tok), scanner.TokenString(p.tok), lit) - } - p.next() - return lit -} - -func (p *gcParser) expectSpecial(tok string) { - sep := 'x' // not white space - i := 0 - for i < len(tok) && p.tok == int(tok[i]) && sep > ' ' { - sep = p.scanner.Peek() // if sep <= ' ', there is white space before the next token - p.next() - i++ - } - if i < len(tok) { - p.errorf("expected %q, got %q", tok, tok[0:i]) - } -} - -func (p *gcParser) expectKeyword(keyword string) { - lit := p.expect(scanner.Ident) - if lit != keyword { - p.errorf("expected keyword %s, got %q", keyword, lit) - } -} - -// ---------------------------------------------------------------------------- -// Import declarations - -// ImportPath = string_lit . -// -func (p *gcParser) parsePkgId() *ast.Object { - id, err := strconv.Unquote(p.expect(scanner.String)) - if err != nil { - p.error(err) - } - - switch id { - case "": - // id == "" stands for the imported package id - // (only known at time of package installation) - id = p.id - case "unsafe": - // package unsafe is not in the imports map - handle explicitly - return Unsafe - } - - pkg := p.imports[id] - if pkg == nil { - scope = ast.NewScope(nil) - pkg = ast.NewObj(ast.Pkg, "") - pkg.Data = scope - p.imports[id] = pkg - } - - return pkg -} - -// dotIdentifier = ( ident | '·' ) { ident | int | '·' } . -func (p *gcParser) parseDotIdent() string { - ident := "" - if p.tok != scanner.Int { - sep := 'x' // not white space - for (p.tok == scanner.Ident || p.tok == scanner.Int || p.tok == '·') && sep > ' ' { - ident += p.lit - sep = p.scanner.Peek() // if sep <= ' ', there is white space before the next token - p.next() - } - } - if ident == "" { - p.expect(scanner.Ident) // use expect() for error handling - } - return ident -} - -// ExportedName = ImportPath "." dotIdentifier . -// -func (p *gcParser) parseExportedName(kind ast.ObjKind) *ast.Object { - pkg := p.parsePkgId() - p.expect('.') - name := p.parseDotIdent() - - // a type may have been declared before - if it exists - // already in the respective package scope, return that - // type - scope := pkg.Data.(*ast.Scope) - if kind == ast.Typ { - if obj := scope.Lookup(name); obj != nil { - assert(obj.Kind == ast.Typ) - return obj - } - } - - // any other object must be a newly declared object - - // create it and insert it into the package scope - obj := ast.NewObj(kind, name) - if scope.Insert(obj) != nil { - p.errorf("already declared: %s", obj.Name) - } - - // a new type object is a named type and may be referred - // to before the underlying type is known - set it up - if kind == ast.Typ { - obj.Type = &Name{Obj: obj} - } - - return obj -} - -// ---------------------------------------------------------------------------- -// Types - -// BasicType = identifier . -// -func (p *gcParser) parseBasicType() Type { - obj := Universe.Lookup(p.expect(scanner.Ident)) - if obj == nil || obj.Kind != ast.Typ { - p.errorf("not a basic type: %s", obj.Name) - } - return obj.Type.(Type) -} - -// ArrayType = "[" int_lit "]" Type . -// -func (p *gcParser) parseArrayType() Type { - // "[" already consumed and lookahead known not to be "]" - lit := p.expect(scanner.Int) - p.expect(']') - elt := p.parseType() - n, err := strconv.Atoui64(lit) - if err != nil { - p.error(err) - } - return &Array{Len: n, Elt: elt} -} - -// MapType = "map" "[" Type "]" Type . -// -func (p *gcParser) parseMapType() Type { - p.expectKeyword("map") - p.expect('[') - key := p.parseType() - p.expect(']') - elt := p.parseType() - return &Map{Key: key, Elt: elt} -} - -// Name = identifier | "?" . -// -func (p *gcParser) parseName() (name string) { - switch p.tok { - case scanner.Ident: - name = p.lit - p.next() - case '?': - // anonymous - p.next() - default: - p.error("name expected") - } - return -} - -// Field = Name Type [ ":" string_lit ] . -// -func (p *gcParser) parseField() (fld *ast.Object, tag string) { - name := p.parseName() - ftyp := p.parseType() - if name == "" { - // anonymous field - ftyp must be T or *T and T must be a type name - if _, ok := Deref(ftyp).(*Name); !ok { - p.errorf("anonymous field expected") - } - } - if p.tok == ':' { - p.next() - tag = p.expect(scanner.String) - } - fld = ast.NewObj(ast.Var, name) - fld.Type = ftyp - return -} - -// StructType = "struct" "{" [ FieldList ] "}" . -// FieldList = Field { ";" Field } . -// -func (p *gcParser) parseStructType() Type { - var fields []*ast.Object - var tags []string - - parseField := func() { - fld, tag := p.parseField() - fields = append(fields, fld) - tags = append(tags, tag) - } - - p.expectKeyword("struct") - p.expect('{') - if p.tok != '}' { - parseField() - for p.tok == ';' { - p.next() - parseField() - } - } - p.expect('}') - - return &Struct{Fields: fields, Tags: tags} -} - -// Parameter = ( identifier | "?" ) [ "..." ] Type [ ":" string_lit ] . -// -func (p *gcParser) parseParameter() (par *ast.Object, isVariadic bool) { - name := p.parseName() - if name == "" { - name = "_" // cannot access unnamed identifiers - } - if p.tok == '.' { - p.expectSpecial("...") - isVariadic = true - } - ptyp := p.parseType() - // ignore argument tag - if p.tok == ':' { - p.next() - p.expect(scanner.String) - } - par = ast.NewObj(ast.Var, name) - par.Type = ptyp - return -} - -// Parameters = "(" [ ParameterList ] ")" . -// ParameterList = { Parameter "," } Parameter . -// -func (p *gcParser) parseParameters() (list []*ast.Object, isVariadic bool) { - parseParameter := func() { - par, variadic := p.parseParameter() - list = append(list, par) - if variadic { - if isVariadic { - p.error("... not on final argument") - } - isVariadic = true - } - } - - p.expect('(') - if p.tok != ')' { - parseParameter() - for p.tok == ',' { - p.next() - parseParameter() - } - } - p.expect(')') - - return -} - -// Signature = Parameters [ Result ] . -// Result = Type | Parameters . -// -func (p *gcParser) parseSignature() *Func { - params, isVariadic := p.parseParameters() - - // optional result type - var results []*ast.Object - switch p.tok { - case scanner.Ident, scanner.String, '[', '*', '<': - // single, unnamed result - result := ast.NewObj(ast.Var, "_") - result.Type = p.parseType() - results = []*ast.Object{result} - case '(': - // named or multiple result(s) - var variadic bool - results, variadic = p.parseParameters() - if variadic { - p.error("... not permitted on result type") - } - } - - return &Func{Params: params, Results: results, IsVariadic: isVariadic} -} - -// MethodSpec = identifier Signature . -// -func (p *gcParser) parseMethodSpec() *ast.Object { - if p.tok == scanner.Ident { - p.expect(scanner.Ident) - } else { - // TODO(gri) should this be parseExportedName here? - p.parsePkgId() - p.expect('.') - p.parseDotIdent() - } - p.parseSignature() - - // TODO(gri) compute method object - return ast.NewObj(ast.Fun, "_") -} - -// InterfaceType = "interface" "{" [ MethodList ] "}" . -// MethodList = MethodSpec { ";" MethodSpec } . -// -func (p *gcParser) parseInterfaceType() Type { - var methods ObjList - - parseMethod := func() { - meth := p.parseMethodSpec() - methods = append(methods, meth) - } - - p.expectKeyword("interface") - p.expect('{') - if p.tok != '}' { - parseMethod() - for p.tok == ';' { - p.next() - parseMethod() - } - } - p.expect('}') - - methods.Sort() - return &Interface{Methods: methods} -} - -// ChanType = ( "chan" [ "<-" ] | "<-" "chan" ) Type . -// -func (p *gcParser) parseChanType() Type { - dir := ast.SEND | ast.RECV - if p.tok == scanner.Ident { - p.expectKeyword("chan") - if p.tok == '<' { - p.expectSpecial("<-") - dir = ast.SEND - } - } else { - p.expectSpecial("<-") - p.expectKeyword("chan") - dir = ast.RECV - } - elt := p.parseType() - return &Chan{Dir: dir, Elt: elt} -} - -// Type = -// BasicType | TypeName | ArrayType | SliceType | StructType | -// PointerType | FuncType | InterfaceType | MapType | ChanType | -// "(" Type ")" . -// BasicType = ident . -// TypeName = ExportedName . -// SliceType = "[" "]" Type . -// PointerType = "*" Type . -// FuncType = "func" Signature . -// -func (p *gcParser) parseType() Type { - switch p.tok { - case scanner.Ident: - switch p.lit { - default: - return p.parseBasicType() - case "struct": - return p.parseStructType() - case "func": - // FuncType - p.next() - return p.parseSignature() - case "interface": - return p.parseInterfaceType() - case "map": - return p.parseMapType() - case "chan": - return p.parseChanType() - } - case scanner.String: - // TypeName - return p.parseExportedName(ast.Typ).Type.(Type) - case '[': - p.next() // look ahead - if p.tok == ']' { - // SliceType - p.next() - return &Slice{Elt: p.parseType()} - } - return p.parseArrayType() - case '*': - // PointerType - p.next() - return &Pointer{Base: p.parseType()} - case '<': - return p.parseChanType() - case '(': - // "(" Type ")" - p.next() - typ := p.parseType() - p.expect(')') - return typ - } - p.errorf("expected type, got %s (%q)", scanner.TokenString(p.tok), p.lit) - return nil -} - -// ---------------------------------------------------------------------------- -// Declarations - -// ImportDecl = "import" identifier string_lit . -// -func (p *gcParser) parseImportDecl() { - p.expectKeyword("import") - // The identifier has no semantic meaning in the import data. - // It exists so that error messages can print the real package - // name: binary.ByteOrder instead of "encoding/binary".ByteOrder. - name := p.expect(scanner.Ident) - pkg := p.parsePkgId() - assert(pkg.Name == "" || pkg.Name == name) - pkg.Name = name -} - -// int_lit = [ "+" | "-" ] { "0" ... "9" } . -// -func (p *gcParser) parseInt() (sign, val string) { - switch p.tok { - case '-': - p.next() - sign = "-" - case '+': - p.next() - } - val = p.expect(scanner.Int) - return -} - -// number = int_lit [ "p" int_lit ] . -// -func (p *gcParser) parseNumber() Const { - // mantissa - sign, val := p.parseInt() - mant, ok := new(big.Int).SetString(sign+val, 10) - assert(ok) - - if p.lit == "p" { - // exponent (base 2) - p.next() - sign, val = p.parseInt() - exp, err := strconv.Atoui(val) - if err != nil { - p.error(err) - } - if sign == "-" { - denom := big.NewInt(1) - denom.Lsh(denom, exp) - return Const{new(big.Rat).SetFrac(mant, denom)} - } - if exp > 0 { - mant.Lsh(mant, exp) - } - return Const{new(big.Rat).SetInt(mant)} - } - - return Const{mant} -} - -// ConstDecl = "const" ExportedName [ Type ] "=" Literal . -// Literal = bool_lit | int_lit | float_lit | complex_lit | string_lit . -// bool_lit = "true" | "false" . -// complex_lit = "(" float_lit "+" float_lit ")" . -// string_lit = `"` { unicode_char } `"` . -// -func (p *gcParser) parseConstDecl() { - p.expectKeyword("const") - obj := p.parseExportedName(ast.Con) - var x Const - var typ Type - if p.tok != '=' { - obj.Type = p.parseType() - } - p.expect('=') - switch p.tok { - case scanner.Ident: - // bool_lit - if p.lit != "true" && p.lit != "false" { - p.error("expected true or false") - } - x = Const{p.lit == "true"} - typ = Bool.Underlying - p.next() - case '-', scanner.Int: - // int_lit - x = p.parseNumber() - typ = Int.Underlying - if _, ok := x.val.(*big.Rat); ok { - typ = Float64.Underlying - } - case '(': - // complex_lit - p.next() - re := p.parseNumber() - p.expect('+') - im := p.parseNumber() - p.expect(')') - x = Const{cmplx{re.val.(*big.Rat), im.val.(*big.Rat)}} - typ = Complex128.Underlying - case scanner.String: - // string_lit - x = MakeConst(token.STRING, p.lit) - p.next() - typ = String.Underlying - default: - p.error("expected literal") - } - if obj.Type == nil { - obj.Type = typ - } - obj.Data = x -} - -// TypeDecl = "type" ExportedName Type . -// -func (p *gcParser) parseTypeDecl() { - p.expectKeyword("type") - obj := p.parseExportedName(ast.Typ) - - // The type object may have been imported before and thus already - // have a type associated with it. We still need to parse the type - // structure, but throw it away if the object already has a type. - // This ensures that all imports refer to the same type object for - // a given type declaration. - typ := p.parseType() - - if name := obj.Type.(*Name); name.Underlying == nil { - assert(Underlying(typ) == typ) - name.Underlying = typ - } -} - -// VarDecl = "var" ExportedName Type . -// -func (p *gcParser) parseVarDecl() { - p.expectKeyword("var") - obj := p.parseExportedName(ast.Var) - obj.Type = p.parseType() -} - -// FuncDecl = "func" ExportedName Signature . -// -func (p *gcParser) parseFuncDecl() { - // "func" already consumed - obj := p.parseExportedName(ast.Fun) - obj.Type = p.parseSignature() -} - -// MethodDecl = "func" Receiver identifier Signature . -// Receiver = "(" ( identifier | "?" ) [ "*" ] ExportedName ")" . -// -func (p *gcParser) parseMethodDecl() { - // "func" already consumed - p.expect('(') - p.parseParameter() // receiver - p.expect(')') - p.expect(scanner.Ident) - p.parseSignature() -} - -// Decl = [ ImportDecl | ConstDecl | TypeDecl | VarDecl | FuncDecl | MethodDecl ] "\n" . -// -func (p *gcParser) parseDecl() { - switch p.lit { - case "import": - p.parseImportDecl() - case "const": - p.parseConstDecl() - case "type": - p.parseTypeDecl() - case "var": - p.parseVarDecl() - case "func": - p.next() // look ahead - if p.tok == '(' { - p.parseMethodDecl() - } else { - p.parseFuncDecl() - } - } - p.expect('\n') -} - -// ---------------------------------------------------------------------------- -// Export - -// Export = "PackageClause { Decl } "$$" . -// PackageClause = "package" identifier [ "safe" ] "\n" . -// -func (p *gcParser) parseExport() *ast.Object { - p.expectKeyword("package") - name := p.expect(scanner.Ident) - if p.tok != '\n' { - // A package is safe if it was compiled with the -u flag, - // which disables the unsafe package. - // TODO(gri) remember "safe" package - p.expectKeyword("safe") - } - p.expect('\n') - - assert(p.imports[p.id] == nil) - pkg := ast.NewObj(ast.Pkg, name) - pkg.Data = ast.NewScope(nil) - p.imports[p.id] = pkg - - for p.tok != '$' && p.tok != scanner.EOF { - p.parseDecl() - } - - if ch := p.scanner.Peek(); p.tok != '$' || ch != '$' { - // don't call next()/expect() since reading past the - // export data may cause scanner errors (e.g. NUL chars) - p.errorf("expected '$$', got %s %c", scanner.TokenString(p.tok), ch) - } - - if n := p.scanner.ErrorCount; n != 0 { - p.errorf("expected no scanner errors, got %d", n) - } - - return pkg -} diff --git a/src/pkg/go/types/gcimporter_test.go b/src/pkg/go/types/gcimporter_test.go deleted file mode 100644 index ec87f5d51..000000000 --- a/src/pkg/go/types/gcimporter_test.go +++ /dev/null @@ -1,101 +0,0 @@ -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package types - -import ( - "exec" - "go/ast" - "io/ioutil" - "path/filepath" - "runtime" - "strings" - "testing" - "time" -) - -var gcName, gcPath string // compiler name and path - -func init() { - // determine compiler - switch runtime.GOARCH { - case "386": - gcName = "8g" - case "amd64": - gcName = "6g" - case "arm": - gcName = "5g" - default: - gcName = "unknown-GOARCH-compiler" - gcPath = gcName - return - } - gcPath, _ = exec.LookPath(gcName) -} - -func compile(t *testing.T, dirname, filename string) { - cmd := exec.Command(gcPath, filename) - cmd.Dir = dirname - out, err := cmd.CombinedOutput() - if err != nil { - t.Errorf("%s %s failed: %s", gcName, filename, err) - return - } - t.Logf("%s", string(out)) -} - -// Use the same global imports map for all tests. The effect is -// as if all tested packages were imported into a single package. -var imports = make(map[string]*ast.Object) - -func testPath(t *testing.T, path string) bool { - _, err := GcImporter(imports, path) - if err != nil { - t.Errorf("testPath(%s): %s", path, err) - return false - } - return true -} - -const maxTime = 3e9 // maximum allotted testing time in ns - -func testDir(t *testing.T, dir string, endTime int64) (nimports int) { - dirname := filepath.Join(pkgRoot, dir) - list, err := ioutil.ReadDir(dirname) - if err != nil { - t.Errorf("testDir(%s): %s", dirname, err) - } - for _, f := range list { - if time.Nanoseconds() >= endTime { - t.Log("testing time used up") - return - } - switch { - case f.IsRegular(): - // try extensions - for _, ext := range pkgExts { - if strings.HasSuffix(f.Name, ext) { - name := f.Name[0 : len(f.Name)-len(ext)] // remove extension - if testPath(t, filepath.Join(dir, name)) { - nimports++ - } - } - } - case f.IsDirectory(): - nimports += testDir(t, filepath.Join(dir, f.Name), endTime) - } - } - return -} - -func TestGcImport(t *testing.T) { - compile(t, "testdata", "exports.go") - - nimports := 0 - if testPath(t, "./testdata/exports") { - nimports++ - } - nimports += testDir(t, "", time.Nanoseconds()+maxTime) // installed packages - t.Logf("tested %d imports", nimports) -} diff --git a/src/pkg/go/types/testdata/exports.go b/src/pkg/go/types/testdata/exports.go deleted file mode 100644 index ed63bf9ad..000000000 --- a/src/pkg/go/types/testdata/exports.go +++ /dev/null @@ -1,84 +0,0 @@ -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// This file is used to generate an object file which -// serves as test file for gcimporter_test.go. - -package exports - -import ( - "go/ast" -) - -const ( - C0 int = 0 - C1 = 3.14159265 - C2 = 2.718281828i - C3 = -123.456e-789 - C4 = +123.456E+789 - C5 = 1234i - C6 = "foo\n" - C7 = `bar\n` -) - -type ( - T1 int - T2 [10]int - T3 []int - T4 *int - T5 chan int - T6a chan<- int - T6b chan (<-chan int) - T6c chan<- (chan int) - T7 <-chan *ast.File - T8 struct{} - T9 struct { - a int - b, c float32 - d []string `go:"tag"` - } - T10 struct { - T8 - T9 - _ *T10 - } - T11 map[int]string - T12 interface{} - T13 interface { - m1() - m2(int) float32 - } - T14 interface { - T12 - T13 - m3(x ...struct{}) []T9 - } - T15 func() - T16 func(int) - T17 func(x int) - T18 func() float32 - T19 func() (x float32) - T20 func(...interface{}) - T21 struct{ next *T21 } - T22 struct{ link *T23 } - T23 struct{ link *T22 } - T24 *T24 - T25 *T26 - T26 *T27 - T27 *T25 - T28 func(T28) T28 -) - -var ( - V0 int - V1 = -991.0 -) - -func F1() {} -func F2(x int) {} -func F3() int { return 0 } -func F4() float32 { return 0 } -func F5(a, b, c int, u, v, w struct{ x, y T1 }, more ...interface{}) (p, q, r chan<- T10) - -func (p *T1) M1() diff --git a/src/pkg/go/types/testdata/test0.src b/src/pkg/go/types/testdata/test0.src deleted file mode 100644 index 84a1abe27..000000000 --- a/src/pkg/go/types/testdata/test0.src +++ /dev/null @@ -1,154 +0,0 @@ -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// type declarations - -package test0 - -import "unsafe" - -const pi = 3.1415 - -type ( - N undeclared /* ERROR "undeclared" */ - B bool - I int32 - A [10]P - T struct { - x, y P - } - P *T - R (*R) - F func(A) I - Y interface { - f(A) I - } - S [](((P))) - M map[I]F - C chan<- I -) - - -type ( - p1 pi /* ERROR "not a package" */ .foo - p2 unsafe.Pointer -) - - -type ( - Pi pi /* ERROR "not a type" */ - - a /* DISABLED "illegal cycle" */ a - a /* ERROR "redeclared" */ int - - // where the cycle error appears depends on the - // order in which declarations are processed - // (which depends on the order in which a map - // is iterated through) - b c - c /* DISABLED "illegal cycle" */ d - d e - e b - - t *t - - U V - V *W - W U - - P1 *S2 - P2 P1 - - S0 struct { - } - S1 struct { - a, b, c int - u, v, a /* ERROR "redeclared" */ float32 - } - S2 struct { - U // anonymous field - // TODO(gri) recognize double-declaration below - // U /* ERROR "redeclared" */ int - } - S3 struct { - x S2 - } - S4/* DISABLED "illegal cycle" */ struct { - S4 - } - S5 struct { - S6 - } - S6 /* DISABLED "illegal cycle" */ struct { - field S7 - } - S7 struct { - S5 - } - - L1 []L1 - L2 []int - - A1 [10]int - A2 /* DISABLED "illegal cycle" */ [10]A2 - A3 /* DISABLED "illegal cycle" */ [10]struct { - x A4 - } - A4 [10]A3 - - F1 func() - F2 func(x, y, z float32) - F3 func(x, y, x /* ERROR "redeclared" */ float32) - F4 func() (x, y, x /* ERROR "redeclared" */ float32) - F5 func(x int) (x /* ERROR "redeclared" */ float32) - F6 func(x ...int) - - I1 interface{} - I2 interface { - m1() - } - I3 interface { - m1() - m1 /* ERROR "redeclared" */ () - } - I4 interface { - m1(x, y, x /* ERROR "redeclared" */ float32) - m2() (x, y, x /* ERROR "redeclared" */ float32) - m3(x int) (x /* ERROR "redeclared" */ float32) - } - I5 interface { - m1(I5) - } - I6 interface { - S0 /* ERROR "non-interface" */ - } - I7 interface { - I1 - I1 - } - I8 /* DISABLED "illegal cycle" */ interface { - I8 - } - I9 /* DISABLED "illegal cycle" */ interface { - I10 - } - I10 interface { - I11 - } - I11 interface { - I9 - } - - C1 chan int - C2 <-chan int - C3 chan<- C3 - C4 chan C5 - C5 chan C6 - C6 chan C4 - - M1 map[Last]string - M2 map[string]M2 - - Last int -) diff --git a/src/pkg/go/types/types.go b/src/pkg/go/types/types.go deleted file mode 100644 index 3aa896892..000000000 --- a/src/pkg/go/types/types.go +++ /dev/null @@ -1,255 +0,0 @@ -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// PACKAGE UNDER CONSTRUCTION. ANY AND ALL PARTS MAY CHANGE. -// Package types declares the types used to represent Go types. -// -package types - -import ( - "go/ast" - "sort" -) - -// All types implement the Type interface. -type Type interface { - isType() -} - -// All concrete types embed ImplementsType which -// ensures that all types implement the Type interface. -type ImplementsType struct{} - -func (t *ImplementsType) isType() {} - -// A Bad type is a non-nil placeholder type when we don't know a type. -type Bad struct { - ImplementsType - Msg string // for better error reporting/debugging -} - -// A Basic represents a (unnamed) basic type. -type Basic struct { - ImplementsType - // TODO(gri) need a field specifying the exact basic type -} - -// An Array represents an array type [Len]Elt. -type Array struct { - ImplementsType - Len uint64 - Elt Type -} - -// A Slice represents a slice type []Elt. -type Slice struct { - ImplementsType - Elt Type -} - -// A Struct represents a struct type struct{...}. -// Anonymous fields are represented by objects with empty names. -type Struct struct { - ImplementsType - Fields ObjList // struct fields; or nil - Tags []string // corresponding tags; or nil - // TODO(gri) This type needs some rethinking: - // - at the moment anonymous fields are marked with "" object names, - // and their names have to be reconstructed - // - there is no scope for fast lookup (but the parser creates one) -} - -// A Pointer represents a pointer type *Base. -type Pointer struct { - ImplementsType - Base Type -} - -// A Func represents a function type func(...) (...). -// Unnamed parameters are represented by objects with empty names. -type Func struct { - ImplementsType - Recv *ast.Object // nil if not a method - Params ObjList // (incoming) parameters from left to right; or nil - Results ObjList // (outgoing) results from left to right; or nil - IsVariadic bool // true if the last parameter's type is of the form ...T -} - -// An Interface represents an interface type interface{...}. -type Interface struct { - ImplementsType - Methods ObjList // interface methods sorted by name; or nil -} - -// A Map represents a map type map[Key]Elt. -type Map struct { - ImplementsType - Key, Elt Type -} - -// A Chan represents a channel type chan Elt, <-chan Elt, or chan<-Elt. -type Chan struct { - ImplementsType - Dir ast.ChanDir - Elt Type -} - -// A Name represents a named type as declared in a type declaration. -type Name struct { - ImplementsType - Underlying Type // nil if not fully declared - Obj *ast.Object // corresponding declared object - // TODO(gri) need to remember fields and methods. -} - -// If typ is a pointer type, Deref returns the pointer's base type; -// otherwise it returns typ. -func Deref(typ Type) Type { - if typ, ok := typ.(*Pointer); ok { - return typ.Base - } - return typ -} - -// Underlying returns the underlying type of a type. -func Underlying(typ Type) Type { - if typ, ok := typ.(*Name); ok { - utyp := typ.Underlying - if _, ok := utyp.(*Basic); !ok { - return utyp - } - // the underlying type of a type name referring - // to an (untyped) basic type is the basic type - // name - } - return typ -} - -// An ObjList represents an ordered (in some fashion) list of objects. -type ObjList []*ast.Object - -// ObjList implements sort.Interface. -func (list ObjList) Len() int { return len(list) } -func (list ObjList) Less(i, j int) bool { return list[i].Name < list[j].Name } -func (list ObjList) Swap(i, j int) { list[i], list[j] = list[j], list[i] } - -// Sort sorts an object list by object name. -func (list ObjList) Sort() { sort.Sort(list) } - -// identicalTypes returns true if both lists a and b have the -// same length and corresponding objects have identical types. -func identicalTypes(a, b ObjList) bool { - if len(a) == len(b) { - for i, x := range a { - y := b[i] - if !Identical(x.Type.(Type), y.Type.(Type)) { - return false - } - } - return true - } - return false -} - -// Identical returns true if two types are identical. -func Identical(x, y Type) bool { - if x == y { - return true - } - - switch x := x.(type) { - case *Bad: - // A Bad type is always identical to any other type - // (to avoid spurious follow-up errors). - return true - - case *Basic: - if y, ok := y.(*Basic); ok { - panic("unimplemented") - _ = y - } - - case *Array: - // Two array types are identical if they have identical element types - // and the same array length. - if y, ok := y.(*Array); ok { - return x.Len == y.Len && Identical(x.Elt, y.Elt) - } - - case *Slice: - // Two slice types are identical if they have identical element types. - if y, ok := y.(*Slice); ok { - return Identical(x.Elt, y.Elt) - } - - case *Struct: - // Two struct types are identical if they have the same sequence of fields, - // and if corresponding fields have the same names, and identical types, - // and identical tags. Two anonymous fields are considered to have the same - // name. Lower-case field names from different packages are always different. - if y, ok := y.(*Struct); ok { - // TODO(gri) handle structs from different packages - if identicalTypes(x.Fields, y.Fields) { - for i, f := range x.Fields { - g := y.Fields[i] - if f.Name != g.Name || x.Tags[i] != y.Tags[i] { - return false - } - } - return true - } - } - - case *Pointer: - // Two pointer types are identical if they have identical base types. - if y, ok := y.(*Pointer); ok { - return Identical(x.Base, y.Base) - } - - case *Func: - // Two function types are identical if they have the same number of parameters - // and result values, corresponding parameter and result types are identical, - // and either both functions are variadic or neither is. Parameter and result - // names are not required to match. - if y, ok := y.(*Func); ok { - return identicalTypes(x.Params, y.Params) && - identicalTypes(x.Results, y.Results) && - x.IsVariadic == y.IsVariadic - } - - case *Interface: - // Two interface types are identical if they have the same set of methods with - // the same names and identical function types. Lower-case method names from - // different packages are always different. The order of the methods is irrelevant. - if y, ok := y.(*Interface); ok { - return identicalTypes(x.Methods, y.Methods) // methods are sorted - } - - case *Map: - // Two map types are identical if they have identical key and value types. - if y, ok := y.(*Map); ok { - return Identical(x.Key, y.Key) && Identical(x.Elt, y.Elt) - } - - case *Chan: - // Two channel types are identical if they have identical value types - // and the same direction. - if y, ok := y.(*Chan); ok { - return x.Dir == y.Dir && Identical(x.Elt, y.Elt) - } - - case *Name: - // Two named types are identical if their type names originate - // in the same type declaration. - if y, ok := y.(*Name); ok { - return x.Obj == y.Obj || - // permit bad objects to be equal to avoid - // follow up errors - x.Obj != nil && x.Obj.Kind == ast.Bad || - y.Obj != nil && y.Obj.Kind == ast.Bad - } - } - - return false -} diff --git a/src/pkg/go/types/universe.go b/src/pkg/go/types/universe.go deleted file mode 100644 index 6ae88e5f9..000000000 --- a/src/pkg/go/types/universe.go +++ /dev/null @@ -1,108 +0,0 @@ -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// FILE UNDER CONSTRUCTION. ANY AND ALL PARTS MAY CHANGE. -// This file implements the universe and unsafe package scopes. - -package types - -import "go/ast" - -var ( - scope *ast.Scope // current scope to use for initialization - Universe *ast.Scope - Unsafe *ast.Object // package unsafe -) - -func define(kind ast.ObjKind, name string) *ast.Object { - obj := ast.NewObj(kind, name) - if scope.Insert(obj) != nil { - panic("types internal error: double declaration") - } - return obj -} - -func defType(name string) *Name { - obj := define(ast.Typ, name) - typ := &Name{Underlying: &Basic{}, Obj: obj} - obj.Type = typ - return typ -} - -func defConst(name string) { - obj := define(ast.Con, name) - _ = obj // TODO(gri) fill in other properties -} - -func defFun(name string) { - obj := define(ast.Fun, name) - _ = obj // TODO(gri) fill in other properties -} - -var ( - Bool, - Int, - Float64, - Complex128, - String *Name -) - -func init() { - scope = ast.NewScope(nil) - Universe = scope - - Bool = defType("bool") - defType("byte") // TODO(gri) should be an alias for uint8 - defType("complex64") - Complex128 = defType("complex128") - defType("float32") - Float64 = defType("float64") - defType("int8") - defType("int16") - defType("int32") - defType("int64") - String = defType("string") - defType("uint8") - defType("uint16") - defType("uint32") - defType("uint64") - Int = defType("int") - defType("uint") - defType("uintptr") - - defConst("true") - defConst("false") - defConst("iota") - defConst("nil") - - defFun("append") - defFun("cap") - defFun("close") - defFun("complex") - defFun("copy") - defFun("imag") - defFun("len") - defFun("make") - defFun("new") - defFun("panic") - defFun("print") - defFun("println") - defFun("real") - defFun("recover") - - scope = ast.NewScope(nil) - Unsafe = ast.NewObj(ast.Pkg, "unsafe") - Unsafe.Data = scope - - defType("Pointer") - - defFun("Alignof") - defFun("New") - defFun("NewArray") - defFun("Offsetof") - defFun("Reflect") - defFun("Sizeof") - defFun("Typeof") - defFun("Unreflect") -} |