diff options
Diffstat (limited to 'src/pkg/go/token')
| -rw-r--r-- | src/pkg/go/token/Makefile | 12 | ||||
| -rw-r--r-- | src/pkg/go/token/position.go | 277 | ||||
| -rw-r--r-- | src/pkg/go/token/position_test.go | 9 | ||||
| -rw-r--r-- | src/pkg/go/token/serialize.go | 56 | ||||
| -rw-r--r-- | src/pkg/go/token/serialize_test.go | 111 | ||||
| -rw-r--r-- | src/pkg/go/token/token.go | 12 |
6 files changed, 321 insertions, 156 deletions
diff --git a/src/pkg/go/token/Makefile b/src/pkg/go/token/Makefile deleted file mode 100644 index 4a4e64dc8..000000000 --- a/src/pkg/go/token/Makefile +++ /dev/null @@ -1,12 +0,0 @@ -# Copyright 2009 The Go Authors. All rights reserved. -# Use of this source code is governed by a BSD-style -# license that can be found in the LICENSE file. - -include ../../../Make.inc - -TARG=go/token -GOFILES=\ - position.go\ - token.go\ - -include ../../../Make.pkg diff --git a/src/pkg/go/token/position.go b/src/pkg/go/token/position.go index c559e19f8..647d1b770 100644 --- a/src/pkg/go/token/position.go +++ b/src/pkg/go/token/position.go @@ -12,6 +12,9 @@ import ( "sync" ) +// ----------------------------------------------------------------------------- +// Positions + // Position describes an arbitrary source position // including the file, line, and column location. // A Position is valid if the line number is > 0. @@ -81,82 +84,8 @@ func (p Pos) IsValid() bool { return p != NoPos } -func searchFiles(a []*File, x int) int { - return sort.Search(len(a), func(i int) bool { return a[i].base > x }) - 1 -} - -func (s *FileSet) file(p Pos) *File { - if f := s.last; f != nil && f.base <= int(p) && int(p) <= f.base+f.size { - return f - } - if i := searchFiles(s.files, int(p)); i >= 0 { - f := s.files[i] - // f.base <= int(p) by definition of searchFiles - if int(p) <= f.base+f.size { - s.last = f - return f - } - } - return nil -} - -// File returns the file which contains the position p. -// If no such file is found (for instance for p == NoPos), -// the result is nil. -// -func (s *FileSet) File(p Pos) (f *File) { - if p != NoPos { - s.mutex.RLock() - f = s.file(p) - s.mutex.RUnlock() - } - return -} - -func (f *File) position(p Pos) (pos Position) { - offset := int(p) - f.base - pos.Offset = offset - pos.Filename, pos.Line, pos.Column = f.info(offset) - return -} - -// Position converts a Pos in the fileset into a general Position. -func (s *FileSet) Position(p Pos) (pos Position) { - if p != NoPos { - // TODO(gri) consider optimizing the case where p - // is in the last file added, or perhaps - // looked at - will eliminate one level - // of search - s.mutex.RLock() - if f := s.file(p); f != nil { - pos = f.position(p) - } - s.mutex.RUnlock() - } - return -} - -type lineInfo struct { - offset int - filename string - line int -} - -// AddLineInfo adds alternative file and line number information for -// a given file offset. The offset must be larger than the offset for -// the previously added alternative line info and smaller than the -// file size; otherwise the information is ignored. -// -// AddLineInfo is typically used to register alternative position -// information for //line filename:line comments in source files. -// -func (f *File) AddLineInfo(offset int, filename string, line int) { - f.set.mutex.Lock() - if i := len(f.infos); i == 0 || f.infos[i-1].offset < offset && offset < f.size { - f.infos = append(f.infos, lineInfo{offset, filename, line}) - } - f.set.mutex.Unlock() -} +// ----------------------------------------------------------------------------- +// File // A File is a handle for a file belonging to a FileSet. // A File has a name, size, and line offset table. @@ -251,6 +180,32 @@ func (f *File) SetLinesForContent(content []byte) { f.set.mutex.Unlock() } +// A lineInfo object describes alternative file and line number +// information (such as provided via a //line comment in a .go +// file) for a given file offset. +type lineInfo struct { + // fields are exported to make them accessible to gob + Offset int + Filename string + Line int +} + +// AddLineInfo adds alternative file and line number information for +// a given file offset. The offset must be larger than the offset for +// the previously added alternative line info and smaller than the +// file size; otherwise the information is ignored. +// +// AddLineInfo is typically used to register alternative position +// information for //line filename:line comments in source files. +// +func (f *File) AddLineInfo(offset int, filename string, line int) { + f.set.mutex.Lock() + if i := len(f.infos); i == 0 || f.infos[i-1].Offset < offset && offset < f.size { + f.infos = append(f.infos, lineInfo{offset, filename, line}) + } + f.set.mutex.Unlock() +} + // Pos returns the Pos value for the given file offset; // the offset must be <= f.Size(). // f.Pos(f.Offset(p)) == p. @@ -281,43 +236,8 @@ func (f *File) Line(p Pos) int { return f.Position(p).Line } -// Position returns the Position value for the given file position p; -// p must be a Pos value in that file or NoPos. -// -func (f *File) Position(p Pos) (pos Position) { - if p != NoPos { - if int(p) < f.base || int(p) > f.base+f.size { - panic("illegal Pos value") - } - pos = f.position(p) - } - return -} - -func searchInts(a []int, x int) int { - // This function body is a manually inlined version of: - // - // return sort.Search(len(a), func(i int) bool { return a[i] > x }) - 1 - // - // With better compiler optimizations, this may not be needed in the - // future, but at the moment this change improves the go/printer - // benchmark performance by ~30%. This has a direct impact on the - // speed of gofmt and thus seems worthwhile (2011-04-29). - i, j := 0, len(a) - for i < j { - h := i + (j-i)/2 // avoid overflow when computing h - // i ≤ h < j - if a[h] <= x { - i = h + 1 - } else { - j = h - } - } - return i - 1 -} - func searchLineInfos(a []lineInfo, x int) int { - return sort.Search(len(a), func(i int) bool { return a[i].offset > x }) - 1 + return sort.Search(len(a), func(i int) bool { return a[i].Offset > x }) - 1 } // info returns the file name, line, and column number for a file offset. @@ -330,15 +250,38 @@ func (f *File) info(offset int) (filename string, line, column int) { // almost no files have extra line infos if i := searchLineInfos(f.infos, offset); i >= 0 { alt := &f.infos[i] - filename = alt.filename - if i := searchInts(f.lines, alt.offset); i >= 0 { - line += alt.line - i - 1 + filename = alt.Filename + if i := searchInts(f.lines, alt.Offset); i >= 0 { + line += alt.Line - i - 1 } } } return } +func (f *File) position(p Pos) (pos Position) { + offset := int(p) - f.base + pos.Offset = offset + pos.Filename, pos.Line, pos.Column = f.info(offset) + return +} + +// Position returns the Position value for the given file position p; +// p must be a Pos value in that file or NoPos. +// +func (f *File) Position(p Pos) (pos Position) { + if p != NoPos { + if int(p) < f.base || int(p) > f.base+f.size { + panic("illegal Pos value") + } + pos = f.position(p) + } + return +} + +// ----------------------------------------------------------------------------- +// FileSet + // A FileSet represents a set of source files. // Methods of file sets are synchronized; multiple goroutines // may invoke them concurrently. @@ -402,23 +345,91 @@ func (s *FileSet) AddFile(filename string, base, size int) *File { return f } -// Files returns the files added to the file set. -func (s *FileSet) Files() <-chan *File { - ch := make(chan *File) - go func() { - for i := 0; ; i++ { - var f *File - s.mutex.RLock() - if i < len(s.files) { - f = s.files[i] - } - s.mutex.RUnlock() - if f == nil { - break - } - ch <- f +// Iterate calls f for the files in the file set in the order they were added +// until f returns false. +// +func (s *FileSet) Iterate(f func(*File) bool) { + for i := 0; ; i++ { + var file *File + s.mutex.RLock() + if i < len(s.files) { + file = s.files[i] } - close(ch) - }() - return ch + s.mutex.RUnlock() + if file == nil || !f(file) { + break + } + } +} + +func searchFiles(a []*File, x int) int { + return sort.Search(len(a), func(i int) bool { return a[i].base > x }) - 1 +} + +func (s *FileSet) file(p Pos) *File { + // common case: p is in last file + if f := s.last; f != nil && f.base <= int(p) && int(p) <= f.base+f.size { + return f + } + // p is not in last file - search all files + if i := searchFiles(s.files, int(p)); i >= 0 { + f := s.files[i] + // f.base <= int(p) by definition of searchFiles + if int(p) <= f.base+f.size { + s.last = f + return f + } + } + return nil +} + +// File returns the file that contains the position p. +// If no such file is found (for instance for p == NoPos), +// the result is nil. +// +func (s *FileSet) File(p Pos) (f *File) { + if p != NoPos { + s.mutex.RLock() + f = s.file(p) + s.mutex.RUnlock() + } + return +} + +// Position converts a Pos in the fileset into a general Position. +func (s *FileSet) Position(p Pos) (pos Position) { + if p != NoPos { + s.mutex.RLock() + if f := s.file(p); f != nil { + pos = f.position(p) + } + s.mutex.RUnlock() + } + return +} + +// ----------------------------------------------------------------------------- +// Helper functions + +func searchInts(a []int, x int) int { + // This function body is a manually inlined version of: + // + // return sort.Search(len(a), func(i int) bool { return a[i] > x }) - 1 + // + // With better compiler optimizations, this may not be needed in the + // future, but at the moment this change improves the go/printer + // benchmark performance by ~30%. This has a direct impact on the + // speed of gofmt and thus seems worthwhile (2011-04-29). + // TODO(gri): Remove this when compilers have caught up. + i, j := 0, len(a) + for i < j { + h := i + (j-i)/2 // avoid overflow when computing h + // i ≤ h < j + if a[h] <= x { + i = h + 1 + } else { + j = h + } + } + return i - 1 } diff --git a/src/pkg/go/token/position_test.go b/src/pkg/go/token/position_test.go index 30bec5991..160107df4 100644 --- a/src/pkg/go/token/position_test.go +++ b/src/pkg/go/token/position_test.go @@ -167,12 +167,13 @@ func TestFiles(t *testing.T) { for i, test := range tests { fset.AddFile(test.filename, fset.Base(), test.size) j := 0 - for g := range fset.Files() { - if g.Name() != tests[j].filename { - t.Errorf("expected filename = %s; got %s", tests[j].filename, g.Name()) + fset.Iterate(func(f *File) bool { + if f.Name() != tests[j].filename { + t.Errorf("expected filename = %s; got %s", tests[j].filename, f.Name()) } j++ - } + return true + }) if j != i+1 { t.Errorf("expected %d files; got %d", i+1, j) } diff --git a/src/pkg/go/token/serialize.go b/src/pkg/go/token/serialize.go new file mode 100644 index 000000000..4adc8f9e3 --- /dev/null +++ b/src/pkg/go/token/serialize.go @@ -0,0 +1,56 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package token + +type serializedFile struct { + // fields correspond 1:1 to fields with same (lower-case) name in File + Name string + Base int + Size int + Lines []int + Infos []lineInfo +} + +type serializedFileSet struct { + Base int + Files []serializedFile +} + +// Read calls decode to deserialize a file set into s; s must not be nil. +func (s *FileSet) Read(decode func(interface{}) error) error { + var ss serializedFileSet + if err := decode(&ss); err != nil { + return err + } + + s.mutex.Lock() + s.base = ss.Base + files := make([]*File, len(ss.Files)) + for i := 0; i < len(ss.Files); i++ { + f := &ss.Files[i] + files[i] = &File{s, f.Name, f.Base, f.Size, f.Lines, f.Infos} + } + s.files = files + s.last = nil + s.mutex.Unlock() + + return nil +} + +// Write calls encode to serialize the file set s. +func (s *FileSet) Write(encode func(interface{}) error) error { + var ss serializedFileSet + + s.mutex.Lock() + ss.Base = s.base + files := make([]serializedFile, len(s.files)) + for i, f := range s.files { + files[i] = serializedFile{f.name, f.base, f.size, f.lines, f.infos} + } + ss.Files = files + s.mutex.Unlock() + + return encode(ss) +} diff --git a/src/pkg/go/token/serialize_test.go b/src/pkg/go/token/serialize_test.go new file mode 100644 index 000000000..4e925adb6 --- /dev/null +++ b/src/pkg/go/token/serialize_test.go @@ -0,0 +1,111 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package token + +import ( + "bytes" + "encoding/gob" + "fmt" + "testing" +) + +// equal returns nil if p and q describe the same file set; +// otherwise it returns an error describing the discrepancy. +func equal(p, q *FileSet) error { + if p == q { + // avoid deadlock if p == q + return nil + } + + // not strictly needed for the test + p.mutex.Lock() + q.mutex.Lock() + defer q.mutex.Unlock() + defer p.mutex.Unlock() + + if p.base != q.base { + return fmt.Errorf("different bases: %d != %d", p.base, q.base) + } + + if len(p.files) != len(q.files) { + return fmt.Errorf("different number of files: %d != %d", len(p.files), len(q.files)) + } + + for i, f := range p.files { + g := q.files[i] + if f.set != p { + return fmt.Errorf("wrong fileset for %q", f.name) + } + if g.set != q { + return fmt.Errorf("wrong fileset for %q", g.name) + } + if f.name != g.name { + return fmt.Errorf("different filenames: %q != %q", f.name, g.name) + } + if f.base != g.base { + return fmt.Errorf("different base for %q: %d != %d", f.name, f.base, g.base) + } + if f.size != g.size { + return fmt.Errorf("different size for %q: %d != %d", f.name, f.size, g.size) + } + for j, l := range f.lines { + m := g.lines[j] + if l != m { + return fmt.Errorf("different offsets for %q", f.name) + } + } + for j, l := range f.infos { + m := g.infos[j] + if l.Offset != m.Offset || l.Filename != m.Filename || l.Line != m.Line { + return fmt.Errorf("different infos for %q", f.name) + } + } + } + + // we don't care about .last - it's just a cache + return nil +} + +func checkSerialize(t *testing.T, p *FileSet) { + var buf bytes.Buffer + encode := func(x interface{}) error { + return gob.NewEncoder(&buf).Encode(x) + } + if err := p.Write(encode); err != nil { + t.Errorf("writing fileset failed: %s", err) + return + } + q := NewFileSet() + decode := func(x interface{}) error { + return gob.NewDecoder(&buf).Decode(x) + } + if err := q.Read(decode); err != nil { + t.Errorf("reading fileset failed: %s", err) + return + } + if err := equal(p, q); err != nil { + t.Errorf("filesets not identical: %s", err) + } +} + +func TestSerialization(t *testing.T) { + p := NewFileSet() + checkSerialize(t, p) + // add some files + for i := 0; i < 10; i++ { + f := p.AddFile(fmt.Sprintf("file%d", i), p.Base()+i, i*100) + checkSerialize(t, p) + // add some lines and alternative file infos + line := 1000 + for offs := 0; offs < f.Size(); offs += 40 + i { + f.AddLine(offs) + if offs%7 == 0 { + f.AddLineInfo(offs, fmt.Sprintf("file%d", offs), line) + line += 33 + } + } + checkSerialize(t, p) + } +} diff --git a/src/pkg/go/token/token.go b/src/pkg/go/token/token.go index 557374052..84b6314d5 100644 --- a/src/pkg/go/token/token.go +++ b/src/pkg/go/token/token.go @@ -283,10 +283,8 @@ func init() { // Lookup maps an identifier to its keyword token or IDENT (if not a keyword). // -func Lookup(ident []byte) Token { - // TODO Maps with []byte key are illegal because []byte does not - // support == . Should find a more efficient solution eventually. - if tok, is_keyword := keywords[string(ident)]; is_keyword { +func Lookup(ident string) Token { + if tok, is_keyword := keywords[ident]; is_keyword { return tok } return IDENT @@ -295,16 +293,16 @@ func Lookup(ident []byte) Token { // Predicates // IsLiteral returns true for tokens corresponding to identifiers -// and basic type literals; returns false otherwise. +// and basic type literals; it returns false otherwise. // func (tok Token) IsLiteral() bool { return literal_beg < tok && tok < literal_end } // IsOperator returns true for tokens corresponding to operators and -// delimiters; returns false otherwise. +// delimiters; it returns false otherwise. // func (tok Token) IsOperator() bool { return operator_beg < tok && tok < operator_end } // IsKeyword returns true for tokens corresponding to keywords; -// returns false otherwise. +// it returns false otherwise. // func (tok Token) IsKeyword() bool { return keyword_beg < tok && tok < keyword_end } |