summaryrefslogtreecommitdiff
path: root/src/pkg/go/scanner
diff options
context:
space:
mode:
Diffstat (limited to 'src/pkg/go/scanner')
-rw-r--r--src/pkg/go/scanner/scanner.go26
-rw-r--r--src/pkg/go/scanner/scanner_test.go60
2 files changed, 49 insertions, 37 deletions
diff --git a/src/pkg/go/scanner/scanner.go b/src/pkg/go/scanner/scanner.go
index 153707f59..2f949ad25 100644
--- a/src/pkg/go/scanner/scanner.go
+++ b/src/pkg/go/scanner/scanner.go
@@ -177,11 +177,11 @@ var prefix = []byte("//line ")
func (S *Scanner) interpretLineComment(text []byte) {
if bytes.HasPrefix(text, prefix) {
// get filename and line number, if any
- if i := bytes.Index(text, []byte{':'}); i > 0 {
+ if i := bytes.LastIndex(text, []byte{':'}); i > 0 {
if line, err := strconv.Atoi(string(text[i+1:])); err == nil && line > 0 {
// valid //line filename:line comment;
filename := filepath.Clean(string(text[len(prefix):i]))
- if filename[0] != '/' {
+ if !filepath.IsAbs(filename) {
// make filename relative to current directory
filename = filepath.Join(S.dir, filename)
}
@@ -538,14 +538,12 @@ func (S *Scanner) switch4(tok0, tok1 token.Token, ch2 int, tok2, tok3 token.Toke
}
-var newline = []byte{'\n'}
-
-// Scan scans the next token and returns the token position pos,
-// the token tok, and the literal text lit corresponding to the
+// Scan scans the next token and returns the token position,
+// the token, and the literal string corresponding to the
// token. The source end is indicated by token.EOF.
//
// If the returned token is token.SEMICOLON, the corresponding
-// literal value is ";" if the semicolon was present in the source,
+// literal string is ";" if the semicolon was present in the source,
// and "\n" if the semicolon was inserted because of a newline or
// at EOF.
//
@@ -560,7 +558,7 @@ var newline = []byte{'\n'}
// set with Init. Token positions are relative to that file
// and thus relative to the file set.
//
-func (S *Scanner) Scan() (token.Pos, token.Token, []byte) {
+func (S *Scanner) Scan() (token.Pos, token.Token, string) {
scanAgain:
S.skipWhitespace()
@@ -586,7 +584,7 @@ scanAgain:
case -1:
if S.insertSemi {
S.insertSemi = false // EOF consumed
- return S.file.Pos(offs), token.SEMICOLON, newline
+ return S.file.Pos(offs), token.SEMICOLON, "\n"
}
tok = token.EOF
case '\n':
@@ -594,7 +592,7 @@ scanAgain:
// set in the first place and exited early
// from S.skipWhitespace()
S.insertSemi = false // newline consumed
- return S.file.Pos(offs), token.SEMICOLON, newline
+ return S.file.Pos(offs), token.SEMICOLON, "\n"
case '"':
insertSemi = true
tok = token.STRING
@@ -662,7 +660,7 @@ scanAgain:
S.offset = offs
S.rdOffset = offs + 1
S.insertSemi = false // newline consumed
- return S.file.Pos(offs), token.SEMICOLON, newline
+ return S.file.Pos(offs), token.SEMICOLON, "\n"
}
S.scanComment()
if S.mode&ScanComments == 0 {
@@ -711,5 +709,9 @@ scanAgain:
if S.mode&InsertSemis != 0 {
S.insertSemi = insertSemi
}
- return S.file.Pos(offs), tok, S.src[offs:S.offset]
+
+ // TODO(gri): The scanner API should change such that the literal string
+ // is only valid if an actual literal was scanned. This will
+ // permit a more efficient implementation.
+ return S.file.Pos(offs), tok, string(S.src[offs:S.offset])
}
diff --git a/src/pkg/go/scanner/scanner_test.go b/src/pkg/go/scanner/scanner_test.go
index c622ff482..8afb00ee5 100644
--- a/src/pkg/go/scanner/scanner_test.go
+++ b/src/pkg/go/scanner/scanner_test.go
@@ -7,6 +7,8 @@ package scanner
import (
"go/token"
"os"
+ "path/filepath"
+ "runtime"
"testing"
)
@@ -232,12 +234,11 @@ func TestScan(t *testing.T) {
index := 0
epos := token.Position{"", 0, 1, 1} // expected position
for {
- pos, tok, litb := s.Scan()
+ pos, tok, lit := s.Scan()
e := elt{token.EOF, "", special}
if index < len(tokens) {
e = tokens[index]
}
- lit := string(litb)
if tok == token.EOF {
lit = "<EOF>"
epos.Line = src_linecount
@@ -255,7 +256,7 @@ func TestScan(t *testing.T) {
}
epos.Offset += len(lit) + len(whitespace)
epos.Line += newlineCount(lit) + whitespace_linecount
- if tok == token.COMMENT && litb[1] == '/' {
+ if tok == token.COMMENT && lit[1] == '/' {
// correct for unaccounted '/n' in //-style comment
epos.Offset++
epos.Line++
@@ -290,7 +291,7 @@ func checkSemi(t *testing.T, line string, mode uint) {
semiPos.Column++
pos, tok, lit = S.Scan()
if tok == token.SEMICOLON {
- if string(lit) != semiLit {
+ if lit != semiLit {
t.Errorf(`bad literal for %q: got %q, expected %q`, line, lit, semiLit)
}
checkPos(t, line, pos, semiPos)
@@ -443,32 +444,41 @@ func TestSemis(t *testing.T) {
}
}
-
-var segments = []struct {
+type segment struct {
srcline string // a line of source text
filename string // filename for current token
line int // line number for current token
-}{
+}
+
+var segments = []segment{
// exactly one token per line since the test consumes one token per segment
- {" line1", "dir/TestLineComments", 1},
- {"\nline2", "dir/TestLineComments", 2},
- {"\nline3 //line File1.go:100", "dir/TestLineComments", 3}, // bad line comment, ignored
- {"\nline4", "dir/TestLineComments", 4},
- {"\n//line File1.go:100\n line100", "dir/File1.go", 100},
- {"\n//line File2.go:200\n line200", "dir/File2.go", 200},
+ {" line1", filepath.Join("dir", "TestLineComments"), 1},
+ {"\nline2", filepath.Join("dir", "TestLineComments"), 2},
+ {"\nline3 //line File1.go:100", filepath.Join("dir", "TestLineComments"), 3}, // bad line comment, ignored
+ {"\nline4", filepath.Join("dir", "TestLineComments"), 4},
+ {"\n//line File1.go:100\n line100", filepath.Join("dir", "File1.go"), 100},
+ {"\n//line File2.go:200\n line200", filepath.Join("dir", "File2.go"), 200},
{"\n//line :1\n line1", "dir", 1},
- {"\n//line foo:42\n line42", "dir/foo", 42},
- {"\n //line foo:42\n line44", "dir/foo", 44}, // bad line comment, ignored
- {"\n//line foo 42\n line46", "dir/foo", 46}, // bad line comment, ignored
- {"\n//line foo:42 extra text\n line48", "dir/foo", 48}, // bad line comment, ignored
- {"\n//line /bar:42\n line42", "/bar", 42},
- {"\n//line ./foo:42\n line42", "dir/foo", 42},
- {"\n//line a/b/c/File1.go:100\n line100", "dir/a/b/c/File1.go", 100},
+ {"\n//line foo:42\n line42", filepath.Join("dir", "foo"), 42},
+ {"\n //line foo:42\n line44", filepath.Join("dir", "foo"), 44}, // bad line comment, ignored
+ {"\n//line foo 42\n line46", filepath.Join("dir", "foo"), 46}, // bad line comment, ignored
+ {"\n//line foo:42 extra text\n line48", filepath.Join("dir", "foo"), 48}, // bad line comment, ignored
+ {"\n//line /bar:42\n line42", string(filepath.Separator) + "bar", 42},
+ {"\n//line ./foo:42\n line42", filepath.Join("dir", "foo"), 42},
+ {"\n//line a/b/c/File1.go:100\n line100", filepath.Join("dir", "a", "b", "c", "File1.go"), 100},
+}
+
+var winsegments = []segment{
+ {"\n//line c:\\dir\\File1.go:100\n line100", "c:\\dir\\File1.go", 100},
}
// Verify that comments of the form "//line filename:line" are interpreted correctly.
func TestLineComments(t *testing.T) {
+ if runtime.GOOS == "windows" {
+ segments = append(segments, winsegments...)
+ }
+
// make source
var src string
for _, e := range segments {
@@ -477,12 +487,12 @@ func TestLineComments(t *testing.T) {
// verify scan
var S Scanner
- file := fset.AddFile("dir/TestLineComments", fset.Base(), len(src))
+ file := fset.AddFile(filepath.Join("dir", "TestLineComments"), fset.Base(), len(src))
S.Init(file, []byte(src), nil, 0)
for _, s := range segments {
p, _, lit := S.Scan()
pos := file.Position(p)
- checkPos(t, string(lit), p, token.Position{s.filename, pos.Offset, s.line, pos.Column})
+ checkPos(t, lit, p, token.Position{s.filename, pos.Offset, s.line, pos.Column})
}
if S.ErrorCount != 0 {
@@ -536,10 +546,10 @@ func TestIllegalChars(t *testing.T) {
for offs, ch := range src {
pos, tok, lit := s.Scan()
if poffs := file.Offset(pos); poffs != offs {
- t.Errorf("bad position for %s: got %d, expected %d", string(lit), poffs, offs)
+ t.Errorf("bad position for %s: got %d, expected %d", lit, poffs, offs)
}
- if tok == token.ILLEGAL && string(lit) != string(ch) {
- t.Errorf("bad token: got %s, expected %s", string(lit), string(ch))
+ if tok == token.ILLEGAL && lit != string(ch) {
+ t.Errorf("bad token: got %s, expected %s", lit, string(ch))
}
}
generated by cgit v1.2.3 (git 2.46.0) at 2025-11-02 07:16:49 +0000