summaryrefslogtreecommitdiff
path: root/src/pkg/go/scanner/scanner_test.go
diff options
context:
space:
mode:
Diffstat (limited to 'src/pkg/go/scanner/scanner_test.go')
-rw-r--r--src/pkg/go/scanner/scanner_test.go155
1 files changed, 89 insertions, 66 deletions
diff --git a/src/pkg/go/scanner/scanner_test.go b/src/pkg/go/scanner/scanner_test.go
index eb9e1cb81..06223e23b 100644
--- a/src/pkg/go/scanner/scanner_test.go
+++ b/src/pkg/go/scanner/scanner_test.go
@@ -83,6 +83,8 @@ var tokens = [...]elt{
"`",
literal,
},
+ {token.STRING, "`\r`", literal},
+ {token.STRING, "`foo\r\nbar`", literal},
// Operators and delimiters
{token.ADD, "+", operator},
@@ -175,13 +177,14 @@ var tokens = [...]elt{
const whitespace = " \t \n\n\n" // to separate tokens
-type testErrorHandler struct {
- t *testing.T
-}
-
-func (h *testErrorHandler) Error(pos token.Position, msg string) {
- h.t.Errorf("Error() called (msg = %s)", msg)
-}
+var source = func() []byte {
+ var src []byte
+ for _, t := range tokens {
+ src = append(src, t.lit...)
+ src = append(src, whitespace...)
+ }
+ return src
+}()
func newlineCount(s string) int {
n := 0
@@ -212,20 +215,31 @@ func checkPos(t *testing.T, lit string, p token.Pos, expected token.Position) {
// Verify that calling Scan() provides the correct results.
func TestScan(t *testing.T) {
// make source
- var src string
- for _, e := range tokens {
- src += e.lit + whitespace
- }
- src_linecount := newlineCount(src)
+ src_linecount := newlineCount(string(source))
whitespace_linecount := newlineCount(whitespace)
+ // error handler
+ eh := func(_ token.Position, msg string) {
+ t.Errorf("error handler called (msg = %s)", msg)
+ }
+
// verify scan
var s Scanner
- s.Init(fset.AddFile("", fset.Base(), len(src)), []byte(src), &testErrorHandler{t}, ScanComments)
+ s.Init(fset.AddFile("", fset.Base(), len(source)), source, eh, ScanComments|dontInsertSemis)
index := 0
- epos := token.Position{"", 0, 1, 1} // expected position
+ // epos is the expected position
+ epos := token.Position{
+ Filename: "",
+ Offset: 0,
+ Line: 1,
+ Column: 1,
+ }
for {
pos, tok, lit := s.Scan()
+ if lit == "" {
+ // no literal value for non-literal tokens
+ lit = tok.String()
+ }
e := elt{token.EOF, "", special}
if index < len(tokens) {
e = tokens[index]
@@ -237,10 +251,18 @@ func TestScan(t *testing.T) {
}
checkPos(t, lit, pos, epos)
if tok != e.tok {
- t.Errorf("bad token for %q: got %s, expected %s", lit, tok.String(), e.tok.String())
+ t.Errorf("bad token for %q: got %s, expected %s", lit, tok, e.tok)
}
- if e.tok.IsLiteral() && lit != e.lit {
- t.Errorf("bad literal for %q: got %q, expected %q", lit, lit, e.lit)
+ if e.tok.IsLiteral() {
+ // no CRs in raw string literals
+ elit := e.lit
+ if elit[0] == '`' {
+ elit = string(stripCR([]byte(elit)))
+ epos.Offset += len(e.lit) - len(lit) // correct position
+ }
+ if lit != elit {
+ t.Errorf("bad literal for %q: got %q, expected %q", lit, lit, elit)
+ }
}
if tokenclass(tok) != e.class {
t.Errorf("bad class for %q: got %d, expected %d", lit, tokenclass(tok), e.class)
@@ -262,7 +284,7 @@ func TestScan(t *testing.T) {
}
}
-func checkSemi(t *testing.T, line string, mode uint) {
+func checkSemi(t *testing.T, line string, mode Mode) {
var S Scanner
file := fset.AddFile("TestSemis", fset.Base(), len(line))
S.Init(file, []byte(line), nil, mode)
@@ -286,7 +308,7 @@ func checkSemi(t *testing.T, line string, mode uint) {
}
checkPos(t, line, pos, semiPos)
} else {
- t.Errorf("bad token for %q: got %s, expected ;", line, tok.String())
+ t.Errorf("bad token for %q: got %s, expected ;", line, tok)
}
} else if tok == token.SEMICOLON {
t.Errorf("bad token for %q: got ;, expected no ;", line)
@@ -420,14 +442,14 @@ var lines = []string{
func TestSemis(t *testing.T) {
for _, line := range lines {
- checkSemi(t, line, AllowIllegalChars|InsertSemis)
- checkSemi(t, line, AllowIllegalChars|InsertSemis|ScanComments)
+ checkSemi(t, line, 0)
+ checkSemi(t, line, ScanComments)
// if the input ended in newlines, the input must tokenize the
// same with or without those newlines
for i := len(line) - 1; i >= 0 && line[i] == '\n'; i-- {
- checkSemi(t, line[0:i], AllowIllegalChars|InsertSemis)
- checkSemi(t, line[0:i], AllowIllegalChars|InsertSemis|ScanComments)
+ checkSemi(t, line[0:i], 0)
+ checkSemi(t, line[0:i], ScanComments)
}
}
}
@@ -482,11 +504,16 @@ func TestLineComments(t *testing.T) {
// verify scan
var S Scanner
file := fset.AddFile(filepath.Join("dir", "TestLineComments"), fset.Base(), len(src))
- S.Init(file, []byte(src), nil, 0)
+ S.Init(file, []byte(src), nil, dontInsertSemis)
for _, s := range segs {
p, _, lit := S.Scan()
pos := file.Position(p)
- checkPos(t, lit, p, token.Position{s.filename, pos.Offset, s.line, pos.Column})
+ checkPos(t, lit, p, token.Position{
+ Filename: s.filename,
+ Offset: pos.Offset,
+ Line: s.line,
+ Column: pos.Column,
+ })
}
if S.ErrorCount != 0 {
@@ -501,7 +528,7 @@ func TestInit(t *testing.T) {
// 1st init
src1 := "if true { }"
f1 := fset.AddFile("src1", fset.Base(), len(src1))
- s.Init(f1, []byte(src1), nil, 0)
+ s.Init(f1, []byte(src1), nil, dontInsertSemis)
if f1.Size() != len(src1) {
t.Errorf("bad file size: got %d, expected %d", f1.Size(), len(src1))
}
@@ -509,40 +536,19 @@ func TestInit(t *testing.T) {
s.Scan() // true
_, tok, _ := s.Scan() // {
if tok != token.LBRACE {
- t.Errorf("bad token: got %s, expected %s", tok.String(), token.LBRACE)
+ t.Errorf("bad token: got %s, expected %s", tok, token.LBRACE)
}
// 2nd init
src2 := "go true { ]"
f2 := fset.AddFile("src2", fset.Base(), len(src2))
- s.Init(f2, []byte(src2), nil, 0)
+ s.Init(f2, []byte(src2), nil, dontInsertSemis)
if f2.Size() != len(src2) {
t.Errorf("bad file size: got %d, expected %d", f2.Size(), len(src2))
}
_, tok, _ = s.Scan() // go
if tok != token.GO {
- t.Errorf("bad token: got %s, expected %s", tok.String(), token.GO)
- }
-
- if s.ErrorCount != 0 {
- t.Errorf("found %d errors", s.ErrorCount)
- }
-}
-
-func TestIllegalChars(t *testing.T) {
- var s Scanner
-
- const src = "*?*$*@*"
- file := fset.AddFile("", fset.Base(), len(src))
- s.Init(file, []byte(src), &testErrorHandler{t}, AllowIllegalChars)
- for offs, ch := range src {
- pos, tok, lit := s.Scan()
- if poffs := file.Offset(pos); poffs != offs {
- t.Errorf("bad position for %s: got %d, expected %d", lit, poffs, offs)
- }
- if tok == token.ILLEGAL && lit != string(ch) {
- t.Errorf("bad token: got %s, expected %s", lit, string(ch))
- }
+ t.Errorf("bad token: got %s, expected %s", tok, token.GO)
}
if s.ErrorCount != 0 {
@@ -560,36 +566,37 @@ func TestStdErrorHander(t *testing.T) {
"//line File1:1\n" +
"@ @ @" // original file, line 1 again
- v := new(ErrorVector)
+ var list ErrorList
+ eh := func(pos token.Position, msg string) { list.Add(pos, msg) }
+
var s Scanner
- s.Init(fset.AddFile("File1", fset.Base(), len(src)), []byte(src), v, 0)
+ s.Init(fset.AddFile("File1", fset.Base(), len(src)), []byte(src), eh, dontInsertSemis)
for {
if _, tok, _ := s.Scan(); tok == token.EOF {
break
}
}
- list := v.GetErrorList(Raw)
+ if len(list) != s.ErrorCount {
+ t.Errorf("found %d errors, expected %d", len(list), s.ErrorCount)
+ }
+
if len(list) != 9 {
t.Errorf("found %d raw errors, expected 9", len(list))
PrintError(os.Stderr, list)
}
- list = v.GetErrorList(Sorted)
+ list.Sort()
if len(list) != 9 {
t.Errorf("found %d sorted errors, expected 9", len(list))
PrintError(os.Stderr, list)
}
- list = v.GetErrorList(NoMultiples)
+ list.RemoveMultiples()
if len(list) != 4 {
t.Errorf("found %d one-per-line errors, expected 4", len(list))
PrintError(os.Stderr, list)
}
-
- if v.ErrorCount() != s.ErrorCount {
- t.Errorf("found %d errors, expected %d", v.ErrorCount(), s.ErrorCount)
- }
}
type errorCollector struct {
@@ -598,16 +605,15 @@ type errorCollector struct {
pos token.Position // last error position encountered
}
-func (h *errorCollector) Error(pos token.Position, msg string) {
- h.cnt++
- h.msg = msg
- h.pos = pos
-}
-
func checkError(t *testing.T, src string, tok token.Token, pos int, err string) {
var s Scanner
var h errorCollector
- s.Init(fset.AddFile("", fset.Base(), len(src)), []byte(src), &h, ScanComments)
+ eh := func(pos token.Position, msg string) {
+ h.cnt++
+ h.msg = msg
+ h.pos = pos
+ }
+ s.Init(fset.AddFile("", fset.Base(), len(src)), []byte(src), eh, ScanComments|dontInsertSemis)
_, tok0, _ := s.Scan()
_, tok1, _ := s.Scan()
if tok0 != tok {
@@ -670,3 +676,20 @@ func TestScanErrors(t *testing.T) {
checkError(t, e.src, e.tok, e.pos, e.err)
}
}
+
+func BenchmarkScan(b *testing.B) {
+ b.StopTimer()
+ fset := token.NewFileSet()
+ file := fset.AddFile("", fset.Base(), len(source))
+ var s Scanner
+ b.StartTimer()
+ for i := b.N - 1; i >= 0; i-- {
+ s.Init(file, source, nil, ScanComments)
+ for {
+ _, tok, _ := s.Scan()
+ if tok == token.EOF {
+ break
+ }
+ }
+ }
+}
generated by cgit v1.2.3 (git 2.46.0) at 2025-10-12 07:38:43 +0000