summaryrefslogtreecommitdiff
path: root/src/pkg/go/scanner
diff options
context:
space:
mode:
Diffstat (limited to 'src/pkg/go/scanner')
-rw-r--r--src/pkg/go/scanner/Makefile12
-rw-r--r--src/pkg/go/scanner/errors.go168
-rw-r--r--src/pkg/go/scanner/scanner.go670
-rw-r--r--src/pkg/go/scanner/scanner_test.go672
4 files changed, 1522 insertions, 0 deletions
diff --git a/src/pkg/go/scanner/Makefile b/src/pkg/go/scanner/Makefile
new file mode 100644
index 000000000..453faac00
--- /dev/null
+++ b/src/pkg/go/scanner/Makefile
@@ -0,0 +1,12 @@
+# Copyright 2009 The Go Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+include ../../../Make.inc
+
+TARG=go/scanner
+GOFILES=\
+ errors.go\
+ scanner.go\
+
+include ../../../Make.pkg
diff --git a/src/pkg/go/scanner/errors.go b/src/pkg/go/scanner/errors.go
new file mode 100644
index 000000000..a0927e416
--- /dev/null
+++ b/src/pkg/go/scanner/errors.go
@@ -0,0 +1,168 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package scanner
+
+import (
+ "fmt"
+ "go/token"
+ "io"
+ "os"
+ "sort"
+)
+
+// An implementation of an ErrorHandler may be provided to the Scanner.
+// If a syntax error is encountered and a handler was installed, Error
+// is called with a position and an error message. The position points
+// to the beginning of the offending token.
+//
+type ErrorHandler interface {
+ Error(pos token.Position, msg string)
+}
+
+// ErrorVector implements the ErrorHandler interface. It maintains a list
+// of errors which can be retrieved with GetErrorList and GetError. The
+// zero value for an ErrorVector is an empty ErrorVector ready to use.
+//
+// A common usage pattern is to embed an ErrorVector alongside a
+// scanner in a data structure that uses the scanner. By passing a
+// reference to an ErrorVector to the scanner's Init call, default
+// error handling is obtained.
+//
+type ErrorVector struct {
+ errors []*Error
+}
+
+// Reset resets an ErrorVector to no errors.
+func (h *ErrorVector) Reset() { h.errors = h.errors[:0] }
+
+// ErrorCount returns the number of errors collected.
+func (h *ErrorVector) ErrorCount() int { return len(h.errors) }
+
+// Within ErrorVector, an error is represented by an Error node. The
+// position Pos, if valid, points to the beginning of the offending
+// token, and the error condition is described by Msg.
+//
+type Error struct {
+ Pos token.Position
+ Msg string
+}
+
+func (e *Error) String() string {
+ if e.Pos.Filename != "" || e.Pos.IsValid() {
+ // don't print "<unknown position>"
+ // TODO(gri) reconsider the semantics of Position.IsValid
+ return e.Pos.String() + ": " + e.Msg
+ }
+ return e.Msg
+}
+
+// An ErrorList is a (possibly sorted) list of Errors.
+type ErrorList []*Error
+
+// ErrorList implements the sort Interface.
+func (p ErrorList) Len() int { return len(p) }
+func (p ErrorList) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
+
+func (p ErrorList) Less(i, j int) bool {
+ e := &p[i].Pos
+ f := &p[j].Pos
+ // Note that it is not sufficient to simply compare file offsets because
+ // the offsets do not reflect modified line information (through //line
+ // comments).
+ if e.Filename < f.Filename {
+ return true
+ }
+ if e.Filename == f.Filename {
+ if e.Line < f.Line {
+ return true
+ }
+ if e.Line == f.Line {
+ return e.Column < f.Column
+ }
+ }
+ return false
+}
+
+func (p ErrorList) String() string {
+ switch len(p) {
+ case 0:
+ return "unspecified error"
+ case 1:
+ return p[0].String()
+ }
+ return fmt.Sprintf("%s (and %d more errors)", p[0].String(), len(p)-1)
+}
+
+// These constants control the construction of the ErrorList
+// returned by GetErrors.
+//
+const (
+ Raw = iota // leave error list unchanged
+ Sorted // sort error list by file, line, and column number
+ NoMultiples // sort error list and leave only the first error per line
+)
+
+// GetErrorList returns the list of errors collected by an ErrorVector.
+// The construction of the ErrorList returned is controlled by the mode
+// parameter. If there are no errors, the result is nil.
+//
+func (h *ErrorVector) GetErrorList(mode int) ErrorList {
+ if len(h.errors) == 0 {
+ return nil
+ }
+
+ list := make(ErrorList, len(h.errors))
+ copy(list, h.errors)
+
+ if mode >= Sorted {
+ sort.Sort(list)
+ }
+
+ if mode >= NoMultiples {
+ var last token.Position // initial last.Line is != any legal error line
+ i := 0
+ for _, e := range list {
+ if e.Pos.Filename != last.Filename || e.Pos.Line != last.Line {
+ last = e.Pos
+ list[i] = e
+ i++
+ }
+ }
+ list = list[0:i]
+ }
+
+ return list
+}
+
+// GetError is like GetErrorList, but it returns an os.Error instead
+// so that a nil result can be assigned to an os.Error variable and
+// remains nil.
+//
+func (h *ErrorVector) GetError(mode int) os.Error {
+ if len(h.errors) == 0 {
+ return nil
+ }
+
+ return h.GetErrorList(mode)
+}
+
+// ErrorVector implements the ErrorHandler interface.
+func (h *ErrorVector) Error(pos token.Position, msg string) {
+ h.errors = append(h.errors, &Error{pos, msg})
+}
+
+// PrintError is a utility function that prints a list of errors to w,
+// one error per line, if the err parameter is an ErrorList. Otherwise
+// it prints the err string.
+//
+func PrintError(w io.Writer, err os.Error) {
+ if list, ok := err.(ErrorList); ok {
+ for _, e := range list {
+ fmt.Fprintf(w, "%s\n", e)
+ }
+ } else {
+ fmt.Fprintf(w, "%s\n", err)
+ }
+}
diff --git a/src/pkg/go/scanner/scanner.go b/src/pkg/go/scanner/scanner.go
new file mode 100644
index 000000000..7f3dd2373
--- /dev/null
+++ b/src/pkg/go/scanner/scanner.go
@@ -0,0 +1,670 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package scanner implements a scanner for Go source text. Takes a []byte as
+// source which can then be tokenized through repeated calls to the Scan
+// function. Typical use:
+//
+// var s Scanner
+// fset := token.NewFileSet() // position information is relative to fset
+// file := fset.AddFile(filename, fset.Base(), len(src)) // register file
+// s.Init(file, src, nil /* no error handler */, 0)
+// for {
+// pos, tok, lit := s.Scan()
+// if tok == token.EOF {
+// break
+// }
+// // do something here with pos, tok, and lit
+// }
+//
+package scanner
+
+import (
+ "bytes"
+ "fmt"
+ "go/token"
+ "path/filepath"
+ "strconv"
+ "unicode"
+ "utf8"
+)
+
+// A Scanner holds the scanner's internal state while processing
+// a given text. It can be allocated as part of another data
+// structure but must be initialized via Init before use.
+//
+type Scanner struct {
+ // immutable state
+ file *token.File // source file handle
+ dir string // directory portion of file.Name()
+ src []byte // source
+ err ErrorHandler // error reporting; or nil
+ mode uint // scanning mode
+
+ // scanning state
+ ch int // current character
+ offset int // character offset
+ rdOffset int // reading offset (position after current character)
+ lineOffset int // current line offset
+ insertSemi bool // insert a semicolon before next newline
+
+ // public state - ok to modify
+ ErrorCount int // number of errors encountered
+}
+
+// Read the next Unicode char into S.ch.
+// S.ch < 0 means end-of-file.
+//
+func (S *Scanner) next() {
+ if S.rdOffset < len(S.src) {
+ S.offset = S.rdOffset
+ if S.ch == '\n' {
+ S.lineOffset = S.offset
+ S.file.AddLine(S.offset)
+ }
+ r, w := int(S.src[S.rdOffset]), 1
+ switch {
+ case r == 0:
+ S.error(S.offset, "illegal character NUL")
+ case r >= 0x80:
+ // not ASCII
+ r, w = utf8.DecodeRune(S.src[S.rdOffset:])
+ if r == utf8.RuneError && w == 1 {
+ S.error(S.offset, "illegal UTF-8 encoding")
+ }
+ }
+ S.rdOffset += w
+ S.ch = r
+ } else {
+ S.offset = len(S.src)
+ if S.ch == '\n' {
+ S.lineOffset = S.offset
+ S.file.AddLine(S.offset)
+ }
+ S.ch = -1 // eof
+ }
+}
+
+// The mode parameter to the Init function is a set of flags (or 0).
+// They control scanner behavior.
+//
+const (
+ ScanComments = 1 << iota // return comments as COMMENT tokens
+ AllowIllegalChars // do not report an error for illegal chars
+ InsertSemis // automatically insert semicolons
+)
+
+// Init prepares the scanner S to tokenize the text src by setting the
+// scanner at the beginning of src. The scanner uses the file set file
+// for position information and it adds line information for each line.
+// It is ok to re-use the same file when re-scanning the same file as
+// line information which is already present is ignored. Init causes a
+// panic if the file size does not match the src size.
+//
+// Calls to Scan will use the error handler err if they encounter a
+// syntax error and err is not nil. Also, for each error encountered,
+// the Scanner field ErrorCount is incremented by one. The mode parameter
+// determines how comments, illegal characters, and semicolons are handled.
+//
+// Note that Init may call err if there is an error in the first character
+// of the file.
+//
+func (S *Scanner) Init(file *token.File, src []byte, err ErrorHandler, mode uint) {
+ // Explicitly initialize all fields since a scanner may be reused.
+ if file.Size() != len(src) {
+ panic("file size does not match src len")
+ }
+ S.file = file
+ S.dir, _ = filepath.Split(file.Name())
+ S.src = src
+ S.err = err
+ S.mode = mode
+
+ S.ch = ' '
+ S.offset = 0
+ S.rdOffset = 0
+ S.lineOffset = 0
+ S.insertSemi = false
+ S.ErrorCount = 0
+
+ S.next()
+}
+
+func (S *Scanner) error(offs int, msg string) {
+ if S.err != nil {
+ S.err.Error(S.file.Position(S.file.Pos(offs)), msg)
+ }
+ S.ErrorCount++
+}
+
+var prefix = []byte("//line ")
+
+func (S *Scanner) interpretLineComment(text []byte) {
+ if bytes.HasPrefix(text, prefix) {
+ // get filename and line number, if any
+ if i := bytes.LastIndex(text, []byte{':'}); i > 0 {
+ if line, err := strconv.Atoi(string(text[i+1:])); err == nil && line > 0 {
+ // valid //line filename:line comment;
+ filename := filepath.Clean(string(text[len(prefix):i]))
+ if !filepath.IsAbs(filename) {
+ // make filename relative to current directory
+ filename = filepath.Join(S.dir, filename)
+ }
+ // update scanner position
+ S.file.AddLineInfo(S.lineOffset, filename, line-1) // -1 since comment applies to next line
+ }
+ }
+ }
+}
+
+func (S *Scanner) scanComment() {
+ // initial '/' already consumed; S.ch == '/' || S.ch == '*'
+ offs := S.offset - 1 // position of initial '/'
+
+ if S.ch == '/' {
+ //-style comment
+ S.next()
+ for S.ch != '\n' && S.ch >= 0 {
+ S.next()
+ }
+ if offs == S.lineOffset {
+ // comment starts at the beginning of the current line
+ S.interpretLineComment(S.src[offs:S.offset])
+ }
+ return
+ }
+
+ /*-style comment */
+ S.next()
+ for S.ch >= 0 {
+ ch := S.ch
+ S.next()
+ if ch == '*' && S.ch == '/' {
+ S.next()
+ return
+ }
+ }
+
+ S.error(offs, "comment not terminated")
+}
+
+func (S *Scanner) findLineEnd() bool {
+ // initial '/' already consumed
+
+ defer func(offs int) {
+ // reset scanner state to where it was upon calling findLineEnd
+ S.ch = '/'
+ S.offset = offs
+ S.rdOffset = offs + 1
+ S.next() // consume initial '/' again
+ }(S.offset - 1)
+
+ // read ahead until a newline, EOF, or non-comment token is found
+ for S.ch == '/' || S.ch == '*' {
+ if S.ch == '/' {
+ //-style comment always contains a newline
+ return true
+ }
+ /*-style comment: look for newline */
+ S.next()
+ for S.ch >= 0 {
+ ch := S.ch
+ if ch == '\n' {
+ return true
+ }
+ S.next()
+ if ch == '*' && S.ch == '/' {
+ S.next()
+ break
+ }
+ }
+ S.skipWhitespace() // S.insertSemi is set
+ if S.ch < 0 || S.ch == '\n' {
+ return true
+ }
+ if S.ch != '/' {
+ // non-comment token
+ return false
+ }
+ S.next() // consume '/'
+ }
+
+ return false
+}
+
+func isLetter(ch int) bool {
+ return 'a' <= ch && ch <= 'z' || 'A' <= ch && ch <= 'Z' || ch == '_' || ch >= 0x80 && unicode.IsLetter(ch)
+}
+
+func isDigit(ch int) bool {
+ return '0' <= ch && ch <= '9' || ch >= 0x80 && unicode.IsDigit(ch)
+}
+
+func (S *Scanner) scanIdentifier() token.Token {
+ offs := S.offset
+ for isLetter(S.ch) || isDigit(S.ch) {
+ S.next()
+ }
+ return token.Lookup(S.src[offs:S.offset])
+}
+
+func digitVal(ch int) int {
+ switch {
+ case '0' <= ch && ch <= '9':
+ return ch - '0'
+ case 'a' <= ch && ch <= 'f':
+ return ch - 'a' + 10
+ case 'A' <= ch && ch <= 'F':
+ return ch - 'A' + 10
+ }
+ return 16 // larger than any legal digit val
+}
+
+func (S *Scanner) scanMantissa(base int) {
+ for digitVal(S.ch) < base {
+ S.next()
+ }
+}
+
+func (S *Scanner) scanNumber(seenDecimalPoint bool) token.Token {
+ // digitVal(S.ch) < 10
+ tok := token.INT
+
+ if seenDecimalPoint {
+ tok = token.FLOAT
+ S.scanMantissa(10)
+ goto exponent
+ }
+
+ if S.ch == '0' {
+ // int or float
+ offs := S.offset
+ S.next()
+ if S.ch == 'x' || S.ch == 'X' {
+ // hexadecimal int
+ S.next()
+ S.scanMantissa(16)
+ if S.offset-offs <= 2 {
+ // only scanned "0x" or "0X"
+ S.error(offs, "illegal hexadecimal number")
+ }
+ } else {
+ // octal int or float
+ seenDecimalDigit := false
+ S.scanMantissa(8)
+ if S.ch == '8' || S.ch == '9' {
+ // illegal octal int or float
+ seenDecimalDigit = true
+ S.scanMantissa(10)
+ }
+ if S.ch == '.' || S.ch == 'e' || S.ch == 'E' || S.ch == 'i' {
+ goto fraction
+ }
+ // octal int
+ if seenDecimalDigit {
+ S.error(offs, "illegal octal number")
+ }
+ }
+ goto exit
+ }
+
+ // decimal int or float
+ S.scanMantissa(10)
+
+fraction:
+ if S.ch == '.' {
+ tok = token.FLOAT
+ S.next()
+ S.scanMantissa(10)
+ }
+
+exponent:
+ if S.ch == 'e' || S.ch == 'E' {
+ tok = token.FLOAT
+ S.next()
+ if S.ch == '-' || S.ch == '+' {
+ S.next()
+ }
+ S.scanMantissa(10)
+ }
+
+ if S.ch == 'i' {
+ tok = token.IMAG
+ S.next()
+ }
+
+exit:
+ return tok
+}
+
+func (S *Scanner) scanEscape(quote int) {
+ offs := S.offset
+
+ var i, base, max uint32
+ switch S.ch {
+ case 'a', 'b', 'f', 'n', 'r', 't', 'v', '\\', quote:
+ S.next()
+ return
+ case '0', '1', '2', '3', '4', '5', '6', '7':
+ i, base, max = 3, 8, 255
+ case 'x':
+ S.next()
+ i, base, max = 2, 16, 255
+ case 'u':
+ S.next()
+ i, base, max = 4, 16, unicode.MaxRune
+ case 'U':
+ S.next()
+ i, base, max = 8, 16, unicode.MaxRune
+ default:
+ S.next() // always make progress
+ S.error(offs, "unknown escape sequence")
+ return
+ }
+
+ var x uint32
+ for ; i > 0 && S.ch != quote && S.ch >= 0; i-- {
+ d := uint32(digitVal(S.ch))
+ if d >= base {
+ S.error(S.offset, "illegal character in escape sequence")
+ break
+ }
+ x = x*base + d
+ S.next()
+ }
+ // in case of an error, consume remaining chars
+ for ; i > 0 && S.ch != quote && S.ch >= 0; i-- {
+ S.next()
+ }
+ if x > max || 0xd800 <= x && x < 0xe000 {
+ S.error(offs, "escape sequence is invalid Unicode code point")
+ }
+}
+
+func (S *Scanner) scanChar() {
+ // '\'' opening already consumed
+ offs := S.offset - 1
+
+ n := 0
+ for S.ch != '\'' {
+ ch := S.ch
+ n++
+ S.next()
+ if ch == '\n' || ch < 0 {
+ S.error(offs, "character literal not terminated")
+ n = 1
+ break
+ }
+ if ch == '\\' {
+ S.scanEscape('\'')
+ }
+ }
+
+ S.next()
+
+ if n != 1 {
+ S.error(offs, "illegal character literal")
+ }
+}
+
+func (S *Scanner) scanString() {
+ // '"' opening already consumed
+ offs := S.offset - 1
+
+ for S.ch != '"' {
+ ch := S.ch
+ S.next()
+ if ch == '\n' || ch < 0 {
+ S.error(offs, "string not terminated")
+ break
+ }
+ if ch == '\\' {
+ S.scanEscape('"')
+ }
+ }
+
+ S.next()
+}
+
+func (S *Scanner) scanRawString() {
+ // '`' opening already consumed
+ offs := S.offset - 1
+
+ for S.ch != '`' {
+ ch := S.ch
+ S.next()
+ if ch < 0 {
+ S.error(offs, "string not terminated")
+ break
+ }
+ }
+
+ S.next()
+}
+
+func (S *Scanner) skipWhitespace() {
+ for S.ch == ' ' || S.ch == '\t' || S.ch == '\n' && !S.insertSemi || S.ch == '\r' {
+ S.next()
+ }
+}
+
+// Helper functions for scanning multi-byte tokens such as >> += >>= .
+// Different routines recognize different length tok_i based on matches
+// of ch_i. If a token ends in '=', the result is tok1 or tok3
+// respectively. Otherwise, the result is tok0 if there was no other
+// matching character, or tok2 if the matching character was ch2.
+
+func (S *Scanner) switch2(tok0, tok1 token.Token) token.Token {
+ if S.ch == '=' {
+ S.next()
+ return tok1
+ }
+ return tok0
+}
+
+func (S *Scanner) switch3(tok0, tok1 token.Token, ch2 int, tok2 token.Token) token.Token {
+ if S.ch == '=' {
+ S.next()
+ return tok1
+ }
+ if S.ch == ch2 {
+ S.next()
+ return tok2
+ }
+ return tok0
+}
+
+func (S *Scanner) switch4(tok0, tok1 token.Token, ch2 int, tok2, tok3 token.Token) token.Token {
+ if S.ch == '=' {
+ S.next()
+ return tok1
+ }
+ if S.ch == ch2 {
+ S.next()
+ if S.ch == '=' {
+ S.next()
+ return tok3
+ }
+ return tok2
+ }
+ return tok0
+}
+
+// Scan scans the next token and returns the token position,
+// the token, and the literal string corresponding to the
+// token. The source end is indicated by token.EOF.
+//
+// If the returned token is token.SEMICOLON, the corresponding
+// literal string is ";" if the semicolon was present in the source,
+// and "\n" if the semicolon was inserted because of a newline or
+// at EOF.
+//
+// For more tolerant parsing, Scan will return a valid token if
+// possible even if a syntax error was encountered. Thus, even
+// if the resulting token sequence contains no illegal tokens,
+// a client may not assume that no error occurred. Instead it
+// must check the scanner's ErrorCount or the number of calls
+// of the error handler, if there was one installed.
+//
+// Scan adds line information to the file added to the file
+// set with Init. Token positions are relative to that file
+// and thus relative to the file set.
+//
+func (S *Scanner) Scan() (token.Pos, token.Token, string) {
+scanAgain:
+ S.skipWhitespace()
+
+ // current token start
+ insertSemi := false
+ offs := S.offset
+ tok := token.ILLEGAL
+
+ // determine token value
+ switch ch := S.ch; {
+ case isLetter(ch):
+ tok = S.scanIdentifier()
+ switch tok {
+ case token.IDENT, token.BREAK, token.CONTINUE, token.FALLTHROUGH, token.RETURN:
+ insertSemi = true
+ }
+ case digitVal(ch) < 10:
+ insertSemi = true
+ tok = S.scanNumber(false)
+ default:
+ S.next() // always make progress
+ switch ch {
+ case -1:
+ if S.insertSemi {
+ S.insertSemi = false // EOF consumed
+ return S.file.Pos(offs), token.SEMICOLON, "\n"
+ }
+ tok = token.EOF
+ case '\n':
+ // we only reach here if S.insertSemi was
+ // set in the first place and exited early
+ // from S.skipWhitespace()
+ S.insertSemi = false // newline consumed
+ return S.file.Pos(offs), token.SEMICOLON, "\n"
+ case '"':
+ insertSemi = true
+ tok = token.STRING
+ S.scanString()
+ case '\'':
+ insertSemi = true
+ tok = token.CHAR
+ S.scanChar()
+ case '`':
+ insertSemi = true
+ tok = token.STRING
+ S.scanRawString()
+ case ':':
+ tok = S.switch2(token.COLON, token.DEFINE)
+ case '.':
+ if digitVal(S.ch) < 10 {
+ insertSemi = true
+ tok = S.scanNumber(true)
+ } else if S.ch == '.' {
+ S.next()
+ if S.ch == '.' {
+ S.next()
+ tok = token.ELLIPSIS
+ }
+ } else {
+ tok = token.PERIOD
+ }
+ case ',':
+ tok = token.COMMA
+ case ';':
+ tok = token.SEMICOLON
+ case '(':
+ tok = token.LPAREN
+ case ')':
+ insertSemi = true
+ tok = token.RPAREN
+ case '[':
+ tok = token.LBRACK
+ case ']':
+ insertSemi = true
+ tok = token.RBRACK
+ case '{':
+ tok = token.LBRACE
+ case '}':
+ insertSemi = true
+ tok = token.RBRACE
+ case '+':
+ tok = S.switch3(token.ADD, token.ADD_ASSIGN, '+', token.INC)
+ if tok == token.INC {
+ insertSemi = true
+ }
+ case '-':
+ tok = S.switch3(token.SUB, token.SUB_ASSIGN, '-', token.DEC)
+ if tok == token.DEC {
+ insertSemi = true
+ }
+ case '*':
+ tok = S.switch2(token.MUL, token.MUL_ASSIGN)
+ case '/':
+ if S.ch == '/' || S.ch == '*' {
+ // comment
+ if S.insertSemi && S.findLineEnd() {
+ // reset position to the beginning of the comment
+ S.ch = '/'
+ S.offset = offs
+ S.rdOffset = offs + 1
+ S.insertSemi = false // newline consumed
+ return S.file.Pos(offs), token.SEMICOLON, "\n"
+ }
+ S.scanComment()
+ if S.mode&ScanComments == 0 {
+ // skip comment
+ S.insertSemi = false // newline consumed
+ goto scanAgain
+ }
+ tok = token.COMMENT
+ } else {
+ tok = S.switch2(token.QUO, token.QUO_ASSIGN)
+ }
+ case '%':
+ tok = S.switch2(token.REM, token.REM_ASSIGN)
+ case '^':
+ tok = S.switch2(token.XOR, token.XOR_ASSIGN)
+ case '<':
+ if S.ch == '-' {
+ S.next()
+ tok = token.ARROW
+ } else {
+ tok = S.switch4(token.LSS, token.LEQ, '<', token.SHL, token.SHL_ASSIGN)
+ }
+ case '>':
+ tok = S.switch4(token.GTR, token.GEQ, '>', token.SHR, token.SHR_ASSIGN)
+ case '=':
+ tok = S.switch2(token.ASSIGN, token.EQL)
+ case '!':
+ tok = S.switch2(token.NOT, token.NEQ)
+ case '&':
+ if S.ch == '^' {
+ S.next()
+ tok = S.switch2(token.AND_NOT, token.AND_NOT_ASSIGN)
+ } else {
+ tok = S.switch3(token.AND, token.AND_ASSIGN, '&', token.LAND)
+ }
+ case '|':
+ tok = S.switch3(token.OR, token.OR_ASSIGN, '|', token.LOR)
+ default:
+ if S.mode&AllowIllegalChars == 0 {
+ S.error(offs, fmt.Sprintf("illegal character %#U", ch))
+ }
+ insertSemi = S.insertSemi // preserve insertSemi info
+ }
+ }
+
+ if S.mode&InsertSemis != 0 {
+ S.insertSemi = insertSemi
+ }
+
+ // TODO(gri): The scanner API should change such that the literal string
+ // is only valid if an actual literal was scanned. This will
+ // permit a more efficient implementation.
+ return S.file.Pos(offs), tok, string(S.src[offs:S.offset])
+}
diff --git a/src/pkg/go/scanner/scanner_test.go b/src/pkg/go/scanner/scanner_test.go
new file mode 100644
index 000000000..eb9e1cb81
--- /dev/null
+++ b/src/pkg/go/scanner/scanner_test.go
@@ -0,0 +1,672 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package scanner
+
+import (
+ "go/token"
+ "os"
+ "path/filepath"
+ "runtime"
+ "testing"
+)
+
+var fset = token.NewFileSet()
+
+const /* class */ (
+ special = iota
+ literal
+ operator
+ keyword
+)
+
+func tokenclass(tok token.Token) int {
+ switch {
+ case tok.IsLiteral():
+ return literal
+ case tok.IsOperator():
+ return operator
+ case tok.IsKeyword():
+ return keyword
+ }
+ return special
+}
+
+type elt struct {
+ tok token.Token
+ lit string
+ class int
+}
+
+var tokens = [...]elt{
+ // Special tokens
+ {token.COMMENT, "/* a comment */", special},
+ {token.COMMENT, "// a comment \n", special},
+
+ // Identifiers and basic type literals
+ {token.IDENT, "foobar", literal},
+ {token.IDENT, "a۰۱۸", literal},
+ {token.IDENT, "foo६४", literal},
+ {token.IDENT, "bar9876", literal},
+ {token.INT, "0", literal},
+ {token.INT, "1", literal},
+ {token.INT, "123456789012345678890", literal},
+ {token.INT, "01234567", literal},
+ {token.INT, "0xcafebabe", literal},
+ {token.FLOAT, "0.", literal},
+ {token.FLOAT, ".0", literal},
+ {token.FLOAT, "3.14159265", literal},
+ {token.FLOAT, "1e0", literal},
+ {token.FLOAT, "1e+100", literal},
+ {token.FLOAT, "1e-100", literal},
+ {token.FLOAT, "2.71828e-1000", literal},
+ {token.IMAG, "0i", literal},
+ {token.IMAG, "1i", literal},
+ {token.IMAG, "012345678901234567889i", literal},
+ {token.IMAG, "123456789012345678890i", literal},
+ {token.IMAG, "0.i", literal},
+ {token.IMAG, ".0i", literal},
+ {token.IMAG, "3.14159265i", literal},
+ {token.IMAG, "1e0i", literal},
+ {token.IMAG, "1e+100i", literal},
+ {token.IMAG, "1e-100i", literal},
+ {token.IMAG, "2.71828e-1000i", literal},
+ {token.CHAR, "'a'", literal},
+ {token.CHAR, "'\\000'", literal},
+ {token.CHAR, "'\\xFF'", literal},
+ {token.CHAR, "'\\uff16'", literal},
+ {token.CHAR, "'\\U0000ff16'", literal},
+ {token.STRING, "`foobar`", literal},
+ {token.STRING, "`" + `foo
+ bar` +
+ "`",
+ literal,
+ },
+
+ // Operators and delimiters
+ {token.ADD, "+", operator},
+ {token.SUB, "-", operator},
+ {token.MUL, "*", operator},
+ {token.QUO, "/", operator},
+ {token.REM, "%", operator},
+
+ {token.AND, "&", operator},
+ {token.OR, "|", operator},
+ {token.XOR, "^", operator},
+ {token.SHL, "<<", operator},
+ {token.SHR, ">>", operator},
+ {token.AND_NOT, "&^", operator},
+
+ {token.ADD_ASSIGN, "+=", operator},
+ {token.SUB_ASSIGN, "-=", operator},
+ {token.MUL_ASSIGN, "*=", operator},
+ {token.QUO_ASSIGN, "/=", operator},
+ {token.REM_ASSIGN, "%=", operator},
+
+ {token.AND_ASSIGN, "&=", operator},
+ {token.OR_ASSIGN, "|=", operator},
+ {token.XOR_ASSIGN, "^=", operator},
+ {token.SHL_ASSIGN, "<<=", operator},
+ {token.SHR_ASSIGN, ">>=", operator},
+ {token.AND_NOT_ASSIGN, "&^=", operator},
+
+ {token.LAND, "&&", operator},
+ {token.LOR, "||", operator},
+ {token.ARROW, "<-", operator},
+ {token.INC, "++", operator},
+ {token.DEC, "--", operator},
+
+ {token.EQL, "==", operator},
+ {token.LSS, "<", operator},
+ {token.GTR, ">", operator},
+ {token.ASSIGN, "=", operator},
+ {token.NOT, "!", operator},
+
+ {token.NEQ, "!=", operator},
+ {token.LEQ, "<=", operator},
+ {token.GEQ, ">=", operator},
+ {token.DEFINE, ":=", operator},
+ {token.ELLIPSIS, "...", operator},
+
+ {token.LPAREN, "(", operator},
+ {token.LBRACK, "[", operator},
+ {token.LBRACE, "{", operator},
+ {token.COMMA, ",", operator},
+ {token.PERIOD, ".", operator},
+
+ {token.RPAREN, ")", operator},
+ {token.RBRACK, "]", operator},
+ {token.RBRACE, "}", operator},
+ {token.SEMICOLON, ";", operator},
+ {token.COLON, ":", operator},
+
+ // Keywords
+ {token.BREAK, "break", keyword},
+ {token.CASE, "case", keyword},
+ {token.CHAN, "chan", keyword},
+ {token.CONST, "const", keyword},
+ {token.CONTINUE, "continue", keyword},
+
+ {token.DEFAULT, "default", keyword},
+ {token.DEFER, "defer", keyword},
+ {token.ELSE, "else", keyword},
+ {token.FALLTHROUGH, "fallthrough", keyword},
+ {token.FOR, "for", keyword},
+
+ {token.FUNC, "func", keyword},
+ {token.GO, "go", keyword},
+ {token.GOTO, "goto", keyword},
+ {token.IF, "if", keyword},
+ {token.IMPORT, "import", keyword},
+
+ {token.INTERFACE, "interface", keyword},
+ {token.MAP, "map", keyword},
+ {token.PACKAGE, "package", keyword},
+ {token.RANGE, "range", keyword},
+ {token.RETURN, "return", keyword},
+
+ {token.SELECT, "select", keyword},
+ {token.STRUCT, "struct", keyword},
+ {token.SWITCH, "switch", keyword},
+ {token.TYPE, "type", keyword},
+ {token.VAR, "var", keyword},
+}
+
+const whitespace = " \t \n\n\n" // to separate tokens
+
+type testErrorHandler struct {
+ t *testing.T
+}
+
+func (h *testErrorHandler) Error(pos token.Position, msg string) {
+ h.t.Errorf("Error() called (msg = %s)", msg)
+}
+
+func newlineCount(s string) int {
+ n := 0
+ for i := 0; i < len(s); i++ {
+ if s[i] == '\n' {
+ n++
+ }
+ }
+ return n
+}
+
+func checkPos(t *testing.T, lit string, p token.Pos, expected token.Position) {
+ pos := fset.Position(p)
+ if pos.Filename != expected.Filename {
+ t.Errorf("bad filename for %q: got %s, expected %s", lit, pos.Filename, expected.Filename)
+ }
+ if pos.Offset != expected.Offset {
+ t.Errorf("bad position for %q: got %d, expected %d", lit, pos.Offset, expected.Offset)
+ }
+ if pos.Line != expected.Line {
+ t.Errorf("bad line for %q: got %d, expected %d", lit, pos.Line, expected.Line)
+ }
+ if pos.Column != expected.Column {
+ t.Errorf("bad column for %q: got %d, expected %d", lit, pos.Column, expected.Column)
+ }
+}
+
+// Verify that calling Scan() provides the correct results.
+func TestScan(t *testing.T) {
+ // make source
+ var src string
+ for _, e := range tokens {
+ src += e.lit + whitespace
+ }
+ src_linecount := newlineCount(src)
+ whitespace_linecount := newlineCount(whitespace)
+
+ // verify scan
+ var s Scanner
+ s.Init(fset.AddFile("", fset.Base(), len(src)), []byte(src), &testErrorHandler{t}, ScanComments)
+ index := 0
+ epos := token.Position{"", 0, 1, 1} // expected position
+ for {
+ pos, tok, lit := s.Scan()
+ e := elt{token.EOF, "", special}
+ if index < len(tokens) {
+ e = tokens[index]
+ }
+ if tok == token.EOF {
+ lit = "<EOF>"
+ epos.Line = src_linecount
+ epos.Column = 2
+ }
+ checkPos(t, lit, pos, epos)
+ if tok != e.tok {
+ t.Errorf("bad token for %q: got %s, expected %s", lit, tok.String(), e.tok.String())
+ }
+ if e.tok.IsLiteral() && lit != e.lit {
+ t.Errorf("bad literal for %q: got %q, expected %q", lit, lit, e.lit)
+ }
+ if tokenclass(tok) != e.class {
+ t.Errorf("bad class for %q: got %d, expected %d", lit, tokenclass(tok), e.class)
+ }
+ epos.Offset += len(lit) + len(whitespace)
+ epos.Line += newlineCount(lit) + whitespace_linecount
+ if tok == token.COMMENT && lit[1] == '/' {
+ // correct for unaccounted '/n' in //-style comment
+ epos.Offset++
+ epos.Line++
+ }
+ index++
+ if tok == token.EOF {
+ break
+ }
+ }
+ if s.ErrorCount != 0 {
+ t.Errorf("found %d errors", s.ErrorCount)
+ }
+}
+
+func checkSemi(t *testing.T, line string, mode uint) {
+ var S Scanner
+ file := fset.AddFile("TestSemis", fset.Base(), len(line))
+ S.Init(file, []byte(line), nil, mode)
+ pos, tok, lit := S.Scan()
+ for tok != token.EOF {
+ if tok == token.ILLEGAL {
+ // the illegal token literal indicates what
+ // kind of semicolon literal to expect
+ semiLit := "\n"
+ if lit[0] == '#' {
+ semiLit = ";"
+ }
+ // next token must be a semicolon
+ semiPos := file.Position(pos)
+ semiPos.Offset++
+ semiPos.Column++
+ pos, tok, lit = S.Scan()
+ if tok == token.SEMICOLON {
+ if lit != semiLit {
+ t.Errorf(`bad literal for %q: got %q, expected %q`, line, lit, semiLit)
+ }
+ checkPos(t, line, pos, semiPos)
+ } else {
+ t.Errorf("bad token for %q: got %s, expected ;", line, tok.String())
+ }
+ } else if tok == token.SEMICOLON {
+ t.Errorf("bad token for %q: got ;, expected no ;", line)
+ }
+ pos, tok, lit = S.Scan()
+ }
+}
+
+var lines = []string{
+ // # indicates a semicolon present in the source
+ // $ indicates an automatically inserted semicolon
+ "",
+ "#;",
+ "foo$\n",
+ "123$\n",
+ "1.2$\n",
+ "'x'$\n",
+ `"x"` + "$\n",
+ "`x`$\n",
+
+ "+\n",
+ "-\n",
+ "*\n",
+ "/\n",
+ "%\n",
+
+ "&\n",
+ "|\n",
+ "^\n",
+ "<<\n",
+ ">>\n",
+ "&^\n",
+
+ "+=\n",
+ "-=\n",
+ "*=\n",
+ "/=\n",
+ "%=\n",
+
+ "&=\n",
+ "|=\n",
+ "^=\n",
+ "<<=\n",
+ ">>=\n",
+ "&^=\n",
+
+ "&&\n",
+ "||\n",
+ "<-\n",
+ "++$\n",
+ "--$\n",
+
+ "==\n",
+ "<\n",
+ ">\n",
+ "=\n",
+ "!\n",
+
+ "!=\n",
+ "<=\n",
+ ">=\n",
+ ":=\n",
+ "...\n",
+
+ "(\n",
+ "[\n",
+ "{\n",
+ ",\n",
+ ".\n",
+
+ ")$\n",
+ "]$\n",
+ "}$\n",
+ "#;\n",
+ ":\n",
+
+ "break$\n",
+ "case\n",
+ "chan\n",
+ "const\n",
+ "continue$\n",
+
+ "default\n",
+ "defer\n",
+ "else\n",
+ "fallthrough$\n",
+ "for\n",
+
+ "func\n",
+ "go\n",
+ "goto\n",
+ "if\n",
+ "import\n",
+
+ "interface\n",
+ "map\n",
+ "package\n",
+ "range\n",
+ "return$\n",
+
+ "select\n",
+ "struct\n",
+ "switch\n",
+ "type\n",
+ "var\n",
+
+ "foo$//comment\n",
+ "foo$//comment",
+ "foo$/*comment*/\n",
+ "foo$/*\n*/",
+ "foo$/*comment*/ \n",
+ "foo$/*\n*/ ",
+
+ "foo $// comment\n",
+ "foo $// comment",
+ "foo $/*comment*/\n",
+ "foo $/*\n*/",
+ "foo $/* */ /* \n */ bar$/**/\n",
+ "foo $/*0*/ /*1*/ /*2*/\n",
+
+ "foo $/*comment*/ \n",
+ "foo $/*0*/ /*1*/ /*2*/ \n",
+ "foo $/**/ /*-------------*/ /*----\n*/bar $/* \n*/baa$\n",
+ "foo $/* an EOF terminates a line */",
+ "foo $/* an EOF terminates a line */ /*",
+ "foo $/* an EOF terminates a line */ //",
+
+ "package main$\n\nfunc main() {\n\tif {\n\t\treturn /* */ }$\n}$\n",
+ "package main$",
+}
+
+func TestSemis(t *testing.T) {
+ for _, line := range lines {
+ checkSemi(t, line, AllowIllegalChars|InsertSemis)
+ checkSemi(t, line, AllowIllegalChars|InsertSemis|ScanComments)
+
+ // if the input ended in newlines, the input must tokenize the
+ // same with or without those newlines
+ for i := len(line) - 1; i >= 0 && line[i] == '\n'; i-- {
+ checkSemi(t, line[0:i], AllowIllegalChars|InsertSemis)
+ checkSemi(t, line[0:i], AllowIllegalChars|InsertSemis|ScanComments)
+ }
+ }
+}
+
+type segment struct {
+ srcline string // a line of source text
+ filename string // filename for current token
+ line int // line number for current token
+}
+
+var segments = []segment{
+ // exactly one token per line since the test consumes one token per segment
+ {" line1", filepath.Join("dir", "TestLineComments"), 1},
+ {"\nline2", filepath.Join("dir", "TestLineComments"), 2},
+ {"\nline3 //line File1.go:100", filepath.Join("dir", "TestLineComments"), 3}, // bad line comment, ignored
+ {"\nline4", filepath.Join("dir", "TestLineComments"), 4},
+ {"\n//line File1.go:100\n line100", filepath.Join("dir", "File1.go"), 100},
+ {"\n//line File2.go:200\n line200", filepath.Join("dir", "File2.go"), 200},
+ {"\n//line :1\n line1", "dir", 1},
+ {"\n//line foo:42\n line42", filepath.Join("dir", "foo"), 42},
+ {"\n //line foo:42\n line44", filepath.Join("dir", "foo"), 44}, // bad line comment, ignored
+ {"\n//line foo 42\n line46", filepath.Join("dir", "foo"), 46}, // bad line comment, ignored
+ {"\n//line foo:42 extra text\n line48", filepath.Join("dir", "foo"), 48}, // bad line comment, ignored
+ {"\n//line ./foo:42\n line42", filepath.Join("dir", "foo"), 42},
+ {"\n//line a/b/c/File1.go:100\n line100", filepath.Join("dir", "a", "b", "c", "File1.go"), 100},
+}
+
+var unixsegments = []segment{
+ {"\n//line /bar:42\n line42", "/bar", 42},
+}
+
+var winsegments = []segment{
+ {"\n//line c:\\bar:42\n line42", "c:\\bar", 42},
+ {"\n//line c:\\dir\\File1.go:100\n line100", "c:\\dir\\File1.go", 100},
+}
+
+// Verify that comments of the form "//line filename:line" are interpreted correctly.
+func TestLineComments(t *testing.T) {
+ segs := segments
+ if runtime.GOOS == "windows" {
+ segs = append(segs, winsegments...)
+ } else {
+ segs = append(segs, unixsegments...)
+ }
+
+ // make source
+ var src string
+ for _, e := range segs {
+ src += e.srcline
+ }
+
+ // verify scan
+ var S Scanner
+ file := fset.AddFile(filepath.Join("dir", "TestLineComments"), fset.Base(), len(src))
+ S.Init(file, []byte(src), nil, 0)
+ for _, s := range segs {
+ p, _, lit := S.Scan()
+ pos := file.Position(p)
+ checkPos(t, lit, p, token.Position{s.filename, pos.Offset, s.line, pos.Column})
+ }
+
+ if S.ErrorCount != 0 {
+ t.Errorf("found %d errors", S.ErrorCount)
+ }
+}
+
+// Verify that initializing the same scanner more then once works correctly.
+func TestInit(t *testing.T) {
+ var s Scanner
+
+ // 1st init
+ src1 := "if true { }"
+ f1 := fset.AddFile("src1", fset.Base(), len(src1))
+ s.Init(f1, []byte(src1), nil, 0)
+ if f1.Size() != len(src1) {
+ t.Errorf("bad file size: got %d, expected %d", f1.Size(), len(src1))
+ }
+ s.Scan() // if
+ s.Scan() // true
+ _, tok, _ := s.Scan() // {
+ if tok != token.LBRACE {
+ t.Errorf("bad token: got %s, expected %s", tok.String(), token.LBRACE)
+ }
+
+ // 2nd init
+ src2 := "go true { ]"
+ f2 := fset.AddFile("src2", fset.Base(), len(src2))
+ s.Init(f2, []byte(src2), nil, 0)
+ if f2.Size() != len(src2) {
+ t.Errorf("bad file size: got %d, expected %d", f2.Size(), len(src2))
+ }
+ _, tok, _ = s.Scan() // go
+ if tok != token.GO {
+ t.Errorf("bad token: got %s, expected %s", tok.String(), token.GO)
+ }
+
+ if s.ErrorCount != 0 {
+ t.Errorf("found %d errors", s.ErrorCount)
+ }
+}
+
+func TestIllegalChars(t *testing.T) {
+ var s Scanner
+
+ const src = "*?*$*@*"
+ file := fset.AddFile("", fset.Base(), len(src))
+ s.Init(file, []byte(src), &testErrorHandler{t}, AllowIllegalChars)
+ for offs, ch := range src {
+ pos, tok, lit := s.Scan()
+ if poffs := file.Offset(pos); poffs != offs {
+ t.Errorf("bad position for %s: got %d, expected %d", lit, poffs, offs)
+ }
+ if tok == token.ILLEGAL && lit != string(ch) {
+ t.Errorf("bad token: got %s, expected %s", lit, string(ch))
+ }
+ }
+
+ if s.ErrorCount != 0 {
+ t.Errorf("found %d errors", s.ErrorCount)
+ }
+}
+
+func TestStdErrorHander(t *testing.T) {
+ const src = "@\n" + // illegal character, cause an error
+ "@ @\n" + // two errors on the same line
+ "//line File2:20\n" +
+ "@\n" + // different file, but same line
+ "//line File2:1\n" +
+ "@ @\n" + // same file, decreasing line number
+ "//line File1:1\n" +
+ "@ @ @" // original file, line 1 again
+
+ v := new(ErrorVector)
+ var s Scanner
+ s.Init(fset.AddFile("File1", fset.Base(), len(src)), []byte(src), v, 0)
+ for {
+ if _, tok, _ := s.Scan(); tok == token.EOF {
+ break
+ }
+ }
+
+ list := v.GetErrorList(Raw)
+ if len(list) != 9 {
+ t.Errorf("found %d raw errors, expected 9", len(list))
+ PrintError(os.Stderr, list)
+ }
+
+ list = v.GetErrorList(Sorted)
+ if len(list) != 9 {
+ t.Errorf("found %d sorted errors, expected 9", len(list))
+ PrintError(os.Stderr, list)
+ }
+
+ list = v.GetErrorList(NoMultiples)
+ if len(list) != 4 {
+ t.Errorf("found %d one-per-line errors, expected 4", len(list))
+ PrintError(os.Stderr, list)
+ }
+
+ if v.ErrorCount() != s.ErrorCount {
+ t.Errorf("found %d errors, expected %d", v.ErrorCount(), s.ErrorCount)
+ }
+}
+
+type errorCollector struct {
+ cnt int // number of errors encountered
+ msg string // last error message encountered
+ pos token.Position // last error position encountered
+}
+
+func (h *errorCollector) Error(pos token.Position, msg string) {
+ h.cnt++
+ h.msg = msg
+ h.pos = pos
+}
+
+func checkError(t *testing.T, src string, tok token.Token, pos int, err string) {
+ var s Scanner
+ var h errorCollector
+ s.Init(fset.AddFile("", fset.Base(), len(src)), []byte(src), &h, ScanComments)
+ _, tok0, _ := s.Scan()
+ _, tok1, _ := s.Scan()
+ if tok0 != tok {
+ t.Errorf("%q: got %s, expected %s", src, tok0, tok)
+ }
+ if tok1 != token.EOF {
+ t.Errorf("%q: got %s, expected EOF", src, tok1)
+ }
+ cnt := 0
+ if err != "" {
+ cnt = 1
+ }
+ if h.cnt != cnt {
+ t.Errorf("%q: got cnt %d, expected %d", src, h.cnt, cnt)
+ }
+ if h.msg != err {
+ t.Errorf("%q: got msg %q, expected %q", src, h.msg, err)
+ }
+ if h.pos.Offset != pos {
+ t.Errorf("%q: got offset %d, expected %d", src, h.pos.Offset, pos)
+ }
+}
+
+var errors = []struct {
+ src string
+ tok token.Token
+ pos int
+ err string
+}{
+ {"\a", token.ILLEGAL, 0, "illegal character U+0007"},
+ {`#`, token.ILLEGAL, 0, "illegal character U+0023 '#'"},
+ {`…`, token.ILLEGAL, 0, "illegal character U+2026 '…'"},
+ {`' '`, token.CHAR, 0, ""},
+ {`''`, token.CHAR, 0, "illegal character literal"},
+ {`'\8'`, token.CHAR, 2, "unknown escape sequence"},
+ {`'\08'`, token.CHAR, 3, "illegal character in escape sequence"},
+ {`'\x0g'`, token.CHAR, 4, "illegal character in escape sequence"},
+ {`'\Uffffffff'`, token.CHAR, 2, "escape sequence is invalid Unicode code point"},
+ {`'`, token.CHAR, 0, "character literal not terminated"},
+ {`""`, token.STRING, 0, ""},
+ {`"`, token.STRING, 0, "string not terminated"},
+ {"``", token.STRING, 0, ""},
+ {"`", token.STRING, 0, "string not terminated"},
+ {"/**/", token.COMMENT, 0, ""},
+ {"/*", token.COMMENT, 0, "comment not terminated"},
+ {"077", token.INT, 0, ""},
+ {"078.", token.FLOAT, 0, ""},
+ {"07801234567.", token.FLOAT, 0, ""},
+ {"078e0", token.FLOAT, 0, ""},
+ {"078", token.INT, 0, "illegal octal number"},
+ {"07800000009", token.INT, 0, "illegal octal number"},
+ {"0x", token.INT, 0, "illegal hexadecimal number"},
+ {"0X", token.INT, 0, "illegal hexadecimal number"},
+ {"\"abc\x00def\"", token.STRING, 4, "illegal character NUL"},
+ {"\"abc\x80def\"", token.STRING, 4, "illegal UTF-8 encoding"},
+}
+
+func TestScanErrors(t *testing.T) {
+ for _, e := range errors {
+ checkError(t, e.src, e.tok, e.pos, e.err)
+ }
+}