summaryrefslogtreecommitdiff
path: root/src/pkg/go/scanner/scanner.go
diff options
context:
space:
mode:
Diffstat (limited to 'src/pkg/go/scanner/scanner.go')
-rw-r--r--src/pkg/go/scanner/scanner.go342
1 files changed, 171 insertions, 171 deletions
diff --git a/src/pkg/go/scanner/scanner.go b/src/pkg/go/scanner/scanner.go
index 026ae9976..fad3c0f75 100644
--- a/src/pkg/go/scanner/scanner.go
+++ b/src/pkg/go/scanner/scanner.go
@@ -9,11 +9,11 @@
package scanner
import (
- "bytes";
- "go/token";
- "strconv";
- "unicode";
- "utf8";
+ "bytes"
+ "go/token"
+ "strconv"
+ "unicode"
+ "utf8"
)
@@ -24,18 +24,18 @@ import (
//
type Scanner struct {
// immutable state
- src []byte; // source
- err ErrorHandler; // error reporting; or nil
- mode uint; // scanning mode
+ src []byte // source
+ err ErrorHandler // error reporting; or nil
+ mode uint // scanning mode
// scanning state
- pos token.Position; // previous reading position (position before ch)
- offset int; // current reading offset (position after ch)
- ch int; // one char look-ahead
- insertSemi bool; // insert a semicolon before next newline
+ pos token.Position // previous reading position (position before ch)
+ offset int // current reading offset (position after ch)
+ ch int // one char look-ahead
+ insertSemi bool // insert a semicolon before next newline
// public state - ok to modify
- ErrorCount int; // number of errors encountered
+ ErrorCount int // number of errors encountered
}
@@ -44,22 +44,22 @@ type Scanner struct {
//
func (S *Scanner) next() {
if S.offset < len(S.src) {
- S.pos.Offset = S.offset;
- S.pos.Column++;
- r, w := int(S.src[S.offset]), 1;
+ S.pos.Offset = S.offset
+ S.pos.Column++
+ r, w := int(S.src[S.offset]), 1
switch {
case r == '\n':
- S.pos.Line++;
- S.pos.Column = 0;
+ S.pos.Line++
+ S.pos.Column = 0
case r >= 0x80:
// not ASCII
r, w = utf8.DecodeRune(S.src[S.offset:])
}
- S.offset += w;
- S.ch = r;
+ S.offset += w
+ S.ch = r
} else {
- S.pos.Offset = len(S.src);
- S.ch = -1; // eof
+ S.pos.Offset = len(S.src)
+ S.ch = -1 // eof
}
}
@@ -68,9 +68,9 @@ func (S *Scanner) next() {
// They control scanner behavior.
//
const (
- ScanComments = 1 << iota; // return comments as COMMENT tokens
- AllowIllegalChars; // do not report an error for illegal chars
- InsertSemis; // automatically insert semicolons
+ ScanComments = 1 << iota // return comments as COMMENT tokens
+ AllowIllegalChars // do not report an error for illegal chars
+ InsertSemis // automatically insert semicolons
)
@@ -84,18 +84,18 @@ const (
//
func (S *Scanner) Init(filename string, src []byte, err ErrorHandler, mode uint) {
// Explicitly initialize all fields since a scanner may be reused.
- S.src = src;
- S.err = err;
- S.mode = mode;
- S.pos = token.Position{filename, 0, 1, 0};
- S.offset = 0;
- S.ErrorCount = 0;
- S.next();
+ S.src = src
+ S.err = err
+ S.mode = mode
+ S.pos = token.Position{filename, 0, 1, 0}
+ S.offset = 0
+ S.ErrorCount = 0
+ S.next()
}
func charString(ch int) string {
- var s string;
+ var s string
switch ch {
case -1:
return `EOF`
@@ -120,7 +120,7 @@ func charString(ch int) string {
default:
s = string(ch)
}
- return "'" + s + "' (U+" + strconv.Itob(ch, 16) + ")";
+ return "'" + s + "' (U+" + strconv.Itob(ch, 16) + ")"
}
@@ -128,7 +128,7 @@ func (S *Scanner) error(pos token.Position, msg string) {
if S.err != nil {
S.err.Error(pos, msg)
}
- S.ErrorCount++;
+ S.ErrorCount++
}
@@ -136,11 +136,11 @@ func (S *Scanner) expect(ch int) {
if S.ch != ch {
S.error(S.pos, "expected "+charString(ch)+", found "+charString(S.ch))
}
- S.next(); // always make progress
+ S.next() // always make progress
}
-var prefix = []byte{'l', 'i', 'n', 'e', ' '} // "line "
+var prefix = []byte{'l', 'i', 'n', 'e', ' '} // "line "
func (S *Scanner) scanComment(pos token.Position) {
// first '/' already consumed
@@ -148,44 +148,44 @@ func (S *Scanner) scanComment(pos token.Position) {
if S.ch == '/' {
//-style comment
for S.ch >= 0 {
- S.next();
+ S.next()
if S.ch == '\n' {
// '\n' is not part of the comment for purposes of scanning
// (the comment ends on the same line where it started)
if pos.Column == 1 {
- text := S.src[pos.Offset+2 : S.pos.Offset];
+ text := S.src[pos.Offset+2 : S.pos.Offset]
if bytes.HasPrefix(text, prefix) {
// comment starts at beginning of line with "//line ";
// get filename and line number, if any
- i := bytes.Index(text, []byte{':'});
+ i := bytes.Index(text, []byte{':'})
if i >= 0 {
if line, err := strconv.Atoi(string(text[i+1:])); err == nil && line > 0 {
// valid //line filename:line comment;
// update scanner position
- S.pos.Filename = string(text[len(prefix):i]);
- S.pos.Line = line;
+ S.pos.Filename = string(text[len(prefix):i])
+ S.pos.Line = line
}
}
}
}
- return;
+ return
}
}
} else {
/*-style comment */
- S.expect('*');
+ S.expect('*')
for S.ch >= 0 {
- ch := S.ch;
- S.next();
+ ch := S.ch
+ S.next()
if ch == '*' && S.ch == '/' {
- S.next();
- return;
+ S.next()
+ return
}
}
}
- S.error(pos, "comment not terminated");
+ S.error(pos, "comment not terminated")
}
@@ -193,30 +193,30 @@ func (S *Scanner) findNewline(pos token.Position) bool {
// first '/' already consumed; assume S.ch == '/' || S.ch == '*'
// read ahead until a newline or non-comment token is found
- newline := false;
+ newline := false
for pos1 := pos; S.ch >= 0; {
if S.ch == '/' {
//-style comment always contains a newline
- newline = true;
- break;
+ newline = true
+ break
}
- S.scanComment(pos1);
+ S.scanComment(pos1)
if pos1.Line < S.pos.Line {
/*-style comment contained a newline */
- newline = true;
- break;
+ newline = true
+ break
}
- S.skipWhitespace();
+ S.skipWhitespace()
if S.ch == '\n' {
- newline = true;
- break;
+ newline = true
+ break
}
if S.ch != '/' {
// non-comment token
break
}
- pos1 = S.pos;
- S.next();
+ pos1 = S.pos
+ S.next()
if S.ch != '/' && S.ch != '*' {
// non-comment token
break
@@ -224,11 +224,11 @@ func (S *Scanner) findNewline(pos token.Position) bool {
}
// reset position
- S.pos = pos;
- S.offset = pos.Offset + 1;
- S.ch = '/';
+ S.pos = pos
+ S.offset = pos.Offset + 1
+ S.ch = '/'
- return newline;
+ return newline
}
@@ -243,11 +243,11 @@ func isDigit(ch int) bool {
func (S *Scanner) scanIdentifier() token.Token {
- pos := S.pos.Offset;
+ pos := S.pos.Offset
for isLetter(S.ch) || isDigit(S.ch) {
S.next()
}
- return token.Lookup(S.src[pos:S.pos.Offset]);
+ return token.Lookup(S.src[pos:S.pos.Offset])
}
@@ -260,7 +260,7 @@ func digitVal(ch int) int {
case 'A' <= ch && ch <= 'F':
return ch - 'A' + 10
}
- return 16; // larger than any legal digit val
+ return 16 // larger than any legal digit val
}
@@ -272,65 +272,65 @@ func (S *Scanner) scanMantissa(base int) {
func (S *Scanner) scanNumber(seen_decimal_point bool) token.Token {
- tok := token.INT;
+ tok := token.INT
if seen_decimal_point {
- tok = token.FLOAT;
- S.scanMantissa(10);
- goto exponent;
+ tok = token.FLOAT
+ S.scanMantissa(10)
+ goto exponent
}
if S.ch == '0' {
// int or float
- S.next();
+ S.next()
if S.ch == 'x' || S.ch == 'X' {
// hexadecimal int
- S.next();
- S.scanMantissa(16);
+ S.next()
+ S.scanMantissa(16)
} else {
// octal int or float
- S.scanMantissa(8);
+ S.scanMantissa(8)
if digitVal(S.ch) < 10 || S.ch == '.' || S.ch == 'e' || S.ch == 'E' {
// float
- tok = token.FLOAT;
- goto mantissa;
+ tok = token.FLOAT
+ goto mantissa
}
// octal int
}
- goto exit;
+ goto exit
}
mantissa:
// decimal int or float
- S.scanMantissa(10);
+ S.scanMantissa(10)
if S.ch == '.' {
// float
- tok = token.FLOAT;
- S.next();
- S.scanMantissa(10);
+ tok = token.FLOAT
+ S.next()
+ S.scanMantissa(10)
}
exponent:
if S.ch == 'e' || S.ch == 'E' {
// float
- tok = token.FLOAT;
- S.next();
+ tok = token.FLOAT
+ S.next()
if S.ch == '-' || S.ch == '+' {
S.next()
}
- S.scanMantissa(10);
+ S.scanMantissa(10)
}
exit:
- return tok;
+ return tok
}
func (S *Scanner) scanDigits(base, length int) {
for length > 0 && digitVal(S.ch) < base {
- S.next();
- length--;
+ S.next()
+ length--
}
if length > 0 {
S.error(S.pos, "illegal char escape")
@@ -339,14 +339,14 @@ func (S *Scanner) scanDigits(base, length int) {
func (S *Scanner) scanEscape(quote int) {
- pos := S.pos;
- ch := S.ch;
- S.next();
+ pos := S.pos
+ ch := S.ch
+ S.next()
switch ch {
case 'a', 'b', 'f', 'n', 'r', 't', 'v', '\\', quote:
// nothing to do
case '0', '1', '2', '3', '4', '5', '6', '7':
- S.scanDigits(8, 3-1) // 1 char read already
+ S.scanDigits(8, 3-1) // 1 char read already
case 'x':
S.scanDigits(16, 2)
case 'u':
@@ -362,22 +362,22 @@ func (S *Scanner) scanEscape(quote int) {
func (S *Scanner) scanChar(pos token.Position) {
// '\'' already consumed
- n := 0;
+ n := 0
for S.ch != '\'' {
- ch := S.ch;
- n++;
- S.next();
+ ch := S.ch
+ n++
+ S.next()
if ch == '\n' || ch < 0 {
- S.error(pos, "character literal not terminated");
- n = 1;
- break;
+ S.error(pos, "character literal not terminated")
+ n = 1
+ break
}
if ch == '\\' {
S.scanEscape('\'')
}
}
- S.next();
+ S.next()
if n != 1 {
S.error(pos, "illegal character literal")
@@ -389,18 +389,18 @@ func (S *Scanner) scanString(pos token.Position) {
// '"' already consumed
for S.ch != '"' {
- ch := S.ch;
- S.next();
+ ch := S.ch
+ S.next()
if ch == '\n' || ch < 0 {
- S.error(pos, "string not terminated");
- break;
+ S.error(pos, "string not terminated")
+ break
}
if ch == '\\' {
S.scanEscape('"')
}
}
- S.next();
+ S.next()
}
@@ -408,15 +408,15 @@ func (S *Scanner) scanRawString(pos token.Position) {
// '`' already consumed
for S.ch != '`' {
- ch := S.ch;
- S.next();
+ ch := S.ch
+ S.next()
if ch < 0 {
- S.error(pos, "string not terminated");
- break;
+ S.error(pos, "string not terminated")
+ break
}
}
- S.next();
+ S.next()
}
@@ -435,40 +435,40 @@ func (S *Scanner) skipWhitespace() {
func (S *Scanner) switch2(tok0, tok1 token.Token) token.Token {
if S.ch == '=' {
- S.next();
- return tok1;
+ S.next()
+ return tok1
}
- return tok0;
+ return tok0
}
func (S *Scanner) switch3(tok0, tok1 token.Token, ch2 int, tok2 token.Token) token.Token {
if S.ch == '=' {
- S.next();
- return tok1;
+ S.next()
+ return tok1
}
if S.ch == ch2 {
- S.next();
- return tok2;
+ S.next()
+ return tok2
}
- return tok0;
+ return tok0
}
func (S *Scanner) switch4(tok0, tok1 token.Token, ch2 int, tok2, tok3 token.Token) token.Token {
if S.ch == '=' {
- S.next();
- return tok1;
+ S.next()
+ return tok1
}
if S.ch == ch2 {
- S.next();
+ S.next()
if S.ch == '=' {
- S.next();
- return tok3;
+ S.next()
+ return tok3
}
- return tok2;
+ return tok2
}
- return tok0;
+ return tok0
}
@@ -487,25 +487,25 @@ var semicolon = []byte{';'}
//
func (S *Scanner) Scan() (pos token.Position, tok token.Token, lit []byte) {
scanAgain:
- S.skipWhitespace();
+ S.skipWhitespace()
// current token start
- insertSemi := false;
- pos, tok = S.pos, token.ILLEGAL;
+ insertSemi := false
+ pos, tok = S.pos, token.ILLEGAL
// determine token value
switch ch := S.ch; {
case isLetter(ch):
- tok = S.scanIdentifier();
+ tok = S.scanIdentifier()
switch tok {
case token.IDENT, token.BREAK, token.CONTINUE, token.FALLTHROUGH, token.RETURN:
insertSemi = true
}
case digitVal(ch) < 10:
- insertSemi = true;
- tok = S.scanNumber(false);
+ insertSemi = true
+ tok = S.scanNumber(false)
default:
- S.next(); // always make progress
+ S.next() // always make progress
switch ch {
case -1:
tok = token.EOF
@@ -513,31 +513,31 @@ scanAgain:
// we only reach here of S.insertSemi was
// set in the first place and exited early
// from S.skipWhitespace()
- S.insertSemi = false; // newline consumed
- return pos, token.SEMICOLON, semicolon;
+ S.insertSemi = false // newline consumed
+ return pos, token.SEMICOLON, semicolon
case '"':
- insertSemi = true;
- tok = token.STRING;
- S.scanString(pos);
+ insertSemi = true
+ tok = token.STRING
+ S.scanString(pos)
case '\'':
- insertSemi = true;
- tok = token.CHAR;
- S.scanChar(pos);
+ insertSemi = true
+ tok = token.CHAR
+ S.scanChar(pos)
case '`':
- insertSemi = true;
- tok = token.STRING;
- S.scanRawString(pos);
+ insertSemi = true
+ tok = token.STRING
+ S.scanRawString(pos)
case ':':
tok = S.switch2(token.COLON, token.DEFINE)
case '.':
if digitVal(S.ch) < 10 {
- insertSemi = true;
- tok = S.scanNumber(true);
+ insertSemi = true
+ tok = S.scanNumber(true)
} else if S.ch == '.' {
- S.next();
+ S.next()
if S.ch == '.' {
- S.next();
- tok = token.ELLIPSIS;
+ S.next()
+ tok = token.ELLIPSIS
}
} else {
tok = token.PERIOD
@@ -549,25 +549,25 @@ scanAgain:
case '(':
tok = token.LPAREN
case ')':
- insertSemi = true;
- tok = token.RPAREN;
+ insertSemi = true
+ tok = token.RPAREN
case '[':
tok = token.LBRACK
case ']':
- insertSemi = true;
- tok = token.RBRACK;
+ insertSemi = true
+ tok = token.RBRACK
case '{':
tok = token.LBRACE
case '}':
- insertSemi = true;
- tok = token.RBRACE;
+ insertSemi = true
+ tok = token.RBRACE
case '+':
- tok = S.switch3(token.ADD, token.ADD_ASSIGN, '+', token.INC);
+ tok = S.switch3(token.ADD, token.ADD_ASSIGN, '+', token.INC)
if tok == token.INC {
insertSemi = true
}
case '-':
- tok = S.switch3(token.SUB, token.SUB_ASSIGN, '-', token.DEC);
+ tok = S.switch3(token.SUB, token.SUB_ASSIGN, '-', token.DEC)
if tok == token.DEC {
insertSemi = true
}
@@ -577,16 +577,16 @@ scanAgain:
if S.ch == '/' || S.ch == '*' {
// comment
if S.insertSemi && S.findNewline(pos) {
- S.insertSemi = false; // newline consumed
- return pos, token.SEMICOLON, semicolon;
+ S.insertSemi = false // newline consumed
+ return pos, token.SEMICOLON, semicolon
}
- S.scanComment(pos);
+ S.scanComment(pos)
if S.mode&ScanComments == 0 {
// skip comment
- S.insertSemi = false; // newline consumed
- goto scanAgain;
+ S.insertSemi = false // newline consumed
+ goto scanAgain
}
- tok = token.COMMENT;
+ tok = token.COMMENT
} else {
tok = S.switch2(token.QUO, token.QUO_ASSIGN)
}
@@ -596,8 +596,8 @@ scanAgain:
tok = S.switch2(token.XOR, token.XOR_ASSIGN)
case '<':
if S.ch == '-' {
- S.next();
- tok = token.ARROW;
+ S.next()
+ tok = token.ARROW
} else {
tok = S.switch4(token.LSS, token.LEQ, '<', token.SHL, token.SHL_ASSIGN)
}
@@ -609,8 +609,8 @@ scanAgain:
tok = S.switch2(token.NOT, token.NEQ)
case '&':
if S.ch == '^' {
- S.next();
- tok = S.switch2(token.AND_NOT, token.AND_NOT_ASSIGN);
+ S.next()
+ tok = S.switch2(token.AND_NOT, token.AND_NOT_ASSIGN)
} else {
tok = S.switch3(token.AND, token.AND_ASSIGN, '&', token.LAND)
}
@@ -620,14 +620,14 @@ scanAgain:
if S.mode&AllowIllegalChars == 0 {
S.error(pos, "illegal character "+charString(ch))
}
- insertSemi = S.insertSemi; // preserve insertSemi info
+ insertSemi = S.insertSemi // preserve insertSemi info
}
}
if S.mode&InsertSemis != 0 {
S.insertSemi = insertSemi
}
- return pos, tok, S.src[pos.Offset:S.pos.Offset];
+ return pos, tok, S.src[pos.Offset:S.pos.Offset]
}
@@ -638,10 +638,10 @@ scanAgain:
// of errors encountered.
//
func Tokenize(filename string, src []byte, err ErrorHandler, mode uint, f func(pos token.Position, tok token.Token, lit []byte) bool) int {
- var s Scanner;
- s.Init(filename, src, err, mode);
+ var s Scanner
+ s.Init(filename, src, err, mode)
for f(s.Scan()) {
// action happens in f
}
- return s.ErrorCount;
+ return s.ErrorCount
}
generated by cgit v1.2.3 (git 2.46.0) at 2025-10-16 14:12:48 +0000