summaryrefslogtreecommitdiff
path: root/src/pkg/scanner
diff options
context:
space:
mode:
Diffstat (limited to 'src/pkg/scanner')
-rw-r--r--src/pkg/scanner/scanner.go30
-rw-r--r--src/pkg/scanner/scanner_test.go23
2 files changed, 0 insertions, 53 deletions
diff --git a/src/pkg/scanner/scanner.go b/src/pkg/scanner/scanner.go
index e79d392f7..d0c32e70a 100644
--- a/src/pkg/scanner/scanner.go
+++ b/src/pkg/scanner/scanner.go
@@ -34,7 +34,6 @@ import (
"utf8"
)
-
// TODO(gri): Consider changing this to use the new (token) Position package.
// A source position is represented by a Position value.
@@ -46,11 +45,9 @@ type Position struct {
Column int // column number, starting at 1 (character count per line)
}
-
// IsValid returns true if the position is valid.
func (pos *Position) IsValid() bool { return pos.Line > 0 }
-
func (pos Position) String() string {
s := pos.Filename
if pos.IsValid() {
@@ -65,7 +62,6 @@ func (pos Position) String() string {
return s
}
-
// Predefined mode bits to control recognition of tokens. For instance,
// to configure a Scanner such that it only recognizes (Go) identifiers,
// integers, and skips comments, set the Scanner's Mode field to:
@@ -84,7 +80,6 @@ const (
GoTokens = ScanIdents | ScanFloats | ScanChars | ScanStrings | ScanRawStrings | ScanComments | SkipComments
)
-
// The result of Scan is one of the following tokens or a Unicode character.
const (
EOF = -(iota + 1)
@@ -98,7 +93,6 @@ const (
skipComment
)
-
var tokenString = map[int]string{
EOF: "EOF",
Ident: "Ident",
@@ -110,7 +104,6 @@ var tokenString = map[int]string{
Comment: "Comment",
}
-
// TokenString returns a (visible) string for a token or Unicode character.
func TokenString(tok int) string {
if s, found := tokenString[tok]; found {
@@ -119,12 +112,10 @@ func TokenString(tok int) string {
return fmt.Sprintf("%q", string(tok))
}
-
// GoWhitespace is the default value for the Scanner's Whitespace field.
// Its value selects Go's white space characters.
const GoWhitespace = 1<<'\t' | 1<<'\n' | 1<<'\r' | 1<<' '
-
const bufLen = 1024 // at least utf8.UTFMax
// A Scanner implements reading of Unicode characters and tokens from an io.Reader.
@@ -179,7 +170,6 @@ type Scanner struct {
Position
}
-
// Init initializes a Scanner with a new source and returns s.
// Error is set to nil, ErrorCount is set to 0, Mode is set to GoTokens,
// and Whitespace is set to GoWhitespace.
@@ -215,7 +205,6 @@ func (s *Scanner) Init(src io.Reader) *Scanner {
return s
}
-
// TODO(gri): The code for next() and the internal scanner state could benefit
// from a rethink. While next() is optimized for the common ASCII
// case, the "corrections" needed for proper position tracking undo
@@ -300,7 +289,6 @@ func (s *Scanner) next() int {
return ch
}
-
// Next reads and returns the next Unicode character.
// It returns EOF at the end of the source. It reports
// a read error by calling s.Error, if not nil; otherwise
@@ -314,7 +302,6 @@ func (s *Scanner) Next() int {
return ch
}
-
// Peek returns the next Unicode character in the source without advancing
// the scanner. It returns EOF if the scanner's position is at the last
// character of the source.
@@ -325,7 +312,6 @@ func (s *Scanner) Peek() int {
return s.ch
}
-
func (s *Scanner) error(msg string) {
s.ErrorCount++
if s.Error != nil {
@@ -335,7 +321,6 @@ func (s *Scanner) error(msg string) {
fmt.Fprintf(os.Stderr, "%s: %s\n", s.Position, msg)
}
-
func (s *Scanner) scanIdentifier() int {
ch := s.next() // read character after first '_' or letter
for ch == '_' || unicode.IsLetter(ch) || unicode.IsDigit(ch) {
@@ -344,7 +329,6 @@ func (s *Scanner) scanIdentifier() int {
return ch
}
-
func digitVal(ch int) int {
switch {
case '0' <= ch && ch <= '9':
@@ -357,10 +341,8 @@ func digitVal(ch int) int {
return 16 // larger than any legal digit val
}
-
func isDecimal(ch int) bool { return '0' <= ch && ch <= '9' }
-
func (s *Scanner) scanMantissa(ch int) int {
for isDecimal(ch) {
ch = s.next()
@@ -368,7 +350,6 @@ func (s *Scanner) scanMantissa(ch int) int {
return ch
}
-
func (s *Scanner) scanFraction(ch int) int {
if ch == '.' {
ch = s.scanMantissa(s.next())
@@ -376,7 +357,6 @@ func (s *Scanner) scanFraction(ch int) int {
return ch
}
-
func (s *Scanner) scanExponent(ch int) int {
if ch == 'e' || ch == 'E' {
ch = s.next()
@@ -388,7 +368,6 @@ func (s *Scanner) scanExponent(ch int) int {
return ch
}
-
func (s *Scanner) scanNumber(ch int) (int, int) {
// isDecimal(ch)
if ch == '0' {
@@ -433,7 +412,6 @@ func (s *Scanner) scanNumber(ch int) (int, int) {
return Int, ch
}
-
func (s *Scanner) scanDigits(ch, base, n int) int {
for n > 0 && digitVal(ch) < base {
ch = s.next()
@@ -445,7 +423,6 @@ func (s *Scanner) scanDigits(ch, base, n int) int {
return ch
}
-
func (s *Scanner) scanEscape(quote int) int {
ch := s.next() // read character after '/'
switch ch {
@@ -466,7 +443,6 @@ func (s *Scanner) scanEscape(quote int) int {
return ch
}
-
func (s *Scanner) scanString(quote int) (n int) {
ch := s.next() // read character after quote
for ch != quote {
@@ -484,7 +460,6 @@ func (s *Scanner) scanString(quote int) (n int) {
return
}
-
func (s *Scanner) scanRawString() {
ch := s.next() // read character after '`'
for ch != '`' {
@@ -496,14 +471,12 @@ func (s *Scanner) scanRawString() {
}
}
-
func (s *Scanner) scanChar() {
if s.scanString('\'') != 1 {
s.error("illegal char literal")
}
}
-
func (s *Scanner) scanComment(ch int) int {
// ch == '/' || ch == '*'
if ch == '/' {
@@ -532,7 +505,6 @@ func (s *Scanner) scanComment(ch int) int {
return ch
}
-
// Scan reads the next token or Unicode character from source and returns it.
// It only recognizes tokens t for which the respective Mode bit (1<<-t) is set.
// It returns EOF at the end of the source. It reports scanner errors (read and
@@ -635,7 +607,6 @@ redo:
return tok
}
-
// Pos returns the position of the character immediately after
// the character or token returned by the last call to Next or Scan.
func (s *Scanner) Pos() (pos Position) {
@@ -658,7 +629,6 @@ func (s *Scanner) Pos() (pos Position) {
return
}
-
// TokenText returns the string corresponding to the most recently scanned token.
// Valid after calling Scan().
func (s *Scanner) TokenText() string {
diff --git a/src/pkg/scanner/scanner_test.go b/src/pkg/scanner/scanner_test.go
index cf9ad0111..4ba1587e8 100644
--- a/src/pkg/scanner/scanner_test.go
+++ b/src/pkg/scanner/scanner_test.go
@@ -13,14 +13,12 @@ import (
"utf8"
)
-
// A StringReader delivers its data one string segment at a time via Read.
type StringReader struct {
data []string
step int
}
-
func (r *StringReader) Read(p []byte) (n int, err os.Error) {
if r.step < len(r.data) {
s := r.data[r.step]
@@ -32,7 +30,6 @@ func (r *StringReader) Read(p []byte) (n int, err os.Error) {
return
}
-
func readRuneSegments(t *testing.T, segments []string) {
got := ""
want := strings.Join(segments, "")
@@ -49,7 +46,6 @@ func readRuneSegments(t *testing.T, segments []string) {
}
}
-
var segmentList = [][]string{
{},
{""},
@@ -61,14 +57,12 @@ var segmentList = [][]string{
{"Hello", ", ", "", "World", "!"},
}
-
func TestNext(t *testing.T) {
for _, s := range segmentList {
readRuneSegments(t, s)
}
}
-
type token struct {
tok int
text string
@@ -234,7 +228,6 @@ var tokenList = []token{
{'(', "("},
}
-
func makeSource(pattern string) *bytes.Buffer {
var buf bytes.Buffer
for _, k := range tokenList {
@@ -243,7 +236,6 @@ func makeSource(pattern string) *bytes.Buffer {
return &buf
}
-
func checkTok(t *testing.T, s *Scanner, line, got, want int, text string) {
if got != want {
t.Fatalf("tok = %s, want %s for %q", TokenString(got), TokenString(want), text)
@@ -263,7 +255,6 @@ func checkTok(t *testing.T, s *Scanner, line, got, want int, text string) {
}
}
-
func countNewlines(s string) int {
n := 0
for _, ch := range s {
@@ -274,7 +265,6 @@ func countNewlines(s string) int {
return n
}
-
func testScan(t *testing.T, mode uint) {
s := new(Scanner).Init(makeSource(" \t%s\n"))
s.Mode = mode
@@ -290,13 +280,11 @@ func testScan(t *testing.T, mode uint) {
checkTok(t, s, line, tok, EOF, "")
}
-
func TestScan(t *testing.T) {
testScan(t, GoTokens)
testScan(t, GoTokens&^SkipComments)
}
-
func TestPosition(t *testing.T) {
src := makeSource("\t\t\t\t%s\n")
s := new(Scanner).Init(src)
@@ -323,7 +311,6 @@ func TestPosition(t *testing.T) {
}
}
-
func TestScanZeroMode(t *testing.T) {
src := makeSource("%s\n")
str := src.String()
@@ -345,7 +332,6 @@ func TestScanZeroMode(t *testing.T) {
}
}
-
func testScanSelectedMode(t *testing.T, mode uint, class int) {
src := makeSource("%s\n")
s := new(Scanner).Init(src)
@@ -362,7 +348,6 @@ func testScanSelectedMode(t *testing.T, mode uint, class int) {
}
}
-
func TestScanSelectedMask(t *testing.T) {
testScanSelectedMode(t, 0, 0)
testScanSelectedMode(t, ScanIdents, Ident)
@@ -375,7 +360,6 @@ func TestScanSelectedMask(t *testing.T) {
testScanSelectedMode(t, ScanComments, Comment)
}
-
func TestScanNext(t *testing.T) {
s := new(Scanner).Init(bytes.NewBufferString("if a == bcd /* comment */ {\n\ta += c\n} // line comment ending in eof"))
checkTok(t, s, 1, s.Scan(), Ident, "if")
@@ -397,7 +381,6 @@ func TestScanNext(t *testing.T) {
}
}
-
func TestScanWhitespace(t *testing.T) {
var buf bytes.Buffer
var ws uint64
@@ -418,7 +401,6 @@ func TestScanWhitespace(t *testing.T) {
}
}
-
func testError(t *testing.T, src, msg string, tok int) {
s := new(Scanner).Init(bytes.NewBufferString(src))
errorCalled := false
@@ -443,7 +425,6 @@ func testError(t *testing.T, src, msg string, tok int) {
}
}
-
func TestError(t *testing.T) {
testError(t, "\x00", "illegal character NUL", 0)
testError(t, "\xff", "illegal UTF-8 encoding", utf8.RuneError)
@@ -459,7 +440,6 @@ func TestError(t *testing.T) {
testError(t, `"abc`+"\xff"+`def"`, "illegal UTF-8 encoding", String)
}
-
func checkPos(t *testing.T, got, want Position) {
if got.Offset != want.Offset || got.Line != want.Line || got.Column != want.Column {
t.Errorf("got offset, line, column = %d, %d, %d; want %d, %d, %d",
@@ -467,7 +447,6 @@ func checkPos(t *testing.T, got, want Position) {
}
}
-
func checkNextPos(t *testing.T, s *Scanner, offset, line, column, char int) {
if ch := s.Next(); ch != char {
t.Errorf("ch = %s, want %s", TokenString(ch), TokenString(char))
@@ -476,7 +455,6 @@ func checkNextPos(t *testing.T, s *Scanner, offset, line, column, char int) {
checkPos(t, s.Pos(), want)
}
-
func checkScanPos(t *testing.T, s *Scanner, offset, line, column, char int) {
want := Position{Offset: offset, Line: line, Column: column}
checkPos(t, s.Pos(), want)
@@ -489,7 +467,6 @@ func checkScanPos(t *testing.T, s *Scanner, offset, line, column, char int) {
checkPos(t, s.Position, want)
}
-
func TestPos(t *testing.T) {
// corner case: empty source
s := new(Scanner).Init(bytes.NewBufferString(""))
generated by cgit v1.2.3 (git 2.46.0) at 2025-10-23 01:02:12 +0000