mv src/lib to src/pkg

tests: all.bash passes, gobuild still works, godoc still works.

R=rsc
OCL=30096
CL=30102

1 files changed, 347 insertions, 0 deletions

diff --git a/src/pkg/go/token/token.go b/src/pkg/go/token/token.go
new file mode 100644
index 000000000..a70a75a54
--- /dev/null
+++ b/src/pkg/go/token/token.go
@@ -0,0 +1,347 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// This package defines constants representing the lexical
+// tokens of the Go programming language and basic operations
+// on tokens (printing, predicates).
+//
+package token
+
+import "strconv"
+
+// Token is the set of lexical tokens of the Go programming language.
+type Token int
+
+// The list of tokens.
+const (
+	// Special tokens
+	ILLEGAL Token = iota;
+	EOF;
+	COMMENT;
+
+	// Identifiers and basic type literals
+	// (these tokens stand for classes of literals)
+	literal_beg;
+	IDENT;
+	INT;
+	FLOAT;
+	CHAR;
+	STRING;
+	literal_end;
+
+	// Operators and delimiters
+	operator_beg;
+	ADD;
+	SUB;
+	MUL;
+	QUO;
+	REM;
+
+	AND;
+	OR;
+	XOR;
+	SHL;
+	SHR;
+	AND_NOT;
+
+	ADD_ASSIGN;
+	SUB_ASSIGN;
+	MUL_ASSIGN;
+	QUO_ASSIGN;
+	REM_ASSIGN;
+
+	AND_ASSIGN;
+	OR_ASSIGN;
+	XOR_ASSIGN;
+	SHL_ASSIGN;
+	SHR_ASSIGN;
+	AND_NOT_ASSIGN;
+
+	LAND;
+	LOR;
+	ARROW;
+	INC;
+	DEC;
+
+	EQL;
+	LSS;
+	GTR;
+	ASSIGN;
+	NOT;
+
+	NEQ;
+	LEQ;
+	GEQ;
+	DEFINE;
+	ELLIPSIS;
+
+	LPAREN;
+	LBRACK;
+	LBRACE;
+	COMMA;
+	PERIOD;
+
+	RPAREN;
+	RBRACK;
+	RBRACE;
+	SEMICOLON;
+	COLON;
+	operator_end;
+
+	// Keywords
+	keyword_beg;
+	BREAK;
+	CASE;
+	CHAN;
+	CONST;
+	CONTINUE;
+
+	DEFAULT;
+	DEFER;
+	ELSE;
+	FALLTHROUGH;
+	FOR;
+
+	FUNC;
+	GO;
+	GOTO;
+	IF;
+	IMPORT;
+
+	INTERFACE;
+	MAP;
+	PACKAGE;
+	RANGE;
+	RETURN;
+
+	SELECT;
+	STRUCT;
+	SWITCH;
+	TYPE;
+	VAR;
+	keyword_end;
+)
+
+
+// At the moment we have no array literal syntax that lets us describe
+// the index for each element - use a map for now to make sure they are
+// in sync.
+var tokens = map [Token] string {
+	ILLEGAL : "ILLEGAL",
+
+	EOF : "EOF",
+	COMMENT : "COMMENT",
+
+	IDENT : "IDENT",
+	INT : "INT",
+	FLOAT : "FLOAT",
+	CHAR : "CHAR",
+	STRING : "STRING",
+
+	ADD : "+",
+	SUB : "-",
+	MUL : "*",
+	QUO : "/",
+	REM : "%",
+
+	AND : "&",
+	OR : "|",
+	XOR : "^",
+	SHL : "<<",
+	SHR : ">>",
+	AND_NOT : "&^",
+
+	ADD_ASSIGN : "+=",
+	SUB_ASSIGN : "-=",
+	MUL_ASSIGN : "+=",
+	QUO_ASSIGN : "/=",
+	REM_ASSIGN : "%=",
+
+	AND_ASSIGN : "&=",
+	OR_ASSIGN : "|=",
+	XOR_ASSIGN : "^=",
+	SHL_ASSIGN : "<<=",
+	SHR_ASSIGN : ">>=",
+	AND_NOT_ASSIGN : "&^=",
+
+	LAND : "&&",
+	LOR : "||",
+	ARROW : "<-",
+	INC : "++",
+	DEC : "--",
+
+	EQL : "==",
+	LSS : "<",
+	GTR : ">",
+	ASSIGN : "=",
+	NOT : "!",
+
+	NEQ : "!=",
+	LEQ : "<=",
+	GEQ : ">=",
+	DEFINE : ":=",
+	ELLIPSIS : "...",
+
+	LPAREN : "(",
+	LBRACK : "[",
+	LBRACE : "{",
+	COMMA : ",",
+	PERIOD : ".",
+
+	RPAREN : ")",
+	RBRACK : "]",
+	RBRACE : "}",
+	SEMICOLON : ";",
+	COLON : ":",
+
+	BREAK : "break",
+	CASE : "case",
+	CHAN : "chan",
+	CONST : "const",
+	CONTINUE : "continue",
+
+	DEFAULT : "default",
+	DEFER : "defer",
+	ELSE : "else",
+	FALLTHROUGH : "fallthrough",
+	FOR : "for",
+
+	FUNC : "func",
+	GO : "go",
+	GOTO : "goto",
+	IF : "if",
+	IMPORT : "import",
+
+	INTERFACE : "interface",
+	MAP : "map",
+	PACKAGE : "package",
+	RANGE : "range",
+	RETURN : "return",
+
+	SELECT : "select",
+	STRUCT : "struct",
+	SWITCH : "switch",
+	TYPE : "type",
+	VAR : "var",
+}
+
+
+// String returns the string corresponding to the token tok.
+// For operators, delimiters, and keywords the string is the actual
+// token character sequence (e.g., for the token ADD, the string is
+// "+"). For all other tokens the string corresponds to the token
+// constant name (e.g. for the token IDENT, the string is "IDENT").
+//
+func (tok Token) String() string {
+	if str, exists := tokens[tok]; exists {
+		return str;
+	}
+	return "token(" + strconv.Itoa(int(tok)) + ")";
+}
+
+
+// A set of constants for precedence-based expression parsing.
+// Non-operators have lowest precedence, followed by operators
+// starting with precedence 1 up to unary operators. The highest
+// precedence corresponds serves as "catch-all" precedence for
+// selector, indexing, and other operator and delimiter tokens.
+//
+const (
+	LowestPrec = 0;  // non-operators
+	UnaryPrec = 7;
+	HighestPrec = 8;
+)
+
+
+// Precedence returns the operator precedence of the binary
+// operator op. If op is not a binary operator, the result
+// is LowestPrecedence.
+//
+func (op Token) Precedence() int {
+	switch op {
+	case LOR:
+		return 1;
+	case LAND:
+		return 2;
+	case ARROW:
+		return 3;
+	case EQL, NEQ, LSS, LEQ, GTR, GEQ:
+		return 4;
+	case ADD, SUB, OR, XOR:
+		return 5;
+	case MUL, QUO, REM, SHL, SHR, AND, AND_NOT:
+		return 6;
+	}
+	return LowestPrec;
+}
+
+
+var keywords map [string] Token;
+
+func init() {
+	keywords = make(map [string] Token);
+	for i := keyword_beg + 1; i < keyword_end; i++ {
+		keywords[tokens[i]] = i;
+	}
+}
+
+
+// Lookup maps an identifier to its keyword token or IDENT (if not a keyword).
+//
+func Lookup(ident []byte) Token {
+	// TODO Maps with []byte key are illegal because []byte does not
+	//      support == . Should find a more efficient solution eventually.
+	if tok, is_keyword := keywords[string(ident)]; is_keyword {
+		return tok;
+	}
+	return IDENT;
+}
+
+
+// Predicates
+
+// IsLiteral returns true for tokens corresponding to identifiers
+// and basic type literals; returns false otherwise.
+//
+func (tok Token) IsLiteral() bool {
+	return literal_beg < tok && tok < literal_end;
+}
+
+// IsOperator returns true for tokens corresponding to operators and
+// delimiters; returns false otherwise.
+//
+func (tok Token) IsOperator() bool {
+	return operator_beg < tok && tok < operator_end;
+}
+
+// IsKeyword returns true for tokens corresponding to keywords;
+// returns false otherwise.
+//
+func (tok Token) IsKeyword() bool {
+	return keyword_beg < tok && tok < keyword_end;
+}
+
+
+// Token source positions are represented by a Position value.
+// A Position is valid if the line number is > 0.
+//
+type Position struct {
+	Offset int;  // byte offset, starting at 0
+	Line int;  // line number, starting at 1
+	Column int;  // column number, starting at 1 (character count)
+}
+
+
+// Pos is an accessor method for anonymous Position fields.
+// It returns its receiver.
+//
+func (pos *Position) Pos() Position {
+	return *pos;
+}
+
+
+// IsValid returns true if the position is valid.
+func (pos *Position) IsValid() bool {
+	return pos.Line > 0
+}