diff options
author | John Hodge <tpg@mutabah.net> | 2016-11-02 19:38:45 +0800 |
---|---|---|
committer | John Hodge <tpg@mutabah.net> | 2016-11-02 19:38:45 +0800 |
commit | 983ff1c26e15a13e6a1e0e4bb1e964ec496a4777 (patch) | |
tree | bc39e1e44e63b9440dd186c2781e4ad02f82c2c5 | |
parent | 739c2f5a2124e9aa4fcc01e8fc1dbfe147a37254 (diff) | |
download | mrust-983ff1c26e15a13e6a1e0e4bb1e964ec496a4777.tar.gz |
Parse - Rework hygiene (with spelling fix) for take 2
-rw-r--r-- | src/ast/path.cpp | 2 | ||||
-rw-r--r-- | src/ast/path.hpp | 6 | ||||
-rw-r--r-- | src/ast/pattern.hpp | 2 | ||||
-rw-r--r-- | src/expand/env.cpp | 2 | ||||
-rw-r--r-- | src/expand/format_args.cpp | 22 | ||||
-rw-r--r-- | src/ident.cpp | 45 | ||||
-rw-r--r-- | src/include/ident.hpp | 48 | ||||
-rw-r--r-- | src/macro_rules/eval.cpp | 20 | ||||
-rw-r--r-- | src/macro_rules/parse.cpp | 6 | ||||
-rw-r--r-- | src/parse/eTokenType.enum.h | 1 | ||||
-rw-r--r-- | src/parse/expr.cpp | 8 | ||||
-rw-r--r-- | src/parse/lex.cpp | 62 | ||||
-rw-r--r-- | src/parse/lex.hpp | 13 | ||||
-rw-r--r-- | src/parse/paths.cpp | 7 | ||||
-rw-r--r-- | src/parse/pattern.cpp | 19 | ||||
-rw-r--r-- | src/parse/root.cpp | 88 | ||||
-rw-r--r-- | src/parse/token.cpp | 20 | ||||
-rw-r--r-- | src/parse/token.hpp | 8 | ||||
-rw-r--r-- | src/parse/tokentree.hpp | 16 | ||||
-rw-r--r-- | src/resolve/absolute.cpp | 12 |
20 files changed, 196 insertions, 211 deletions
diff --git a/src/ast/path.cpp b/src/ast/path.cpp index 25c0c208..ac023c8f 100644 --- a/src/ast/path.cpp +++ b/src/ast/path.cpp @@ -162,7 +162,7 @@ AST::Path::Path(const Path& x): m_class = Class::make_Local({ent.name}); ), (Relative, - m_class = Class::make_Relative({ent.hygine, ent.nodes}); + m_class = Class::make_Relative({ent.hygiene, ent.nodes}); ), (Self, m_class = Class::make_Self({ent.nodes}); diff --git a/src/ast/path.hpp b/src/ast/path.hpp index 5c2481a0..b45767a2 100644 --- a/src/ast/path.hpp +++ b/src/ast/path.hpp @@ -162,7 +162,7 @@ public: ::std::string name; } ), (Relative, struct { // General relative - Ident::Hygine hygine; // Taken from the first ident + Ident::Hygiene hygiene; ::std::vector<PathNode> nodes; } ), (Self, struct { // Module-relative @@ -226,8 +226,8 @@ public: // RELATIVE struct TagRelative {}; - Path(TagRelative, Ident::Hygine hygine, ::std::vector<PathNode> nodes): - m_class( Class::make_Relative({ mv$(hygine), mv$(nodes) }) ) + Path(TagRelative, Ident::Hygiene hygiene, ::std::vector<PathNode> nodes): + m_class( Class::make_Relative({ mv$(hygiene), mv$(nodes) }) ) {} // SELF struct TagSelf {}; diff --git a/src/ast/pattern.hpp b/src/ast/pattern.hpp index 0dff4071..a2df7b59 100644 --- a/src/ast/pattern.hpp +++ b/src/ast/pattern.hpp @@ -35,7 +35,7 @@ public: unsigned int m_slot; PatternBinding(): - m_name({0,{}}, ""), + m_name({}, ""), m_type(Type::MOVE), m_mutable(false), m_slot( ~0u ) diff --git a/src/expand/env.cpp b/src/expand/env.cpp index 989fbb7a..85742549 100644 --- a/src/expand/env.cpp +++ b/src/expand/env.cpp @@ -67,7 +67,7 @@ class CExpanderOptionEnv: rv.push_back( Token(TOK_PAREN_OPEN) ); rv.push_back( Token(TOK_STRING, var_val_cstr) ); rv.push_back( Token(TOK_PAREN_CLOSE) ); - return box$( TTStreamO(TokenTree( mv$(rv) )) ); + return box$( TTStreamO(TokenTree( {}, mv$(rv) )) ); } } }; diff --git a/src/expand/format_args.cpp b/src/expand/format_args.cpp index 977d7d69..4857a6c2 100644 --- a/src/expand/format_args.cpp +++ b/src/expand/format_args.cpp @@ -365,17 +365,17 @@ namespace { break; case ::AST::Crate::LOAD_CORE: toks.push_back( TokenTree(TOK_DOUBLE_COLON) ); - toks.push_back( Token(Ident("core")) ); + toks.push_back( Token(TOK_IDENT, "core") ); break; case ::AST::Crate::LOAD_STD: toks.push_back( TokenTree(TOK_DOUBLE_COLON) ); - toks.push_back( Token(Ident("std")) ); + toks.push_back( Token(TOK_IDENT, "std") ); break; } for(auto ent : il) { toks.push_back( TokenTree(TOK_DOUBLE_COLON) ); - toks.push_back( Token(Ident(ent)) ); + toks.push_back( Token(TOK_IDENT, ent) ); } } } @@ -417,7 +417,7 @@ class CFormatArgsExpander: if( lex.lookahead(0) == TOK_IDENT && lex.lookahead(1) == TOK_EQUAL ) { GET_CHECK_TOK(tok, lex, TOK_IDENT); - auto name = mv$(tok.ident_str()); + auto name = mv$(tok.str()); GET_CHECK_TOK(tok, lex, TOK_EQUAL); @@ -478,7 +478,7 @@ class CFormatArgsExpander: toks.push_back( TokenTree(TOK_PAREN_OPEN) ); for(unsigned int i = 0; i < free_args.size() + named_args.size(); i ++ ) { - toks.push_back( Token(Ident(FMT("a" << i))) ); + toks.push_back( Token(TOK_IDENT, FMT("a" << i)) ); toks.push_back( TokenTree(TOK_COMMA) ); } toks.push_back( TokenTree(TOK_PAREN_CLOSE) ); @@ -490,13 +490,13 @@ class CFormatArgsExpander: // - Contains N+1 entries, where N is the number of fragments { toks.push_back( TokenTree(TOK_RWORD_STATIC) ); - toks.push_back( Token(Ident("FRAGMENTS")) ); + toks.push_back( Token(TOK_IDENT, "FRAGMENTS") ); toks.push_back( TokenTree(TOK_COLON) ); toks.push_back( TokenTree(TOK_SQUARE_OPEN) ); toks.push_back( Token(TOK_AMP) ); toks.push_back( Token(TOK_LIFETIME, "static") ); - toks.push_back( Token(Ident("str")) ); + toks.push_back( Token(TOK_IDENT, "str") ); toks.push_back( Token(TOK_SEMICOLON) ); toks.push_back( Token(fragments.size() + 1, CORETYPE_UINT) ); toks.push_back( TokenTree(TOK_SQUARE_CLOSE) ); @@ -522,7 +522,7 @@ class CFormatArgsExpander: toks.push_back( TokenTree(TOK_PAREN_OPEN) ); { toks.push_back( TokenTree(TOK_AMP) ); - toks.push_back( Token(Ident("FRAGMENTS")) ); + toks.push_back( Token(TOK_IDENT, "FRAGMENTS") ); toks.push_back( TokenTree(TOK_COMMA) ); toks.push_back( TokenTree(TOK_AMP) ); @@ -531,7 +531,7 @@ class CFormatArgsExpander: { push_path(toks, crate, {"fmt", "ArgumentV1", "new"}); toks.push_back( Token(TOK_PAREN_OPEN) ); - toks.push_back( Token(Ident(FMT("a" << frag.arg_index))) ); + toks.push_back( Token(TOK_IDENT, FMT("a" << frag.arg_index)) ); toks.push_back( TokenTree(TOK_COMMA) ); @@ -554,7 +554,7 @@ class CFormatArgsExpander: toks.push_back( TokenTree(TOK_PAREN_OPEN) ); { toks.push_back( TokenTree(TOK_AMP) ); - toks.push_back( Token(Ident("FRAGMENTS")) ); + toks.push_back( Token(TOK_IDENT, "FRAGMENTS") ); toks.push_back( TokenTree(TOK_COMMA) ); // 1. Generate a set of arguments+formatters @@ -574,7 +574,7 @@ class CFormatArgsExpander: toks.push_back( TokenTree(TOK_BRACE_CLOSE) ); toks.push_back( TokenTree(TOK_BRACE_CLOSE) ); - return box$( TTStreamO(TokenTree(mv$(toks))) ); + return box$( TTStreamO(TokenTree(Ident::Hygiene::new_scope(), mv$(toks))) ); } }; diff --git a/src/ident.cpp b/src/ident.cpp index 5d0fe00c..a5a6bdbb 100644 --- a/src/ident.cpp +++ b/src/ident.cpp @@ -3,49 +3,20 @@ * - By John Hodge (Mutabah/thePowersGang) * * include/ident.cpp - * - Identifiers with hygine + * - Identifiers with hygiene */ #include <iostream> #include <ident.hpp> #include <debug.hpp> #include <common.hpp> // vector print -bool Ident::Hygine::is_visible(const Hygine& src) const +unsigned int Ident::Hygiene::g_next_scope = 0; + +bool Ident::Hygiene::is_visible(const Hygiene& src) const { - // HACK: Disable hygine for now + // HACK: Disable hygiene for now return true; - - DEBUG("*this = " << *this << ", src=" << src); - if( this->file_num != src.file_num ) { - DEBUG("- Different file"); - return false; - } - - // `this` is the item, `src` is the ident used to access it - - // If this is from a deeper point than the source, it won't work. - if( this->indexes.size() > src.indexes.size() ) { - DEBUG("- Not subset: len"); - return false; - } - - // If this hygine list (barring the last) is a subset of the source - for(unsigned int i = 0; i < this->indexes.size()-1 - 1; i ++) - { - if( this->indexes[i] != src.indexes[i] ) { - DEBUG("- Not subset: " << i); - return false; - } - } - - unsigned int end = this->indexes.size()-1-1; - // Allow match if this ident is from before the addressing ident - if( this->indexes[end] < src.indexes[end] ) { - return true; - } - - DEBUG("- Not before"); - return false; + //return x.scope_index == src.scope_index; } ::std::ostream& operator<<(::std::ostream& os, const Ident& x) { @@ -53,8 +24,8 @@ bool Ident::Hygine::is_visible(const Hygine& src) const return os; } -::std::ostream& operator<<(::std::ostream& os, const Ident::Hygine& x) { - os << "{" << x.file_num << ": [" << x.indexes << "]}"; +::std::ostream& operator<<(::std::ostream& os, const Ident::Hygiene& x) { + os << "{" << x.scope_index << "}"; return os; } diff --git a/src/include/ident.hpp b/src/include/ident.hpp index d5ce2e77..cff918ca 100644 --- a/src/include/ident.hpp +++ b/src/include/ident.hpp @@ -3,7 +3,7 @@ * - By John Hodge (Mutabah/thePowersGang) * * include/ident.hpp - * - Identifiers with hygine + * - Identifiers with hygiene */ #pragma once #include <vector> @@ -11,42 +11,50 @@ struct Ident { - struct Hygine + class Hygiene { - unsigned int file_num; - ::std::vector<unsigned int> indexes; + static unsigned g_next_scope; + unsigned int scope_index; - Hygine(unsigned int file, ::std::vector<unsigned int> indexes): - file_num(file), - indexes(::std::move(indexes)) + Hygiene(unsigned int index): + scope_index(index) {} + public: + Hygiene(): + scope_index(0) + {} + + static Hygiene new_scope() + { + return Hygiene(++g_next_scope); + } - Hygine(Hygine&& x) = default; - Hygine(const Hygine& x) = default; - Hygine& operator=(Hygine&& x) = default; - Hygine& operator=(const Hygine& x) = default; + Hygiene(Hygiene&& x) = default; + Hygiene(const Hygiene& x) = default; + Hygiene& operator=(Hygiene&& x) = default; + Hygiene& operator=(const Hygiene& x) = default; // Returns true if an ident with hygine `souce` can see an ident with this hygine - bool is_visible(const Hygine& source) const; - bool operator==(const Hygine& x) const { return file_num == x.file_num && indexes == x.indexes; } - bool operator!=(const Hygine& x) const { return file_num != x.file_num || indexes != x.indexes; } + bool is_visible(const Hygiene& source) const; + bool operator==(const Hygiene& x) const { return scope_index == x.scope_index; } + bool operator!=(const Hygiene& x) const { return scope_index != x.scope_index; } - friend ::std::ostream& operator<<(::std::ostream& os, const Hygine& v); + friend ::std::ostream& operator<<(::std::ostream& os, const Hygiene& v); }; - Hygine hygine; + Hygiene hygiene; ::std::string name; Ident(const char* name): - hygine(~0u, {}), + hygiene(), name(name) { } Ident(::std::string name): - hygine(~0u, {}), + hygiene(), name(::std::move(name)) { } - Ident(Hygine hygine, ::std::string name): - hygine(::std::move(hygine)), name(::std::move(name)) + Ident(Hygiene hygiene, ::std::string name): + hygiene(::std::move(hygiene)), name(::std::move(name)) { } Ident(Ident&& x) = default; diff --git a/src/macro_rules/eval.cpp b/src/macro_rules/eval.cpp index 31665a25..110e8525 100644 --- a/src/macro_rules/eval.cpp +++ b/src/macro_rules/eval.cpp @@ -566,6 +566,7 @@ class MacroExpander: Token m_next_token; // used for inserting a single token into the stream ::std::unique_ptr<TTStreamO> m_ttstream; + Ident::Hygiene m_hygiene; public: MacroExpander(const MacroExpander& x) = delete; @@ -574,12 +575,14 @@ public: m_macro_filename( FMT("Macro:" << macro_name) ), m_crate_name( mv$(crate_name) ), m_mappings( mv$(mappings) ), - m_state( contents, m_mappings ) + m_state( contents, m_mappings ), + m_hygiene( Ident::Hygiene::new_scope() ) { } - virtual Position getPosition() const override; - virtual Token realGetToken() override; + Position getPosition() const override; + Ident::Hygiene getHygiene() const override; + Token realGetToken() override; }; void Macro_InitDefaults() @@ -1044,6 +1047,17 @@ Position MacroExpander::getPosition() const // TODO: Return a far better span - invocaion location? return Position(m_macro_filename, 0, m_state.top_pos()); } +Ident::Hygiene MacroExpander::getHygiene() const +{ + if( m_ttstream ) + { + return m_ttstream->getHygiene(); + } + else + { + return m_hygiene; + } +} Token MacroExpander::realGetToken() { // Use m_next_token first diff --git a/src/macro_rules/parse.cpp b/src/macro_rules/parse.cpp index bd54c67d..c7a6ef89 100644 --- a/src/macro_rules/parse.cpp +++ b/src/macro_rules/parse.cpp @@ -49,10 +49,10 @@ public: switch( GET_TOK(tok, lex) ) { case TOK_IDENT: { - ::std::string name = mv$(tok.ident_str()); + ::std::string name = mv$(tok.str()); GET_CHECK_TOK(tok, lex, TOK_COLON); GET_CHECK_TOK(tok, lex, TOK_IDENT); - ::std::string type = mv$(tok.ident_str()); + ::std::string type = mv$(tok.str()); unsigned int idx = ::std::find( names.begin(), names.end(), name ) - names.begin(); if( idx == names.size() ) @@ -202,7 +202,7 @@ public: else if( tok.type() == TOK_IDENT ) { // Look up the named parameter in the list of param names for this arm - const auto& name = tok.ident_str(); + const auto& name = tok.str(); unsigned int idx = ::std::find(var_names.begin(), var_names.end(), name) - var_names.begin(); if( idx == var_names.size() ) ERROR(lex.getPosition(), E0000, "Macro variable $" << name << " not found"); diff --git a/src/parse/eTokenType.enum.h b/src/parse/eTokenType.enum.h index 9caa3aba..8d410b0c 100644 --- a/src/parse/eTokenType.enum.h +++ b/src/parse/eTokenType.enum.h @@ -5,6 +5,7 @@ _(TOK_NEWLINE) _(TOK_WHITESPACE) _(TOK_COMMENT) +_(TOK_INTERPOLATED_IDENT) _(TOK_INTERPOLATED_PATH) _(TOK_INTERPOLATED_TYPE) _(TOK_INTERPOLATED_PATTERN) diff --git a/src/parse/expr.cpp b/src/parse/expr.cpp index f6ebca1f..832e6a4a 100644 --- a/src/parse/expr.cpp +++ b/src/parse/expr.cpp @@ -889,7 +889,7 @@ ExprNodeP Parse_ExprFC(TokenStream& lex) switch(GET_TOK(tok, lex))
{
case TOK_IDENT: {
- AST::PathNode path( mv$(tok.ident_str()) , {});
+ AST::PathNode path( mv$(tok.str()) , {});
switch( GET_TOK(tok, lex) )
{
case TOK_PAREN_OPEN:
@@ -965,7 +965,7 @@ ExprNodeP Parse_ExprVal_StructLiteral(TokenStream& lex, AST::Path path) ::std::vector< ::std::pair< ::std::string, ::std::unique_ptr<AST::ExprNode>> > items;
while( GET_TOK(tok, lex) == TOK_IDENT )
{
- auto name = mv$(tok.ident_str());
+ auto name = mv$(tok.str());
GET_CHECK_TOK(tok, lex, TOK_COLON);
ExprNodeP val = Parse_Stmt(lex);
items.push_back( ::std::make_pair(::std::move(name), ::std::move(val)) );
@@ -1224,7 +1224,7 @@ ExprNodeP Parse_ExprMacro(TokenStream& lex, Token tok) ::std::string name = tok.str();
::std::string ident;
if( GET_TOK(tok, lex) == TOK_IDENT ) {
- ident = mv$(tok.ident_str());
+ ident = mv$(tok.str());
}
else {
PUTBACK(tok, lex);
@@ -1285,5 +1285,5 @@ TokenTree Parse_TT(TokenStream& lex, bool unwrapped) }
if( !unwrapped )
items.push_back( mv$(tok) );
- return TokenTree(mv$(items));
+ return TokenTree(lex.getHygiene(), mv$(items));
}
diff --git a/src/parse/lex.cpp b/src/parse/lex.cpp index a5c4e372..9acb56e0 100644 --- a/src/parse/lex.cpp +++ b/src/parse/lex.cpp @@ -29,7 +29,8 @@ Lexer::Lexer(const ::std::string& filename): m_line(1), m_line_ofs(0), m_istream(filename.c_str()), - m_last_char_valid(false) + m_last_char_valid(false), + m_hygiene( Ident::Hygiene::new_scope() ) { if( !m_istream.is_open() ) { @@ -243,6 +244,10 @@ Position Lexer::getPosition() const { return Position(m_path, m_line, m_line_ofs); } +Ident::Hygiene Lexer::getHygiene() const +{ + return m_hygiene; +} Token Lexer::realGetToken() { while(true) @@ -251,29 +256,6 @@ Token Lexer::realGetToken() //::std::cout << "getTokenInt: tok = " << tok << ::std::endl; switch(tok.type()) { - case TOK_IDENT: - if( m_hygine_stack.size() > 0 ) - { - m_hygine_stack.back() ++; - } - return tok; - - case TOK_PAREN_OPEN: - case TOK_SQUARE_OPEN: - case TOK_BRACE_OPEN: - case TOK_ATTR_OPEN: - case TOK_CATTR_OPEN: - m_hygine_stack.push_back(0); - return tok; - case TOK_PAREN_CLOSE: - case TOK_SQUARE_CLOSE: - case TOK_BRACE_CLOSE: - m_hygine_stack.pop_back(); - if( m_hygine_stack.size() > 0 ) { - m_hygine_stack.back() ++; - } - return tok; - case TOK_NEWLINE: m_line ++; m_line_ofs = 0; @@ -790,7 +772,7 @@ Token Lexer::getTokenInt_Identifier(Codepoint leader, Codepoint leader2) if( str < RWORDS[i].chars ) break; if( str == RWORDS[i].chars ) return Token((enum eTokenType)RWORDS[i].type); } - return Token(Ident( Ident::Hygine(m_file_index, m_hygine_stack), mv$(str) )); + return Token(TOK_IDENT, mv$(str)); } } @@ -1022,22 +1004,29 @@ Token TTStream::realGetToken() const TokenTree& subtree = tree[idx]; idx ++; if( subtree.size() == 0 ) { + m_hygiene_ptr = &m_stack.back().second->hygiene(); return subtree.tok().clone(); } else { - m_stack.push_back( ::std::make_pair(0, &subtree ) ); + m_stack.push_back( ::std::make_pair(0, &subtree) ); } } else { m_stack.pop_back(); } } + //m_hygiene = nullptr; return Token(TOK_EOF); } Position TTStream::getPosition() const { return Position("TTStream", 0,0); } +Ident::Hygiene TTStream::getHygiene() const +{ + //assert( m_hygiene ); + return *m_hygiene_ptr; +} TTStreamO::TTStreamO(TokenTree input_tt): @@ -1071,7 +1060,7 @@ Token TTStreamO::realGetToken() return mv$( subtree.tok() ); } else { - m_stack.push_back( ::std::make_pair(0, &subtree ) ); + m_stack.push_back( ::std::make_pair(0, &subtree) ); } } else { @@ -1084,6 +1073,10 @@ Position TTStreamO::getPosition() const { return m_last_pos; } +Ident::Hygiene TTStreamO::getHygiene() const +{ + return (m_stack.back().second ? m_stack.back().second->hygiene() : m_input_tt.hygiene()); +} TokenStream::TokenStream(): @@ -1183,7 +1176,18 @@ Span TokenStream::end_span(ProtoSpan ps) const p.line, p.ofs ); } - +Ident TokenStream::get_ident(Token tok) const +{ + if(tok.type() == TOK_IDENT) { + return Ident(getHygiene(), tok.str()); + } + else if( tok.type() == TOK_INTERPOLATED_IDENT ) { + TODO(getPosition(), ""); + } + else { + throw ParseError::Unexpected(*this, tok); + } +} TokenTree TokenTree::clone() const { @@ -1195,7 +1199,7 @@ TokenTree TokenTree::clone() const ents.reserve( m_subtrees.size() ); for(const auto& sub : m_subtrees) ents.push_back( sub.clone() ); - return TokenTree( mv$(ents) ); + return TokenTree( m_hygiene, mv$(ents) ); } } diff --git a/src/parse/lex.hpp b/src/parse/lex.hpp index 208fa670..d147d79d 100644 --- a/src/parse/lex.hpp +++ b/src/parse/lex.hpp @@ -62,13 +62,17 @@ public: Token getToken(); void putback(Token tok); eTokenType lookahead(unsigned int count); + virtual Position getPosition() const = 0; + virtual Ident::Hygiene getHygiene() const = 0; ParseState& parse_state() { return m_parse_state; } ProtoSpan start_span() const; Span end_span(ProtoSpan ps) const; + Ident get_ident(Token tok) const; + protected: virtual Token realGetToken() = 0; private: @@ -126,14 +130,13 @@ class Lexer: Codepoint m_last_char; Token m_next_token; // Used when lexing generated two tokens - unsigned int m_file_index; - // Incremented on every ident and TT close, pushed on TT open - ::std::vector<unsigned int> m_hygine_stack; + Ident::Hygiene m_hygiene; public: Lexer(const ::std::string& filename); - virtual Position getPosition() const override; - virtual Token realGetToken() override; + Position getPosition() const override; + Ident::Hygiene getHygiene() const override; + Token realGetToken() override; private: Token getTokenInt(); diff --git a/src/parse/paths.cpp b/src/parse/paths.cpp index 5e0faf90..e600e51e 100644 --- a/src/parse/paths.cpp +++ b/src/parse/paths.cpp @@ -91,8 +91,9 @@ AST::Path Parse_Path(TokenStream& lex, bool is_abs, eParsePathGenericMode generi } } else { + // TODO: TOK_INTERPOLATED_IDENT? GET_CHECK_TOK(tok, lex, TOK_IDENT); - auto hygine = mv$( tok.ident().hygine ); + auto hygine = lex.getHygiene(); PUTBACK(tok, lex); return AST::Path(AST::Path::TagRelative(), mv$(hygine), Parse_PathNodes(lex, generic_mode)); } @@ -111,7 +112,7 @@ AST::Path Parse_Path(TokenStream& lex, bool is_abs, eParsePathGenericMode generi ::AST::PathParams params; CHECK_TOK(tok, TOK_IDENT); - auto component = mv$( tok.ident_str() ); + auto component = mv$( tok.str() ); GET_TOK(tok, lex); if( generic_mode == PATH_GENERIC_TYPE ) @@ -218,7 +219,7 @@ AST::Path Parse_Path(TokenStream& lex, bool is_abs, eParsePathGenericMode generi case TOK_IDENT: if( LOOK_AHEAD(lex) == TOK_EQUAL ) { - ::std::string name = mv$(tok.ident_str()); + ::std::string name = mv$(tok.str()); GET_CHECK_TOK(tok, lex, TOK_EQUAL); assoc_bounds.push_back( ::std::make_pair( mv$(name), Parse_Type(lex,false) ) ); break; diff --git a/src/parse/pattern.cpp b/src/parse/pattern.cpp index df251bd3..f43a1205 100644 --- a/src/parse/pattern.cpp +++ b/src/parse/pattern.cpp @@ -85,7 +85,7 @@ AST::Pattern Parse_Pattern(TokenStream& lex, bool is_refutable) if( expect_bind ) { CHECK_TOK(tok, TOK_IDENT); - auto bind_name = tok.take_ident(); + auto bind_name = Ident(lex.getHygiene(), mv$(tok.str())); // If there's no '@' after it, it's a name binding only (_ pattern) if( GET_TOK(tok, lex) != TOK_AT ) { @@ -114,21 +114,23 @@ AST::Pattern Parse_Pattern(TokenStream& lex, bool is_refutable) break; // Known binding `ident @` case TOK_AT: - binding = AST::PatternBinding( tok.take_ident(), bind_type/*MOVE*/, is_mut/*false*/ ); + binding = AST::PatternBinding( Ident(lex.getHygiene(), mv$(tok.str())), bind_type/*MOVE*/, is_mut/*false*/ ); GET_TOK(tok, lex); // '@' GET_TOK(tok, lex); // Match lex.putback() below break; - default: // Maybe bind + default: { // Maybe bind + Ident name = Ident(lex.getHygiene(), mv$(tok.str())); // if the pattern can be refuted (i.e this could be an enum variant), return MaybeBind if( is_refutable ) { assert(bind_type == ::AST::PatternBinding::Type::MOVE); assert(is_mut == false); - return AST::Pattern(AST::Pattern::TagMaybeBind(), tok.take_ident()); + return AST::Pattern(AST::Pattern::TagMaybeBind(), mv$(name)); } // Otherwise, it IS a binding else { - return AST::Pattern(AST::Pattern::TagBind(), tok.take_ident(), bind_type, is_mut); + return AST::Pattern(AST::Pattern::TagBind(), mv$(name), bind_type, is_mut); } + break;} } } else @@ -138,7 +140,6 @@ AST::Pattern Parse_Pattern(TokenStream& lex, bool is_refutable) PUTBACK(tok, lex); auto pat = Parse_PatternReal(lex, is_refutable); - //pat.set_bind( mv$(binding) ); pat.binding() = mv$(binding); return pat; } @@ -276,10 +277,10 @@ AST::Pattern Parse_PatternReal_Slice(TokenStream& lex, bool is_refutable) ::AST::PatternBinding binding; if( tok.type() == TOK_RWORD_REF && lex.lookahead(0) == TOK_IDENT && lex.lookahead(1) == TOK_DOUBLE_DOT ) { GET_TOK(tok, lex); - binding = ::AST::PatternBinding( tok.take_ident(), ::AST::PatternBinding::Type::REF, false ); + binding = ::AST::PatternBinding( lex.get_ident(mv$(tok)), ::AST::PatternBinding::Type::REF, false ); } else if( tok.type() == TOK_IDENT && lex.lookahead(0) == TOK_DOUBLE_DOT) { - binding = ::AST::PatternBinding( tok.take_ident(), ::AST::PatternBinding::Type::MOVE, false ); + binding = ::AST::PatternBinding( lex.get_ident(mv$(tok)), ::AST::PatternBinding::Type::MOVE, false ); } else if( tok.type() == TOK_UNDERSCORE && lex.lookahead(0) == TOK_DOUBLE_DOT) { // No binding, but switching to trailing @@ -466,7 +467,7 @@ AST::Pattern Parse_PatternStruct(TokenStream& lex, AST::Path path, bool is_refut } CHECK_TOK(tok, TOK_IDENT); - auto field_ident = tok.take_ident(); + auto field_ident = lex.get_ident(mv$(tok)); ::std::string field_name; GET_TOK(tok, lex); diff --git a/src/parse/root.cpp b/src/parse/root.cpp index 6c9acc14..2f5114ed 100644 --- a/src/parse/root.cpp +++ b/src/parse/root.cpp @@ -133,7 +133,7 @@ AST::GenericParams Parse_GenericParams(TokenStream& lex) if( tok.type() == TOK_IDENT )
{
// TODO: Hygine
- ::std::string param_name = mv$(tok.ident_str());
+ ::std::string param_name = mv$(tok.str());
ret.add_ty_param( AST::TypeParam( param_name ) );
auto param_ty = TypeRef(lex.getPosition(), param_name);
@@ -529,7 +529,7 @@ AST::Struct Parse_Struct(TokenStream& lex, const AST::MetaItems& meta_items) }
CHECK_TOK(tok, TOK_IDENT);
- auto name = mv$(tok.ident_str());
+ auto name = mv$(tok.str());
GET_CHECK_TOK(tok, lex, TOK_COLON);
TypeRef type = Parse_Type(lex);
@@ -615,7 +615,7 @@ AST::Trait Parse_TraitDef(TokenStream& lex, const AST::MetaItems& meta_items) }
bool is_specialisable = false;
- if( tok.type() == TOK_IDENT && tok.ident_str() == "default" ) {
+ if( tok.type() == TOK_IDENT && tok.str() == "default" ) {
is_specialisable = true;
GET_TOK(tok, lex);
}
@@ -629,7 +629,7 @@ AST::Trait Parse_TraitDef(TokenStream& lex, const AST::MetaItems& meta_items) {
case TOK_RWORD_STATIC: {
GET_CHECK_TOK(tok, lex, TOK_IDENT);
- auto name = mv$(tok.ident_str());
+ auto name = mv$(tok.str());
GET_CHECK_TOK(tok, lex, TOK_COLON);
auto ty = Parse_Type(lex);
GET_CHECK_TOK(tok, lex, TOK_SEMICOLON);
@@ -646,7 +646,7 @@ AST::Trait Parse_TraitDef(TokenStream& lex, const AST::MetaItems& meta_items) break; }
case TOK_RWORD_CONST: {
GET_CHECK_TOK(tok, lex, TOK_IDENT);
- auto name = mv$(tok.ident_str());
+ auto name = mv$(tok.str());
GET_CHECK_TOK(tok, lex, TOK_COLON);
auto ty = Parse_Type(lex);
@@ -664,7 +664,7 @@ AST::Trait Parse_TraitDef(TokenStream& lex, const AST::MetaItems& meta_items) case TOK_RWORD_TYPE: {
auto atype_params = ::AST::GenericParams { };
GET_CHECK_TOK(tok, lex, TOK_IDENT);
- auto name = mv$(tok.ident_str());
+ auto name = mv$(tok.str());
if( GET_TOK(tok, lex) == TOK_COLON )
{
// Bounded associated type
@@ -704,7 +704,7 @@ AST::Trait Parse_TraitDef(TokenStream& lex, const AST::MetaItems& meta_items) CHECK_TOK(tok, TOK_RWORD_FN);
case TOK_RWORD_FN: {
GET_CHECK_TOK(tok, lex, TOK_IDENT);
- ::std::string name = tok.take_ident().into_string();
+ ::std::string name = mv$(tok.str());
// Self allowed, prototype-form allowed (optional names and no code)
auto fcn = Parse_FunctionDef(lex, abi, true, true, fn_is_unsafe, fn_is_const);
if( GET_TOK(tok, lex) == TOK_BRACE_OPEN )
@@ -768,7 +768,7 @@ AST::Enum Parse_EnumDef(TokenStream& lex, const AST::MetaItems& meta_items) SET_ATTRS(lex, item_attrs);
CHECK_TOK(tok, TOK_IDENT);
- ::std::string name = tok.take_ident().into_string();
+ ::std::string name = mv$(tok.str());
if( GET_TOK(tok, lex) == TOK_PAREN_OPEN )
{
::std::vector<TypeRef> types;
@@ -815,7 +815,7 @@ AST::Enum Parse_EnumDef(TokenStream& lex, const AST::MetaItems& meta_items) }
GET_CHECK_TOK(tok, lex, TOK_IDENT);
- auto name = tok.take_ident().into_string();
+ auto name = mv$(tok.str());
GET_CHECK_TOK(tok, lex, TOK_COLON);
auto ty = Parse_Type(lex);
fields.push_back( ::AST::StructItem(mv$(field_attrs), true, mv$(name), mv$(ty)) );
@@ -888,7 +888,7 @@ AST::Enum Parse_EnumDef(TokenStream& lex, const AST::MetaItems& meta_items) }
GET_CHECK_TOK(tok, lex, TOK_IDENT);
- auto name = tok.take_ident().into_string();
+ auto name = mv$(tok.str());
GET_CHECK_TOK(tok, lex, TOK_COLON);
auto ty = Parse_Type(lex);
@@ -913,7 +913,7 @@ AST::MetaItem Parse_MetaItem(TokenStream& lex) }
CHECK_TOK(tok, TOK_IDENT);
- ::std::string name = tok.take_ident().into_string();
+ ::std::string name = mv$(tok.str());
switch(GET_TOK(tok, lex))
{
case TOK_EQUAL:
@@ -1071,7 +1071,7 @@ void Parse_Impl_Item(TokenStream& lex, AST::Impl& impl) }
bool is_specialisable = false;
- if( tok.type() == TOK_IDENT && tok.ident() == "default" ) {
+ if( tok.type() == TOK_IDENT && tok.str() == "default" ) {
is_specialisable = true;
GET_TOK(tok, lex);
}
@@ -1083,7 +1083,7 @@ void Parse_Impl_Item(TokenStream& lex, AST::Impl& impl) {
case TOK_RWORD_TYPE: {
GET_CHECK_TOK(tok, lex, TOK_IDENT);
- auto name = tok.take_ident().into_string();
+ auto name = mv$(tok.str());
GET_CHECK_TOK(tok, lex, TOK_EQUAL);
impl.add_type(is_public, is_specialisable, name, Parse_Type(lex));
GET_CHECK_TOK(tok, lex, TOK_SEMICOLON);
@@ -1098,7 +1098,7 @@ void Parse_Impl_Item(TokenStream& lex, AST::Impl& impl) if( tok.type() != TOK_RWORD_FN && tok.type() != TOK_RWORD_UNSAFE && !fn_is_unsafe )
{
CHECK_TOK(tok, TOK_IDENT);
- auto name = tok.take_ident().into_string();
+ auto name = mv$(tok.str());
GET_CHECK_TOK(tok, lex, TOK_COLON);
auto ty = Parse_Type(lex);
GET_CHECK_TOK(tok, lex, TOK_EQUAL);
@@ -1134,7 +1134,7 @@ void Parse_Impl_Item(TokenStream& lex, AST::Impl& impl) case TOK_RWORD_FN: {
GET_CHECK_TOK(tok, lex, TOK_IDENT);
// TODO: Hygine on function names? - Not in impl blocks?
- ::std::string name = tok.take_ident().into_string();
+ ::std::string name = mv$(tok.str());
DEBUG("Function " << name);
// - Self allowed, can't be prototype-form
auto fcn = Parse_FunctionDefWithCode(lex, abi, true, fn_is_unsafe, fn_is_const);
@@ -1185,7 +1185,7 @@ AST::ExternBlock Parse_ExternBlock(TokenStream& lex, ::std::string abi, ::AST::M {
case TOK_RWORD_FN: {
GET_CHECK_TOK(tok, lex, TOK_IDENT);
- auto name = tok.take_ident().into_string();
+ auto name = mv$(tok.str());
// parse function as prototype
// - no self, "safe" and not const
auto i = ::AST::Item( Parse_FunctionDef(lex, abi, false, true, false,false) );
@@ -1203,7 +1203,7 @@ AST::ExternBlock Parse_ExternBlock(TokenStream& lex, ::std::string abi, ::AST::M else
PUTBACK(tok, lex);
GET_CHECK_TOK(tok, lex, TOK_IDENT);
- auto name = tok.take_ident().into_string();
+ auto name = mv$(tok.str());
GET_CHECK_TOK(tok, lex, TOK_COLON);
auto type = Parse_Type(lex);
GET_CHECK_TOK(tok, lex, TOK_SEMICOLON);
@@ -1242,13 +1242,13 @@ void Parse_Use_Set(TokenStream& lex, const ProtoSpan& ps, const AST::Path& base_ }
else {
CHECK_TOK(tok, TOK_IDENT);
- path = base_path + AST::PathNode(tok.ident_str(), {});
- name = mv$(tok.ident_str());
+ path = base_path + AST::PathNode(tok.str(), {});
+ name = mv$(tok.str());
}
if( GET_TOK(tok, lex) == TOK_RWORD_AS ) {
GET_CHECK_TOK(tok, lex, TOK_IDENT);
- name = mv$(tok.ident_str());
+ name = mv$(tok.str());
}
else {
PUTBACK(tok, lex);
@@ -1282,7 +1282,7 @@ void Parse_Use(TokenStream& lex, ::std::function<void(AST::UseStmt, ::std::strin path = AST::Path( AST::Path::TagSuper(), count, {} );
break; }
case TOK_IDENT:
- path.append( AST::PathNode(tok.take_ident().into_string(), {}) );
+ path.append( AST::PathNode(mv$(tok.str()), {}) );
break;
// Leading :: is allowed and ignored for the $crate feature
case TOK_DOUBLE_COLON:
@@ -1309,7 +1309,7 @@ void Parse_Use(TokenStream& lex, ::std::function<void(AST::UseStmt, ::std::strin {
if( GET_TOK(tok, lex) == TOK_IDENT )
{
- path.append( AST::PathNode(tok.take_ident().into_string(), {}) );
+ path.append( AST::PathNode( mv$(tok.str()), {}) );
}
else
{
@@ -1338,7 +1338,7 @@ void Parse_Use(TokenStream& lex, ::std::function<void(AST::UseStmt, ::std::strin if( tok.type() == TOK_RWORD_AS )
{
GET_CHECK_TOK(tok, lex, TOK_IDENT);
- name = tok.take_ident().into_string();
+ name = mv$(tok.str());
}
else
{
@@ -1356,7 +1356,7 @@ void Parse_Use(TokenStream& lex, ::std::function<void(AST::UseStmt, ::std::strin Token tok;
::std::string ident;
if( GET_TOK(tok, lex) == TOK_IDENT ) {
- ident = tok.take_ident().into_string();
+ ident = mv$(tok.str());
}
else {
PUTBACK(tok, lex);
@@ -1447,7 +1447,7 @@ void Parse_Use(TokenStream& lex, ::std::function<void(AST::UseStmt, ::std::strin // `extern "<ABI>" fn ...`
case TOK_RWORD_FN: {
GET_CHECK_TOK(tok, lex, TOK_IDENT);
- item_name = mv$(tok.ident_str());
+ item_name = mv$(tok.str());
item_data = ::AST::Item( Parse_FunctionDefWithCode(lex, abi, false, false,false) );
break; }
// `extern "ABI" {`
@@ -1462,7 +1462,7 @@ void Parse_Use(TokenStream& lex, ::std::function<void(AST::UseStmt, ::std::strin // `extern fn ...`
case TOK_RWORD_FN:
GET_CHECK_TOK(tok, lex, TOK_IDENT);
- item_name = mv$(tok.ident_str());
+ item_name = mv$(tok.str());
item_data = ::AST::Item( Parse_FunctionDefWithCode(lex, "C", false, false,false) );
break;
@@ -1484,17 +1484,17 @@ void Parse_Use(TokenStream& lex, ::std::function<void(AST::UseStmt, ::std::strin item_data = ::AST::Item::make_Crate({ tok.str() });
GET_CHECK_TOK(tok, lex, TOK_RWORD_AS);
GET_CHECK_TOK(tok, lex, TOK_IDENT);
- item_name = mv$(tok.ident_str());
+ item_name = mv$(tok.str());
break;
// `extern crate crate_name;`
// `extern crate crate_name as other_name;`
case TOK_IDENT:
- item_name = mv$(tok.ident_str());
+ item_name = mv$(tok.str());
if(GET_TOK(tok, lex) == TOK_RWORD_AS) {
item_data = ::AST::Item::make_Crate({ mv$(item_name) });
GET_CHECK_TOK(tok, lex, TOK_IDENT);
- item_name = mv$(tok.ident_str());
+ item_name = mv$(tok.str());
}
else {
PUTBACK(tok, lex);
@@ -1517,7 +1517,7 @@ void Parse_Use(TokenStream& lex, ::std::function<void(AST::UseStmt, ::std::strin switch( GET_TOK(tok, lex) )
{
case TOK_IDENT: {
- item_name = mv$(tok.ident_str());
+ item_name = mv$(tok.str());
GET_CHECK_TOK(tok, lex, TOK_COLON);
TypeRef type = Parse_Type(lex);
@@ -1529,12 +1529,12 @@ void Parse_Use(TokenStream& lex, ::std::function<void(AST::UseStmt, ::std::strin case TOK_RWORD_UNSAFE:
GET_CHECK_TOK(tok, lex, TOK_RWORD_FN);
GET_CHECK_TOK(tok, lex, TOK_IDENT);
- item_name = mv$(tok.ident_str());
+ item_name = mv$(tok.str());
item_data = ::AST::Item( Parse_FunctionDefWithCode(lex, ABI_RUST, false, true,true/*unsafe,const*/) );
break;
case TOK_RWORD_FN:
GET_CHECK_TOK(tok, lex, TOK_IDENT);
- item_name = mv$(tok.ident_str());
+ item_name = mv$(tok.str());
// - self not allowed, not prototype
item_data = ::AST::Item( Parse_FunctionDefWithCode(lex, ABI_RUST, false, false,true/*unsafe,const*/) );
break;
@@ -1551,7 +1551,7 @@ void Parse_Use(TokenStream& lex, ::std::function<void(AST::UseStmt, ::std::strin GET_TOK(tok, lex);
}
CHECK_TOK(tok, TOK_IDENT);
- item_name = mv$(tok.ident_str());
+ item_name = mv$(tok.str());
GET_CHECK_TOK(tok, lex, TOK_COLON);
TypeRef type = Parse_Type(lex);
@@ -1582,20 +1582,20 @@ void Parse_Use(TokenStream& lex, ::std::function<void(AST::UseStmt, ::std::strin }
GET_CHECK_TOK(tok, lex, TOK_RWORD_FN);
GET_CHECK_TOK(tok, lex, TOK_IDENT);
- item_name = mv$(tok.ident_str());
+ item_name = mv$(tok.str());
item_data = ::AST::Item( Parse_FunctionDefWithCode(lex, abi, false, true,false/*unsafe,const*/) );
break; }
// `unsafe fn`
case TOK_RWORD_FN:
GET_CHECK_TOK(tok, lex, TOK_IDENT);
- item_name = mv$(tok.ident_str());
+ item_name = mv$(tok.str());
// - self not allowed, not prototype
item_data = ::AST::Item( Parse_FunctionDefWithCode(lex, ABI_RUST, false, true,false/*unsafe,const*/) );
break;
// `unsafe trait`
case TOK_RWORD_TRAIT:
GET_CHECK_TOK(tok, lex, TOK_IDENT);
- item_name = mv$(tok.ident_str());
+ item_name = mv$(tok.str());
// TODO: Mark as unsafe
meta_items.push_back( AST::MetaItem("#UNSAFE") );
item_data = ::AST::Item( Parse_TraitDef(lex, meta_items) );
@@ -1610,33 +1610,33 @@ void Parse_Use(TokenStream& lex, ::std::function<void(AST::UseStmt, ::std::strin // `fn`
case TOK_RWORD_FN:
GET_CHECK_TOK(tok, lex, TOK_IDENT);
- item_name = mv$(tok.ident_str());
+ item_name = mv$(tok.str());
// - self not allowed, not prototype
item_data = ::AST::Item( Parse_FunctionDefWithCode(lex, ABI_RUST, false, false,false/*unsafe,const*/) );
break;
// `type`
case TOK_RWORD_TYPE:
GET_CHECK_TOK(tok, lex, TOK_IDENT);
- item_name = mv$(tok.ident_str());
+ item_name = mv$(tok.str());
item_data = ::AST::Item( Parse_TypeAlias(lex) );
break;
// `struct`
case TOK_RWORD_STRUCT:
GET_CHECK_TOK(tok, lex, TOK_IDENT);
- item_name = mv$(tok.ident_str());
+ item_name = mv$(tok.str());
item_data = ::AST::Item( Parse_Struct(lex, meta_items) );
break;
// `enum`
case TOK_RWORD_ENUM:
GET_CHECK_TOK(tok, lex, TOK_IDENT);
- item_name = mv$(tok.ident_str());
+ item_name = mv$(tok.str());
item_data = ::AST::Item( Parse_EnumDef(lex, meta_items) );
break;
// Contextual keywords
case TOK_IDENT:
- if( tok.ident_str() == "union" ) {
+ if( tok.str() == "union" ) {
GET_CHECK_TOK(tok, lex, TOK_IDENT);
- item_name = mv$(tok.ident_str());
+ item_name = mv$(tok.str());
item_data = ::AST::Item( Parse_Union(lex, meta_items) );
}
else {
@@ -1649,13 +1649,13 @@ void Parse_Use(TokenStream& lex, ::std::function<void(AST::UseStmt, ::std::strin // `trait`
case TOK_RWORD_TRAIT:
GET_CHECK_TOK(tok, lex, TOK_IDENT);
- item_name = mv$(tok.ident_str());
+ item_name = mv$(tok.str());
item_data = ::AST::Item( Parse_TraitDef(lex, meta_items) );
break;
case TOK_RWORD_MOD: {
GET_CHECK_TOK(tok, lex, TOK_IDENT);
- auto name = mv$(tok.ident_str());
+ auto name = mv$(tok.str());
DEBUG("Sub module '" << name << "'");
AST::Module submod( mod_path + name );
diff --git a/src/parse/token.cpp b/src/parse/token.cpp index 9f84df79..95adf594 100644 --- a/src/parse/token.cpp +++ b/src/parse/token.cpp @@ -53,12 +53,7 @@ Token::Token(enum eTokenType type, ::std::string str): m_type(type), m_data(Data::make_String(mv$(str))) { - assert( m_type != TOK_IDENT ); } -Token::Token(Ident ident): - m_type(TOK_IDENT), - m_data(Data( mv$(ident) )) -{} Token::Token(uint64_t val, enum eCoreType datatype): m_type(TOK_INTEGER), m_data( Data::make_Integer({datatype, val}) ) @@ -157,7 +152,6 @@ Token::Token(const Token& t): (None, ), (String, m_data = Data::make_String(e); ), (Integer, m_data = Data::make_Integer(e);), - (Ident , m_data = Data(e);), (Float, m_data = Data::make_Float(e);), (Fragment, BUG(t.m_pos, "Attempted to copy a fragment - " << t);) ) @@ -177,7 +171,6 @@ Token Token::clone() const (Integer, rv.m_data = Data::make_Integer(e); ), - (Ident, rv.m_data = Data(e); ), (Float, rv.m_data = Data::make_Float(e); ), @@ -305,8 +298,9 @@ struct EscapedString { case TOK_INTERPOLATED_BLOCK: return "/*:block*/"; case TOK_INTERPOLATED_META: return "/*:meta*/"; case TOK_INTERPOLATED_ITEM: return "/*:item*/"; + case TOK_INTERPOLATED_IDENT: return "/*:ident*/"; // Value tokens - case TOK_IDENT: return m_data.as_Ident().name; + case TOK_IDENT: return m_data.as_String(); case TOK_MACRO: return m_data.as_String() + "!"; case TOK_LIFETIME: return "'" + m_data.as_String(); case TOK_INTEGER: return FMT(m_data.as_Integer().m_intval); // TODO: suffix for type @@ -471,10 +465,6 @@ SERIALISE_TYPE(Token::, "Token", { s % e.m_datatype; s.item( e.m_intval ); ), - (Ident, - // Hygine contex for serialised idents is left for the loader - s << e.name; - ), (Float, s % e.m_datatype; s.item( e.m_floatval ); @@ -507,12 +497,6 @@ SERIALISE_TYPE(Token::, "Token", { s.item( v ); m_data = Token::Data::make_Integer({dt, v}); break; } - case Token::Data::TAG_Ident: { - ::std::string str; - s.item( str ); - // - Hygine is populated by caller, fill with dummy values for now - m_data = Token::Data::make_Ident(Ident { {~0u,{}}, mv$(str) }); - break; } case Token::Data::TAG_Float: { enum eCoreType dt; double v; diff --git a/src/parse/token.hpp b/src/parse/token.hpp index 94458f1f..3e7f7c24 100644 --- a/src/parse/token.hpp +++ b/src/parse/token.hpp @@ -61,7 +61,6 @@ class Token: TAGGED_UNION(Data, None, (None, struct {}), (String, ::std::string), - (Ident, Ident), (Integer, struct { enum eCoreType m_datatype; uint64_t m_intval; @@ -99,7 +98,6 @@ public: Token(enum eTokenType type); Token(enum eTokenType type, ::std::string str); - Token(Ident ident); Token(uint64_t val, enum eCoreType datatype); Token(double val, enum eCoreType datatype); Token(const InterpolatedFragment& ); @@ -113,11 +111,6 @@ public: uint64_t intval() const { return m_data.as_Integer().m_intval; } double floatval() const { return m_data.as_Float().m_floatval; } - const Ident& ident() const { return m_data.as_Ident(); } - Ident take_ident() { return ::std::move(m_data.as_Ident()); } - ::std::string& ident_str() { return m_data.as_Ident().name; } - const ::std::string& ident_str() const { return m_data.as_Ident().name; } - TypeRef& frag_type() { assert(m_type == TOK_INTERPOLATED_TYPE); return *reinterpret_cast<TypeRef*>( m_data.as_Fragment() ); } AST::Path& frag_path() { assert(m_type == TOK_INTERPOLATED_PATH); return *reinterpret_cast<AST::Path*>( m_data.as_Fragment() ); } AST::Pattern& frag_pattern() { assert(m_type == TOK_INTERPOLATED_PATTERN); return *reinterpret_cast<AST::Pattern*>( m_data.as_Fragment() ); } @@ -131,7 +124,6 @@ public: TU_MATCH(Data, (m_data, r.m_data), (e, re), (None, return true;), (String, return e == re; ), - (Ident, return e == re; ), (Integer, return e.m_datatype == re.m_datatype && e.m_intval == re.m_intval;), (Float, return e.m_datatype == re.m_datatype && e.m_floatval == re.m_floatval;), (Fragment, assert(!"Token equality on Fragment");) diff --git a/src/parse/tokentree.hpp b/src/parse/tokentree.hpp index 82c4f97d..627d11e3 100644 --- a/src/parse/tokentree.hpp +++ b/src/parse/tokentree.hpp @@ -5,6 +5,7 @@ class TokenTree
{
+ Ident::Hygiene m_hygiene;
Token m_tok;
::std::vector<TokenTree> m_subtrees;
public:
@@ -16,7 +17,8 @@ public: m_tok( ::std::move(tok) )
{
}
- TokenTree(::std::vector<TokenTree> subtrees):
+ TokenTree(Ident::Hygiene hygiene, ::std::vector<TokenTree> subtrees):
+ m_hygiene( ::std::move(hygiene) ),
m_subtrees( ::std::move(subtrees) )
{
}
@@ -33,6 +35,7 @@ public: TokenTree& operator[](unsigned int idx) { return m_subtrees[idx]; }
const Token& tok() const { return m_tok; }
Token& tok() { return m_tok; }
+ const Ident::Hygiene& hygiene() const { return m_hygiene; }
friend ::std::ostream& operator<<(::std::ostream& os, const TokenTree& tt) {
if( tt.m_subtrees.size() == 0 )
@@ -56,16 +59,18 @@ class TTStream: public TokenStream
{
::std::vector< ::std::pair<unsigned int, const TokenTree*> > m_stack;
+ const Ident::Hygiene* m_hygiene_ptr = nullptr;
public:
TTStream(const TokenTree& input_tt);
~TTStream();
TTStream& operator=(const TTStream& x) { m_stack = x.m_stack; return *this; }
- virtual Position getPosition() const override;
+ Position getPosition() const override;
+ Ident::Hygiene getHygiene() const override;
protected:
- virtual Token realGetToken() override;
+ Token realGetToken() override;
};
class TTStreamO:
@@ -82,10 +87,11 @@ public: TTStreamO& operator=(const TTStreamO& x) { m_stack = x.m_stack; return *this; }
TTStreamO& operator=(TTStreamO&& x) = default;
- virtual Position getPosition() const override;
+ Position getPosition() const override;
+ Ident::Hygiene getHygiene() const override;
protected:
- virtual Token realGetToken() override;
+ Token realGetToken() override;
};
// unwrapped = Exclude the enclosing brackets (used by macro parse code)
diff --git a/src/resolve/absolute.cpp b/src/resolve/absolute.cpp index a37e0fce..38f7b27b 100644 --- a/src/resolve/absolute.cpp +++ b/src/resolve/absolute.cpp @@ -259,7 +259,7 @@ struct Context } return ""; } - AST::Path lookup(const Span& sp, const ::std::string& name, const Ident::Hygine& src_context, LookupMode mode) const { + AST::Path lookup(const Span& sp, const ::std::string& name, const Ident::Hygiene& src_context, LookupMode mode) const { auto rv = this->lookup_opt(name, src_context, mode); if( !rv.is_valid() ) { switch(mode) @@ -352,7 +352,7 @@ struct Context } return false; } - AST::Path lookup_opt(const ::std::string& name, const Ident::Hygine& src_context, LookupMode mode) const { + AST::Path lookup_opt(const ::std::string& name, const Ident::Hygiene& src_context, LookupMode mode) const { for(auto it = m_name_context.rbegin(); it != m_name_context.rend(); ++ it) { @@ -379,7 +379,7 @@ struct Context else { for( auto it2 = e.variables.rbegin(); it2 != e.variables.rend(); ++ it2 ) { - if( it2->first.name == name && it2->first.hygine.is_visible(src_context) ) { + if( it2->first.name == name && it2->first.hygiene.is_visible(src_context) ) { ::AST::Path rv(name); rv.bind_variable( it2->second ); return rv; @@ -1244,7 +1244,7 @@ void Resolve_Absolute_Path(/*const*/ Context& context, const Span& sp, Context:: if(e.nodes.size() > 1) { // Look up type/module name - auto p = context.lookup(sp, e.nodes[0].name(), e.hygine, Context::LookupMode::Namespace); + auto p = context.lookup(sp, e.nodes[0].name(), e.hygiene, Context::LookupMode::Namespace); DEBUG("Found type/mod - " << p); // HACK: If this is a primitive name, and resolved to a module. // - If the next component isn't found in the located module @@ -1333,7 +1333,7 @@ void Resolve_Absolute_Path(/*const*/ Context& context, const Span& sp, Context:: } else { // Look up value - auto p = context.lookup(sp, e.nodes[0].name(), e.hygine, mode); + auto p = context.lookup(sp, e.nodes[0].name(), e.hygiene, mode); //DEBUG("Found path " << p << " for " << path); if( p.is_absolute() ) { assert( !p.nodes().empty() ); @@ -1715,7 +1715,7 @@ void Resolve_Absolute_Pattern(Context& context, bool allow_refutable, ::AST::Pa if( allow_refutable ) { auto name = mv$( e.name ); // Attempt to resolve the name in the current namespace, and if it fails, it's a binding - auto p = context.lookup_opt( name.name, name.hygine, Context::LookupMode::Pattern ); + auto p = context.lookup_opt( name.name, name.hygiene, Context::LookupMode::Pattern ); if( p.is_valid() ) { Resolve_Absolute_Path(context, pat.span(), Context::LookupMode::Pattern, p); pat = ::AST::Pattern(::AST::Pattern::TagValue(), ::AST::Pattern::Value::make_Named(mv$(p))); |