From 983ff1c26e15a13e6a1e0e4bb1e964ec496a4777 Mon Sep 17 00:00:00 2001 From: John Hodge Date: Wed, 2 Nov 2016 19:38:45 +0800 Subject: Parse - Rework hygiene (with spelling fix) for take 2 --- src/ast/path.cpp | 2 +- src/ast/path.hpp | 6 ++-- src/ast/pattern.hpp | 2 +- src/expand/env.cpp | 2 +- src/expand/format_args.cpp | 22 ++++++------ src/ident.cpp | 45 +++++------------------ src/include/ident.hpp | 48 ++++++++++++++----------- src/macro_rules/eval.cpp | 20 +++++++++-- src/macro_rules/parse.cpp | 6 ++-- src/parse/eTokenType.enum.h | 1 + src/parse/expr.cpp | 8 ++--- src/parse/lex.cpp | 62 +++++++++++++++++--------------- src/parse/lex.hpp | 13 ++++--- src/parse/paths.cpp | 7 ++-- src/parse/pattern.cpp | 19 +++++----- src/parse/root.cpp | 88 ++++++++++++++++++++++----------------------- src/parse/token.cpp | 20 ++--------- src/parse/token.hpp | 8 ----- src/parse/tokentree.hpp | 16 ++++++--- src/resolve/absolute.cpp | 12 +++---- 20 files changed, 196 insertions(+), 211 deletions(-) diff --git a/src/ast/path.cpp b/src/ast/path.cpp index 25c0c208..ac023c8f 100644 --- a/src/ast/path.cpp +++ b/src/ast/path.cpp @@ -162,7 +162,7 @@ AST::Path::Path(const Path& x): m_class = Class::make_Local({ent.name}); ), (Relative, - m_class = Class::make_Relative({ent.hygine, ent.nodes}); + m_class = Class::make_Relative({ent.hygiene, ent.nodes}); ), (Self, m_class = Class::make_Self({ent.nodes}); diff --git a/src/ast/path.hpp b/src/ast/path.hpp index 5c2481a0..b45767a2 100644 --- a/src/ast/path.hpp +++ b/src/ast/path.hpp @@ -162,7 +162,7 @@ public: ::std::string name; } ), (Relative, struct { // General relative - Ident::Hygine hygine; // Taken from the first ident + Ident::Hygiene hygiene; ::std::vector nodes; } ), (Self, struct { // Module-relative @@ -226,8 +226,8 @@ public: // RELATIVE struct TagRelative {}; - Path(TagRelative, Ident::Hygine hygine, ::std::vector nodes): - m_class( Class::make_Relative({ mv$(hygine), mv$(nodes) }) ) + Path(TagRelative, Ident::Hygiene hygiene, ::std::vector nodes): + m_class( Class::make_Relative({ mv$(hygiene), mv$(nodes) }) ) {} // SELF struct TagSelf {}; diff --git a/src/ast/pattern.hpp b/src/ast/pattern.hpp index 0dff4071..a2df7b59 100644 --- a/src/ast/pattern.hpp +++ b/src/ast/pattern.hpp @@ -35,7 +35,7 @@ public: unsigned int m_slot; PatternBinding(): - m_name({0,{}}, ""), + m_name({}, ""), m_type(Type::MOVE), m_mutable(false), m_slot( ~0u ) diff --git a/src/expand/env.cpp b/src/expand/env.cpp index 989fbb7a..85742549 100644 --- a/src/expand/env.cpp +++ b/src/expand/env.cpp @@ -67,7 +67,7 @@ class CExpanderOptionEnv: rv.push_back( Token(TOK_PAREN_OPEN) ); rv.push_back( Token(TOK_STRING, var_val_cstr) ); rv.push_back( Token(TOK_PAREN_CLOSE) ); - return box$( TTStreamO(TokenTree( mv$(rv) )) ); + return box$( TTStreamO(TokenTree( {}, mv$(rv) )) ); } } }; diff --git a/src/expand/format_args.cpp b/src/expand/format_args.cpp index 977d7d69..4857a6c2 100644 --- a/src/expand/format_args.cpp +++ b/src/expand/format_args.cpp @@ -365,17 +365,17 @@ namespace { break; case ::AST::Crate::LOAD_CORE: toks.push_back( TokenTree(TOK_DOUBLE_COLON) ); - toks.push_back( Token(Ident("core")) ); + toks.push_back( Token(TOK_IDENT, "core") ); break; case ::AST::Crate::LOAD_STD: toks.push_back( TokenTree(TOK_DOUBLE_COLON) ); - toks.push_back( Token(Ident("std")) ); + toks.push_back( Token(TOK_IDENT, "std") ); break; } for(auto ent : il) { toks.push_back( TokenTree(TOK_DOUBLE_COLON) ); - toks.push_back( Token(Ident(ent)) ); + toks.push_back( Token(TOK_IDENT, ent) ); } } } @@ -417,7 +417,7 @@ class CFormatArgsExpander: if( lex.lookahead(0) == TOK_IDENT && lex.lookahead(1) == TOK_EQUAL ) { GET_CHECK_TOK(tok, lex, TOK_IDENT); - auto name = mv$(tok.ident_str()); + auto name = mv$(tok.str()); GET_CHECK_TOK(tok, lex, TOK_EQUAL); @@ -478,7 +478,7 @@ class CFormatArgsExpander: toks.push_back( TokenTree(TOK_PAREN_OPEN) ); for(unsigned int i = 0; i < free_args.size() + named_args.size(); i ++ ) { - toks.push_back( Token(Ident(FMT("a" << i))) ); + toks.push_back( Token(TOK_IDENT, FMT("a" << i)) ); toks.push_back( TokenTree(TOK_COMMA) ); } toks.push_back( TokenTree(TOK_PAREN_CLOSE) ); @@ -490,13 +490,13 @@ class CFormatArgsExpander: // - Contains N+1 entries, where N is the number of fragments { toks.push_back( TokenTree(TOK_RWORD_STATIC) ); - toks.push_back( Token(Ident("FRAGMENTS")) ); + toks.push_back( Token(TOK_IDENT, "FRAGMENTS") ); toks.push_back( TokenTree(TOK_COLON) ); toks.push_back( TokenTree(TOK_SQUARE_OPEN) ); toks.push_back( Token(TOK_AMP) ); toks.push_back( Token(TOK_LIFETIME, "static") ); - toks.push_back( Token(Ident("str")) ); + toks.push_back( Token(TOK_IDENT, "str") ); toks.push_back( Token(TOK_SEMICOLON) ); toks.push_back( Token(fragments.size() + 1, CORETYPE_UINT) ); toks.push_back( TokenTree(TOK_SQUARE_CLOSE) ); @@ -522,7 +522,7 @@ class CFormatArgsExpander: toks.push_back( TokenTree(TOK_PAREN_OPEN) ); { toks.push_back( TokenTree(TOK_AMP) ); - toks.push_back( Token(Ident("FRAGMENTS")) ); + toks.push_back( Token(TOK_IDENT, "FRAGMENTS") ); toks.push_back( TokenTree(TOK_COMMA) ); toks.push_back( TokenTree(TOK_AMP) ); @@ -531,7 +531,7 @@ class CFormatArgsExpander: { push_path(toks, crate, {"fmt", "ArgumentV1", "new"}); toks.push_back( Token(TOK_PAREN_OPEN) ); - toks.push_back( Token(Ident(FMT("a" << frag.arg_index))) ); + toks.push_back( Token(TOK_IDENT, FMT("a" << frag.arg_index)) ); toks.push_back( TokenTree(TOK_COMMA) ); @@ -554,7 +554,7 @@ class CFormatArgsExpander: toks.push_back( TokenTree(TOK_PAREN_OPEN) ); { toks.push_back( TokenTree(TOK_AMP) ); - toks.push_back( Token(Ident("FRAGMENTS")) ); + toks.push_back( Token(TOK_IDENT, "FRAGMENTS") ); toks.push_back( TokenTree(TOK_COMMA) ); // 1. Generate a set of arguments+formatters @@ -574,7 +574,7 @@ class CFormatArgsExpander: toks.push_back( TokenTree(TOK_BRACE_CLOSE) ); toks.push_back( TokenTree(TOK_BRACE_CLOSE) ); - return box$( TTStreamO(TokenTree(mv$(toks))) ); + return box$( TTStreamO(TokenTree(Ident::Hygiene::new_scope(), mv$(toks))) ); } }; diff --git a/src/ident.cpp b/src/ident.cpp index 5d0fe00c..a5a6bdbb 100644 --- a/src/ident.cpp +++ b/src/ident.cpp @@ -3,49 +3,20 @@ * - By John Hodge (Mutabah/thePowersGang) * * include/ident.cpp - * - Identifiers with hygine + * - Identifiers with hygiene */ #include #include #include #include // vector print -bool Ident::Hygine::is_visible(const Hygine& src) const +unsigned int Ident::Hygiene::g_next_scope = 0; + +bool Ident::Hygiene::is_visible(const Hygiene& src) const { - // HACK: Disable hygine for now + // HACK: Disable hygiene for now return true; - - DEBUG("*this = " << *this << ", src=" << src); - if( this->file_num != src.file_num ) { - DEBUG("- Different file"); - return false; - } - - // `this` is the item, `src` is the ident used to access it - - // If this is from a deeper point than the source, it won't work. - if( this->indexes.size() > src.indexes.size() ) { - DEBUG("- Not subset: len"); - return false; - } - - // If this hygine list (barring the last) is a subset of the source - for(unsigned int i = 0; i < this->indexes.size()-1 - 1; i ++) - { - if( this->indexes[i] != src.indexes[i] ) { - DEBUG("- Not subset: " << i); - return false; - } - } - - unsigned int end = this->indexes.size()-1-1; - // Allow match if this ident is from before the addressing ident - if( this->indexes[end] < src.indexes[end] ) { - return true; - } - - DEBUG("- Not before"); - return false; + //return x.scope_index == src.scope_index; } ::std::ostream& operator<<(::std::ostream& os, const Ident& x) { @@ -53,8 +24,8 @@ bool Ident::Hygine::is_visible(const Hygine& src) const return os; } -::std::ostream& operator<<(::std::ostream& os, const Ident::Hygine& x) { - os << "{" << x.file_num << ": [" << x.indexes << "]}"; +::std::ostream& operator<<(::std::ostream& os, const Ident::Hygiene& x) { + os << "{" << x.scope_index << "}"; return os; } diff --git a/src/include/ident.hpp b/src/include/ident.hpp index d5ce2e77..cff918ca 100644 --- a/src/include/ident.hpp +++ b/src/include/ident.hpp @@ -3,7 +3,7 @@ * - By John Hodge (Mutabah/thePowersGang) * * include/ident.hpp - * - Identifiers with hygine + * - Identifiers with hygiene */ #pragma once #include @@ -11,42 +11,50 @@ struct Ident { - struct Hygine + class Hygiene { - unsigned int file_num; - ::std::vector indexes; + static unsigned g_next_scope; + unsigned int scope_index; - Hygine(unsigned int file, ::std::vector indexes): - file_num(file), - indexes(::std::move(indexes)) + Hygiene(unsigned int index): + scope_index(index) {} + public: + Hygiene(): + scope_index(0) + {} + + static Hygiene new_scope() + { + return Hygiene(++g_next_scope); + } - Hygine(Hygine&& x) = default; - Hygine(const Hygine& x) = default; - Hygine& operator=(Hygine&& x) = default; - Hygine& operator=(const Hygine& x) = default; + Hygiene(Hygiene&& x) = default; + Hygiene(const Hygiene& x) = default; + Hygiene& operator=(Hygiene&& x) = default; + Hygiene& operator=(const Hygiene& x) = default; // Returns true if an ident with hygine `souce` can see an ident with this hygine - bool is_visible(const Hygine& source) const; - bool operator==(const Hygine& x) const { return file_num == x.file_num && indexes == x.indexes; } - bool operator!=(const Hygine& x) const { return file_num != x.file_num || indexes != x.indexes; } + bool is_visible(const Hygiene& source) const; + bool operator==(const Hygiene& x) const { return scope_index == x.scope_index; } + bool operator!=(const Hygiene& x) const { return scope_index != x.scope_index; } - friend ::std::ostream& operator<<(::std::ostream& os, const Hygine& v); + friend ::std::ostream& operator<<(::std::ostream& os, const Hygiene& v); }; - Hygine hygine; + Hygiene hygiene; ::std::string name; Ident(const char* name): - hygine(~0u, {}), + hygiene(), name(name) { } Ident(::std::string name): - hygine(~0u, {}), + hygiene(), name(::std::move(name)) { } - Ident(Hygine hygine, ::std::string name): - hygine(::std::move(hygine)), name(::std::move(name)) + Ident(Hygiene hygiene, ::std::string name): + hygiene(::std::move(hygiene)), name(::std::move(name)) { } Ident(Ident&& x) = default; diff --git a/src/macro_rules/eval.cpp b/src/macro_rules/eval.cpp index 31665a25..110e8525 100644 --- a/src/macro_rules/eval.cpp +++ b/src/macro_rules/eval.cpp @@ -566,6 +566,7 @@ class MacroExpander: Token m_next_token; // used for inserting a single token into the stream ::std::unique_ptr m_ttstream; + Ident::Hygiene m_hygiene; public: MacroExpander(const MacroExpander& x) = delete; @@ -574,12 +575,14 @@ public: m_macro_filename( FMT("Macro:" << macro_name) ), m_crate_name( mv$(crate_name) ), m_mappings( mv$(mappings) ), - m_state( contents, m_mappings ) + m_state( contents, m_mappings ), + m_hygiene( Ident::Hygiene::new_scope() ) { } - virtual Position getPosition() const override; - virtual Token realGetToken() override; + Position getPosition() const override; + Ident::Hygiene getHygiene() const override; + Token realGetToken() override; }; void Macro_InitDefaults() @@ -1044,6 +1047,17 @@ Position MacroExpander::getPosition() const // TODO: Return a far better span - invocaion location? return Position(m_macro_filename, 0, m_state.top_pos()); } +Ident::Hygiene MacroExpander::getHygiene() const +{ + if( m_ttstream ) + { + return m_ttstream->getHygiene(); + } + else + { + return m_hygiene; + } +} Token MacroExpander::realGetToken() { // Use m_next_token first diff --git a/src/macro_rules/parse.cpp b/src/macro_rules/parse.cpp index bd54c67d..c7a6ef89 100644 --- a/src/macro_rules/parse.cpp +++ b/src/macro_rules/parse.cpp @@ -49,10 +49,10 @@ public: switch( GET_TOK(tok, lex) ) { case TOK_IDENT: { - ::std::string name = mv$(tok.ident_str()); + ::std::string name = mv$(tok.str()); GET_CHECK_TOK(tok, lex, TOK_COLON); GET_CHECK_TOK(tok, lex, TOK_IDENT); - ::std::string type = mv$(tok.ident_str()); + ::std::string type = mv$(tok.str()); unsigned int idx = ::std::find( names.begin(), names.end(), name ) - names.begin(); if( idx == names.size() ) @@ -202,7 +202,7 @@ public: else if( tok.type() == TOK_IDENT ) { // Look up the named parameter in the list of param names for this arm - const auto& name = tok.ident_str(); + const auto& name = tok.str(); unsigned int idx = ::std::find(var_names.begin(), var_names.end(), name) - var_names.begin(); if( idx == var_names.size() ) ERROR(lex.getPosition(), E0000, "Macro variable $" << name << " not found"); diff --git a/src/parse/eTokenType.enum.h b/src/parse/eTokenType.enum.h index 9caa3aba..8d410b0c 100644 --- a/src/parse/eTokenType.enum.h +++ b/src/parse/eTokenType.enum.h @@ -5,6 +5,7 @@ _(TOK_NEWLINE) _(TOK_WHITESPACE) _(TOK_COMMENT) +_(TOK_INTERPOLATED_IDENT) _(TOK_INTERPOLATED_PATH) _(TOK_INTERPOLATED_TYPE) _(TOK_INTERPOLATED_PATTERN) diff --git a/src/parse/expr.cpp b/src/parse/expr.cpp index f6ebca1f..832e6a4a 100644 --- a/src/parse/expr.cpp +++ b/src/parse/expr.cpp @@ -889,7 +889,7 @@ ExprNodeP Parse_ExprFC(TokenStream& lex) switch(GET_TOK(tok, lex)) { case TOK_IDENT: { - AST::PathNode path( mv$(tok.ident_str()) , {}); + AST::PathNode path( mv$(tok.str()) , {}); switch( GET_TOK(tok, lex) ) { case TOK_PAREN_OPEN: @@ -965,7 +965,7 @@ ExprNodeP Parse_ExprVal_StructLiteral(TokenStream& lex, AST::Path path) ::std::vector< ::std::pair< ::std::string, ::std::unique_ptr> > items; while( GET_TOK(tok, lex) == TOK_IDENT ) { - auto name = mv$(tok.ident_str()); + auto name = mv$(tok.str()); GET_CHECK_TOK(tok, lex, TOK_COLON); ExprNodeP val = Parse_Stmt(lex); items.push_back( ::std::make_pair(::std::move(name), ::std::move(val)) ); @@ -1224,7 +1224,7 @@ ExprNodeP Parse_ExprMacro(TokenStream& lex, Token tok) ::std::string name = tok.str(); ::std::string ident; if( GET_TOK(tok, lex) == TOK_IDENT ) { - ident = mv$(tok.ident_str()); + ident = mv$(tok.str()); } else { PUTBACK(tok, lex); @@ -1285,5 +1285,5 @@ TokenTree Parse_TT(TokenStream& lex, bool unwrapped) } if( !unwrapped ) items.push_back( mv$(tok) ); - return TokenTree(mv$(items)); + return TokenTree(lex.getHygiene(), mv$(items)); } diff --git a/src/parse/lex.cpp b/src/parse/lex.cpp index a5c4e372..9acb56e0 100644 --- a/src/parse/lex.cpp +++ b/src/parse/lex.cpp @@ -29,7 +29,8 @@ Lexer::Lexer(const ::std::string& filename): m_line(1), m_line_ofs(0), m_istream(filename.c_str()), - m_last_char_valid(false) + m_last_char_valid(false), + m_hygiene( Ident::Hygiene::new_scope() ) { if( !m_istream.is_open() ) { @@ -243,6 +244,10 @@ Position Lexer::getPosition() const { return Position(m_path, m_line, m_line_ofs); } +Ident::Hygiene Lexer::getHygiene() const +{ + return m_hygiene; +} Token Lexer::realGetToken() { while(true) @@ -251,29 +256,6 @@ Token Lexer::realGetToken() //::std::cout << "getTokenInt: tok = " << tok << ::std::endl; switch(tok.type()) { - case TOK_IDENT: - if( m_hygine_stack.size() > 0 ) - { - m_hygine_stack.back() ++; - } - return tok; - - case TOK_PAREN_OPEN: - case TOK_SQUARE_OPEN: - case TOK_BRACE_OPEN: - case TOK_ATTR_OPEN: - case TOK_CATTR_OPEN: - m_hygine_stack.push_back(0); - return tok; - case TOK_PAREN_CLOSE: - case TOK_SQUARE_CLOSE: - case TOK_BRACE_CLOSE: - m_hygine_stack.pop_back(); - if( m_hygine_stack.size() > 0 ) { - m_hygine_stack.back() ++; - } - return tok; - case TOK_NEWLINE: m_line ++; m_line_ofs = 0; @@ -790,7 +772,7 @@ Token Lexer::getTokenInt_Identifier(Codepoint leader, Codepoint leader2) if( str < RWORDS[i].chars ) break; if( str == RWORDS[i].chars ) return Token((enum eTokenType)RWORDS[i].type); } - return Token(Ident( Ident::Hygine(m_file_index, m_hygine_stack), mv$(str) )); + return Token(TOK_IDENT, mv$(str)); } } @@ -1022,22 +1004,29 @@ Token TTStream::realGetToken() const TokenTree& subtree = tree[idx]; idx ++; if( subtree.size() == 0 ) { + m_hygiene_ptr = &m_stack.back().second->hygiene(); return subtree.tok().clone(); } else { - m_stack.push_back( ::std::make_pair(0, &subtree ) ); + m_stack.push_back( ::std::make_pair(0, &subtree) ); } } else { m_stack.pop_back(); } } + //m_hygiene = nullptr; return Token(TOK_EOF); } Position TTStream::getPosition() const { return Position("TTStream", 0,0); } +Ident::Hygiene TTStream::getHygiene() const +{ + //assert( m_hygiene ); + return *m_hygiene_ptr; +} TTStreamO::TTStreamO(TokenTree input_tt): @@ -1071,7 +1060,7 @@ Token TTStreamO::realGetToken() return mv$( subtree.tok() ); } else { - m_stack.push_back( ::std::make_pair(0, &subtree ) ); + m_stack.push_back( ::std::make_pair(0, &subtree) ); } } else { @@ -1084,6 +1073,10 @@ Position TTStreamO::getPosition() const { return m_last_pos; } +Ident::Hygiene TTStreamO::getHygiene() const +{ + return (m_stack.back().second ? m_stack.back().second->hygiene() : m_input_tt.hygiene()); +} TokenStream::TokenStream(): @@ -1183,7 +1176,18 @@ Span TokenStream::end_span(ProtoSpan ps) const p.line, p.ofs ); } - +Ident TokenStream::get_ident(Token tok) const +{ + if(tok.type() == TOK_IDENT) { + return Ident(getHygiene(), tok.str()); + } + else if( tok.type() == TOK_INTERPOLATED_IDENT ) { + TODO(getPosition(), ""); + } + else { + throw ParseError::Unexpected(*this, tok); + } +} TokenTree TokenTree::clone() const { @@ -1195,7 +1199,7 @@ TokenTree TokenTree::clone() const ents.reserve( m_subtrees.size() ); for(const auto& sub : m_subtrees) ents.push_back( sub.clone() ); - return TokenTree( mv$(ents) ); + return TokenTree( m_hygiene, mv$(ents) ); } } diff --git a/src/parse/lex.hpp b/src/parse/lex.hpp index 208fa670..d147d79d 100644 --- a/src/parse/lex.hpp +++ b/src/parse/lex.hpp @@ -62,13 +62,17 @@ public: Token getToken(); void putback(Token tok); eTokenType lookahead(unsigned int count); + virtual Position getPosition() const = 0; + virtual Ident::Hygiene getHygiene() const = 0; ParseState& parse_state() { return m_parse_state; } ProtoSpan start_span() const; Span end_span(ProtoSpan ps) const; + Ident get_ident(Token tok) const; + protected: virtual Token realGetToken() = 0; private: @@ -126,14 +130,13 @@ class Lexer: Codepoint m_last_char; Token m_next_token; // Used when lexing generated two tokens - unsigned int m_file_index; - // Incremented on every ident and TT close, pushed on TT open - ::std::vector m_hygine_stack; + Ident::Hygiene m_hygiene; public: Lexer(const ::std::string& filename); - virtual Position getPosition() const override; - virtual Token realGetToken() override; + Position getPosition() const override; + Ident::Hygiene getHygiene() const override; + Token realGetToken() override; private: Token getTokenInt(); diff --git a/src/parse/paths.cpp b/src/parse/paths.cpp index 5e0faf90..e600e51e 100644 --- a/src/parse/paths.cpp +++ b/src/parse/paths.cpp @@ -91,8 +91,9 @@ AST::Path Parse_Path(TokenStream& lex, bool is_abs, eParsePathGenericMode generi } } else { + // TODO: TOK_INTERPOLATED_IDENT? GET_CHECK_TOK(tok, lex, TOK_IDENT); - auto hygine = mv$( tok.ident().hygine ); + auto hygine = lex.getHygiene(); PUTBACK(tok, lex); return AST::Path(AST::Path::TagRelative(), mv$(hygine), Parse_PathNodes(lex, generic_mode)); } @@ -111,7 +112,7 @@ AST::Path Parse_Path(TokenStream& lex, bool is_abs, eParsePathGenericMode generi ::AST::PathParams params; CHECK_TOK(tok, TOK_IDENT); - auto component = mv$( tok.ident_str() ); + auto component = mv$( tok.str() ); GET_TOK(tok, lex); if( generic_mode == PATH_GENERIC_TYPE ) @@ -218,7 +219,7 @@ AST::Path Parse_Path(TokenStream& lex, bool is_abs, eParsePathGenericMode generi case TOK_IDENT: if( LOOK_AHEAD(lex) == TOK_EQUAL ) { - ::std::string name = mv$(tok.ident_str()); + ::std::string name = mv$(tok.str()); GET_CHECK_TOK(tok, lex, TOK_EQUAL); assoc_bounds.push_back( ::std::make_pair( mv$(name), Parse_Type(lex,false) ) ); break; diff --git a/src/parse/pattern.cpp b/src/parse/pattern.cpp index df251bd3..f43a1205 100644 --- a/src/parse/pattern.cpp +++ b/src/parse/pattern.cpp @@ -85,7 +85,7 @@ AST::Pattern Parse_Pattern(TokenStream& lex, bool is_refutable) if( expect_bind ) { CHECK_TOK(tok, TOK_IDENT); - auto bind_name = tok.take_ident(); + auto bind_name = Ident(lex.getHygiene(), mv$(tok.str())); // If there's no '@' after it, it's a name binding only (_ pattern) if( GET_TOK(tok, lex) != TOK_AT ) { @@ -114,21 +114,23 @@ AST::Pattern Parse_Pattern(TokenStream& lex, bool is_refutable) break; // Known binding `ident @` case TOK_AT: - binding = AST::PatternBinding( tok.take_ident(), bind_type/*MOVE*/, is_mut/*false*/ ); + binding = AST::PatternBinding( Ident(lex.getHygiene(), mv$(tok.str())), bind_type/*MOVE*/, is_mut/*false*/ ); GET_TOK(tok, lex); // '@' GET_TOK(tok, lex); // Match lex.putback() below break; - default: // Maybe bind + default: { // Maybe bind + Ident name = Ident(lex.getHygiene(), mv$(tok.str())); // if the pattern can be refuted (i.e this could be an enum variant), return MaybeBind if( is_refutable ) { assert(bind_type == ::AST::PatternBinding::Type::MOVE); assert(is_mut == false); - return AST::Pattern(AST::Pattern::TagMaybeBind(), tok.take_ident()); + return AST::Pattern(AST::Pattern::TagMaybeBind(), mv$(name)); } // Otherwise, it IS a binding else { - return AST::Pattern(AST::Pattern::TagBind(), tok.take_ident(), bind_type, is_mut); + return AST::Pattern(AST::Pattern::TagBind(), mv$(name), bind_type, is_mut); } + break;} } } else @@ -138,7 +140,6 @@ AST::Pattern Parse_Pattern(TokenStream& lex, bool is_refutable) PUTBACK(tok, lex); auto pat = Parse_PatternReal(lex, is_refutable); - //pat.set_bind( mv$(binding) ); pat.binding() = mv$(binding); return pat; } @@ -276,10 +277,10 @@ AST::Pattern Parse_PatternReal_Slice(TokenStream& lex, bool is_refutable) ::AST::PatternBinding binding; if( tok.type() == TOK_RWORD_REF && lex.lookahead(0) == TOK_IDENT && lex.lookahead(1) == TOK_DOUBLE_DOT ) { GET_TOK(tok, lex); - binding = ::AST::PatternBinding( tok.take_ident(), ::AST::PatternBinding::Type::REF, false ); + binding = ::AST::PatternBinding( lex.get_ident(mv$(tok)), ::AST::PatternBinding::Type::REF, false ); } else if( tok.type() == TOK_IDENT && lex.lookahead(0) == TOK_DOUBLE_DOT) { - binding = ::AST::PatternBinding( tok.take_ident(), ::AST::PatternBinding::Type::MOVE, false ); + binding = ::AST::PatternBinding( lex.get_ident(mv$(tok)), ::AST::PatternBinding::Type::MOVE, false ); } else if( tok.type() == TOK_UNDERSCORE && lex.lookahead(0) == TOK_DOUBLE_DOT) { // No binding, but switching to trailing @@ -466,7 +467,7 @@ AST::Pattern Parse_PatternStruct(TokenStream& lex, AST::Path path, bool is_refut } CHECK_TOK(tok, TOK_IDENT); - auto field_ident = tok.take_ident(); + auto field_ident = lex.get_ident(mv$(tok)); ::std::string field_name; GET_TOK(tok, lex); diff --git a/src/parse/root.cpp b/src/parse/root.cpp index 6c9acc14..2f5114ed 100644 --- a/src/parse/root.cpp +++ b/src/parse/root.cpp @@ -133,7 +133,7 @@ AST::GenericParams Parse_GenericParams(TokenStream& lex) if( tok.type() == TOK_IDENT ) { // TODO: Hygine - ::std::string param_name = mv$(tok.ident_str()); + ::std::string param_name = mv$(tok.str()); ret.add_ty_param( AST::TypeParam( param_name ) ); auto param_ty = TypeRef(lex.getPosition(), param_name); @@ -529,7 +529,7 @@ AST::Struct Parse_Struct(TokenStream& lex, const AST::MetaItems& meta_items) } CHECK_TOK(tok, TOK_IDENT); - auto name = mv$(tok.ident_str()); + auto name = mv$(tok.str()); GET_CHECK_TOK(tok, lex, TOK_COLON); TypeRef type = Parse_Type(lex); @@ -615,7 +615,7 @@ AST::Trait Parse_TraitDef(TokenStream& lex, const AST::MetaItems& meta_items) } bool is_specialisable = false; - if( tok.type() == TOK_IDENT && tok.ident_str() == "default" ) { + if( tok.type() == TOK_IDENT && tok.str() == "default" ) { is_specialisable = true; GET_TOK(tok, lex); } @@ -629,7 +629,7 @@ AST::Trait Parse_TraitDef(TokenStream& lex, const AST::MetaItems& meta_items) { case TOK_RWORD_STATIC: { GET_CHECK_TOK(tok, lex, TOK_IDENT); - auto name = mv$(tok.ident_str()); + auto name = mv$(tok.str()); GET_CHECK_TOK(tok, lex, TOK_COLON); auto ty = Parse_Type(lex); GET_CHECK_TOK(tok, lex, TOK_SEMICOLON); @@ -646,7 +646,7 @@ AST::Trait Parse_TraitDef(TokenStream& lex, const AST::MetaItems& meta_items) break; } case TOK_RWORD_CONST: { GET_CHECK_TOK(tok, lex, TOK_IDENT); - auto name = mv$(tok.ident_str()); + auto name = mv$(tok.str()); GET_CHECK_TOK(tok, lex, TOK_COLON); auto ty = Parse_Type(lex); @@ -664,7 +664,7 @@ AST::Trait Parse_TraitDef(TokenStream& lex, const AST::MetaItems& meta_items) case TOK_RWORD_TYPE: { auto atype_params = ::AST::GenericParams { }; GET_CHECK_TOK(tok, lex, TOK_IDENT); - auto name = mv$(tok.ident_str()); + auto name = mv$(tok.str()); if( GET_TOK(tok, lex) == TOK_COLON ) { // Bounded associated type @@ -704,7 +704,7 @@ AST::Trait Parse_TraitDef(TokenStream& lex, const AST::MetaItems& meta_items) CHECK_TOK(tok, TOK_RWORD_FN); case TOK_RWORD_FN: { GET_CHECK_TOK(tok, lex, TOK_IDENT); - ::std::string name = tok.take_ident().into_string(); + ::std::string name = mv$(tok.str()); // Self allowed, prototype-form allowed (optional names and no code) auto fcn = Parse_FunctionDef(lex, abi, true, true, fn_is_unsafe, fn_is_const); if( GET_TOK(tok, lex) == TOK_BRACE_OPEN ) @@ -768,7 +768,7 @@ AST::Enum Parse_EnumDef(TokenStream& lex, const AST::MetaItems& meta_items) SET_ATTRS(lex, item_attrs); CHECK_TOK(tok, TOK_IDENT); - ::std::string name = tok.take_ident().into_string(); + ::std::string name = mv$(tok.str()); if( GET_TOK(tok, lex) == TOK_PAREN_OPEN ) { ::std::vector types; @@ -815,7 +815,7 @@ AST::Enum Parse_EnumDef(TokenStream& lex, const AST::MetaItems& meta_items) } GET_CHECK_TOK(tok, lex, TOK_IDENT); - auto name = tok.take_ident().into_string(); + auto name = mv$(tok.str()); GET_CHECK_TOK(tok, lex, TOK_COLON); auto ty = Parse_Type(lex); fields.push_back( ::AST::StructItem(mv$(field_attrs), true, mv$(name), mv$(ty)) ); @@ -888,7 +888,7 @@ AST::Enum Parse_EnumDef(TokenStream& lex, const AST::MetaItems& meta_items) } GET_CHECK_TOK(tok, lex, TOK_IDENT); - auto name = tok.take_ident().into_string(); + auto name = mv$(tok.str()); GET_CHECK_TOK(tok, lex, TOK_COLON); auto ty = Parse_Type(lex); @@ -913,7 +913,7 @@ AST::MetaItem Parse_MetaItem(TokenStream& lex) } CHECK_TOK(tok, TOK_IDENT); - ::std::string name = tok.take_ident().into_string(); + ::std::string name = mv$(tok.str()); switch(GET_TOK(tok, lex)) { case TOK_EQUAL: @@ -1071,7 +1071,7 @@ void Parse_Impl_Item(TokenStream& lex, AST::Impl& impl) } bool is_specialisable = false; - if( tok.type() == TOK_IDENT && tok.ident() == "default" ) { + if( tok.type() == TOK_IDENT && tok.str() == "default" ) { is_specialisable = true; GET_TOK(tok, lex); } @@ -1083,7 +1083,7 @@ void Parse_Impl_Item(TokenStream& lex, AST::Impl& impl) { case TOK_RWORD_TYPE: { GET_CHECK_TOK(tok, lex, TOK_IDENT); - auto name = tok.take_ident().into_string(); + auto name = mv$(tok.str()); GET_CHECK_TOK(tok, lex, TOK_EQUAL); impl.add_type(is_public, is_specialisable, name, Parse_Type(lex)); GET_CHECK_TOK(tok, lex, TOK_SEMICOLON); @@ -1098,7 +1098,7 @@ void Parse_Impl_Item(TokenStream& lex, AST::Impl& impl) if( tok.type() != TOK_RWORD_FN && tok.type() != TOK_RWORD_UNSAFE && !fn_is_unsafe ) { CHECK_TOK(tok, TOK_IDENT); - auto name = tok.take_ident().into_string(); + auto name = mv$(tok.str()); GET_CHECK_TOK(tok, lex, TOK_COLON); auto ty = Parse_Type(lex); GET_CHECK_TOK(tok, lex, TOK_EQUAL); @@ -1134,7 +1134,7 @@ void Parse_Impl_Item(TokenStream& lex, AST::Impl& impl) case TOK_RWORD_FN: { GET_CHECK_TOK(tok, lex, TOK_IDENT); // TODO: Hygine on function names? - Not in impl blocks? - ::std::string name = tok.take_ident().into_string(); + ::std::string name = mv$(tok.str()); DEBUG("Function " << name); // - Self allowed, can't be prototype-form auto fcn = Parse_FunctionDefWithCode(lex, abi, true, fn_is_unsafe, fn_is_const); @@ -1185,7 +1185,7 @@ AST::ExternBlock Parse_ExternBlock(TokenStream& lex, ::std::string abi, ::AST::M { case TOK_RWORD_FN: { GET_CHECK_TOK(tok, lex, TOK_IDENT); - auto name = tok.take_ident().into_string(); + auto name = mv$(tok.str()); // parse function as prototype // - no self, "safe" and not const auto i = ::AST::Item( Parse_FunctionDef(lex, abi, false, true, false,false) ); @@ -1203,7 +1203,7 @@ AST::ExternBlock Parse_ExternBlock(TokenStream& lex, ::std::string abi, ::AST::M else PUTBACK(tok, lex); GET_CHECK_TOK(tok, lex, TOK_IDENT); - auto name = tok.take_ident().into_string(); + auto name = mv$(tok.str()); GET_CHECK_TOK(tok, lex, TOK_COLON); auto type = Parse_Type(lex); GET_CHECK_TOK(tok, lex, TOK_SEMICOLON); @@ -1242,13 +1242,13 @@ void Parse_Use_Set(TokenStream& lex, const ProtoSpan& ps, const AST::Path& base_ } else { CHECK_TOK(tok, TOK_IDENT); - path = base_path + AST::PathNode(tok.ident_str(), {}); - name = mv$(tok.ident_str()); + path = base_path + AST::PathNode(tok.str(), {}); + name = mv$(tok.str()); } if( GET_TOK(tok, lex) == TOK_RWORD_AS ) { GET_CHECK_TOK(tok, lex, TOK_IDENT); - name = mv$(tok.ident_str()); + name = mv$(tok.str()); } else { PUTBACK(tok, lex); @@ -1282,7 +1282,7 @@ void Parse_Use(TokenStream& lex, ::std::function" fn ...` case TOK_RWORD_FN: { GET_CHECK_TOK(tok, lex, TOK_IDENT); - item_name = mv$(tok.ident_str()); + item_name = mv$(tok.str()); item_data = ::AST::Item( Parse_FunctionDefWithCode(lex, abi, false, false,false) ); break; } // `extern "ABI" {` @@ -1462,7 +1462,7 @@ void Parse_Use(TokenStream& lex, ::std::function( m_data.as_Fragment() ); } AST::Path& frag_path() { assert(m_type == TOK_INTERPOLATED_PATH); return *reinterpret_cast( m_data.as_Fragment() ); } AST::Pattern& frag_pattern() { assert(m_type == TOK_INTERPOLATED_PATTERN); return *reinterpret_cast( m_data.as_Fragment() ); } @@ -131,7 +124,6 @@ public: TU_MATCH(Data, (m_data, r.m_data), (e, re), (None, return true;), (String, return e == re; ), - (Ident, return e == re; ), (Integer, return e.m_datatype == re.m_datatype && e.m_intval == re.m_intval;), (Float, return e.m_datatype == re.m_datatype && e.m_floatval == re.m_floatval;), (Fragment, assert(!"Token equality on Fragment");) diff --git a/src/parse/tokentree.hpp b/src/parse/tokentree.hpp index 82c4f97d..627d11e3 100644 --- a/src/parse/tokentree.hpp +++ b/src/parse/tokentree.hpp @@ -5,6 +5,7 @@ class TokenTree { + Ident::Hygiene m_hygiene; Token m_tok; ::std::vector m_subtrees; public: @@ -16,7 +17,8 @@ public: m_tok( ::std::move(tok) ) { } - TokenTree(::std::vector subtrees): + TokenTree(Ident::Hygiene hygiene, ::std::vector subtrees): + m_hygiene( ::std::move(hygiene) ), m_subtrees( ::std::move(subtrees) ) { } @@ -33,6 +35,7 @@ public: TokenTree& operator[](unsigned int idx) { return m_subtrees[idx]; } const Token& tok() const { return m_tok; } Token& tok() { return m_tok; } + const Ident::Hygiene& hygiene() const { return m_hygiene; } friend ::std::ostream& operator<<(::std::ostream& os, const TokenTree& tt) { if( tt.m_subtrees.size() == 0 ) @@ -56,16 +59,18 @@ class TTStream: public TokenStream { ::std::vector< ::std::pair > m_stack; + const Ident::Hygiene* m_hygiene_ptr = nullptr; public: TTStream(const TokenTree& input_tt); ~TTStream(); TTStream& operator=(const TTStream& x) { m_stack = x.m_stack; return *this; } - virtual Position getPosition() const override; + Position getPosition() const override; + Ident::Hygiene getHygiene() const override; protected: - virtual Token realGetToken() override; + Token realGetToken() override; }; class TTStreamO: @@ -82,10 +87,11 @@ public: TTStreamO& operator=(const TTStreamO& x) { m_stack = x.m_stack; return *this; } TTStreamO& operator=(TTStreamO&& x) = default; - virtual Position getPosition() const override; + Position getPosition() const override; + Ident::Hygiene getHygiene() const override; protected: - virtual Token realGetToken() override; + Token realGetToken() override; }; // unwrapped = Exclude the enclosing brackets (used by macro parse code) diff --git a/src/resolve/absolute.cpp b/src/resolve/absolute.cpp index a37e0fce..38f7b27b 100644 --- a/src/resolve/absolute.cpp +++ b/src/resolve/absolute.cpp @@ -259,7 +259,7 @@ struct Context } return ""; } - AST::Path lookup(const Span& sp, const ::std::string& name, const Ident::Hygine& src_context, LookupMode mode) const { + AST::Path lookup(const Span& sp, const ::std::string& name, const Ident::Hygiene& src_context, LookupMode mode) const { auto rv = this->lookup_opt(name, src_context, mode); if( !rv.is_valid() ) { switch(mode) @@ -352,7 +352,7 @@ struct Context } return false; } - AST::Path lookup_opt(const ::std::string& name, const Ident::Hygine& src_context, LookupMode mode) const { + AST::Path lookup_opt(const ::std::string& name, const Ident::Hygiene& src_context, LookupMode mode) const { for(auto it = m_name_context.rbegin(); it != m_name_context.rend(); ++ it) { @@ -379,7 +379,7 @@ struct Context else { for( auto it2 = e.variables.rbegin(); it2 != e.variables.rend(); ++ it2 ) { - if( it2->first.name == name && it2->first.hygine.is_visible(src_context) ) { + if( it2->first.name == name && it2->first.hygiene.is_visible(src_context) ) { ::AST::Path rv(name); rv.bind_variable( it2->second ); return rv; @@ -1244,7 +1244,7 @@ void Resolve_Absolute_Path(/*const*/ Context& context, const Span& sp, Context:: if(e.nodes.size() > 1) { // Look up type/module name - auto p = context.lookup(sp, e.nodes[0].name(), e.hygine, Context::LookupMode::Namespace); + auto p = context.lookup(sp, e.nodes[0].name(), e.hygiene, Context::LookupMode::Namespace); DEBUG("Found type/mod - " << p); // HACK: If this is a primitive name, and resolved to a module. // - If the next component isn't found in the located module @@ -1333,7 +1333,7 @@ void Resolve_Absolute_Path(/*const*/ Context& context, const Span& sp, Context:: } else { // Look up value - auto p = context.lookup(sp, e.nodes[0].name(), e.hygine, mode); + auto p = context.lookup(sp, e.nodes[0].name(), e.hygiene, mode); //DEBUG("Found path " << p << " for " << path); if( p.is_absolute() ) { assert( !p.nodes().empty() ); @@ -1715,7 +1715,7 @@ void Resolve_Absolute_Pattern(Context& context, bool allow_refutable, ::AST::Pa if( allow_refutable ) { auto name = mv$( e.name ); // Attempt to resolve the name in the current namespace, and if it fails, it's a binding - auto p = context.lookup_opt( name.name, name.hygine, Context::LookupMode::Pattern ); + auto p = context.lookup_opt( name.name, name.hygiene, Context::LookupMode::Pattern ); if( p.is_valid() ) { Resolve_Absolute_Path(context, pat.span(), Context::LookupMode::Pattern, p); pat = ::AST::Pattern(::AST::Pattern::TagValue(), ::AST::Pattern::Value::make_Named(mv$(p))); -- cgit v1.2.3