From 150a481100bba025bc5132338ae3d37f19de5bfc Mon Sep 17 00:00:00 2001 From: John Hodge Date: Sun, 6 Mar 2016 22:07:41 +0800 Subject: Move macro_rules parsing and expansion to expand/synexts --- src/parse/common.hpp | 2 +- src/parse/expr.cpp | 48 ++++----- src/parse/macro_rules.cpp | 245 ++++++++++++++++++++++++++++++++++++++++++ src/parse/root.cpp | 266 +--------------------------------------------- 4 files changed, 272 insertions(+), 289 deletions(-) create mode 100644 src/parse/macro_rules.cpp (limited to 'src/parse') diff --git a/src/parse/common.hpp b/src/parse/common.hpp index e9b55eb2..def67ef4 100644 --- a/src/parse/common.hpp +++ b/src/parse/common.hpp @@ -48,7 +48,7 @@ extern AST::Struct Parse_Struct(TokenStream& lex, const AST::MetaItems& meta_ite extern AST::Enum Parse_EnumDef(TokenStream& lex, const AST::MetaItems& meta_items); extern AST::Trait Parse_TraitDef(TokenStream& lex, const AST::MetaItems& meta_items); extern void Parse_Impl(TokenStream& lex, AST::Module& mod, AST::MetaItems attrs, bool is_unsafe=false); -extern void Parse_MacroRules(TokenStream& lex, AST::Module& mod, AST::MetaItems meta_items); +extern MacroRules Parse_MacroRules(TokenStream& lex); extern void Parse_ExternCrate(TokenStream& lex, AST::Module& mod, AST::MetaItems meta_items); extern void Parse_Mod_Item(TokenStream& lex, LList& modstack, bool file_controls_dir, const ::std::string& file_path, AST::Module& mod, bool is_public, AST::MetaItems meta_items); diff --git a/src/parse/expr.cpp b/src/parse/expr.cpp index e082dfa3..11818a45 100644 --- a/src/parse/expr.cpp +++ b/src/parse/expr.cpp @@ -30,6 +30,7 @@ ExprNodeP Parse_WhileStmt(TokenStream& lex, ::std::string lifetime); ExprNodeP Parse_ForStmt(TokenStream& lex, ::std::string lifetime); ExprNodeP Parse_Expr_Match(TokenStream& lex); ExprNodeP Parse_Expr1(TokenStream& lex); +ExprNodeP Parse_ExprMacro(TokenStream& lex, Token tok); AST::Expr Parse_Expr(TokenStream& lex, bool const_only) { @@ -105,15 +106,6 @@ ExprNodeP Parse_ExprBlockNode(TokenStream& lex) Parse_Mod_Item(lex, modstack, false,"!", *local_mod, false, mv$(item_attrs)); break; } - if(0) - // Macros - If not macro_rules, fall though to expression - case TOK_MACRO: - if( tok.str() == "macro_rules" ) - { - keep_mod = true; - Parse_MacroRules(lex, *local_mod, ::std::move(item_attrs)); - break; - } // fall default: { lex.putback(tok); @@ -224,6 +216,11 @@ ExprNodeP Parse_ExprBlockLine(TokenStream& lex, bool *expect_end) return rv; } + case TOK_MACRO: + // If a braced macro invocation is the first part of a statement, don't expect a semicolon + if( LOOK_AHEAD(lex) == TOK_BRACE_OPEN || (lex.lookahead(0) == TOK_IDENT && lex.lookahead(1) == TOK_BRACE_OPEN) ) { + return Parse_ExprMacro(lex, tok); + } // Fall through to the statement code default: { lex.putback(tok); @@ -1173,25 +1170,28 @@ ExprNodeP Parse_ExprVal(TokenStream& lex) } } throw ParseError::BugCheck(lex, "Array literal fell"); - case TOK_MACRO: { - ::std::string name = tok.str(); - ::std::string ident; - if( GET_TOK(tok, lex) == TOK_IDENT ) { - ident = mv$(tok.str()); - } - else { - lex.putback(tok); - } - TokenTree tt = Parse_TT(lex, true); - if( tt.is_token() ) { - throw ParseError::Unexpected(lex, tt.tok()); - } - return NEWNODE(AST::ExprNode_Macro, mv$(name), mv$(ident), mv$(tt)); - } + case TOK_MACRO: + return Parse_ExprMacro(lex, tok); default: throw ParseError::Unexpected(lex, tok); } } +ExprNodeP Parse_ExprMacro(TokenStream& lex, Token tok) +{ + ::std::string name = tok.str(); + ::std::string ident; + if( GET_TOK(tok, lex) == TOK_IDENT ) { + ident = mv$(tok.str()); + } + else { + lex.putback(tok); + } + TokenTree tt = Parse_TT(lex, true); + if( tt.is_token() ) { + throw ParseError::Unexpected(lex, tt.tok()); + } + return NEWNODE(AST::ExprNode_Macro, mv$(name), mv$(ident), mv$(tt)); +} // Token Tree Parsing TokenTree Parse_TT(TokenStream& lex, bool unwrapped) diff --git a/src/parse/macro_rules.cpp b/src/parse/macro_rules.cpp new file mode 100644 index 00000000..b4698fa7 --- /dev/null +++ b/src/parse/macro_rules.cpp @@ -0,0 +1,245 @@ +#include +#include "../parse/common.hpp" +#include "../parse/parseerror.hpp" +#include "../macros.hpp" + +MacroRules Parse_MacroRules(TokenStream& lex); + +::std::vector Parse_MacroRules_Pat(TokenStream& lex, bool allow_sub, enum eTokenType open, enum eTokenType close) +{ + TRACE_FUNCTION; + Token tok; + + ::std::vector ret; + + int depth = 0; + while( GET_TOK(tok, lex) != close || depth > 0 ) + { + if( tok.type() == open ) + { + depth ++; + } + else if( tok.type() == close ) + { + if(depth == 0) + throw ParseError::Generic(FMT("Unmatched " << Token(close) << " in macro pattern")); + depth --; + } + + switch(tok.type()) + { + case TOK_DOLLAR: + switch( GET_TOK(tok, lex) ) + { + case TOK_IDENT: { + ::std::string name = tok.str(); + GET_CHECK_TOK(tok, lex, TOK_COLON); + GET_CHECK_TOK(tok, lex, TOK_IDENT); + ::std::string type = tok.str(); + if(0) + ; + else if( type == "tt" ) + ret.push_back( MacroPatEnt(name, MacroPatEnt::PAT_TT) ); + else if( type == "pat" ) + ret.push_back( MacroPatEnt(name, MacroPatEnt::PAT_PAT) ); + else if( type == "ident" ) + ret.push_back( MacroPatEnt(name, MacroPatEnt::PAT_IDENT) ); + else if( type == "path" ) + ret.push_back( MacroPatEnt(name, MacroPatEnt::PAT_PATH) ); + else if( type == "expr" ) + ret.push_back( MacroPatEnt(name, MacroPatEnt::PAT_EXPR) ); + else if( type == "ty" ) + ret.push_back( MacroPatEnt(name, MacroPatEnt::PAT_TYPE) ); + else if( type == "meta" ) + ret.push_back( MacroPatEnt(name, MacroPatEnt::PAT_META) ); + else if( type == "block" ) + ret.push_back( MacroPatEnt(name, MacroPatEnt::PAT_BLOCK) ); + else + throw ParseError::Generic(lex, FMT("Unknown fragment type '" << type << "'")); + break; } + case TOK_PAREN_OPEN: + if( allow_sub ) + { + auto subpat = Parse_MacroRules_Pat(lex, true, TOK_PAREN_OPEN, TOK_PAREN_CLOSE); + enum eTokenType joiner = TOK_NULL; + GET_TOK(tok, lex); + if( tok.type() != TOK_PLUS && tok.type() != TOK_STAR ) + { + DEBUG("Joiner = " << tok); + joiner = tok.type(); + GET_TOK(tok, lex); + } + DEBUG("tok = " << tok); + switch(tok.type()) + { + case TOK_PLUS: + DEBUG("$()+ " << subpat); + ret.push_back( MacroPatEnt(Token(joiner), true, ::std::move(subpat)) ); + break; + case TOK_STAR: + DEBUG("$()* " << subpat); + ret.push_back( MacroPatEnt(Token(joiner), false, ::std::move(subpat)) ); + break; + default: + throw ParseError::Unexpected(lex, tok); + } + } + else + { + throw ParseError::Generic(lex, FMT("Nested repetitions in macro")); + } + break; + default: + throw ParseError::Unexpected(lex, tok); + } + break; + case TOK_EOF: + throw ParseError::Unexpected(lex, tok); + default: + ret.push_back( MacroPatEnt(tok) ); + break; + } + } + + return ret; +} + +::std::vector Parse_MacroRules_Cont(TokenStream& lex, bool allow_sub, enum eTokenType open, enum eTokenType close) +{ + TRACE_FUNCTION; + + Token tok; + ::std::vector ret; + + int depth = 0; + while( GET_TOK(tok, lex) != close || depth > 0 ) + { + if( tok.type() == TOK_EOF ) { + throw ParseError::Unexpected(lex, tok); + } + if( tok.type() == TOK_NULL ) continue ; + + if( tok.type() == open ) + { + DEBUG("depth++"); + depth ++; + } + else if( tok.type() == close ) + { + DEBUG("depth--"); + if(depth == 0) + throw ParseError::Generic(FMT("Unmatched " << Token(close) << " in macro content")); + depth --; + } + + if( tok.type() == TOK_DOLLAR ) + { + GET_TOK(tok, lex); + + if( tok.type() == TOK_PAREN_OPEN ) + { + if( !allow_sub ) + throw ParseError::Unexpected(lex, tok); + + auto content = Parse_MacroRules_Cont(lex, true, TOK_PAREN_OPEN, TOK_PAREN_CLOSE); + + GET_TOK(tok, lex); + enum eTokenType joiner = TOK_NULL; + if( tok.type() != TOK_PLUS && tok.type() != TOK_STAR ) + { + joiner = tok.type(); + GET_TOK(tok, lex); + } + DEBUG("joiner = " << Token(joiner) << ", content = " << content); + switch(tok.type()) + { + case TOK_STAR: + ret.push_back( MacroRuleEnt(joiner, ::std::move(content)) ); + break; + case TOK_PLUS: + // TODO: Ensure that the plusses match + ret.push_back( MacroRuleEnt(joiner, ::std::move(content)) ); + break; + default: + throw ParseError::Unexpected(lex, tok); + } + + } + else if( tok.type() == TOK_IDENT ) + { + ret.push_back( MacroRuleEnt(tok.str()) ); + } + else if( tok.type() == TOK_RWORD_CRATE ) + { + ret.push_back( MacroRuleEnt("*crate") ); + } + else + { + throw ParseError::Unexpected(lex, tok); + } + } + else + { + ret.push_back( MacroRuleEnt(tok) ); + } + } + + return ret; +} + +MacroRule Parse_MacroRules_Var(TokenStream& lex) +{ + TRACE_FUNCTION; + Token tok; + + MacroRule rule; + + // Pattern + enum eTokenType close; + switch(GET_TOK(tok, lex)) + { + case TOK_BRACE_OPEN: close = TOK_BRACE_CLOSE; break; + case TOK_PAREN_OPEN: close = TOK_PAREN_CLOSE; break; + default: + throw ParseError::Unexpected(lex, tok); + } + // - Pattern entries + rule.m_pattern = Parse_MacroRules_Pat(lex, true, tok.type(), close); + + GET_CHECK_TOK(tok, lex, TOK_FATARROW); + + // Replacement + switch(GET_TOK(tok, lex)) + { + case TOK_BRACE_OPEN: close = TOK_BRACE_CLOSE; break; + case TOK_PAREN_OPEN: close = TOK_PAREN_CLOSE; break; + default: + throw ParseError::Unexpected(lex, tok); + } + rule.m_contents = Parse_MacroRules_Cont(lex, true, tok.type(), close); + + DEBUG("Rule - ["< "< rules; + while( GET_TOK(tok, lex) != TOK_EOF ) + { + lex.putback(tok); + + rules.push_back( Parse_MacroRules_Var(lex) ); + if(GET_TOK(tok, lex) != TOK_SEMICOLON) { + CHECK_TOK(tok, TOK_EOF); + break; + } + } + + return MacroRules(mv$(rules)); +} diff --git a/src/parse/root.cpp b/src/parse/root.cpp index 9b910504..faa2b9bb 100644 --- a/src/parse/root.cpp +++ b/src/parse/root.cpp @@ -1205,261 +1205,6 @@ void Parse_Use(TokenStream& lex, ::std::function fcn(path, name); } -::std::vector Parse_MacroRules_Pat(TokenStream& lex, bool allow_sub, enum eTokenType open, enum eTokenType close) -{ - TRACE_FUNCTION; - Token tok; - - ::std::vector ret; - - int depth = 0; - while( GET_TOK(tok, lex) != close || depth > 0 ) - { - if( tok.type() == open ) - { - depth ++; - } - else if( tok.type() == close ) - { - if(depth == 0) - throw ParseError::Generic(FMT("Unmatched " << Token(close) << " in macro pattern")); - depth --; - } - - switch(tok.type()) - { - case TOK_DOLLAR: - switch( GET_TOK(tok, lex) ) - { - case TOK_IDENT: { - ::std::string name = tok.str(); - GET_CHECK_TOK(tok, lex, TOK_COLON); - GET_CHECK_TOK(tok, lex, TOK_IDENT); - ::std::string type = tok.str(); - if(0) - ; - else if( type == "tt" ) - ret.push_back( MacroPatEnt(name, MacroPatEnt::PAT_TT) ); - else if( type == "pat" ) - ret.push_back( MacroPatEnt(name, MacroPatEnt::PAT_PAT) ); - else if( type == "ident" ) - ret.push_back( MacroPatEnt(name, MacroPatEnt::PAT_IDENT) ); - else if( type == "path" ) - ret.push_back( MacroPatEnt(name, MacroPatEnt::PAT_PATH) ); - else if( type == "expr" ) - ret.push_back( MacroPatEnt(name, MacroPatEnt::PAT_EXPR) ); - else if( type == "ty" ) - ret.push_back( MacroPatEnt(name, MacroPatEnt::PAT_TYPE) ); - else if( type == "meta" ) - ret.push_back( MacroPatEnt(name, MacroPatEnt::PAT_META) ); - else if( type == "block" ) - ret.push_back( MacroPatEnt(name, MacroPatEnt::PAT_BLOCK) ); - else - throw ParseError::Generic(lex, FMT("Unknown fragment type '" << type << "'")); - break; } - case TOK_PAREN_OPEN: - if( allow_sub ) - { - auto subpat = Parse_MacroRules_Pat(lex, true, TOK_PAREN_OPEN, TOK_PAREN_CLOSE); - enum eTokenType joiner = TOK_NULL; - GET_TOK(tok, lex); - if( tok.type() != TOK_PLUS && tok.type() != TOK_STAR ) - { - DEBUG("Joiner = " << tok); - joiner = tok.type(); - GET_TOK(tok, lex); - } - DEBUG("tok = " << tok); - switch(tok.type()) - { - case TOK_PLUS: - DEBUG("$()+ " << subpat); - ret.push_back( MacroPatEnt(Token(joiner), true, ::std::move(subpat)) ); - break; - case TOK_STAR: - DEBUG("$()* " << subpat); - ret.push_back( MacroPatEnt(Token(joiner), false, ::std::move(subpat)) ); - break; - default: - throw ParseError::Unexpected(lex, tok); - } - } - else - { - throw ParseError::Generic(lex, FMT("Nested repetitions in macro")); - } - break; - default: - throw ParseError::Unexpected(lex, tok); - } - break; - case TOK_EOF: - throw ParseError::Unexpected(lex, tok); - default: - ret.push_back( MacroPatEnt(tok) ); - break; - } - } - - return ret; -} - -::std::vector Parse_MacroRules_Cont(TokenStream& lex, bool allow_sub, enum eTokenType open, enum eTokenType close) -{ - TRACE_FUNCTION; - - Token tok; - ::std::vector ret; - - int depth = 0; - while( GET_TOK(tok, lex) != close || depth > 0 ) - { - if( tok.type() == TOK_EOF ) { - throw ParseError::Unexpected(lex, tok); - } - if( tok.type() == TOK_NULL ) continue ; - - if( tok.type() == open ) - { - DEBUG("depth++"); - depth ++; - } - else if( tok.type() == close ) - { - DEBUG("depth--"); - if(depth == 0) - throw ParseError::Generic(FMT("Unmatched " << Token(close) << " in macro content")); - depth --; - } - - if( tok.type() == TOK_DOLLAR ) - { - GET_TOK(tok, lex); - - if( tok.type() == TOK_PAREN_OPEN ) - { - if( !allow_sub ) - throw ParseError::Unexpected(lex, tok); - - auto content = Parse_MacroRules_Cont(lex, true, TOK_PAREN_OPEN, TOK_PAREN_CLOSE); - - GET_TOK(tok, lex); - enum eTokenType joiner = TOK_NULL; - if( tok.type() != TOK_PLUS && tok.type() != TOK_STAR ) - { - joiner = tok.type(); - GET_TOK(tok, lex); - } - DEBUG("joiner = " << Token(joiner) << ", content = " << content); - switch(tok.type()) - { - case TOK_STAR: - ret.push_back( MacroRuleEnt(joiner, ::std::move(content)) ); - break; - case TOK_PLUS: - // TODO: Ensure that the plusses match - ret.push_back( MacroRuleEnt(joiner, ::std::move(content)) ); - break; - default: - throw ParseError::Unexpected(lex, tok); - } - - } - else if( tok.type() == TOK_IDENT ) - { - ret.push_back( MacroRuleEnt(tok.str()) ); - } - else if( tok.type() == TOK_RWORD_CRATE ) - { - ret.push_back( MacroRuleEnt("*crate") ); - } - else - { - throw ParseError::Unexpected(lex, tok); - } - } - else - { - ret.push_back( MacroRuleEnt(tok) ); - } - } - - return ret; -} - -MacroRule Parse_MacroRules_Var(TokenStream& lex) -{ - TRACE_FUNCTION; - Token tok; - - MacroRule rule; - - // Pattern - enum eTokenType close; - switch(GET_TOK(tok, lex)) - { - case TOK_BRACE_OPEN: close = TOK_BRACE_CLOSE; break; - case TOK_PAREN_OPEN: close = TOK_PAREN_CLOSE; break; - default: - throw ParseError::Unexpected(lex, tok); - } - // - Pattern entries - rule.m_pattern = Parse_MacroRules_Pat(lex, true, tok.type(), close); - - GET_CHECK_TOK(tok, lex, TOK_FATARROW); - - // Replacement - switch(GET_TOK(tok, lex)) - { - case TOK_BRACE_OPEN: close = TOK_BRACE_CLOSE; break; - case TOK_PAREN_OPEN: close = TOK_PAREN_CLOSE; break; - default: - throw ParseError::Unexpected(lex, tok); - } - rule.m_contents = Parse_MacroRules_Cont(lex, true, tok.type(), close); - - DEBUG("Rule - ["< "< if( GET_TOK(tok, lex) == TOK_MACRO ) { ::std::string name = mv$(tok.str()); - // `macro_rules! ...` - //if( name == "macro_rules" ) - //{ - // Parse_MacroRules(lex, mod, mv$(meta_items)); - //} - //else - //{ - mod.add_macro_invocation( Parse_MacroInvocation( mv$(meta_items), mv$(name), lex ) ); - //} + mod.add_macro_invocation( Parse_MacroInvocation( mv$(meta_items), mv$(name), lex ) ); // - Silently consume ';' after the macro + // TODO: Check the tt next token before parsing to tell if this is needed if( GET_TOK(tok, lex) != TOK_SEMICOLON ) lex.putback(tok); continue ; -- cgit v1.2.3