Move macro_rules parsing and expansion to expand/synexts

4 files changed, 272 insertions, 289 deletions

diff --git a/src/parse/common.hpp b/src/parse/common.hpp
index e9b55eb2..def67ef4 100644
--- a/src/parse/common.hpp
+++ b/src/parse/common.hpp
@@ -48,7 +48,7 @@ extern AST::Struct Parse_Struct(TokenStream& lex, const AST::MetaItems& meta_ite
 extern AST::Enum Parse_EnumDef(TokenStream& lex, const AST::MetaItems& meta_items);

 extern AST::Trait Parse_TraitDef(TokenStream& lex, const AST::MetaItems& meta_items);

 extern void Parse_Impl(TokenStream& lex, AST::Module& mod, AST::MetaItems attrs, bool is_unsafe=false);

-extern void Parse_MacroRules(TokenStream& lex, AST::Module& mod, AST::MetaItems meta_items);

+extern MacroRules Parse_MacroRules(TokenStream& lex);

 extern void Parse_ExternCrate(TokenStream& lex, AST::Module& mod, AST::MetaItems meta_items);

 

 extern void Parse_Mod_Item(TokenStream& lex, LList<AST::Module*>& modstack, bool file_controls_dir, const ::std::string& file_path, AST::Module& mod, bool is_public, AST::MetaItems meta_items);

diff --git a/src/parse/expr.cpp b/src/parse/expr.cpp
index e082dfa3..11818a45 100644
--- a/src/parse/expr.cpp
+++ b/src/parse/expr.cpp
@@ -30,6 +30,7 @@ ExprNodeP Parse_WhileStmt(TokenStream& lex, ::std::string lifetime);
 ExprNodeP Parse_ForStmt(TokenStream& lex, ::std::string lifetime);

 ExprNodeP Parse_Expr_Match(TokenStream& lex);

 ExprNodeP Parse_Expr1(TokenStream& lex);

+ExprNodeP Parse_ExprMacro(TokenStream& lex, Token tok);

 

 AST::Expr Parse_Expr(TokenStream& lex, bool const_only)

 {

@@ -105,15 +106,6 @@ ExprNodeP Parse_ExprBlockNode(TokenStream& lex)
                 Parse_Mod_Item(lex, modstack, false,"!", *local_mod, false, mv$(item_attrs));

                 break;

             }

-            if(0)

-        // Macros - If not macro_rules, fall though to expression

-        case TOK_MACRO:

-            if( tok.str() == "macro_rules" )

-            {

-                keep_mod = true;

-                Parse_MacroRules(lex, *local_mod, ::std::move(item_attrs));

-                break;

-            }

             // fall

         default: {

             lex.putback(tok);

@@ -224,6 +216,11 @@ ExprNodeP Parse_ExprBlockLine(TokenStream& lex, bool *expect_end)
             return rv;

             }

         

+        case TOK_MACRO:

+            // If a braced macro invocation is the first part of a statement, don't expect a semicolon

+            if( LOOK_AHEAD(lex) == TOK_BRACE_OPEN || (lex.lookahead(0) == TOK_IDENT && lex.lookahead(1) == TOK_BRACE_OPEN) ) {

+                return Parse_ExprMacro(lex, tok);

+            }

         // Fall through to the statement code

         default: {

             lex.putback(tok);

@@ -1173,25 +1170,28 @@ ExprNodeP Parse_ExprVal(TokenStream& lex)
             }

         }

         throw ParseError::BugCheck(lex, "Array literal fell");

-    case TOK_MACRO: {

-        ::std::string name = tok.str();

-        ::std::string ident;

-        if( GET_TOK(tok, lex) == TOK_IDENT ) {

-            ident = mv$(tok.str());

-        }

-        else {

-            lex.putback(tok);

-        }

-        TokenTree tt = Parse_TT(lex, true);

-        if( tt.is_token() ) {

-            throw ParseError::Unexpected(lex, tt.tok());

-        }

-        return NEWNODE(AST::ExprNode_Macro, mv$(name), mv$(ident), mv$(tt));

-        }

+    case TOK_MACRO:

+        return Parse_ExprMacro(lex, tok);

     default:

         throw ParseError::Unexpected(lex, tok);

     }

 }

+ExprNodeP Parse_ExprMacro(TokenStream& lex, Token tok)

+{

+    ::std::string name = tok.str();

+    ::std::string ident;

+    if( GET_TOK(tok, lex) == TOK_IDENT ) {

+        ident = mv$(tok.str());

+    }

+    else {

+        lex.putback(tok);

+    }

+    TokenTree tt = Parse_TT(lex, true);

+    if( tt.is_token() ) {

+        throw ParseError::Unexpected(lex, tt.tok());

+    }

+    return NEWNODE(AST::ExprNode_Macro, mv$(name), mv$(ident), mv$(tt));

+}

 

 // Token Tree Parsing

 TokenTree Parse_TT(TokenStream& lex, bool unwrapped)

diff --git a/src/parse/macro_rules.cpp b/src/parse/macro_rules.cpp
new file mode 100644
index 00000000..b4698fa7
--- /dev/null
+++ b/src/parse/macro_rules.cpp
@@ -0,0 +1,245 @@
+#include <common.hpp>
+#include "../parse/common.hpp"
+#include "../parse/parseerror.hpp"
+#include "../macros.hpp"
+
+MacroRules Parse_MacroRules(TokenStream& lex);
+
+::std::vector<MacroPatEnt> Parse_MacroRules_Pat(TokenStream& lex, bool allow_sub, enum eTokenType open, enum eTokenType close)
+{
+    TRACE_FUNCTION;
+    Token tok;
+    
+    ::std::vector<MacroPatEnt>  ret;
+    
+     int    depth = 0;
+    while( GET_TOK(tok, lex) != close || depth > 0 )
+    {
+        if( tok.type() == open )
+        {
+            depth ++;
+        }
+        else if( tok.type() == close )
+        {
+            if(depth == 0)
+                throw ParseError::Generic(FMT("Unmatched " << Token(close) << " in macro pattern"));
+            depth --;
+        }
+        
+        switch(tok.type())
+        {
+        case TOK_DOLLAR:
+            switch( GET_TOK(tok, lex) )
+            {
+            case TOK_IDENT: {
+                ::std::string   name = tok.str();
+                GET_CHECK_TOK(tok, lex, TOK_COLON);
+                GET_CHECK_TOK(tok, lex, TOK_IDENT);
+                ::std::string   type = tok.str();
+                if(0)
+                    ;
+                else if( type == "tt" )
+                    ret.push_back( MacroPatEnt(name, MacroPatEnt::PAT_TT) );
+                else if( type == "pat" )
+                    ret.push_back( MacroPatEnt(name, MacroPatEnt::PAT_PAT) );
+                else if( type == "ident" )
+                    ret.push_back( MacroPatEnt(name, MacroPatEnt::PAT_IDENT) );
+                else if( type == "path" )
+                    ret.push_back( MacroPatEnt(name, MacroPatEnt::PAT_PATH) );
+                else if( type == "expr" )
+                    ret.push_back( MacroPatEnt(name, MacroPatEnt::PAT_EXPR) );
+                else if( type == "ty" )
+                    ret.push_back( MacroPatEnt(name, MacroPatEnt::PAT_TYPE) );
+                else if( type == "meta" )
+                    ret.push_back( MacroPatEnt(name, MacroPatEnt::PAT_META) );
+                else if( type == "block" )
+                    ret.push_back( MacroPatEnt(name, MacroPatEnt::PAT_BLOCK) );
+                else
+                    throw ParseError::Generic(lex, FMT("Unknown fragment type '" << type << "'"));
+                break; }
+            case TOK_PAREN_OPEN:
+                if( allow_sub )
+                {
+                    auto subpat = Parse_MacroRules_Pat(lex, true, TOK_PAREN_OPEN, TOK_PAREN_CLOSE);
+                    enum eTokenType joiner = TOK_NULL;
+                    GET_TOK(tok, lex);
+                    if( tok.type() != TOK_PLUS && tok.type() != TOK_STAR )
+                    {
+                        DEBUG("Joiner = " << tok);
+                        joiner = tok.type();
+                        GET_TOK(tok, lex);
+                    }
+                    DEBUG("tok = " << tok);
+                    switch(tok.type())
+                    {
+                    case TOK_PLUS:
+                        DEBUG("$()+ " << subpat);
+                        ret.push_back( MacroPatEnt(Token(joiner), true, ::std::move(subpat)) );
+                        break;
+                    case TOK_STAR:
+                        DEBUG("$()* " << subpat);
+                        ret.push_back( MacroPatEnt(Token(joiner), false, ::std::move(subpat)) );
+                        break;
+                    default:
+                        throw ParseError::Unexpected(lex, tok);
+                    }
+                }
+                else
+                {
+                    throw ParseError::Generic(lex, FMT("Nested repetitions in macro"));
+                }
+                break;
+            default:
+                throw ParseError::Unexpected(lex, tok);
+            }
+            break;
+        case TOK_EOF:
+            throw ParseError::Unexpected(lex, tok);
+        default:
+            ret.push_back( MacroPatEnt(tok) );
+            break;
+        }
+    }
+    
+    return ret;
+}
+
+::std::vector<MacroRuleEnt> Parse_MacroRules_Cont(TokenStream& lex, bool allow_sub, enum eTokenType open, enum eTokenType close)
+{
+    TRACE_FUNCTION;
+    
+    Token tok;
+    ::std::vector<MacroRuleEnt> ret;
+    
+     int    depth = 0;
+    while( GET_TOK(tok, lex) != close || depth > 0 )
+    {
+        if( tok.type() == TOK_EOF ) {
+            throw ParseError::Unexpected(lex, tok);
+        }
+        if( tok.type() == TOK_NULL )    continue ;
+        
+        if( tok.type() == open )
+        {
+            DEBUG("depth++");
+            depth ++;
+        }
+        else if( tok.type() == close )
+        {
+            DEBUG("depth--");
+            if(depth == 0)
+                throw ParseError::Generic(FMT("Unmatched " << Token(close) << " in macro content"));
+            depth --;
+        }
+        
+        if( tok.type() == TOK_DOLLAR )
+        {
+            GET_TOK(tok, lex);
+            
+            if( tok.type() == TOK_PAREN_OPEN )
+            {   
+                if( !allow_sub )
+                    throw ParseError::Unexpected(lex, tok);
+                
+                auto content = Parse_MacroRules_Cont(lex, true, TOK_PAREN_OPEN, TOK_PAREN_CLOSE);
+                
+                GET_TOK(tok, lex);
+                enum eTokenType joiner = TOK_NULL;
+                if( tok.type() != TOK_PLUS && tok.type() != TOK_STAR )
+                {
+                    joiner = tok.type();
+                    GET_TOK(tok, lex);
+                }
+                DEBUG("joiner = " << Token(joiner) << ", content = " << content);
+                switch(tok.type())
+                {
+                case TOK_STAR:
+                    ret.push_back( MacroRuleEnt(joiner, ::std::move(content)) );
+                    break;
+                case TOK_PLUS:
+                    // TODO: Ensure that the plusses match
+                    ret.push_back( MacroRuleEnt(joiner, ::std::move(content)) );
+                    break;
+                default:
+                    throw ParseError::Unexpected(lex, tok);
+                }
+                
+            }
+            else if( tok.type() == TOK_IDENT )
+            {
+                ret.push_back( MacroRuleEnt(tok.str()) );
+            }
+            else if( tok.type() == TOK_RWORD_CRATE )
+            {
+                ret.push_back( MacroRuleEnt("*crate") );
+            }
+            else
+            {
+                throw ParseError::Unexpected(lex, tok);
+            }
+        }
+        else
+        {
+            ret.push_back( MacroRuleEnt(tok) );
+        }
+    }
+    
+    return ret;
+}
+
+MacroRule Parse_MacroRules_Var(TokenStream& lex)
+{
+    TRACE_FUNCTION;
+    Token tok;
+    
+    MacroRule   rule;
+    
+    // Pattern
+    enum eTokenType close;
+    switch(GET_TOK(tok, lex))
+    {
+    case TOK_BRACE_OPEN:    close = TOK_BRACE_CLOSE;    break;
+    case TOK_PAREN_OPEN:    close = TOK_PAREN_CLOSE;    break;
+    default:
+        throw ParseError::Unexpected(lex, tok);
+    }
+    // - Pattern entries
+    rule.m_pattern = Parse_MacroRules_Pat(lex, true, tok.type(), close);
+    
+    GET_CHECK_TOK(tok, lex, TOK_FATARROW);
+
+    // Replacement
+    switch(GET_TOK(tok, lex))
+    {
+    case TOK_BRACE_OPEN:    close = TOK_BRACE_CLOSE;    break;
+    case TOK_PAREN_OPEN:    close = TOK_PAREN_CLOSE;    break;
+    default:
+        throw ParseError::Unexpected(lex, tok);
+    }
+    rule.m_contents = Parse_MacroRules_Cont(lex, true, tok.type(), close);
+
+    DEBUG("Rule - ["<<rule.m_pattern<<"] => "<<rule.m_contents<<"");
+    
+    return rule;
+}
+
+MacroRules Parse_MacroRules(TokenStream& lex)
+{
+    TRACE_FUNCTION_F("");
+    
+    Token tok;
+    
+    ::std::vector<MacroRule>    rules;
+    while( GET_TOK(tok, lex) != TOK_EOF )
+    {
+        lex.putback(tok);
+        
+        rules.push_back( Parse_MacroRules_Var(lex) );
+        if(GET_TOK(tok, lex) != TOK_SEMICOLON) {
+            CHECK_TOK(tok, TOK_EOF);
+            break;
+        }
+    }
+    
+    return MacroRules(mv$(rules));
+}
diff --git a/src/parse/root.cpp b/src/parse/root.cpp
index 9b910504..faa2b9bb 100644
--- a/src/parse/root.cpp
+++ b/src/parse/root.cpp
@@ -1205,261 +1205,6 @@ void Parse_Use(TokenStream& lex, ::std::function<void(AST::Path, ::std::string)>
     fcn(path, name);

 }

 

-::std::vector<MacroPatEnt> Parse_MacroRules_Pat(TokenStream& lex, bool allow_sub, enum eTokenType open, enum eTokenType close)

-{

-    TRACE_FUNCTION;

-    Token tok;

-    

-    ::std::vector<MacroPatEnt>  ret;

-    

-     int    depth = 0;

-    while( GET_TOK(tok, lex) != close || depth > 0 )

-    {

-        if( tok.type() == open )

-        {

-            depth ++;

-        }

-        else if( tok.type() == close )

-        {

-            if(depth == 0)

-                throw ParseError::Generic(FMT("Unmatched " << Token(close) << " in macro pattern"));

-            depth --;

-        }

-        

-        switch(tok.type())

-        {

-        case TOK_DOLLAR:

-            switch( GET_TOK(tok, lex) )

-            {

-            case TOK_IDENT: {

-                ::std::string   name = tok.str();

-                GET_CHECK_TOK(tok, lex, TOK_COLON);

-                GET_CHECK_TOK(tok, lex, TOK_IDENT);

-                ::std::string   type = tok.str();

-                if(0)

-                    ;

-                else if( type == "tt" )

-                    ret.push_back( MacroPatEnt(name, MacroPatEnt::PAT_TT) );

-                else if( type == "pat" )

-                    ret.push_back( MacroPatEnt(name, MacroPatEnt::PAT_PAT) );

-                else if( type == "ident" )

-                    ret.push_back( MacroPatEnt(name, MacroPatEnt::PAT_IDENT) );

-                else if( type == "path" )

-                    ret.push_back( MacroPatEnt(name, MacroPatEnt::PAT_PATH) );

-                else if( type == "expr" )

-                    ret.push_back( MacroPatEnt(name, MacroPatEnt::PAT_EXPR) );

-                else if( type == "ty" )

-                    ret.push_back( MacroPatEnt(name, MacroPatEnt::PAT_TYPE) );

-                else if( type == "meta" )

-                    ret.push_back( MacroPatEnt(name, MacroPatEnt::PAT_META) );

-                else if( type == "block" )

-                    ret.push_back( MacroPatEnt(name, MacroPatEnt::PAT_BLOCK) );

-                else

-                    throw ParseError::Generic(lex, FMT("Unknown fragment type '" << type << "'"));

-                break; }

-            case TOK_PAREN_OPEN:

-                if( allow_sub )

-                {

-                    auto subpat = Parse_MacroRules_Pat(lex, true, TOK_PAREN_OPEN, TOK_PAREN_CLOSE);

-                    enum eTokenType joiner = TOK_NULL;

-                    GET_TOK(tok, lex);

-                    if( tok.type() != TOK_PLUS && tok.type() != TOK_STAR )

-                    {

-                        DEBUG("Joiner = " << tok);

-                        joiner = tok.type();

-                        GET_TOK(tok, lex);

-                    }

-                    DEBUG("tok = " << tok);

-                    switch(tok.type())

-                    {

-                    case TOK_PLUS:

-                        DEBUG("$()+ " << subpat);

-                        ret.push_back( MacroPatEnt(Token(joiner), true, ::std::move(subpat)) );

-                        break;

-                    case TOK_STAR:

-                        DEBUG("$()* " << subpat);

-                        ret.push_back( MacroPatEnt(Token(joiner), false, ::std::move(subpat)) );

-                        break;

-                    default:

-                        throw ParseError::Unexpected(lex, tok);

-                    }

-                }

-                else

-                {

-                    throw ParseError::Generic(lex, FMT("Nested repetitions in macro"));

-                }

-                break;

-            default:

-                throw ParseError::Unexpected(lex, tok);

-            }

-            break;

-        case TOK_EOF:

-            throw ParseError::Unexpected(lex, tok);

-        default:

-            ret.push_back( MacroPatEnt(tok) );

-            break;

-        }

-    }

-    

-    return ret;

-}

-

-::std::vector<MacroRuleEnt> Parse_MacroRules_Cont(TokenStream& lex, bool allow_sub, enum eTokenType open, enum eTokenType close)

-{

-    TRACE_FUNCTION;

-    

-    Token tok;

-    ::std::vector<MacroRuleEnt> ret;

-    

-     int    depth = 0;

-    while( GET_TOK(tok, lex) != close || depth > 0 )

-    {

-        if( tok.type() == TOK_EOF ) {

-            throw ParseError::Unexpected(lex, tok);

-        }

-        if( tok.type() == TOK_NULL )    continue ;

-        

-        if( tok.type() == open )

-        {

-            DEBUG("depth++");

-            depth ++;

-        }

-        else if( tok.type() == close )

-        {

-            DEBUG("depth--");

-            if(depth == 0)

-                throw ParseError::Generic(FMT("Unmatched " << Token(close) << " in macro content"));

-            depth --;

-        }

-        

-        if( tok.type() == TOK_DOLLAR )

-        {

-            GET_TOK(tok, lex);

-            

-            if( tok.type() == TOK_PAREN_OPEN )

-            {   

-                if( !allow_sub )

-                    throw ParseError::Unexpected(lex, tok);

-                

-                auto content = Parse_MacroRules_Cont(lex, true, TOK_PAREN_OPEN, TOK_PAREN_CLOSE);

-                

-                GET_TOK(tok, lex);

-                enum eTokenType joiner = TOK_NULL;

-                if( tok.type() != TOK_PLUS && tok.type() != TOK_STAR )

-                {

-                    joiner = tok.type();

-                    GET_TOK(tok, lex);

-                }

-                DEBUG("joiner = " << Token(joiner) << ", content = " << content);

-                switch(tok.type())

-                {

-                case TOK_STAR:

-                    ret.push_back( MacroRuleEnt(joiner, ::std::move(content)) );

-                    break;

-                case TOK_PLUS:

-                    // TODO: Ensure that the plusses match

-                    ret.push_back( MacroRuleEnt(joiner, ::std::move(content)) );

-                    break;

-                default:

-                    throw ParseError::Unexpected(lex, tok);

-                }

-                

-            }

-            else if( tok.type() == TOK_IDENT )

-            {

-                ret.push_back( MacroRuleEnt(tok.str()) );

-            }

-            else if( tok.type() == TOK_RWORD_CRATE )

-            {

-                ret.push_back( MacroRuleEnt("*crate") );

-            }

-            else

-            {

-                throw ParseError::Unexpected(lex, tok);

-            }

-        }

-        else

-        {

-            ret.push_back( MacroRuleEnt(tok) );

-        }

-    }

-    

-    return ret;

-}

-

-MacroRule Parse_MacroRules_Var(TokenStream& lex)

-{

-    TRACE_FUNCTION;

-    Token tok;

-    

-    MacroRule   rule;

-    

-    // Pattern

-    enum eTokenType close;

-    switch(GET_TOK(tok, lex))

-    {

-    case TOK_BRACE_OPEN:    close = TOK_BRACE_CLOSE;    break;

-    case TOK_PAREN_OPEN:    close = TOK_PAREN_CLOSE;    break;

-    default:

-        throw ParseError::Unexpected(lex, tok);

-    }

-    // - Pattern entries

-    rule.m_pattern = Parse_MacroRules_Pat(lex, true, tok.type(), close);

-    

-    GET_CHECK_TOK(tok, lex, TOK_FATARROW);

-

-    // Replacement

-    switch(GET_TOK(tok, lex))

-    {

-    case TOK_BRACE_OPEN:    close = TOK_BRACE_CLOSE;    break;

-    case TOK_PAREN_OPEN:    close = TOK_PAREN_CLOSE;    break;

-    default:

-        throw ParseError::Unexpected(lex, tok);

-    }

-    rule.m_contents = Parse_MacroRules_Cont(lex, true, tok.type(), close);

-

-    DEBUG("Rule - ["<<rule.m_pattern<<"] => "<<rule.m_contents<<"");

-    

-    return rule;

-}

-

-void Parse_MacroRules(TokenStream& lex, AST::Module& mod, AST::MetaItems meta_items)

-{

-    TRACE_FUNCTION_F("meta_items="<<meta_items);

-    

-    Token tok;

-    

-    GET_CHECK_TOK(tok, lex, TOK_IDENT);

-    ::std::string name = tok.str();

-    

-    eTokenType  close;

-    switch(GET_TOK(tok,lex))

-    {

-    case TOK_BRACE_OPEN:    close = TOK_BRACE_CLOSE;    break;

-    case TOK_PAREN_OPEN:    close = TOK_PAREN_CLOSE;    break;

-    case TOK_SQUARE_OPEN:   close = TOK_SQUARE_CLOSE;   break;

-    default:

-        // TODO: Synerror

-        throw ParseError::Unexpected(lex, tok, {TOK_BRACE_OPEN, TOK_PAREN_OPEN, TOK_SQUARE_OPEN});

-        break;

-    }

-    

-    ::std::vector<MacroRule>    rules;

-    while( GET_TOK(tok, lex) != close )

-    {

-        lex.putback(tok);

-        

-        rules.push_back( Parse_MacroRules_Var(lex) );

-        if(GET_TOK(tok, lex) != TOK_SEMICOLON) {

-            CHECK_TOK(tok, TOK_BRACE_CLOSE);

-            break;

-        }

-    }

-    

-    bool is_pub = meta_items.has("macro_export");

-    

-    mod.add_macro( is_pub, name, MacroRules(move(rules)) );

-}

 

 ::AST::MacroInvocation Parse_MacroInvocation(::AST::MetaItems meta_items, ::std::string name, TokenStream& lex)

 {

@@ -1856,16 +1601,9 @@ void Parse_ModRoot_Items(TokenStream& lex, AST::Module& mod, LList<AST::Module*>
         if( GET_TOK(tok, lex) == TOK_MACRO )

         {

             ::std::string   name = mv$(tok.str());

-            // `macro_rules! ...`

-            //if( name == "macro_rules" )

-            //{

-            //    Parse_MacroRules(lex, mod, mv$(meta_items));

-            //}

-            //else

-            //{

-                mod.add_macro_invocation( Parse_MacroInvocation( mv$(meta_items), mv$(name), lex ) );

-            //}

+            mod.add_macro_invocation( Parse_MacroInvocation( mv$(meta_items), mv$(name), lex ) );

             // - Silently consume ';' after the macro

+            // TODO: Check the tt next token before parsing to tell if this is needed

             if( GET_TOK(tok, lex) != TOK_SEMICOLON )

                 lex.putback(tok);

             continue ;