(semibroken) Defer macro expansion

10 files changed, 98 insertions, 63 deletions

diff --git a/Makefile b/Makefile
index 64ce9813..80d5aba0 100644
--- a/Makefile
+++ b/Makefile
@@ -60,7 +60,7 @@ output/core.ast: $(RUSTCSRC)src/libcore/lib.rs $(BIN)
 
 .PHONY: rust_tests
 RUST_TESTS_DIR := $(RUSTCSRC)src/test/
-rust_tests: $(patsubst $(RUST_TESTS_DIR)%.rs,.rust_tests_out/%.txt,$(wildcard $(RUST_TESTS_DIR)rpass/*.rs))
+rust_tests: $(patsubst $(RUST_TESTS_DIR)%.rs,output/rust/%.txt,$(wildcard $(RUST_TESTS_DIR)run-pass/*.rs))
 
 output/rust/%.txt: $(RUST_TESTS_DIR)%.rs $(BIN)
 	@mkdir -p $(dir $@)
diff --git a/src/ast/ast.cpp b/src/ast/ast.cpp
index 7d9e16c5..7d6185e3 100644
--- a/src/ast/ast.cpp
+++ b/src/ast/ast.cpp
@@ -500,6 +500,12 @@ SERIALISE_TYPE(ExternCrate::, "AST_ExternCrate", {
 },{

 })

 

+SERIALISE_TYPE_A(MacroItem::, "AST_MacroItem", {

+    s.item(m_macro_name);

+    s.item(m_ident);

+    s.item(m_input);

+})

+

 SERIALISE_TYPE_A(Module::, "AST_Module", {

     s.item(m_name);

     s.item(m_attrs);

diff --git a/src/ast/ast.hpp b/src/ast/ast.hpp
index 8e6f0361..9c644bf6 100644
--- a/src/ast/ast.hpp
+++ b/src/ast/ast.hpp
@@ -615,6 +615,29 @@ class Module;
 

 typedef void fcn_visitor_t(const AST::Crate& crate, const AST::Module& mod, Function& fcn);

 

+class MacroItem:

+    public Serialisable

+{

+    MetaItems   m_attrs;

+    ::std::string   m_macro_name;

+    ::std::string   m_ident;

+    TokenTree   m_input;

+public:

+    MacroItem()

+    {

+    }

+    

+    MacroItem(MetaItems attrs, ::std::string macro, ::std::string ident, TokenTree input):

+        m_attrs( mv$(attrs) ),

+        m_macro_name( mv$(macro) ),

+        m_ident( mv$(ident) ),

+        m_input( mv$(input) )

+    {

+    }

+

+    SERIALISABLE_PROTOTYPES();

+};

+

 /// Representation of a parsed (and being converted) function

 class Module:

     public Serialisable

@@ -641,6 +664,7 @@ class Module:
     itemlist_macros_t   m_macros;

     macro_imports_t m_macro_imports;    // module => macro

     ::std::vector< ItemNS<const MacroRules*> > m_macro_import_res; // Vec of imported macros (not serialised)

+    ::std::vector<MacroItem>    m_macro_invocations;

     

     

     

@@ -704,6 +728,9 @@ public:
         m_macros.push_back( Item<MacroRules>( move(name), move(macro), is_exported ) );

     }

     void add_macro_import(const Crate& crate, ::std::string mod, ::std::string name);

+    void add_macro_invocation(MacroItem item) {

+        m_macro_invocations.push_back( mv$(item) );

+    }

 

     void add_attr(MetaItem item) {

         m_attrs.push_back(item);

diff --git a/src/ast/expr.cpp b/src/ast/expr.cpp
index f42870f8..48e080d6 100644
--- a/src/ast/expr.cpp
+++ b/src/ast/expr.cpp
@@ -102,9 +102,15 @@ NODE(ExprNode_Block, {
 
 NODE(ExprNode_Macro, {
     s.item(m_name);
-    //s.item(m_tokens);
+    s.item(m_ident);
+    s.item(m_tokens);
 },{
-    os << m_name << "!(" << ")";
+    os << m_name << "!";
+    if( m_ident.size() > 0 )
+    {
+        os << " " << m_ident << " ";
+    }
+    os << "(" << ")";
 })
 
 void operator%(::Serialiser& s, const ExprNode_Flow::Type t) {
diff --git a/src/ast/expr.hpp b/src/ast/expr.hpp
index bbfa37ad..2fe9ab36 100644
--- a/src/ast/expr.hpp
+++ b/src/ast/expr.hpp
@@ -70,11 +70,13 @@ struct ExprNode_Macro:
     public ExprNode
 {
     ::std::string   m_name;
+    ::std::string   m_ident;
     ::TokenTree m_tokens;
     
     ExprNode_Macro() {}
-    ExprNode_Macro(::std::string name, ::TokenTree&& tokens):
+    ExprNode_Macro(::std::string name, ::std::string ident, ::TokenTree&& tokens):
         m_name(name),
+        m_ident(ident),
         m_tokens( move(tokens) )
     {}
     
diff --git a/src/main.cpp b/src/main.cpp
index 0e0ad016..2949b4a6 100644
--- a/src/main.cpp
+++ b/src/main.cpp
@@ -84,7 +84,7 @@ int main(int argc, char *argv[])
         // Iterate all items in the AST, applying syntax extensions

         CompilePhaseV("Decorators", [&]() {

             Process_Decorators(crate);

-            Process_Synexts(crate);

+            //Process_Synexts(crate);

             });

             

         // Run a quick post-parse pass

diff --git a/src/parse/expr.cpp b/src/parse/expr.cpp
index 5c7bfda5..47fe444a 100644
--- a/src/parse/expr.cpp
+++ b/src/parse/expr.cpp
@@ -1141,34 +1141,20 @@ ExprNodeP Parse_ExprVal(TokenStream& lex)
             }

         }

         throw ParseError::BugCheck(lex, "Array literal fell");

-    case TOK_MACRO:

-        if( CHECK_PARSE_FLAG(lex, no_expand_macros) )

-        {

-            ::std::string name = tok.str();

-            TokenTree tt = Parse_TT(lex, true);

-            if( tt.is_token() ) {

-                throw ParseError::Unexpected(lex, tt.tok());

-            }

-            return NEWNODE(AST::ExprNode_Macro, ::std::move(name), ::std::move(tt));

+    case TOK_MACRO: {

+        ::std::string name = tok.str();

+        ::std::string ident;

+        if( GET_TOK(tok, lex) == TOK_IDENT ) {

+            ident = mv$(tok.str());

         }

-        else

-        {

-            TokenTree tt = Parse_TT(lex, true);

-            if( tt.is_token() ) {

-                throw ParseError::Unexpected(lex, tt.tok());

-            }

-            ::std::string name = tok.str();

-            

-            if( name == "format_args" )

-            {

-                TTStream    slex(tt);

-                return Parse_FormatArgs(slex);

-            }

-            else

-            {

-                auto expanded_macro = Macro_Invoke(lex, name, tt);

-                return Parse_Expr0(*expanded_macro);

-            }

+        else {

+            lex.putback(tok);

+        }

+        TokenTree tt = Parse_TT(lex, true);

+        if( tt.is_token() ) {

+            throw ParseError::Unexpected(lex, tt.tok());

+        }

+        return NEWNODE(AST::ExprNode_Macro, mv$(name), mv$(ident), mv$(tt));

         }

     default:

         throw ParseError::Unexpected(lex, tok);

diff --git a/src/parse/lex.cpp b/src/parse/lex.cpp
index a3187cda..cb97022d 100644
--- a/src/parse/lex.cpp
+++ b/src/parse/lex.cpp
@@ -1042,3 +1042,9 @@ Span TokenStream::end_span(ProtoSpan ps) const
         );
 }
 
+
+SERIALISE_TYPE_A(TokenTree::, "TokenTree", {
+    s.item(m_tok);
+    s.item(m_subtrees);
+})
+
diff --git a/src/parse/root.cpp b/src/parse/root.cpp
index 5ba66dbc..93b360f1 100644
--- a/src/parse/root.cpp
+++ b/src/parse/root.cpp
@@ -829,25 +829,25 @@ void Parse_Impl_Item(TokenStream& lex, AST::Impl& impl)
     ::std::string   abi = "rust";

     switch(tok.type())

     {

-    case TOK_MACRO:

-        {

-            TokenTree tt = Parse_TT(lex, true);

-            if( tt.is_token() ) {

-                DEBUG("TT was a single token (not a sub-tree)");

-                throw ParseError::Unexpected(lex, tt.tok());

-            }

-            

-            auto expanded_macro = Macro_Invoke(lex, tok.str().c_str(), tt);

-            auto& lex = *expanded_macro;

-            while( GET_TOK(tok, lex) != TOK_EOF )

-            {

-                lex.putback(tok);

-                Parse_Impl_Item(lex, impl);

-            }

-        }

-        if(GET_TOK(tok, lex) != TOK_SEMICOLON)

-            lex.putback(tok);

-        break;

+    //case TOK_MACRO:

+    //    {

+    //        TokenTree tt = Parse_TT(lex, true);

+    //        if( tt.is_token() ) {

+    //            DEBUG("TT was a single token (not a sub-tree)");

+    //            throw ParseError::Unexpected(lex, tt.tok());

+    //        }

+    //        

+    //        auto expanded_macro = Macro_Invoke(lex, tok.str().c_str(), tt);

+    //        auto& lex = *expanded_macro;

+    //        while( GET_TOK(tok, lex) != TOK_EOF )

+    //        {

+    //            lex.putback(tok);

+    //            Parse_Impl_Item(lex, impl);

+    //        }

+    //    }

+    //    if(GET_TOK(tok, lex) != TOK_SEMICOLON)

+    //        lex.putback(tok);

+    //    break;

     case TOK_RWORD_TYPE: {

         GET_CHECK_TOK(tok, lex, TOK_IDENT);

         ::std::string name = tok.str();

@@ -1308,24 +1308,23 @@ void Parse_ModRoot_Items(TokenStream& lex, AST::Crate& crate, AST::Module& mod,
         // root-level macros

         if( GET_TOK(tok, lex) == TOK_MACRO )

         {

+            ::std::string   name = mv$(tok.str());

             // `macro_rules! ...`

-            if( tok.str() == "macro_rules" )

+            if( name == "macro_rules" )

             {

                 Parse_MacroRules(lex, mod, mv$(meta_items));

             }

             else

             {

-                DEBUG("Invoke macro '"<<tok.str()<<"'");

-                TokenTree tt = Parse_TT(lex, true);

-                if( tt.is_token() ) {

-                    DEBUG("TT was a single token (not a sub-tree)");

-                    throw ParseError::Unexpected(lex, tt.tok());

+                ::std::string   ident;

+                if( GET_TOK(tok, lex) == TOK_IDENT ) {

+                    ident = mv$(tok.str());

                 }

-                ::std::string name = tok.str();

-                

-                auto expanded_macro = Macro_Invoke(lex, name.c_str(), tt);

-                // Pass "!" as 'path' to allow termination on EOF

-                Parse_ModRoot_Items(*expanded_macro, crate, mod, modstack, "!");

+                else {

+                    lex.putback(tok);

+                }

+                TokenTree tt = Parse_TT(lex, true);

+                mod.add_macro_invocation( ::AST::MacroItem( mv$(meta_items), mv$(name), mv$(ident), mv$(tt)) );

             }

             // - Silently consume ';' after the macro

             if( GET_TOK(tok, lex) != TOK_SEMICOLON )

diff --git a/src/parse/tokentree.hpp b/src/parse/tokentree.hpp
index cde35c8c..aa72a522 100644
--- a/src/parse/tokentree.hpp
+++ b/src/parse/tokentree.hpp
@@ -4,7 +4,8 @@
 #include "lex.hpp"

 

 

-class TokenTree

+class TokenTree:

+    public Serialisable

 {

     Token   m_tok;

     ::std::vector<TokenTree>    m_subtrees;

@@ -38,6 +39,8 @@ public:
         else

             return os << "TokenTree([" << tt.m_subtrees << "])";

     }

+

+    SERIALISABLE_PROTOTYPES();

 };

 

 class TTStream: