Expand - Add parent spans to TTStream[O]

13 files changed, 51 insertions, 45 deletions

diff --git a/src/expand/asm.cpp b/src/expand/asm.cpp
index 336321f7..51412f99 100644
--- a/src/expand/asm.cpp
+++ b/src/expand/asm.cpp
@@ -38,7 +38,7 @@ class CAsmExpander:
     ::std::unique_ptr<TokenStream> expand(const Span& sp, const ::AST::Crate& crate, const ::std::string& ident, const TokenTree& tt, AST::Module& mod) override
     {
         Token   tok;
-        auto lex = TTStream(tt);
+        auto lex = TTStream(sp, tt);
         if( ident != "" )
             ERROR(sp, E0000, "format_args! doesn't take an ident");
 
@@ -160,9 +160,9 @@ class CAsmExpander:
             ERROR(sp, E0000, "Unexpected token in asm! - " << lex.getToken());
         }
 
+        // Convert this into an AST node and insert as an intepolated expression
         ::AST::ExprNodeP rv = ::AST::ExprNodeP( new ::AST::ExprNode_Asm { mv$(template_text), mv$(outputs), mv$(inputs), mv$(clobbers), mv$(flags) } );
-        // TODO: Convert this into an AST node
-        return box$( TTStreamO(TokenTree(Token( InterpolatedFragment(InterpolatedFragment::EXPR, rv.release()) ))));
+        return box$( TTStreamO(sp, TokenTree(Token( InterpolatedFragment(InterpolatedFragment::EXPR, rv.release()) ))));
     }
 };
 
diff --git a/src/expand/cfg.cpp b/src/expand/cfg.cpp
index 8572b2e5..a2fca312 100644
--- a/src/expand/cfg.cpp
+++ b/src/expand/cfg.cpp
@@ -97,15 +97,15 @@ class CCfgExpander:
             ERROR(sp, E0000, "cfg! doesn't take an identifier");
         }
 
-        auto lex = TTStream(tt);
+        auto lex = TTStream(sp, tt);
         auto attrs = Parse_MetaItem(lex);
         DEBUG("cfg!() - " << attrs);
 
         if( check_cfg(sp, attrs) ) {
-            return box$( TTStreamO(TokenTree({},TOK_RWORD_TRUE )) );
+            return box$( TTStreamO(sp, TokenTree({},TOK_RWORD_TRUE )) );
         }
         else {
-            return box$( TTStreamO(TokenTree({},TOK_RWORD_FALSE)) );
+            return box$( TTStreamO(sp, TokenTree({},TOK_RWORD_FALSE)) );
         }
     }
 };
diff --git a/src/expand/concat.cpp b/src/expand/concat.cpp
index a08e9168..29a066df 100644
--- a/src/expand/concat.cpp
+++ b/src/expand/concat.cpp
@@ -20,7 +20,7 @@ class CConcatExpander:
     {
         Token   tok;
 
-        auto lex = TTStream(tt);
+        auto lex = TTStream(sp, tt);
         if( ident != "" )
             ERROR(sp, E0000, "format_args! doesn't take an ident");
 
@@ -64,7 +64,7 @@ class CConcatExpander:
         if( tok.type() != TOK_EOF )
             throw ParseError::Unexpected(lex, tok, {TOK_COMMA, TOK_EOF});
 
-        return box$( TTStreamO(TokenTree(Token(TOK_STRING, mv$(rv)))) );
+        return box$( TTStreamO(sp, TokenTree(Token(TOK_STRING, mv$(rv)))) );
     }
 };
 
diff --git a/src/expand/env.cpp b/src/expand/env.cpp
index cad2fa7c..f4577ef1 100644
--- a/src/expand/env.cpp
+++ b/src/expand/env.cpp
@@ -14,7 +14,7 @@
 namespace {
     // Read a string out of the input stream
     ::std::string get_string(const Span& sp, const AST::Crate& crate, AST::Module& mod, const TokenTree& tt) {
-        auto lex = TTStream(tt);
+        auto lex = TTStream(sp, tt);
 
         auto n = Parse_ExprVal(lex);
         ASSERT_BUG(sp, n, "No expression returned");
@@ -44,7 +44,7 @@ class CExpanderEnv:
         if( !var_val_cstr ) {
             ERROR(sp, E0000, "Environment variable '" << varname << "' not defined");
         }
-        return box$( TTStreamO(TokenTree(Token(TOK_STRING, var_val_cstr))) );
+        return box$( TTStreamO(sp, TokenTree(Token(TOK_STRING, var_val_cstr))) );
     }
 };
 
@@ -68,7 +68,7 @@ class CExpanderOptionEnv:
             rv.push_back( Token(TOK_LIFETIME, "static") );
             rv.push_back( Token(TOK_IDENT, "str") );
             rv.push_back( Token(TOK_GT) );
-            return box$( TTStreamO(TokenTree( {}, mv$(rv) )) );
+            return box$( TTStreamO(sp, TokenTree( {}, mv$(rv) )) );
         }
         else {
             ::std::vector< TokenTree>   rv;
@@ -77,7 +77,7 @@ class CExpanderOptionEnv:
             rv.push_back( Token(TOK_PAREN_OPEN) );
             rv.push_back( Token(TOK_STRING, var_val_cstr) );
             rv.push_back( Token(TOK_PAREN_CLOSE) );
-            return box$( TTStreamO(TokenTree( {}, mv$(rv) )) );
+            return box$( TTStreamO(sp, TokenTree( {}, mv$(rv) )) );
         }
     }
 };
diff --git a/src/expand/file_line.cpp b/src/expand/file_line.cpp
index 7e117993..7a827ccf 100644
--- a/src/expand/file_line.cpp
+++ b/src/expand/file_line.cpp
@@ -25,7 +25,7 @@ class CExpanderFile:
 {
     ::std::unique_ptr<TokenStream> expand(const Span& sp, const AST::Crate& crate, const ::std::string& ident, const TokenTree& tt, AST::Module& mod) override
     {
-        return box$( TTStreamO(TokenTree(Token(TOK_STRING, get_top_span(sp).filename.c_str()))) );
+        return box$( TTStreamO(sp, TokenTree(Token(TOK_STRING, get_top_span(sp).filename.c_str()))) );
     }
 };
 
@@ -34,7 +34,7 @@ class CExpanderLine:
 {
     ::std::unique_ptr<TokenStream> expand(const Span& sp, const AST::Crate& crate, const ::std::string& ident, const TokenTree& tt, AST::Module& mod) override
     {
-        return box$( TTStreamO(TokenTree(Token((uint64_t)get_top_span(sp).start_line, CORETYPE_U32))) );
+        return box$( TTStreamO(sp, TokenTree(Token((uint64_t)get_top_span(sp).start_line, CORETYPE_U32))) );
     }
 };
 
@@ -43,7 +43,7 @@ class CExpanderColumn:
 {
     ::std::unique_ptr<TokenStream> expand(const Span& sp, const AST::Crate& crate, const ::std::string& ident, const TokenTree& tt, AST::Module& mod) override
     {
-        return box$( TTStreamO(TokenTree(Token((uint64_t)get_top_span(sp).start_ofs, CORETYPE_U32))) );
+        return box$( TTStreamO(sp, TokenTree(Token((uint64_t)get_top_span(sp).start_ofs, CORETYPE_U32))) );
     }
 };
 
@@ -58,7 +58,7 @@ class CExpanderModulePath:
                 path_str += "::";
             path_str += comp.name();
         }
-        return box$( TTStreamO(TokenTree( Token(TOK_STRING, mv$(path_str)) )) );
+        return box$( TTStreamO(sp, TokenTree( Token(TOK_STRING, mv$(path_str)) )) );
     }
 };
 
diff --git a/src/expand/format_args.cpp b/src/expand/format_args.cpp
index e4c02b8b..dfad2397 100644
--- a/src/expand/format_args.cpp
+++ b/src/expand/format_args.cpp
@@ -440,7 +440,7 @@ class CFormatArgsExpander:
     {
         Token   tok;
 
-        auto lex = TTStream(tt);
+        auto lex = TTStream(sp, tt);
         if( ident != "" )
             ERROR(sp, E0000, "format_args! doesn't take an ident");
 
@@ -716,7 +716,7 @@ class CFormatArgsExpander:
         toks.push_back( TokenTree(TOK_BRACE_CLOSE) );
         toks.push_back( TokenTree(TOK_BRACE_CLOSE) );
 
-        return box$( TTStreamO(TokenTree(Ident::Hygiene::new_scope(), mv$(toks))) );
+        return box$( TTStreamO(sp, TokenTree(Ident::Hygiene::new_scope(), mv$(toks))) );
     }
 };
 
diff --git a/src/expand/include.cpp b/src/expand/include.cpp
index e66e07ce..59f33d47 100644
--- a/src/expand/include.cpp
+++ b/src/expand/include.cpp
@@ -69,7 +69,7 @@ class CIncludeExpander:
             ERROR(sp, E0000, "include! doesn't take an ident");
 
         Token   tok;
-        auto lex = TTStream(tt);
+        auto lex = TTStream(sp, tt);
 
         auto path = get_string(sp, lex, crate, mod);
         GET_CHECK_TOK(tok, lex, TOK_EOF);
@@ -95,7 +95,7 @@ class CIncludeBytesExpander:
             ERROR(sp, E0000, "include_bytes! doesn't take an ident");
 
         Token   tok;
-        auto lex = TTStream(tt);
+        auto lex = TTStream(sp, tt);
 
         auto path = get_string(sp, lex, crate, mod);
         GET_CHECK_TOK(tok, lex, TOK_EOF);
@@ -111,7 +111,7 @@ class CIncludeBytesExpander:
 
         ::std::vector<TokenTree>    toks;
         toks.push_back(Token(TOK_BYTESTRING, mv$(ss.str())));
-        return box$( TTStreamO(TokenTree(Ident::Hygiene::new_scope(), mv$(toks))) );
+        return box$( TTStreamO(sp, TokenTree(Ident::Hygiene::new_scope(), mv$(toks))) );
     }
 };
 
@@ -124,7 +124,7 @@ class CIncludeStrExpander:
             ERROR(sp, E0000, "include_str! doesn't take an ident");
 
         Token   tok;
-        auto lex = TTStream(tt);
+        auto lex = TTStream(sp, tt);
 
         auto path = get_string(sp, lex, crate, mod);
         GET_CHECK_TOK(tok, lex, TOK_EOF);
@@ -140,7 +140,7 @@ class CIncludeStrExpander:
 
         ::std::vector<TokenTree>    toks;
         toks.push_back(Token(TOK_STRING, mv$(ss.str())));
-        return box$( TTStreamO(TokenTree(Ident::Hygiene::new_scope(), mv$(toks))) );
+        return box$( TTStreamO(sp, TokenTree(Ident::Hygiene::new_scope(), mv$(toks))) );
     }
 };
 
diff --git a/src/expand/macro_rules.cpp b/src/expand/macro_rules.cpp
index 0375f430..7faa8223 100644
--- a/src/expand/macro_rules.cpp
+++ b/src/expand/macro_rules.cpp
@@ -28,11 +28,11 @@ class CMacroRulesExpander:
             ERROR(sp, E0000, "macro_rules! requires an identifier" );
 
         DEBUG("Parsing macro_rules! " << ident);
-        TTStream    lex(tt);
+        TTStream    lex(sp, tt);
         auto mac = Parse_MacroRules(lex);
         mod.add_macro( false, ident, mv$(mac) );
 
-        return ::std::unique_ptr<TokenStream>( new TTStreamO(TokenTree()) );
+        return ::std::unique_ptr<TokenStream>( new TTStreamO(sp, TokenTree()) );
     }
 };
 
diff --git a/src/expand/rustc_diagnostics.cpp b/src/expand/rustc_diagnostics.cpp
index f2e436c4..0e95bb7c 100644
--- a/src/expand/rustc_diagnostics.cpp
+++ b/src/expand/rustc_diagnostics.cpp
@@ -15,7 +15,7 @@ class CExpanderRegisterDiagnostic:
 {
     ::std::unique_ptr<TokenStream> expand(const Span& sp, const AST::Crate& crate, const ::std::string& ident, const TokenTree& tt, AST::Module& mod) override
     {
-        return box$( TTStreamO(TokenTree()) );
+        return box$( TTStreamO(sp, TokenTree()) );
     }
 };
 class CExpanderDiagnosticUsed:
@@ -23,7 +23,7 @@ class CExpanderDiagnosticUsed:
 {
     ::std::unique_ptr<TokenStream> expand(const Span& sp, const AST::Crate& crate, const ::std::string& ident, const TokenTree& tt, AST::Module& mod) override
     {
-        return box$( TTStreamO(TokenTree()) );
+        return box$( TTStreamO(sp, TokenTree()) );
     }
 };
 class CExpanderBuildDiagnosticArray:
@@ -33,7 +33,7 @@ class CExpanderBuildDiagnosticArray:
     {
         if( ident != "" )
             ERROR(sp, E0000, "__build_diagnostic_array! doesn't take an ident");
-        auto lex = TTStream(tt);
+        auto lex = TTStream(sp, tt);
 
         Token   tok;
 
@@ -64,7 +64,7 @@ class CExpanderBuildDiagnosticArray:
         toks.push_back( TOK_SQUARE_CLOSE );
         toks.push_back( TOK_SEMICOLON );
 
-        return box$( TTStreamO(TokenTree( lex.getHygiene(), mv$(toks) )) );
+        return box$( TTStreamO(sp, TokenTree( lex.getHygiene(), mv$(toks) )) );
     }
 };
 
diff --git a/src/expand/stringify.cpp b/src/expand/stringify.cpp
index cbb5c65c..06c65c02 100644
--- a/src/expand/stringify.cpp
+++ b/src/expand/stringify.cpp
@@ -17,7 +17,7 @@ class CExpander:
         Token   tok;
         ::std::string rv;
 
-        auto lex = TTStream(tt);
+        auto lex = TTStream(sp, tt);
         while( GET_TOK(tok, lex) != TOK_EOF )
         {
             if(!rv.empty())
@@ -25,7 +25,7 @@ class CExpander:
             rv += tok.to_str();
         }
 
-        return box$( TTStreamO(TokenTree(Token(TOK_STRING, mv$(rv)))) );
+        return box$( TTStreamO(sp, TokenTree(Token(TOK_STRING, mv$(rv)))) );
     }
 };
 
diff --git a/src/macro_rules/eval.cpp b/src/macro_rules/eval.cpp
index ebee2019..5b041717 100644
--- a/src/macro_rules/eval.cpp
+++ b/src/macro_rules/eval.cpp
@@ -169,7 +169,7 @@ private:
 };
 
 // === Prototypes ===
-unsigned int Macro_InvokeRules_MatchPattern(const MacroRules& rules, TokenTree input, AST::Module& mod,  ParameterMappings& bound_tts);
+unsigned int Macro_InvokeRules_MatchPattern(const Span& sp, const MacroRules& rules, TokenTree input, AST::Module& mod,  ParameterMappings& bound_tts);
 void Macro_InvokeRules_CountSubstUses(ParameterMappings& bound_tts, const ::std::vector<MacroExpansionEnt>& contents);
 
 // ------------------------------------
@@ -730,7 +730,7 @@ InterpolatedFragment Macro_HandlePatternCap(TokenStream& lex, MacroPatEnt::Type
     TRACE_FUNCTION_F("'" << name << "', " << input);
 
     ParameterMappings   bound_tts;
-    unsigned int    rule_index = Macro_InvokeRules_MatchPattern(rules, mv$(input), mod,  bound_tts);
+    unsigned int    rule_index = Macro_InvokeRules_MatchPattern(sp, rules, mv$(input), mod,  bound_tts);
 
     const auto& rule = rules.m_rules.at(rule_index);
 
@@ -1759,10 +1759,9 @@ namespace
     }
 }
 
-unsigned int Macro_InvokeRules_MatchPattern(const MacroRules& rules, TokenTree input, AST::Module& mod,  ParameterMappings& bound_tts)
+unsigned int Macro_InvokeRules_MatchPattern(const Span& sp, const MacroRules& rules, TokenTree input, AST::Module& mod,  ParameterMappings& bound_tts)
 {
     TRACE_FUNCTION;
-    static Span sp;
 
     struct ActiveArm {
         unsigned int    index;
@@ -1857,7 +1856,7 @@ unsigned int Macro_InvokeRules_MatchPattern(const MacroRules& rules, TokenTree i
         // NOTE: There can be multiple arms active, take the first.
         auto i = matches[0];
 
-        auto lex = TTStreamO(mv$(input));
+        auto lex = TTStreamO(sp, mv$(input));
         SET_MODULE(lex, mod);
         auto arm_stream = MacroPatternStream(rules.m_rules[i].m_pattern);
 
@@ -1930,7 +1929,7 @@ unsigned int Macro_InvokeRules_MatchPattern(const MacroRules& rules, TokenTree i
     }
 }
 #else
-unsigned int Macro_InvokeRules_MatchPattern(const MacroRules& rules, TokenTree input, AST::Module& mod,  ParameterMappings& bound_tts)
+unsigned int Macro_InvokeRules_MatchPattern(const Span& sp, const MacroRules& rules, TokenTree input, AST::Module& mod,  ParameterMappings& bound_tts)
 {
     TRACE_FUNCTION;
     Span    sp;// = input.span();
@@ -1956,7 +1955,7 @@ unsigned int Macro_InvokeRules_MatchPattern(const MacroRules& rules, TokenTree i
     // - List of captured values
     ::std::vector<InterpolatedFragment> captures;
 
-    TTStreamO   lex( mv$(input) );
+    TTStreamO   lex(sp, mv$(input) );
     SET_MODULE(lex, mod);
     while(true)
     {
@@ -2321,11 +2320,11 @@ Token MacroExpander::realGetToken()
                 {
                     if( can_steal )
                     {
-                        m_ttstream.reset( new TTStreamO( mv$(frag->as_tt()) ) );
+                        m_ttstream.reset( new TTStreamO(*this->outerSpan(), mv$(frag->as_tt()) ) );
                     }
                     else
                     {
-                        m_ttstream.reset( new TTStreamO( frag->as_tt().clone() ) );
+                        m_ttstream.reset( new TTStreamO(*this->outerSpan(), frag->as_tt().clone() ) );
                     }
                     return m_ttstream->getToken();
                 }
diff --git a/src/parse/ttstream.cpp b/src/parse/ttstream.cpp
index 4ba4a6f5..32c9a90a 100644
--- a/src/parse/ttstream.cpp
+++ b/src/parse/ttstream.cpp
@@ -8,7 +8,8 @@
 #include "ttstream.hpp"
 #include <common.hpp>
 
-TTStream::TTStream(const TokenTree& input_tt)
+TTStream::TTStream(Span parent, const TokenTree& input_tt):
+    m_parent_span( new Span(mv$(parent)) )
 {
     DEBUG("input_tt = [" << input_tt << "]");
     m_stack.push_back( ::std::make_pair(0, &input_tt) );
@@ -52,6 +53,7 @@ Token TTStream::realGetToken()
 }
 Position TTStream::getPosition() const
 {
+    // TODO: Position associated with the previous/next token?
     return Position("TTStream", 0,0);
 }
 Ident::Hygiene TTStream::realGetHygiene() const
@@ -63,8 +65,9 @@ Ident::Hygiene TTStream::realGetHygiene() const
 }
 
 
-TTStreamO::TTStreamO(TokenTree input_tt):
-    m_input_tt( mv$(input_tt) )
+TTStreamO::TTStreamO(Span parent, TokenTree input_tt):
+    m_input_tt( mv$(input_tt) ),
+    m_parent_span( new Span(mv$(parent)) )
 {
     m_stack.push_back( ::std::make_pair(0, nullptr) );
 }
diff --git a/src/parse/ttstream.hpp b/src/parse/ttstream.hpp
index 946cca5f..530a177b 100644
--- a/src/parse/ttstream.hpp
+++ b/src/parse/ttstream.hpp
@@ -15,14 +15,16 @@ class TTStream:
     public TokenStream
 {
     ::std::vector< ::std::pair<unsigned int, const TokenTree*> > m_stack;
+    ::std::shared_ptr<Span> m_parent_span;
     const Ident::Hygiene*   m_hygiene_ptr = nullptr;
 public:
-    TTStream(const TokenTree& input_tt);
+    TTStream(Span parent, const TokenTree& input_tt);
     ~TTStream();
 
     TTStream& operator=(const TTStream& x) { m_stack = x.m_stack; return *this; }
 
     Position getPosition() const override;
+    ::std::shared_ptr<Span> outerSpan() const override { return m_parent_span; }
 
 protected:
     Ident::Hygiene realGetHygiene() const override;
@@ -38,7 +40,8 @@ class TTStreamO:
     ::std::vector< ::std::pair<unsigned int, TokenTree*> > m_stack;
     const Ident::Hygiene*   m_hygiene_ptr = nullptr;
 public:
-    TTStreamO(TokenTree input_tt);
+    ::std::shared_ptr<Span> m_parent_span;
+    TTStreamO(Span parent, TokenTree input_tt);
     TTStreamO(TTStreamO&& x) = default;
     ~TTStreamO();
 
@@ -46,6 +49,7 @@ public:
     TTStreamO& operator=(TTStreamO&& x) = default;
 
     Position getPosition() const override;
+    ::std::shared_ptr<Span> outerSpan() const override { return m_parent_span; }
 
 protected:
     Ident::Hygiene realGetHygiene() const override;