Parse - Support chaining of spans (for macro expansions)

7 files changed, 48 insertions, 40 deletions

diff --git a/src/parse/expr.cpp b/src/parse/expr.cpp
index 28729b93..547e38e0 100644
--- a/src/parse/expr.cpp
+++ b/src/parse/expr.cpp
@@ -20,7 +20,8 @@
 
 using AST::ExprNode;
 using AST::ExprNodeP;
-static inline ExprNodeP mk_exprnodep(const TokenStream& lex, AST::ExprNode* en){en->set_pos(lex.getPosition()); return ExprNodeP(en); }
+// TODO: Use a ProtoSpan
+static inline ExprNodeP mk_exprnodep(const TokenStream& lex, AST::ExprNode* en){en->set_span(lex.point_span()); return ExprNodeP(en); }
 #define NEWNODE(type, ...)  mk_exprnodep(lex, new type(__VA_ARGS__))
 
 //ExprNodeP Parse_ExprBlockNode(TokenStream& lex, bool is_unsafe=false);    // common.hpp
@@ -552,7 +553,7 @@ ExprNodeP Parse_Stmt_Let(TokenStream& lex)
 {
     Token   tok;
     AST::Pattern pat = Parse_Pattern(lex, false);   // irrefutable
-    TypeRef type { lex.getPosition() };
+    TypeRef type { lex.point_span() };
     if( GET_TOK(tok, lex) == TOK_COLON ) {
         type = Parse_Type(lex);
         GET_TOK(tok, lex);
@@ -969,7 +970,7 @@ ExprNodeP Parse_ExprVal_StructLiteral(TokenStream& lex, AST::Path path)
             GET_CHECK_TOK(tok, lex, TOK_COLON);
             ExprNodeP   val = Parse_Stmt(lex);
             if( ! nodes.insert( ::std::make_pair(ofs, mv$(val)) ).second ) {
-                ERROR(lex.getPosition(), E0000, "Duplicate index");
+                ERROR(lex.point_span(), E0000, "Duplicate index");
             }
 
             if( GET_TOK(tok,lex) == TOK_BRACE_CLOSE )
@@ -983,7 +984,7 @@ ExprNodeP Parse_ExprVal_StructLiteral(TokenStream& lex, AST::Path path)
         for(auto& p : nodes)
         {
             if( p.first != i ) {
-                ERROR(lex.getPosition(), E0000, "Missing index " << i);
+                ERROR(lex.point_span(), E0000, "Missing index " << i);
             }
             items.push_back( mv$(p.second) );
             i ++;
@@ -1041,7 +1042,7 @@ ExprNodeP Parse_ExprVal_Closure(TokenStream& lex, bool is_move)
         // Irrefutable pattern
         AST::Pattern    pat = Parse_Pattern(lex, false);
 
-        TypeRef type { lex.getPosition() };
+        TypeRef type { lex.point_span() };
         if( GET_TOK(tok, lex) == TOK_COLON )
             type = Parse_Type(lex);
         else
@@ -1054,7 +1055,7 @@ ExprNodeP Parse_ExprVal_Closure(TokenStream& lex, bool is_move)
     }
     CHECK_TOK(tok, TOK_PIPE);
 
-    auto rt = TypeRef(lex.getPosition());
+    auto rt = TypeRef(lex.point_span());
     if( GET_TOK(tok, lex) == TOK_THINARROW ) {
 
         if( GET_TOK(tok, lex) == TOK_EXCLAM ) {
@@ -1264,8 +1265,9 @@ ExprNodeP Parse_ExprVal(TokenStream& lex)
 }
 ExprNodeP Parse_ExprMacro(TokenStream& lex, AST::Path path)
 {
+    ASSERT_BUG(lex.point_span(), path.is_trivial(), "TODO: Support path macros - " << path);
+
     Token   tok;
-    ASSERT_BUG(lex.getPosition(), path.is_trivial(), "TODO: Support path macros - " << path);
     ::std::string name = path.m_class.is_Local() ? path.m_class.as_Local().name : path.nodes()[0].name();
     ::std::string ident;
     if( GET_TOK(tok, lex) == TOK_IDENT ) {
diff --git a/src/parse/parseerror.cpp b/src/parse/parseerror.cpp
index 3c5d41fe..1bb30985 100644
--- a/src/parse/parseerror.cpp
+++ b/src/parse/parseerror.cpp
@@ -19,13 +19,13 @@ CompileError::Generic::Generic(::std::string message):
 }
 CompileError::Generic::Generic(const TokenStream& lex, ::std::string message)
 {
-    ::std::cout << lex.getPosition() << ": Generic(" << message << ")" << ::std::endl;
+    ::std::cout << lex.point_span() << ": Generic(" << message << ")" << ::std::endl;
 }
 
 CompileError::BugCheck::BugCheck(const TokenStream& lex, ::std::string message):
     m_message(message)
 {
-    ::std::cout << lex.getPosition() << "BugCheck(" << message << ")" << ::std::endl;
+    ::std::cout << lex.point_span() << "BugCheck(" << message << ")" << ::std::endl;
 }
 CompileError::BugCheck::BugCheck(::std::string message):
     m_message(message)
@@ -41,7 +41,7 @@ CompileError::Todo::Todo(::std::string message):
 CompileError::Todo::Todo(const TokenStream& lex, ::std::string message):
     m_message(message)
 {
-    ::std::cout << lex.getPosition() << ": Todo(" << message << ")" << ::std::endl;
+    ::std::cout << lex.point_span() << ": Todo(" << message << ")" << ::std::endl;
 }
 CompileError::Todo::~Todo() throw()
 {
@@ -49,7 +49,7 @@ CompileError::Todo::~Todo() throw()
 
 ParseError::BadChar::BadChar(const TokenStream& lex, char character)
 {
-    ::std::cout << lex.getPosition() << ": BadChar(" << character << ")" << ::std::endl;
+    ::std::cout << lex.point_span() << ": BadChar(" << character << ")" << ::std::endl;
 }
 ParseError::BadChar::~BadChar() throw()
 {
@@ -58,24 +58,24 @@ ParseError::BadChar::~BadChar() throw()
 ParseError::Unexpected::Unexpected(const TokenStream& lex, const Token& tok)//:
 //    m_tok( mv$(tok) )
 {
-    auto pos = tok.get_pos();
+    Span pos = tok.get_pos();
     if(pos.filename == "")
-        pos = lex.getPosition();
+        pos = lex.point_span();
     ::std::cout << pos << ": Unexpected(" << tok << ")" << ::std::endl;
 }
 ParseError::Unexpected::Unexpected(const TokenStream& lex, const Token& tok, Token exp)//:
 //    m_tok( mv$(tok) )
 {
-    auto pos = tok.get_pos();
+    Span pos = tok.get_pos();
     if(pos.filename == "")
-        pos = lex.getPosition();
+        pos = lex.point_span();
     ::std::cout << pos << ": Unexpected(" << tok << ", " << exp << ")" << ::std::endl;
 }
 ParseError::Unexpected::Unexpected(const TokenStream& lex, const Token& tok, ::std::vector<eTokenType> exp)
 {
-    auto pos = tok.get_pos();
+    Span pos = tok.get_pos();
     if(pos.filename == "")
-        pos = lex.getPosition();
+        pos = lex.point_span();
     ::std::cout << pos << ": Unexpected " << tok << ", expected ";
     bool f = true;
     for(auto v: exp) {
diff --git a/src/parse/pattern.cpp b/src/parse/pattern.cpp
index e9d086f9..974ff5fb 100644
--- a/src/parse/pattern.cpp
+++ b/src/parse/pattern.cpp
@@ -258,7 +258,7 @@ AST::Pattern Parse_PatternReal1(TokenStream& lex, bool is_refutable)
             return AST::Pattern( AST::Pattern::TagValue(), AST::Pattern::Value::make_Float({n->m_datatype, n->m_value}) );
         }
         else {
-            TODO(lex.getPosition(), "Convert :expr into a pattern value - " << *e);
+            TODO(lex.point_span(), "Convert :expr into a pattern value - " << *e);
         }
         } break;
 
@@ -414,7 +414,7 @@ AST::Pattern Parse_PatternStruct(TokenStream& lex, AST::Path path, bool is_refut
             GET_CHECK_TOK(tok, lex, TOK_COLON);
             auto val = Parse_Pattern(lex, is_refutable);
             if( ! pats.insert( ::std::make_pair(ofs, mv$(val)) ).second ) {
-                ERROR(lex.getPosition(), E0000, "Duplicate index");
+                ERROR(lex.point_span(), E0000, "Duplicate index");
             }
 
             if( GET_TOK(tok,lex) == TOK_BRACE_CLOSE )
@@ -435,7 +435,7 @@ AST::Pattern Parse_PatternStruct(TokenStream& lex, AST::Path path, bool is_refut
         {
             if( p.first != i ) {
                 if( has_split || !split_allowed ) {
-                    ERROR(lex.getPosition(), E0000, "Missing index " << i);
+                    ERROR(lex.point_span(), E0000, "Missing index " << i);
                 }
                 has_split = true;
                 i = p.first;
diff --git a/src/parse/root.cpp b/src/parse/root.cpp
index 5f8ed0a3..0790f05b 100644
--- a/src/parse/root.cpp
+++ b/src/parse/root.cpp
@@ -196,7 +196,7 @@ AST::GenericParams Parse_GenericParams(TokenStream& lex)
             ::std::string param_name = mv$(tok.str());
             ret.add_ty_param( AST::TypeParam( param_name ) );
 
-            auto param_ty = TypeRef(lex.getPosition(), param_name);
+            auto param_ty = TypeRef(lex.point_span(), param_name);
             if( GET_TOK(tok, lex) == TOK_COLON )
             {
                 Parse_TypeBound(lex, ret, mv$(param_ty));
@@ -382,7 +382,7 @@ AST::Function Parse_FunctionDef(TokenStream& lex, ::std::string abi, bool allow_
             GET_TOK(tok, lex);
             if( allow_self == false )
                 throw ParseError::Generic(lex, "Self binding not expected");
-            TypeRef ty = TypeRef( lex.getPosition(), "Self", 0xFFFF );
+            TypeRef ty = TypeRef( lex.point_span(), "Self", 0xFFFF );
             if( GET_TOK(tok, lex) == TOK_COLON ) {
                 // Typed mut self
                 ty = Parse_Type(lex);
@@ -399,7 +399,7 @@ AST::Function Parse_FunctionDef(TokenStream& lex, ::std::string abi, bool allow_
         // By-value method
         if( allow_self == false )
             throw ParseError::Generic(lex, "Self binding not expected");
-        TypeRef ty = TypeRef( lex.getPosition(), "Self", 0xFFFF );
+        TypeRef ty = TypeRef( lex.point_span(), "Self", 0xFFFF );
         if( GET_TOK(tok, lex) == TOK_COLON ) {
             // Typed mut self
             ty = Parse_Type(lex);
@@ -723,14 +723,14 @@ AST::Trait Parse_TraitDef(TokenStream& lex, const AST::MetaItems& meta_items)
             if( GET_TOK(tok, lex) == TOK_COLON )
             {
                 // Bounded associated type
-                Parse_TypeBound(lex, atype_params, TypeRef(lex.getPosition(), "Self", 0xFFFF));
+                Parse_TypeBound(lex, atype_params, TypeRef(lex.point_span(), "Self", 0xFFFF));
                 GET_TOK(tok, lex);
             }
             if( tok.type() == TOK_RWORD_WHERE ) {
                 throw ParseError::Todo(lex, "Where clause on associated type");
             }
 
-            TypeRef default_type = TypeRef( lex.getPosition() );
+            TypeRef default_type = TypeRef( lex.point_span() );
             if( tok.type() == TOK_EQUAL ) {
                 default_type = Parse_Type(lex);
                 GET_TOK(tok, lex);
@@ -1049,7 +1049,7 @@ AST::MetaItem Parse_MetaItem(TokenStream& lex)
         if( GET_TOK(tok, lex) == TOK_DOUBLE_DOT )
         {
             // Default impl
-            impl_type = TypeRef(TypeRef::TagInvalid(), lex.getPosition());
+            impl_type = TypeRef(TypeRef::TagInvalid(), lex.point_span());
         }
         else
         {
@@ -1390,7 +1390,7 @@ void Parse_Use(TokenStream& lex, ::std::function<void(AST::UseStmt, ::std::strin
     else
     {
         PUTBACK(tok, lex);
-        ASSERT_BUG(lex.getPosition(), path.nodes().size() > 0, "`use` with no path");
+        ASSERT_BUG(lex.point_span(), path.nodes().size() > 0, "`use` with no path");
         name = path.nodes().back().name();
     }
 
@@ -1490,7 +1490,7 @@ bool Parse_MacroInvocation_Opt(TokenStream& lex,  AST::MacroInvocation& out_inv)
         Parse_Use(lex, [&](AST::UseStmt p, std::string s) {
                 DEBUG(mod_path << " - use " << p << " as '" << s << "'");
                 if( !item_data.is_None() )
-                    TODO(lex.getPosition(), "Encode multi-item use statements as a single Item");
+                    TODO(lex.point_span(), "Encode multi-item use statements as a single Item");
                 item_data = ::AST::Item(mv$(p));
                 item_name = mv$(s);
             });
@@ -1735,7 +1735,7 @@ bool Parse_MacroInvocation_Opt(TokenStream& lex,  AST::MacroInvocation& out_inv)
         bool    sub_file_controls_dir = true;
         if( mod_fileinfo.path == "-" ) {
             if( path_attr.size() ) {
-                ERROR(lex.getPosition(), E0000, "Cannot load module from file when reading stdin");
+                ERROR(lex.point_span(), E0000, "Cannot load module from file when reading stdin");
             }
             sub_path = "-";
         }
@@ -1779,11 +1779,11 @@ bool Parse_MacroInvocation_Opt(TokenStream& lex,  AST::MacroInvocation& out_inv)
             break;
         case TOK_SEMICOLON:
             if( sub_path == "-" ) {
-                ERROR(lex.getPosition(), E0000, "Cannot load module from file when reading stdin");
+                ERROR(lex.point_span(), E0000, "Cannot load module from file when reading stdin");
             }
             else if( path_attr.size() == 0 && ! mod_fileinfo.controls_dir )
             {
-                ERROR(lex.getPosition(), E0000, "Can't load from files outside of mod.rs or crate root");
+                ERROR(lex.point_span(), E0000, "Can't load from files outside of mod.rs or crate root");
             }
             else if( !H::check_item_cfg(meta_items) ) {
                 // Ignore - emit Item::None
@@ -1801,7 +1801,7 @@ bool Parse_MacroInvocation_Opt(TokenStream& lex,  AST::MacroInvocation& out_inv)
                 if( ifs_dir.is_open() && ifs_file.is_open() )
                 {
                     // Collision
-                    ERROR(lex.getPosition(), E0000, "Both modname.rs and modname/mod.rs exist");
+                    ERROR(lex.point_span(), E0000, "Both modname.rs and modname/mod.rs exist");
                 }
                 else if( ifs_dir.is_open() )
                 {
@@ -1815,7 +1815,7 @@ bool Parse_MacroInvocation_Opt(TokenStream& lex,  AST::MacroInvocation& out_inv)
                 else
                 {
                     // Can't find file
-                    ERROR(lex.getPosition(), E0000, "Can't find file for '" << name << "' in '" << mod_fileinfo.path << "'");
+                    ERROR(lex.point_span(), E0000, "Can't find file for '" << name << "' in '" << mod_fileinfo.path << "'");
                 }
                 DEBUG("- path = " << submod.m_file_info.path);
                 Lexer sub_lex(submod.m_file_info.path);
diff --git a/src/parse/tokenstream.cpp b/src/parse/tokenstream.cpp
index 8cb9a910..2975a523 100644
--- a/src/parse/tokenstream.cpp
+++ b/src/parse/tokenstream.cpp
@@ -116,11 +116,15 @@ ProtoSpan TokenStream::start_span() const
 Span TokenStream::end_span(ProtoSpan ps) const
 {
     auto p = this->getPosition();
-    return Span(
-        ps.filename,
-        ps.start_line, ps.start_ofs,
-        p.line, p.ofs
-        );
+    auto rv = Span( ps.filename,  ps.start_line, ps.start_ofs,  p.line, p.ofs );
+    rv.outer_span = this->outerSpan();
+    return rv;
+}
+Span TokenStream::point_span() const
+{
+    Span rv = this->getPosition();
+    rv.outer_span = this->outerSpan();
+    return rv;
 }
 Ident TokenStream::get_ident(Token tok) const
 {
diff --git a/src/parse/tokenstream.hpp b/src/parse/tokenstream.hpp
index 85fc62e2..5f2e0733 100644
--- a/src/parse/tokenstream.hpp
+++ b/src/parse/tokenstream.hpp
@@ -60,17 +60,19 @@ public:
     void    putback(Token tok);
     eTokenType  lookahead(unsigned int count);
 
-    virtual Position getPosition() const = 0;
     Ident::Hygiene getHygiene() const;
 
     ParseState& parse_state() { return m_parse_state; }
 
     ProtoSpan   start_span() const;
     Span    end_span(ProtoSpan ps) const;
+    Span    point_span() const;
 
     Ident get_ident(Token tok) const;
 
 protected:
+    virtual Position getPosition() const = 0;
+    virtual ::std::shared_ptr<Span> outerSpan() const { return ::std::shared_ptr<Span>(0); }
     virtual Token   realGetToken() = 0;
     virtual Ident::Hygiene realGetHygiene() const = 0;
 private:
diff --git a/src/parse/types.cpp b/src/parse/types.cpp
index 905bd935..ff993693 100644
--- a/src/parse/types.cpp
+++ b/src/parse/types.cpp
@@ -218,7 +218,7 @@ TypeRef Parse_Type_Fn(TokenStream& lex, ::std::vector<::std::string> hrls)
         if( GET_TOK(tok, lex) == TOK_STRING ) {
             abi = tok.str();
             if( abi == "" )
-                ERROR(lex.getPosition(), E0000, "Empty ABI");
+                ERROR(lex.point_span(), E0000, "Empty ABI");
             GET_TOK(tok, lex);
         }
         else {