Parse - Remove TOK_MACRO

10 files changed, 89 insertions, 63 deletions

diff --git a/src/macro_rules/mod.cpp b/src/macro_rules/mod.cpp
index f685f6dc..eb0f9be0 100644
--- a/src/macro_rules/mod.cpp
+++ b/src/macro_rules/mod.cpp
@@ -76,7 +76,6 @@ bool is_token_expr(eTokenType tt) {
     case TOK_EXCLAM:    // Invert
     case TOK_RWORD_BOX: // Box
     // Composite values
-    case TOK_MACRO:
     case TOK_PAREN_OPEN:    // Parenthesised
     case TOK_SQUARE_OPEN:   // Array
 
diff --git a/src/macro_rules/parse.cpp b/src/macro_rules/parse.cpp
index fa82e0e0..8264180f 100644
--- a/src/macro_rules/parse.cpp
+++ b/src/macro_rules/parse.cpp
@@ -48,8 +48,9 @@ public:
         case TOK_DOLLAR:
             switch( GET_TOK(tok, lex) )
             {
+            case TOK_RWORD_TYPE:
             case TOK_IDENT: {
-                ::std::string   name = mv$(tok.str());
+                ::std::string   name = tok.type() == TOK_IDENT ? mv$(tok.str()) : FMT(tok);
                 GET_CHECK_TOK(tok, lex, TOK_COLON);
                 GET_CHECK_TOK(tok, lex, TOK_IDENT);
                 ::std::string   type = mv$(tok.str());
@@ -199,10 +200,10 @@ public:
                 }
 
             }
-            else if( tok.type() == TOK_IDENT )
+            else if( tok.type() == TOK_IDENT || tok.type() == TOK_RWORD_TYPE )
             {
                 // Look up the named parameter in the list of param names for this arm
-                const auto& name = tok.str();
+                auto name = tok.type() == TOK_IDENT ? tok.str() : FMT(tok);
                 unsigned int idx = ::std::find(var_names.begin(), var_names.end(), name) - var_names.begin();
                 if( idx == var_names.size() )
                     ERROR(lex.getPosition(), E0000, "Macro variable $" << name << " not found");
diff --git a/src/parse/eTokenType.enum.h b/src/parse/eTokenType.enum.h
index 8d410b0c..2de1000b 100644
--- a/src/parse/eTokenType.enum.h
+++ b/src/parse/eTokenType.enum.h
@@ -17,7 +17,6 @@ _(TOK_INTERPOLATED_ITEM)
 
 // Value tokens
 _(TOK_IDENT)
-_(TOK_MACRO)
 _(TOK_LIFETIME)
 _(TOK_INTEGER)
 _(TOK_CHAR)
diff --git a/src/parse/expr.cpp b/src/parse/expr.cpp
index 93adc3d4..27d122ce 100644
--- a/src/parse/expr.cpp
+++ b/src/parse/expr.cpp
@@ -35,7 +35,7 @@ ExprNodeP Parse_WhileStmt(TokenStream& lex, ::std::string lifetime);
 ExprNodeP Parse_ForStmt(TokenStream& lex, ::std::string lifetime);
 ExprNodeP Parse_Expr_Match(TokenStream& lex);
 ExprNodeP Parse_Expr1(TokenStream& lex);
-ExprNodeP Parse_ExprMacro(TokenStream& lex, Token tok);
+ExprNodeP Parse_ExprMacro(TokenStream& lex, AST::Path tok);
 
 AST::Expr Parse_Expr(TokenStream& lex)
 {
@@ -120,7 +120,7 @@ ExprNodeP Parse_ExprBlockLine_WithItems(TokenStream& lex, ::std::shared_ptr<AST:
         return ExprNodeP();
     }
 
-    if( tok.type() == TOK_MACRO && tok.str() == "macro_rules" )
+    if( tok.type() == TOK_IDENT && tok.str() == "macro_rules" && lex.lookahead(0) == TOK_EXCLAM )
     {
         // Special case - create a local module if macro_rules! is seen
         // - Allows correct scoping of defined macros
@@ -285,10 +285,14 @@ ExprNodeP Parse_ExprBlockLine(TokenStream& lex, bool *add_silence)
             return ret;
             }
 
-        case TOK_MACRO:
-            // If a braced macro invocation is the first part of a statement, don't expect a semicolon
-            if( LOOK_AHEAD(lex) == TOK_BRACE_OPEN || (lex.lookahead(0) == TOK_IDENT && lex.lookahead(1) == TOK_BRACE_OPEN) ) {
-                return Parse_ExprMacro(lex, tok);
+        case TOK_IDENT:
+            if( lex.lookahead(0) == TOK_EXCLAM )
+            {
+                // If a braced macro invocation is the first part of a statement, don't expect a semicolon
+                if( lex.lookahead(1) == TOK_BRACE_OPEN || (lex.lookahead(1) == TOK_IDENT && lex.lookahead(2) == TOK_BRACE_OPEN) ) {
+                    lex.getToken();
+                    return Parse_ExprMacro(lex, tok.str());
+                }
             }
         // Fall through to the statement code
         default:
@@ -682,8 +686,6 @@ bool Parse_IsTokValue(eTokenType tok_type)
     case TOK_INTERPOLATED_PATH:
     case TOK_INTERPOLATED_EXPR:
 
-    case TOK_MACRO:
-
     case TOK_PIPE:
     case TOK_EXCLAM:
     case TOK_DASH:
@@ -1138,9 +1140,12 @@ ExprNodeP Parse_ExprVal(TokenStream& lex)
         if(0)
     case TOK_DOUBLE_COLON:
         path = Parse_Path(lex, true, PATH_GENERIC_EXPR);
+        DEBUG("path = " << path << ", lookahead=" << Token::typestr(lex.lookahead(0)));
         // SKIP TARGET
         switch( GET_TOK(tok, lex) )
         {
+        case TOK_EXCLAM:
+            return Parse_ExprMacro(lex, mv$(path));
         case TOK_PAREN_OPEN:
             // Function call
             PUTBACK(tok, lex);
@@ -1244,15 +1249,15 @@ ExprNodeP Parse_ExprVal(TokenStream& lex)
             }
         }
         throw ParseError::BugCheck(lex, "Array literal fell");
-    case TOK_MACRO:
-        return Parse_ExprMacro(lex, mv$(tok));
     default:
         throw ParseError::Unexpected(lex, tok);
     }
 }
-ExprNodeP Parse_ExprMacro(TokenStream& lex, Token tok)
+ExprNodeP Parse_ExprMacro(TokenStream& lex, AST::Path path)
 {
-    ::std::string name = tok.str();
+    Token   tok;
+    ASSERT_BUG(lex.getPosition(), path.is_trivial(), "TODO: Support path macros - " << path);
+    ::std::string name = path.m_class.is_Local() ? path.m_class.as_Local().name : path.nodes()[0].name();
     ::std::string ident;
     if( GET_TOK(tok, lex) == TOK_IDENT ) {
         ident = mv$(tok.str());
diff --git a/src/parse/lex.cpp b/src/parse/lex.cpp
index 57a0bedc..4cd9954f 100644
--- a/src/parse/lex.cpp
+++ b/src/parse/lex.cpp
@@ -744,20 +744,13 @@ Token Lexer::getTokenInt_Identifier(Codepoint leader, Codepoint leader2)
         ch = this->getc();
     }
 
-    if( ch == '!' )
-    {
-        return Token(TOK_MACRO, str);
-    }
-    else
+    this->ungetc();
+    for( unsigned int i = 0; i < LEN(RWORDS); i ++ )
     {
-        this->ungetc();
-        for( unsigned int i = 0; i < LEN(RWORDS); i ++ )
-        {
-            if( str < RWORDS[i].chars ) break;
-            if( str == RWORDS[i].chars )    return Token((enum eTokenType)RWORDS[i].type);
-        }
-        return Token(TOK_IDENT, mv$(str));
+        if( str < RWORDS[i].chars ) break;
+        if( str == RWORDS[i].chars )    return Token((enum eTokenType)RWORDS[i].type);
     }
+    return Token(TOK_IDENT, mv$(str));
 }
 
 // Takes the VERY lazy way of reading the float into a string then passing to strtod
diff --git a/src/parse/pattern.cpp b/src/parse/pattern.cpp
index 699e7fae..aed2c91d 100644
--- a/src/parse/pattern.cpp
+++ b/src/parse/pattern.cpp
@@ -43,8 +43,9 @@ AST::Pattern Parse_Pattern(TokenStream& lex, bool is_refutable)
     Token   tok;
     tok = lex.getToken();
 
-    if( tok.type() == TOK_MACRO )
+    if( tok.type() == TOK_IDENT && lex.lookahead(0) == TOK_EXCLAM )
     {
+        lex.getToken();
         return AST::Pattern( AST::Pattern::TagMacro(), box$(Parse_MacroInvocation(ps, tok.str(), lex)));
     }
     if( tok.type() == TOK_INTERPOLATED_PATTERN )
diff --git a/src/parse/root.cpp b/src/parse/root.cpp
index d40e1f95..3bdd34b1 100644
--- a/src/parse/root.cpp
+++ b/src/parse/root.cpp
@@ -42,6 +42,7 @@ Spanned<T> get_spanned(TokenStream& lex, ::std::function<T()> f) {
 
 AST::MetaItem   Parse_MetaItem(TokenStream& lex);
 void Parse_ModRoot(TokenStream& lex, AST::Module& mod, AST::MetaItems& mod_attrs);
+bool Parse_MacroInvocation_Opt(TokenStream& lex,  AST::MacroInvocation& out_inv);
 
 //::AST::Path Parse_Publicity(TokenStream& lex)
 bool Parse_Publicity(TokenStream& lex, bool allow_restricted=true)
@@ -657,14 +658,15 @@ AST::Trait Parse_TraitDef(TokenStream& lex, const AST::MetaItems& meta_items)
         SET_ATTRS(lex, item_attrs);
 
         auto ps = lex.start_span();
-        if( tok.type() == TOK_MACRO ) {
-            auto inv = Parse_MacroInvocation( ps, mv$(tok.str()), lex );
-            // - Silently consume ';' after the macro
-            if( GET_TOK(tok, lex) != TOK_SEMICOLON )
-                PUTBACK(tok, lex);
-
-            trait.items().push_back( AST::Named<AST::Item>("", AST::Item(mv$(inv)), false) );
-            continue ;
+        {
+            ::AST::MacroInvocation  inv;
+            PUTBACK(tok, lex);
+            if( Parse_MacroInvocation_Opt(lex, inv) )
+            {
+                trait.items().push_back( AST::Named<AST::Item>("", AST::Item(mv$(inv)), false) );
+                continue ;
+            }
+            GET_TOK(tok, lex);
         }
 
         bool is_specialisable = false;
@@ -1075,22 +1077,20 @@ AST::MetaItem Parse_MetaItem(TokenStream& lex)
     AST::Impl   impl( AST::ImplDef( lex.end_span(ps), mv$(attrs), mv$(params), mv$(trait_path), mv$(impl_type) ) );
 
     // A sequence of method implementations
-    while( GET_TOK(tok, lex) != TOK_BRACE_CLOSE )
+    while( lex.lookahead(0) != TOK_BRACE_CLOSE )
     {
         auto ps = lex.start_span();
-        if( tok.type() == TOK_MACRO )
+        ::AST::MacroInvocation  inv;
+        if( Parse_MacroInvocation_Opt(lex,  inv) )
         {
-            impl.add_macro_invocation( Parse_MacroInvocation( ps, mv$(tok.str()), lex ) );
-            // - Silently consume ';' after the macro
-            if( GET_TOK(tok, lex) != TOK_SEMICOLON )
-                PUTBACK(tok, lex);
+            impl.add_macro_invocation( mv$(inv) );
         }
         else
         {
-            PUTBACK(tok, lex);
             Parse_Impl_Item(lex, impl);
         }
     }
+    GET_CHECK_TOK(tok, lex, TOK_BRACE_CLOSE);
 
     return ::AST::Item::make_Impl( mv$(impl) );
 }
@@ -1409,6 +1409,34 @@ void Parse_Use(TokenStream& lex, ::std::function<void(AST::UseStmt, ::std::strin
     return ::AST::MacroInvocation( lex.end_span(span_start), mv$(name), mv$(ident), mv$(tt));
 }
 
+bool Parse_MacroInvocation_Opt(TokenStream& lex,  AST::MacroInvocation& out_inv)
+{
+    Token   tok;
+    if( lex.lookahead(0) == TOK_IDENT && lex.lookahead(1) == TOK_EXCLAM )
+    {
+        // Good
+    }
+    else
+    {
+        return false;
+    }
+
+    auto ps = lex.start_span();
+    GET_CHECK_TOK(tok, lex, TOK_IDENT);
+    auto name = tok.str();
+    GET_CHECK_TOK(tok, lex, TOK_EXCLAM);
+
+    bool is_braced = (lex.lookahead(0) == TOK_BRACE_OPEN || (lex.lookahead(0) == TOK_IDENT && lex.lookahead(1) == TOK_BRACE_OPEN));
+
+    out_inv = Parse_MacroInvocation(ps, name, lex);
+
+    if(!is_braced )
+    {
+        GET_CHECK_TOK(tok, lex, TOK_SEMICOLON);
+    }
+    return true;
+}
+
 ::AST::Named<::AST::Item> Parse_Mod_Item_S(TokenStream& lex, const AST::Module::FileInfo& mod_fileinfo, const ::AST::Path& mod_path, AST::MetaItems meta_items)
 {
     TRACE_FUNCTION_F("mod_path="<<mod_path<<", meta_items="<<meta_items);
@@ -1436,22 +1464,16 @@ void Parse_Use(TokenStream& lex, ::std::function<void(AST::UseStmt, ::std::strin
     ::std::string   item_name;
     ::AST::Item item_data;
 
-    if( LOOK_AHEAD(lex) == TOK_MACRO ) {
-        GET_TOK(tok, lex);
-
-        ::std::string   name = mv$(tok.str());
-        bool is_braced = (LOOK_AHEAD(lex) == TOK_BRACE_OPEN || LOOKAHEAD2(lex, TOK_IDENT, TOK_BRACE_OPEN));
-        item_data = ::AST::Item( Parse_MacroInvocation( ps, mv$(name), lex ) );
+    {
+        ::AST::MacroInvocation  inv;
+        if( Parse_MacroInvocation_Opt(lex, inv) )
+        {
+            item_data = ::AST::Item( mv$(inv) );
+            item_data.attrs = mv$(meta_items);
+            item_data.span = lex.end_span(ps);
 
-        if( !is_braced ) {
-            // - Consume the ';' after the macro
-            GET_CHECK_TOK(tok, lex, TOK_SEMICOLON);
+            return ::AST::Named< ::AST::Item> { "", mv$(item_data), false };
         }
-
-        item_data.attrs = mv$(meta_items);
-        item_data.span = lex.end_span(ps);
-
-        return ::AST::Named< ::AST::Item> { "", mv$(item_data), false };
     }
 
     bool    is_public = Parse_Publicity(lex);
diff --git a/src/parse/token.cpp b/src/parse/token.cpp
index 84838207..cf0038bd 100644
--- a/src/parse/token.cpp
+++ b/src/parse/token.cpp
@@ -305,7 +305,6 @@ struct EscapedString {
     case TOK_INTERPOLATED_IDENT: return "/*:ident*/";
     // Value tokens
     case TOK_IDENT:     return m_data.as_String();
-    case TOK_MACRO:     return m_data.as_String() + "!";
     case TOK_LIFETIME:  return "'" + m_data.as_String();
     case TOK_INTEGER:   return FMT(m_data.as_Integer().m_intval);    // TODO: suffix for type
     case TOK_CHAR:      return FMT("'\\u{"<< ::std::hex << m_data.as_Integer().m_intval << "}");
@@ -521,7 +520,6 @@ SERIALISE_TYPE(Token::, "Token", {
     case TOK_STRING:
     case TOK_BYTESTRING:
     case TOK_IDENT:
-    case TOK_MACRO:
     case TOK_LIFETIME:
         if( tok.m_data.is_String() )
             os << "\"" << EscapedString(tok.str()) << "\"";
diff --git a/src/parse/tokenstream.cpp b/src/parse/tokenstream.cpp
index 9a961a04..5322822f 100644
--- a/src/parse/tokenstream.cpp
+++ b/src/parse/tokenstream.cpp
@@ -33,6 +33,7 @@ Token TokenStream::getToken()
 {
     if( m_cache_valid )
     {
+        //DEBUG("<<< " << m_cache << " (cache)");
         m_cache_valid = false;
         return mv$(m_cache);
     }
@@ -41,6 +42,7 @@ Token TokenStream::getToken()
         Token ret = mv$( m_lookahead.front().first );
         m_hygiene = m_lookahead.front().second;
         m_lookahead.erase(m_lookahead.begin());
+        //DEBUG("<<< " << ret << " (lookahead)");
         if( DEBUG_PRINT_TOKENS ) {
             ::std::cout << "getToken[" << typeid(*this).name() << "] - " << ret.get_pos() << "-" << ret << ::std::endl;
         }
@@ -50,6 +52,7 @@ Token TokenStream::getToken()
     {
         Token ret = this->innerGetToken();
         m_hygiene = this->realGetHygiene();
+        //DEBUG("<<< " << ret << " (new)");
         if( DEBUG_PRINT_TOKENS ) {
             ::std::cout << "getToken[" << typeid(*this).name() << "] - " << ret.get_pos() << "-" << ret << ::std::endl;
         }
@@ -65,6 +68,7 @@ void TokenStream::putback(Token tok)
     }
     else
     {
+        //DEBUG(">>> " << tok);
         m_cache_valid = true;
         m_cache = mv$(tok);
     }
diff --git a/src/parse/types.cpp b/src/parse/types.cpp
index cb664f79..905bd935 100644
--- a/src/parse/types.cpp
+++ b/src/parse/types.cpp
@@ -37,8 +37,6 @@ TypeRef Parse_Type_Int(TokenStream& lex, bool allow_trait_list)
     {
     case TOK_INTERPOLATED_TYPE:
         return mv$(tok.frag_type());
-    case TOK_MACRO:
-        return TypeRef(TypeRef::TagMacro(), Parse_MacroInvocation(ps, mv$(tok.str()), lex));
     // '!' - Only ever used as part of function prototypes, but is kinda a type... not allowed here though
     case TOK_EXCLAM:
         return TypeRef( Span(tok.get_pos()), TypeData::make_Bang({}) );
@@ -81,6 +79,12 @@ TypeRef Parse_Type_Int(TokenStream& lex, bool allow_trait_list)
         }
     // <ident> - Either a primitive, or a path
     case TOK_IDENT:
+        if( lex.lookahead(0) == TOK_EXCLAM )
+        {
+            lex.getToken();
+            // TODO: path macros
+            return TypeRef(TypeRef::TagMacro(), Parse_MacroInvocation(ps, mv$(tok.str()), lex));
+        }
         // or a primitive
         //if( auto ct = coretype_fromstring(tok.str()) )
         //{