diff options
Diffstat (limited to 'src/parse')
-rw-r--r-- | src/parse/common.hpp | 5 | ||||
-rw-r--r-- | src/parse/eTokenType.enum.h | 8 | ||||
-rw-r--r-- | src/parse/expr.cpp | 131 | ||||
-rw-r--r-- | src/parse/interpolated_fragment.cpp | 76 | ||||
-rw-r--r-- | src/parse/interpolated_fragment.hpp | 49 | ||||
-rw-r--r-- | src/parse/lex.cpp | 20 | ||||
-rw-r--r-- | src/parse/parseerror.cpp | 10 | ||||
-rw-r--r-- | src/parse/parseerror.hpp | 6 | ||||
-rw-r--r-- | src/parse/paths.cpp | 27 | ||||
-rw-r--r-- | src/parse/pattern.cpp | 24 | ||||
-rw-r--r-- | src/parse/root.cpp | 98 | ||||
-rw-r--r-- | src/parse/token.cpp | 148 | ||||
-rw-r--r-- | src/parse/token.hpp | 32 | ||||
-rw-r--r-- | src/parse/tokentree.hpp | 5 | ||||
-rw-r--r-- | src/parse/types.cpp | 97 |
15 files changed, 550 insertions, 186 deletions
diff --git a/src/parse/common.hpp b/src/parse/common.hpp index d7f84315..c42889c3 100644 --- a/src/parse/common.hpp +++ b/src/parse/common.hpp @@ -41,7 +41,7 @@ extern ::std::vector<TypeRef> Parse_Path_GenericList(TokenStream& lex); extern AST::MetaItem Parse_MetaItem(TokenStream& lex);
extern ::AST::MacroInvocation Parse_MacroInvocation(ProtoSpan ps, ::AST::MetaItems meta_items, ::std::string name, TokenStream& lex);
-extern TypeRef Parse_Type(TokenStream& lex);
+extern TypeRef Parse_Type(TokenStream& lex, bool allow_trait_list = false);
extern AST::Pattern Parse_Pattern(TokenStream& lex, bool is_refutable);
extern void Parse_Impl_Item(TokenStream& lex, AST::Impl& impl);
@@ -51,7 +51,8 @@ extern void Parse_ModRoot_Items(TokenStream& lex, AST::Module& mod, bool file_co extern AST::Expr Parse_Expr(TokenStream& lex);
extern AST::Expr Parse_ExprBlock(TokenStream& lex);
-extern AST::ExprNodeP Parse_Expr0(TokenStream& lex);
+extern AST::ExprNodeP Parse_Expr0(TokenStream& lex);
+extern AST::ExprNodeP Parse_ExprVal(TokenStream& lex);
extern bool Parse_IsTokValue(eTokenType tok_type);
diff --git a/src/parse/eTokenType.enum.h b/src/parse/eTokenType.enum.h index 2adfb042..f7157805 100644 --- a/src/parse/eTokenType.enum.h +++ b/src/parse/eTokenType.enum.h @@ -5,6 +5,14 @@ _(TOK_NEWLINE) _(TOK_WHITESPACE) _(TOK_COMMENT) +_(TOK_INTERPOLATED_PATH) +_(TOK_INTERPOLATED_TYPE) +_(TOK_INTERPOLATED_PATTERN) +_(TOK_INTERPOLATED_EXPR) +_(TOK_INTERPOLATED_STMT) +_(TOK_INTERPOLATED_BLOCK) +_(TOK_INTERPOLATED_META) + // Value tokens _(TOK_IDENT) _(TOK_MACRO) diff --git a/src/parse/expr.cpp b/src/parse/expr.cpp index 7094cf96..37bc03e7 100644 --- a/src/parse/expr.cpp +++ b/src/parse/expr.cpp @@ -85,7 +85,7 @@ ExprNodeP Parse_ExprBlockNode(TokenStream& lex) case TOK_RWORD_IMPL:
case TOK_RWORD_FN:
case TOK_RWORD_MOD:
- lex.putback(tok);
+ PUTBACK(tok, lex);
if( !local_mod ) {
local_mod = lex.parse_state().get_current_mod().add_anon();
}
@@ -95,7 +95,7 @@ ExprNodeP Parse_ExprBlockNode(TokenStream& lex) case TOK_RWORD_UNSAFE:
if( LOOK_AHEAD(lex) != TOK_BRACE_OPEN )
{
- lex.putback(tok);
+ PUTBACK(tok, lex);
if( !local_mod ) {
local_mod = lex.parse_state().get_current_mod().add_anon();
}
@@ -104,7 +104,7 @@ ExprNodeP Parse_ExprBlockNode(TokenStream& lex) }
// fall
default: {
- lex.putback(tok);
+ PUTBACK(tok, lex);
bool expect_end = false;
nodes.push_back(Parse_ExprBlockLine(lex, &expect_end));
if( nodes.back() ) {
@@ -121,7 +121,7 @@ ExprNodeP Parse_ExprBlockNode(TokenStream& lex) {
throw ParseError::Unexpected(lex, tok, Token(TOK_BRACE_CLOSE));
}
- lex.putback(tok);
+ PUTBACK(tok, lex);
}
break;
}
@@ -159,7 +159,7 @@ ExprNodeP Parse_ExprBlockLine(TokenStream& lex, bool *expect_end) case TOK_RWORD_MATCH:
return Parse_Expr_Match(lex);
case TOK_BRACE_OPEN:
- lex.putback(tok);
+ PUTBACK(tok, lex);
return Parse_ExprBlockNode(lex);
default:
@@ -173,7 +173,7 @@ ExprNodeP Parse_ExprBlockLine(TokenStream& lex, bool *expect_end) case TOK_SEMICOLON:
return 0;
case TOK_BRACE_OPEN:
- lex.putback(tok);
+ PUTBACK(tok, lex);
return Parse_ExprBlockNode(lex);
// let binding
@@ -215,11 +215,11 @@ ExprNodeP Parse_ExprBlockLine(TokenStream& lex, bool *expect_end) }
// Fall through to the statement code
default: {
- lex.putback(tok);
+ PUTBACK(tok, lex);
auto ret = Parse_Stmt(lex);
if( GET_TOK(tok, lex) != TOK_SEMICOLON )
{
- lex.putback(tok);
+ PUTBACK(tok, lex);
*expect_end = true;
}
return ::std::move(ret);
@@ -245,7 +245,7 @@ ExprNodeP Parse_WhileStmt(TokenStream& lex, ::std::string lifetime) ::std::move(pat), ::std::move(val), Parse_ExprBlockNode(lex) );
}
else {
- lex.putback(tok);
+ PUTBACK(tok, lex);
ExprNodeP cnd;
{
SET_PARSE_FLAG(lex, disallow_struct_literal);
@@ -291,7 +291,7 @@ ExprNodeP Parse_IfStmt(TokenStream& lex) cond = Parse_Expr0(lex);
}
else {
- lex.putback(tok);
+ PUTBACK(tok, lex);
cond = Parse_Expr0(lex);
}
}
@@ -309,13 +309,13 @@ ExprNodeP Parse_IfStmt(TokenStream& lex) }
// - or get block
else {
- lex.putback(tok);
+ PUTBACK(tok, lex);
altcode = Parse_ExprBlockNode(lex);
}
}
// - or nothing
else {
- lex.putback(tok);
+ PUTBACK(tok, lex);
}
if( if_let )
@@ -343,7 +343,7 @@ ExprNodeP Parse_Expr_Match(TokenStream& lex) do {
if( GET_TOK(tok, lex) == TOK_BRACE_CLOSE )
break;
- lex.putback(tok);
+ PUTBACK(tok, lex);
AST::ExprNode_Match_Arm arm;
::AST::MetaItems arm_attrs;
@@ -372,7 +372,7 @@ ExprNodeP Parse_Expr_Match(TokenStream& lex) if( GET_TOK(tok, lex) == TOK_COMMA )
continue;
- lex.putback(tok);
+ PUTBACK(tok, lex);
} while( 1 );
CHECK_TOK(tok, TOK_BRACE_CLOSE);
@@ -431,20 +431,20 @@ ExprNodeP Parse_Stmt(TokenStream& lex) case TOK_BRACE_CLOSE:
case TOK_PAREN_CLOSE:
case TOK_SQUARE_CLOSE:
- lex.putback(tok);
+ PUTBACK(tok, lex);
break;
default:
- lex.putback(tok);
+ PUTBACK(tok, lex);
val = Parse_Expr1(lex);
break;
}
return NEWNODE( AST::ExprNode_Flow, type, lifetime, ::std::move(val) );
}
case TOK_BRACE_OPEN:
- lex.putback(tok);
+ PUTBACK(tok, lex);
return Parse_ExprBlockNode(lex);
default:
- lex.putback(tok);
+ PUTBACK(tok, lex);
return Parse_Expr0(lex);
}
}
@@ -460,7 +460,7 @@ ExprNodeP Parse_Stmt(TokenStream& lex) GET_CHECK_TOK(tok, lex, TOK_PAREN_OPEN);
if( GET_TOK(tok, lex) != TOK_PAREN_CLOSE )
{
- lex.putback(tok);
+ PUTBACK(tok, lex);
do {
if( LOOK_AHEAD(lex) == TOK_PAREN_CLOSE ) {
GET_TOK(tok, lex);
@@ -521,7 +521,7 @@ ExprNodeP Parse_Expr0(TokenStream& lex) return rv;
default:
- lex.putback(tok);
+ PUTBACK(tok, lex);
rv->set_attrs(mv$(expr_attrs));
return rv;
}
@@ -542,7 +542,7 @@ ExprNodeP cur(TokenStream& lex) \ cases \
default: \
/*::std::cout << "<<" << #cur << ::std::endl; */\
- lex.putback(tok); \
+ PUTBACK(tok, lex); \
return rv; \
} \
} \
@@ -585,26 +585,39 @@ ExprNodeP Parse_Expr1(TokenStream& lex) ExprNodeP (*next)(TokenStream&) = Parse_Expr1_1;
ExprNodeP left, right;
+ // Inclusive range to a value
+ if( GET_TOK(tok, lex) == TOK_TRIPLE_DOT ) {
+ right = next(lex);
+ return NEWNODE( AST::ExprNode_BinOp, AST::ExprNode_BinOp::RANGE_INC, nullptr, mv$(right) );
+ }
+ else {
+ PUTBACK(tok, lex);
+ }
+
+ // Exclusive ranges
+ // - If NOT `.. <VAL>`, parse a leading value
if( GET_TOK(tok, lex) != TOK_DOUBLE_DOT )
{
- lex.putback(tok);
+ PUTBACK(tok, lex);
left = next(lex);
+ // - If NOT `<VAL> ..`, return the value
if( GET_TOK(tok, lex) != TOK_DOUBLE_DOT )
{
- lex.putback(tok);
+ PUTBACK(tok, lex);
return ::std::move(left);
}
}
- if( Parse_IsTokValue( GET_TOK(tok, lex) ) )
+ assert( tok.type() == TOK_DOUBLE_DOT );
+ // If the next token is part of a value, parse that value
+ if( Parse_IsTokValue( LOOK_AHEAD(lex) ) )
{
- lex.putback(tok);
right = next(lex);
}
else
{
- lex.putback(tok);
+ // Otherwise, leave `right` as nullptr
}
return NEWNODE( AST::ExprNode_BinOp, AST::ExprNode_BinOp::RANGE, ::std::move(left), ::std::move(right) );
@@ -717,7 +730,7 @@ ExprNodeP Parse_Expr12(TokenStream& lex) }
else
{
- lex.putback(tok);
+ PUTBACK(tok, lex);
}
return rv;
}
@@ -752,11 +765,11 @@ ExprNodeP Parse_Expr13(TokenStream& lex) if( GET_TOK(tok, lex) == TOK_RWORD_MUT )
return NEWNODE( AST::ExprNode_UniOp, AST::ExprNode_UniOp::REFMUT, Parse_Expr12(lex) );
else {
- lex.putback(tok);
+ PUTBACK(tok, lex);
return NEWNODE( AST::ExprNode_UniOp, AST::ExprNode_UniOp::REF, Parse_Expr12(lex) );
}
default:
- lex.putback(tok);
+ PUTBACK(tok, lex);
return Parse_ExprFC(lex);
}
}
@@ -776,7 +789,7 @@ ExprNodeP Parse_ExprFC(TokenStream& lex) case TOK_PAREN_OPEN:
// Expression method call
- lex.putback(tok);
+ PUTBACK(tok, lex);
val = NEWNODE( AST::ExprNode_CallObject, ::std::move(val), Parse_ParenList(lex) );
break;
case TOK_SQUARE_OPEN:
@@ -793,7 +806,7 @@ ExprNodeP Parse_ExprFC(TokenStream& lex) switch( GET_TOK(tok, lex) )
{
case TOK_PAREN_OPEN:
- lex.putback(tok);
+ PUTBACK(tok, lex);
val = NEWNODE( AST::ExprNode_CallMethod, ::std::move(val), ::std::move(path), Parse_ParenList(lex) );
break;
case TOK_DOUBLE_COLON:
@@ -862,7 +875,7 @@ ExprNodeP Parse_ExprVal_Closure(TokenStream& lex, bool is_move) while( GET_TOK(tok, lex) != TOK_PIPE )
{
- lex.putback(tok);
+ PUTBACK(tok, lex);
// Irrefutable pattern
AST::Pattern pat = Parse_Pattern(lex, false);
@@ -870,7 +883,7 @@ ExprNodeP Parse_ExprVal_Closure(TokenStream& lex, bool is_move) if( GET_TOK(tok, lex) == TOK_COLON )
type = Parse_Type(lex);
else
- lex.putback(tok);
+ PUTBACK(tok, lex);
args.push_back( ::std::make_pair( ::std::move(pat), ::std::move(type) ) );
@@ -886,12 +899,12 @@ ExprNodeP Parse_ExprVal_Closure(TokenStream& lex, bool is_move) rt = TypeRef(TypeRef::TagInvalid(), Span(tok.get_pos()));
}
else {
- lex.putback(tok);
+ PUTBACK(tok, lex);
rt = Parse_Type(lex);
}
}
else
- lex.putback(tok);
+ PUTBACK(tok, lex);
auto code = Parse_Expr0(lex);
@@ -932,14 +945,17 @@ ExprNodeP Parse_ExprVal(TokenStream& lex) switch( GET_TOK(tok, lex) )
{
case TOK_BRACE_OPEN:
- lex.putback(tok);
+ PUTBACK(tok, lex);
return Parse_ExprBlockNode(lex);
+ case TOK_INTERPOLATED_EXPR:
+ return tok.take_frag_node();
+
// TODO: Return/break/continue/... here?
case TOK_RWORD_RETURN:
case TOK_RWORD_CONTINUE:
case TOK_RWORD_BREAK:
- lex.putback(tok);
+ PUTBACK(tok, lex);
return Parse_Stmt(lex);
case TOK_RWORD_LOOP:
@@ -960,7 +976,7 @@ ExprNodeP Parse_ExprVal(TokenStream& lex) // UFCS
case TOK_DOUBLE_LT:
- lex.putback(tok);
+ PUTBACK(tok, lex);
case TOK_LT: {
TypeRef ty = Parse_Type(lex);
if( GET_TOK(tok, lex) == TOK_RWORD_AS ) {
@@ -970,7 +986,7 @@ ExprNodeP Parse_ExprVal(TokenStream& lex) path = AST::Path(AST::Path::TagUfcs(), ty, trait, Parse_PathNodes(lex, PATH_GENERIC_EXPR));
}
else {
- lex.putback(tok);
+ PUTBACK(tok, lex);
GET_CHECK_TOK(tok, lex, TOK_GT);
// TODO: Terminating the "path" here is sometimes valid
GET_CHECK_TOK(tok, lex, TOK_DOUBLE_COLON);
@@ -985,31 +1001,36 @@ ExprNodeP Parse_ExprVal(TokenStream& lex) }
else
{
- lex.putback(tok);
+ PUTBACK(tok, lex);
path = Parse_Path(lex, PATH_GENERIC_EXPR);
}
}
if(0)
case TOK_RWORD_SUPER:
{
- lex.putback(tok);
+ PUTBACK(tok, lex);
path = Parse_Path(lex, PATH_GENERIC_EXPR);
}
if(0)
case TOK_IDENT:
// Get path
{
- lex.putback(tok);
+ PUTBACK(tok, lex);
path = Parse_Path(lex, false, PATH_GENERIC_EXPR);
}
if(0)
+ case TOK_INTERPOLATED_PATH:
+ {
+ path = mv$(tok.frag_path());
+ }
+ if(0)
case TOK_DOUBLE_COLON:
path = Parse_Path(lex, true, PATH_GENERIC_EXPR);
switch( GET_TOK(tok, lex) )
{
case TOK_PAREN_OPEN:
// Function call
- lex.putback(tok);
+ PUTBACK(tok, lex);
return NEWNODE( AST::ExprNode_CallPath, ::std::move(path), Parse_ParenList(lex) );
case TOK_BRACE_OPEN:
if( !CHECK_PARSE_FLAG(lex, disallow_struct_literal) )
@@ -1018,7 +1039,7 @@ ExprNodeP Parse_ExprVal(TokenStream& lex) DEBUG("Not parsing struct literal");
default:
// Value
- lex.putback(tok);
+ PUTBACK(tok, lex);
return NEWNODE( AST::ExprNode_NamedValue, ::std::move(path) );
}
case TOK_RWORD_MOVE:
@@ -1065,7 +1086,7 @@ ExprNodeP Parse_ExprVal(TokenStream& lex) else
{
CLEAR_PARSE_FLAG(lex, disallow_struct_literal);
- lex.putback(tok);
+ PUTBACK(tok, lex);
ExprNodeP rv = Parse_Expr0(lex);
if( GET_TOK(tok, lex) == TOK_COMMA ) {
@@ -1074,7 +1095,7 @@ ExprNodeP Parse_ExprVal(TokenStream& lex) do {
if( GET_TOK(tok, lex) == TOK_PAREN_CLOSE )
break;
- lex.putback(tok);
+ PUTBACK(tok, lex);
ents.push_back( Parse_Expr0(lex) );
} while( GET_TOK(tok, lex) == TOK_COMMA );
rv = NEWNODE( AST::ExprNode_Tuple, ::std::move(ents) );
@@ -1090,7 +1111,7 @@ ExprNodeP Parse_ExprVal(TokenStream& lex) }
else
{
- lex.putback(tok);
+ PUTBACK(tok, lex);
auto first = Parse_Expr0(lex);
if( GET_TOK(tok, lex) == TOK_SEMICOLON )
{
@@ -1108,7 +1129,7 @@ ExprNodeP Parse_ExprVal(TokenStream& lex) if( GET_TOK(tok, lex) == TOK_SQUARE_CLOSE )
break;
else
- lex.putback(tok);
+ PUTBACK(tok, lex);
items.push_back( Parse_Expr0(lex) );
GET_TOK(tok, lex);
}
@@ -1118,7 +1139,7 @@ ExprNodeP Parse_ExprVal(TokenStream& lex) }
throw ParseError::BugCheck(lex, "Array literal fell");
case TOK_MACRO:
- return Parse_ExprMacro(lex, tok);
+ return Parse_ExprMacro(lex, mv$(tok));
default:
throw ParseError::Unexpected(lex, tok);
}
@@ -1131,7 +1152,7 @@ ExprNodeP Parse_ExprMacro(TokenStream& lex, Token tok) ident = mv$(tok.str());
}
else {
- lex.putback(tok);
+ PUTBACK(tok, lex);
}
TokenTree tt = Parse_TT(lex, true);
if( tt.is_token() ) {
@@ -1162,21 +1183,21 @@ TokenTree Parse_TT(TokenStream& lex, bool unwrapped) case TOK_NULL:
throw ParseError::Unexpected(lex, tok);
default:
- return TokenTree(tok);
+ return TokenTree( mv$(tok) );
}
::std::vector<TokenTree> items;
if( !unwrapped )
- items.push_back(tok);
+ items.push_back( mv$(tok) );
while(GET_TOK(tok, lex) != closer && tok.type() != TOK_EOF)
{
if( tok.type() == TOK_NULL )
throw ParseError::Unexpected(lex, tok);
- lex.putback(tok);
+ PUTBACK(tok, lex);
items.push_back(Parse_TT(lex, false));
}
if( !unwrapped )
- items.push_back(tok);
+ items.push_back( mv$(tok) );
return TokenTree(mv$(items));
}
@@ -1244,7 +1265,7 @@ TokenTree Parse_TT_Path(TokenStream& lex, bool mode_expr) Parse_Path(wlex, true, (mode_expr ? PATH_GENERIC_EXPR : PATH_GENERIC_TYPE));
}
else {
- wlex.putback(tok);
+ PUTBACK(tok, lex);
Parse_Path(wlex, false, (mode_expr ? PATH_GENERIC_EXPR : PATH_GENERIC_TYPE));
}
diff --git a/src/parse/interpolated_fragment.cpp b/src/parse/interpolated_fragment.cpp new file mode 100644 index 00000000..dd490bf5 --- /dev/null +++ b/src/parse/interpolated_fragment.cpp @@ -0,0 +1,76 @@ +/* + */ +#include <iostream> +#include "interpolated_fragment.hpp" +#include <ast/ast.hpp> + +InterpolatedFragment::~InterpolatedFragment() +{ + if( m_ptr ) + { + switch(m_type) + { + case InterpolatedFragment::TT: delete reinterpret_cast<TokenTree*>(m_ptr); break; + case InterpolatedFragment::PAT: delete reinterpret_cast<AST::Pattern*>(m_ptr); break; + case InterpolatedFragment::PATH:delete reinterpret_cast<AST::Path*>(m_ptr); break; + case InterpolatedFragment::TYPE:delete reinterpret_cast<TypeRef*>(m_ptr); break; + case InterpolatedFragment::EXPR: + case InterpolatedFragment::STMT: + case InterpolatedFragment::BLOCK: + delete reinterpret_cast<AST::ExprNode*>(m_ptr); + break; + case InterpolatedFragment::META: + delete reinterpret_cast<AST::MetaItem*>(m_ptr); + break; + } + } +} + +InterpolatedFragment::InterpolatedFragment(InterpolatedFragment&& x): + m_type( x.m_type ) +{ + m_ptr = x.m_ptr, x.m_ptr = nullptr; +} +InterpolatedFragment& InterpolatedFragment::operator=(InterpolatedFragment&& x) +{ + m_type = x.m_type; + m_ptr = x.m_ptr, x.m_ptr = nullptr; + return *this; +} + +InterpolatedFragment::InterpolatedFragment(InterpolatedFragment::Type type, AST::ExprNode* ptr): + m_type( type ), + m_ptr( ptr ) +{ +} +InterpolatedFragment::InterpolatedFragment(AST::MetaItem v): + m_type( InterpolatedFragment::META ), + m_ptr( new AST::MetaItem(mv$(v)) ) +{ +} +InterpolatedFragment::InterpolatedFragment(TokenTree v): + m_type( InterpolatedFragment::TT ), + m_ptr( new TokenTree(mv$(v)) ) +{ +} +InterpolatedFragment::InterpolatedFragment(AST::Path v): + m_type( InterpolatedFragment::PATH ), + m_ptr( new AST::Path(mv$(v)) ) +{ +} +InterpolatedFragment::InterpolatedFragment(AST::Pattern v): + m_type( InterpolatedFragment::PAT ), + m_ptr( new AST::Pattern(mv$(v)) ) +{ +} +InterpolatedFragment::InterpolatedFragment(TypeRef v): + m_type( InterpolatedFragment::TYPE ), + m_ptr( new TypeRef(mv$(v)) ) +{ +} + +::std::ostream& operator<<(::std::ostream& os, InterpolatedFragment const& x) +{ + return os; +} + diff --git a/src/parse/interpolated_fragment.hpp b/src/parse/interpolated_fragment.hpp new file mode 100644 index 00000000..40aa764a --- /dev/null +++ b/src/parse/interpolated_fragment.hpp @@ -0,0 +1,49 @@ +/* + */ +#pragma once + +#include <cassert> + +class TypeRef; +class TokenTree; +namespace AST { + class Pattern; + class Path; + class ExprNode; + class MetaItem; +}; + +class InterpolatedFragment +{ +public: + enum Type + { + TT, + PAT, + PATH, + TYPE, + + EXPR, + STMT, + BLOCK, + + META, + } m_type; + + void* m_ptr; + + InterpolatedFragment(InterpolatedFragment&& ); + InterpolatedFragment& operator=(InterpolatedFragment&& ); + //InterpolatedFragment(const InterpolatedFragment& ); + InterpolatedFragment(TokenTree ); + InterpolatedFragment(::AST::Pattern); + InterpolatedFragment(::AST::Path); + InterpolatedFragment(::TypeRef); + InterpolatedFragment(::AST::MetaItem ); + ~InterpolatedFragment(); + InterpolatedFragment(Type , ::AST::ExprNode*); + + TokenTree& as_tt() { assert(m_type == TT); return *reinterpret_cast<TokenTree*>(m_ptr); } + + friend ::std::ostream& operator<<(::std::ostream& os, const InterpolatedFragment& x); +}; diff --git a/src/parse/lex.cpp b/src/parse/lex.cpp index fd6044f4..395eee48 100644 --- a/src/parse/lex.cpp +++ b/src/parse/lex.cpp @@ -20,7 +20,8 @@ #include <typeinfo> #include <algorithm> // std::count -const bool DEBUG_PRINT_TOKENS = false; +//const bool DEBUG_PRINT_TOKENS = false; +const bool DEBUG_PRINT_TOKENS = true; Lexer::Lexer(const ::std::string& filename): m_path(filename.c_str()), @@ -956,7 +957,7 @@ Token TTStream::realGetToken() const TokenTree& subtree = tree[idx]; idx ++; if( subtree.size() == 0 ) { - return subtree.tok(); + return subtree.tok().clone(); } else { m_stack.push_back( ::std::make_pair(0, &subtree ) ); @@ -1041,7 +1042,7 @@ Token TokenStream::getToken() if( m_cache_valid ) { m_cache_valid = false; - return m_cache; + return mv$(m_cache); } else if( m_lookahead.size() ) { @@ -1119,6 +1120,19 @@ Span TokenStream::end_span(ProtoSpan ps) const } +TokenTree TokenTree::clone() const +{ + if( m_subtrees.size() == 0 ) { + return TokenTree(m_tok.clone()); + } + else { + ::std::vector< TokenTree> ents; + ents.reserve( m_subtrees.size() ); + for(const auto& sub : m_subtrees) + ents.push_back( sub.clone() ); + return TokenTree( mv$(ents) ); + } +} SERIALISE_TYPE_A(TokenTree::, "TokenTree", { s.item(m_tok); s.item(m_subtrees); diff --git a/src/parse/parseerror.cpp b/src/parse/parseerror.cpp index e05ecb05..7c6f6a21 100644 --- a/src/parse/parseerror.cpp +++ b/src/parse/parseerror.cpp @@ -56,23 +56,23 @@ ParseError::BadChar::~BadChar() throw() {
}
-ParseError::Unexpected::Unexpected(const TokenStream& lex, Token tok):
- m_tok(tok)
+ParseError::Unexpected::Unexpected(const TokenStream& lex, const Token& tok)//:
+// m_tok( mv$(tok) )
{
auto pos = tok.get_pos();
if(pos.filename == "")
pos = lex.getPosition();
::std::cout << pos << ": Unexpected(" << tok << ")" << ::std::endl;
}
-ParseError::Unexpected::Unexpected(const TokenStream& lex, Token tok, Token exp):
- m_tok(tok)
+ParseError::Unexpected::Unexpected(const TokenStream& lex, const Token& tok, Token exp)//:
+// m_tok( mv$(tok) )
{
auto pos = tok.get_pos();
if(pos.filename == "")
pos = lex.getPosition();
::std::cout << pos << ": Unexpected(" << tok << ", " << exp << ")" << ::std::endl;
}
-ParseError::Unexpected::Unexpected(const TokenStream& lex, Token tok, ::std::vector<eTokenType> exp)
+ParseError::Unexpected::Unexpected(const TokenStream& lex, const Token& tok, ::std::vector<eTokenType> exp)
{
auto pos = tok.get_pos();
if(pos.filename == "")
diff --git a/src/parse/parseerror.hpp b/src/parse/parseerror.hpp index 1074bf46..42a89785 100644 --- a/src/parse/parseerror.hpp +++ b/src/parse/parseerror.hpp @@ -26,9 +26,9 @@ class Unexpected: {
Token m_tok;
public:
- Unexpected(const TokenStream& lex, Token tok);
- Unexpected(const TokenStream& lex, Token tok, Token exp);
- Unexpected(const TokenStream& lex, Token tok, ::std::vector<eTokenType> exp);
+ Unexpected(const TokenStream& lex, const Token& tok);
+ Unexpected(const TokenStream& lex, const Token& tok, Token exp);
+ Unexpected(const TokenStream& lex, const Token& tok, ::std::vector<eTokenType> exp);
virtual ~Unexpected() throw ();
};
diff --git a/src/parse/paths.cpp b/src/parse/paths.cpp index 06545b7e..a5d10f74 100644 --- a/src/parse/paths.cpp +++ b/src/parse/paths.cpp @@ -19,6 +19,9 @@ AST::Path Parse_Path(TokenStream& lex, eParsePathGenericMode generic_mode) Token tok; switch( GET_TOK(tok, lex) ) { + case TOK_INTERPOLATED_PATH: + return mv$(tok.frag_path()); + case TOK_RWORD_SELF: GET_CHECK_TOK(tok, lex, TOK_DOUBLE_COLON); return Parse_Path(lex, false, generic_mode); @@ -47,7 +50,7 @@ AST::Path Parse_Path(TokenStream& lex, eParsePathGenericMode generic_mode) trait = Parse_Path(lex, true, PATH_GENERIC_TYPE); } else { - lex.putback(tok); + PUTBACK(tok, lex); trait = Parse_Path(lex, false, PATH_GENERIC_TYPE); } GET_CHECK_TOK(tok, lex, TOK_GT); @@ -55,7 +58,7 @@ AST::Path Parse_Path(TokenStream& lex, eParsePathGenericMode generic_mode) return AST::Path(AST::Path::TagUfcs(), ty, trait, Parse_PathNodes(lex, generic_mode)); } else { - lex.putback(tok); + PUTBACK(tok, lex); GET_CHECK_TOK(tok, lex, TOK_GT); // TODO: Terminating the "path" here is sometimes valid? GET_CHECK_TOK(tok, lex, TOK_DOUBLE_COLON); @@ -64,7 +67,7 @@ AST::Path Parse_Path(TokenStream& lex, eParsePathGenericMode generic_mode) throw ""; } default: - lex.putback(tok); + PUTBACK(tok, lex); return Parse_Path(lex, false, generic_mode); } } @@ -79,7 +82,7 @@ AST::Path Parse_Path(TokenStream& lex, bool is_abs, eParsePathGenericMode generi return AST::Path(cratename, Parse_PathNodes(lex, generic_mode)); } else { - lex.putback(tok); + PUTBACK(tok, lex); return AST::Path("", Parse_PathNodes(lex, generic_mode)); } } @@ -89,7 +92,7 @@ AST::Path Parse_Path(TokenStream& lex, bool is_abs, eParsePathGenericMode generi // return AST::Path( tok.str() ); //} //else { - // lex.putback(tok); + // PUTBACK(tok, lex); return AST::Path(AST::Path::TagRelative(), Parse_PathNodes(lex, generic_mode)); //} } @@ -133,7 +136,7 @@ AST::Path Parse_Path(TokenStream& lex, bool is_abs, eParsePathGenericMode generi } else { - lex.putback(tok); + PUTBACK(tok, lex); do { args.push_back( Parse_Type(lex) ); } while( GET_TOK(tok, lex) == TOK_COMMA ); @@ -145,7 +148,7 @@ AST::Path Parse_Path(TokenStream& lex, bool is_abs, eParsePathGenericMode generi ret_type = Parse_Type(lex); } else { - lex.putback(tok); + PUTBACK(tok, lex); } DEBUG("- Fn("<<args<<")->"<<ret_type<<""); @@ -160,7 +163,7 @@ AST::Path Parse_Path(TokenStream& lex, bool is_abs, eParsePathGenericMode generi } } if( tok.type() != TOK_DOUBLE_COLON ) { - ret.push_back( AST::PathNode(component, params) ); + ret.push_back( AST::PathNode(component, mv$(params)) ); break; } tok = lex.getToken(); @@ -174,14 +177,14 @@ AST::Path Parse_Path(TokenStream& lex, bool is_abs, eParsePathGenericMode generi params = Parse_Path_GenericList(lex); tok = lex.getToken(); if( tok.type() != TOK_DOUBLE_COLON ) { - ret.push_back( AST::PathNode(component, params) ); + ret.push_back( AST::PathNode(component, mv$(params)) ); break; } GET_TOK(tok, lex); } - ret.push_back( AST::PathNode(component, params) ); + ret.push_back( AST::PathNode(component, mv$(params)) ); } - lex.putback(tok); + PUTBACK(tok, lex); //if( path.is_trivial() ) { // path = AST::Path(path[0].name()); //} @@ -217,7 +220,7 @@ AST::Path Parse_Path(TokenStream& lex, bool is_abs, eParsePathGenericMode generi break; } default: - lex.putback(tok); + PUTBACK(tok, lex); types.push_back( Parse_Type(lex) ); break; } diff --git a/src/parse/pattern.cpp b/src/parse/pattern.cpp index 085df108..e67647bd 100644 --- a/src/parse/pattern.cpp +++ b/src/parse/pattern.cpp @@ -46,6 +46,10 @@ AST::Pattern Parse_Pattern(TokenStream& lex, bool is_refutable) { return AST::Pattern( AST::Pattern::TagMacro(), box$(Parse_MacroInvocation(ps, AST::MetaItems(), tok.str(), lex))); } + if( tok.type() == TOK_INTERPOLATED_PATTERN ) + { + return mv$(tok.frag_pattern()); + } bool expect_bind = false; ::AST::Pattern::BindType bind_type = AST::Pattern::BIND_MOVE; @@ -85,7 +89,7 @@ AST::Pattern Parse_Pattern(TokenStream& lex, bool is_refutable) // If there's no '@' after it, it's a name binding only (_ pattern) if( GET_TOK(tok, lex) != TOK_AT ) { - lex.putback(tok); + PUTBACK(tok, lex); return AST::Pattern(AST::Pattern::TagBind(), bind_name); } @@ -126,7 +130,7 @@ AST::Pattern Parse_Pattern(TokenStream& lex, bool is_refutable) // Otherwise, fall through } - lex.putback(tok); + PUTBACK(tok, lex); AST::Pattern pat = Parse_PatternReal(lex, is_refutable); pat.set_bind(bind_name, bind_type, is_mut); return ::std::move(pat); @@ -152,7 +156,7 @@ AST::Pattern Parse_PatternReal(TokenStream& lex, bool is_refutable) } else { - lex.putback(tok); + PUTBACK(tok, lex); return ret; } } @@ -179,12 +183,12 @@ AST::Pattern Parse_PatternReal1(TokenStream& lex, bool is_refutable) if( GET_TOK(tok, lex) == TOK_RWORD_MUT ) // TODO: Actually use mutability return AST::Pattern( AST::Pattern::TagReference(), Parse_Pattern(lex, is_refutable) ); - lex.putback(tok); + PUTBACK(tok, lex); return AST::Pattern( AST::Pattern::TagReference(), Parse_Pattern(lex, is_refutable) ); case TOK_RWORD_SELF: case TOK_RWORD_SUPER: case TOK_IDENT: - lex.putback(tok); + PUTBACK(tok, lex); return Parse_PatternReal_Path( lex, Parse_Path(lex, PATH_GENERIC_EXPR), is_refutable ); case TOK_DOUBLE_COLON: // 2. Paths are enum/struct names @@ -240,7 +244,7 @@ AST::Pattern Parse_PatternReal_Path(TokenStream& lex, AST::Path path, bool is_re case TOK_BRACE_OPEN: return Parse_PatternStruct(lex, ::std::move(path), is_refutable); default: - lex.putback(tok); + PUTBACK(tok, lex); return AST::Pattern( AST::Pattern::TagValue(), AST::Pattern::Value::make_Named(mv$(path)) ); } } @@ -271,7 +275,7 @@ AST::Pattern Parse_PatternReal_Slice(TokenStream& lex, bool is_refutable) } else if( tok.type() == TOK_DOUBLE_DOT ) { binding_name = "_"; - lex.putback(tok); + PUTBACK(tok, lex); } else { } @@ -284,7 +288,7 @@ AST::Pattern Parse_PatternReal_Slice(TokenStream& lex, bool is_refutable) GET_TOK(tok, lex); // TOK_DOUBLE_DOT } else { - lex.putback(tok); + PUTBACK(tok, lex); if(is_trailing) { rv_array.trailing.push_back( Parse_Pattern(lex, is_refutable) ); } @@ -312,7 +316,7 @@ AST::Pattern Parse_PatternReal_Slice(TokenStream& lex, bool is_refutable) if( GET_TOK(tok, lex) == end ) break; else - lex.putback(tok); + PUTBACK(tok, lex); AST::Pattern pat = Parse_Pattern(lex, is_refutable); DEBUG("pat = " << pat); @@ -370,7 +374,7 @@ AST::Pattern Parse_PatternStruct(TokenStream& lex, AST::Path path, bool is_refut AST::Pattern pat; if( is_short_bind || tok.type() != TOK_COLON ) { - lex.putback(tok); + PUTBACK(tok, lex); pat = AST::Pattern(AST::Pattern::TagBind(), field); pat.set_bind(field, bind_type, is_mut); if( is_box ) diff --git a/src/parse/root.cpp b/src/parse/root.cpp index 9deb3446..58708ec7 100644 --- a/src/parse/root.cpp +++ b/src/parse/root.cpp @@ -87,13 +87,13 @@ void Parse_TypeBound(TokenStream& lex, AST::GenericParams& ret, TypeRef checked_ CHECK_TOK(tok, TOK_GT);
}
else {
- lex.putback(tok);
+ PUTBACK(tok, lex);
}
ret.add_bound( AST::GenericBound::make_IsTrait( {type: checked_type, hrls: lifetimes, trait: Parse_Path(lex, PATH_GENERIC_TYPE) }) );
}
} while( GET_TOK(tok, lex) == TOK_PLUS );
- lex.putback(tok);
+ PUTBACK(tok, lex);
}
/// Parse type parameters within '<' and '>' (definition)
@@ -147,7 +147,7 @@ AST::GenericParams Parse_GenericParams(TokenStream& lex) GET_TOK(tok, lex);
}
} while( tok.type() == TOK_COMMA );
- lex.putback(tok);
+ PUTBACK(tok, lex);
return ret;
}
@@ -183,13 +183,13 @@ void Parse_WhereClause(TokenStream& lex, AST::GenericParams& params) }
else
{
- lex.putback(tok);
+ PUTBACK(tok, lex);
TypeRef type = Parse_Type(lex);
GET_CHECK_TOK(tok, lex, TOK_COLON);
Parse_TypeBound(lex, params, type);
}
} while( GET_TOK(tok, lex) == TOK_COMMA );
- lex.putback(tok);
+ PUTBACK(tok, lex);
}
// Parse a single function argument
@@ -238,7 +238,7 @@ AST::Function Parse_FunctionDef(TokenStream& lex, ::std::string abi, AST::MetaIt GET_CHECK_TOK(tok, lex, TOK_GT);
}
else {
- lex.putback(tok);
+ PUTBACK(tok, lex);
}
AST::Function::Arglist args;
@@ -302,7 +302,7 @@ AST::Function Parse_FunctionDef(TokenStream& lex, ::std::string abi, AST::MetaIt ty = Parse_Type(lex);
}
else {
- lex.putback(tok);
+ PUTBACK(tok, lex);
ty = TypeRef("Self");
}
args.push_back( ::std::make_pair( AST::Pattern(AST::Pattern::TagBind(), "self"), ty) );
@@ -320,7 +320,7 @@ AST::Function Parse_FunctionDef(TokenStream& lex, ::std::string abi, AST::MetaIt ty = Parse_Type(lex);
}
else {
- lex.putback(tok);
+ PUTBACK(tok, lex);
ty = TypeRef("Self");
}
args.push_back( ::std::make_pair( AST::Pattern(AST::Pattern::TagBind(), "self"), ty) );
@@ -339,7 +339,7 @@ AST::Function Parse_FunctionDef(TokenStream& lex, ::std::string abi, AST::MetaIt CHECK_TOK(tok, TOK_COMMA);
}
else {
- lex.putback(tok);
+ PUTBACK(tok, lex);
}
// Argument list
@@ -370,13 +370,13 @@ AST::Function Parse_FunctionDef(TokenStream& lex, ::std::string abi, AST::MetaIt ret_type = TypeRef(TypeRef::TagInvalid(), Span(tok.get_pos()));
}
else {
- lex.putback(tok);
+ PUTBACK(tok, lex);
ret_type = Parse_Type(lex);
}
}
else
{
- lex.putback(tok);
+ PUTBACK(tok, lex);
}
if( GET_TOK(tok, lex) == TOK_RWORD_WHERE )
@@ -384,7 +384,7 @@ AST::Function Parse_FunctionDef(TokenStream& lex, ::std::string abi, AST::MetaIt Parse_WhereClause(lex, params);
}
else {
- lex.putback(tok);
+ PUTBACK(tok, lex);
}
return AST::Function(::std::move(params), ::std::move(ret_type), ::std::move(args));
@@ -395,7 +395,7 @@ AST::Function Parse_FunctionDefWithCode(TokenStream& lex, ::std::string abi, AST Token tok;
auto ret = Parse_FunctionDef(lex, abi, attrs, allow_self, false);
GET_CHECK_TOK(tok, lex, TOK_BRACE_OPEN);
- lex.putback(tok);
+ PUTBACK(tok, lex);
ret.set_code( Parse_ExprBlock(lex) );
return ret;
}
@@ -468,7 +468,7 @@ AST::Struct Parse_Struct(TokenStream& lex, const AST::MetaItems& meta_items) if(tok.type() == TOK_RWORD_PUB)
is_pub = true;
else
- lex.putback(tok);
+ PUTBACK(tok, lex);
refs.push_back( AST::TupleItem( mv$(item_attrs), is_pub, Parse_Type(lex) ) );
if( GET_TOK(tok, lex) != TOK_COMMA )
@@ -555,7 +555,7 @@ AST::Trait Parse_TraitDef(TokenStream& lex, AST::Module& mod, const AST::MetaIte supertraits.push_back( make_spanned( Span(tok.get_pos()), AST::Path() ) );
}
else {
- lex.putback(tok);
+ PUTBACK(tok, lex);
supertraits.push_back( GET_SPANNED(::AST::Path, lex, Parse_Path(lex, PATH_GENERIC_TYPE)) );
}
} while( GET_TOK(tok, lex) == TOK_PLUS );
@@ -657,7 +657,7 @@ AST::Trait Parse_TraitDef(TokenStream& lex, AST::Module& mod, const AST::MetaIte if( GET_TOK(tok, lex) == TOK_STRING )
abi = tok.str();
else
- lex.putback(tok);
+ PUTBACK(tok, lex);
GET_TOK(tok, lex);
}
@@ -669,7 +669,7 @@ AST::Trait Parse_TraitDef(TokenStream& lex, AST::Module& mod, const AST::MetaIte auto fcn = Parse_FunctionDef(lex, abi, item_attrs, true, true);
if( GET_TOK(tok, lex) == TOK_BRACE_OPEN )
{
- lex.putback(tok);
+ PUTBACK(tok, lex);
fcn.set_code( Parse_ExprBlock(lex) );
}
else if( tok.type() == TOK_SEMICOLON )
@@ -811,7 +811,13 @@ AST::MetaItem Parse_MetaItem(TokenStream& lex) {
TRACE_FUNCTION;
Token tok;
- GET_CHECK_TOK(tok, lex, TOK_IDENT);
+ GET_TOK(tok, lex);
+
+ if( tok.type() == TOK_INTERPOLATED_META ) {
+ return mv$(tok.frag_meta());
+ }
+
+ CHECK_TOK(tok, TOK_IDENT);
::std::string name = tok.str();
switch(GET_TOK(tok, lex))
{
@@ -830,7 +836,7 @@ AST::MetaItem Parse_MetaItem(TokenStream& lex) CHECK_TOK(tok, TOK_PAREN_CLOSE);
return AST::MetaItem(name, mv$(items)); }
default:
- lex.putback(tok);
+ PUTBACK(tok, lex);
return AST::MetaItem(name);
}
}
@@ -849,7 +855,7 @@ void Parse_Impl(TokenStream& lex, AST::Module& mod, AST::MetaItems attrs, bool i GET_CHECK_TOK(tok, lex, TOK_GT);
}
else {
- lex.putback(tok);
+ PUTBACK(tok, lex);
}
// 2. Either a trait name (with type params), or the type to impl
@@ -861,7 +867,7 @@ void Parse_Impl(TokenStream& lex, AST::Module& mod, AST::MetaItems attrs, bool i {
trait_path = GET_SPANNED(::AST::Path, lex, Parse_Path(lex, PATH_GENERIC_TYPE));
GET_CHECK_TOK(tok, lex, TOK_RWORD_FOR);
- impl_type = Parse_Type(lex);
+ impl_type = Parse_Type(lex, true);
if( GET_TOK(tok, lex) == TOK_RWORD_WHERE )
{
@@ -878,9 +884,9 @@ void Parse_Impl(TokenStream& lex, AST::Module& mod, AST::MetaItems attrs, bool i else
{
// - Don't care which at this stage
- lex.putback(tok);
+ PUTBACK(tok, lex);
- impl_type = Parse_Type(lex);
+ impl_type = Parse_Type(lex, true);
// TODO: Handle the "impl Any + Send" syntax here
if( GET_TOK(tok, lex) == TOK_RWORD_FOR )
@@ -899,12 +905,12 @@ void Parse_Impl(TokenStream& lex, AST::Module& mod, AST::MetaItems attrs, bool i }
else
{
- lex.putback(tok);
- impl_type = Parse_Type(lex);
+ PUTBACK(tok, lex);
+ impl_type = Parse_Type(lex, true);
}
}
else {
- lex.putback(tok);
+ PUTBACK(tok, lex);
}
}
@@ -914,7 +920,7 @@ void Parse_Impl(TokenStream& lex, AST::Module& mod, AST::MetaItems attrs, bool i Parse_WhereClause(lex, params);
}
else {
- lex.putback(tok);
+ PUTBACK(tok, lex);
}
GET_CHECK_TOK(tok, lex, TOK_BRACE_OPEN);
@@ -936,11 +942,11 @@ void Parse_Impl(TokenStream& lex, AST::Module& mod, AST::MetaItems attrs, bool i impl.add_macro_invocation( Parse_MacroInvocation( ps, AST::MetaItems(), mv$(tok.str()), lex ) );
// - Silently consume ';' after the macro
if( GET_TOK(tok, lex) != TOK_SEMICOLON )
- lex.putback(tok);
+ PUTBACK(tok, lex);
}
else
{
- lex.putback(tok);
+ PUTBACK(tok, lex);
Parse_Impl_Item(lex, impl);
}
}
@@ -1020,7 +1026,7 @@ void Parse_Impl_Item(TokenStream& lex, AST::Impl& impl) if( GET_TOK(tok, lex) == TOK_STRING )
abi = tok.str();
else
- lex.putback(tok);
+ PUTBACK(tok, lex);
GET_TOK(tok, lex);
}
@@ -1066,7 +1072,7 @@ void Parse_ExternBlock(TokenStream& lex, AST::Module& mod, ::std::string abi, :: block_attrs.push_back( Parse_MetaItem(lex) );
GET_CHECK_TOK(tok, lex, TOK_SQUARE_CLOSE);
}
- lex.putback(tok);
+ PUTBACK(tok, lex);
// TODO: Use `block_attrs`
while( GET_TOK(tok, lex) != TOK_BRACE_CLOSE )
@@ -1100,7 +1106,7 @@ void Parse_ExternBlock(TokenStream& lex, AST::Module& mod, ::std::string abi, :: if( GET_TOK(tok, lex) == TOK_RWORD_MUT )
is_mut = true;
else
- lex.putback(tok);
+ PUTBACK(tok, lex);
GET_CHECK_TOK(tok, lex, TOK_IDENT);
auto name = mv$(tok.str());
GET_CHECK_TOK(tok, lex, TOK_COLON);
@@ -1146,11 +1152,11 @@ void Parse_Use_Set(TokenStream& lex, const ProtoSpan& ps, const AST::Path& base_ name = mv$(tok.str());
}
else {
- lex.putback(tok);
+ PUTBACK(tok, lex);
}
fcn(AST::UseStmt(lex.end_span(ps), mv$(path)), mv$(name));
} while( GET_TOK(tok, lex) == TOK_COMMA );
- lex.putback(tok);
+ PUTBACK(tok, lex);
}
void Parse_Use(TokenStream& lex, ::std::function<void(AST::UseStmt, ::std::string)> fcn)
@@ -1190,7 +1196,7 @@ void Parse_Use(TokenStream& lex, ::std::function<void(AST::UseStmt, ::std::strin GET_CHECK_TOK(tok, lex, TOK_DOUBLE_COLON);
}
else {
- lex.putback(tok);
+ PUTBACK(tok, lex);
}
break;
case TOK_BRACE_OPEN:
@@ -1237,7 +1243,7 @@ void Parse_Use(TokenStream& lex, ::std::function<void(AST::UseStmt, ::std::strin }
else
{
- lex.putback(tok);
+ PUTBACK(tok, lex);
assert(path.nodes().size() > 0);
name = path.nodes().back().name();
}
@@ -1254,7 +1260,7 @@ void Parse_Use(TokenStream& lex, ::std::function<void(AST::UseStmt, ::std::strin ident = mv$(tok.str());
}
else {
- lex.putback(tok);
+ PUTBACK(tok, lex);
}
TokenTree tt = Parse_TT(lex, true);
return ::AST::MacroInvocation( lex.end_span(span_start), mv$(meta_items), mv$(name), mv$(ident), mv$(tt));
@@ -1284,7 +1290,7 @@ void Parse_ExternCrate(TokenStream& lex, AST::Module& mod, bool is_public, AST:: name = mv$(tok.str());
}
else {
- lex.putback(tok);
+ PUTBACK(tok, lex);
name = path;
}
break;
@@ -1442,7 +1448,7 @@ void Parse_Mod_Item(TokenStream& lex, bool file_controls_dir, const ::std::strin abi = mv$(tok.str());
}
else {
- lex.putback(tok);
+ PUTBACK(tok, lex);
}
GET_CHECK_TOK(tok, lex, TOK_RWORD_FN);
GET_CHECK_TOK(tok, lex, TOK_IDENT);
@@ -1632,10 +1638,10 @@ void Parse_ModRoot_Items(TokenStream& lex, AST::Module& mod, bool file_controls_ {
case TOK_BRACE_CLOSE:
case TOK_EOF:
- lex.putback(tok);
+ PUTBACK(tok, lex);
return;
default:
- lex.putback(tok);
+ PUTBACK(tok, lex);
break;
}
@@ -1646,7 +1652,7 @@ void Parse_ModRoot_Items(TokenStream& lex, AST::Module& mod, bool file_controls_ meta_items.push_back( Parse_MetaItem(lex) );
GET_CHECK_TOK(tok, lex, TOK_SQUARE_CLOSE);
}
- lex.putback(tok);
+ PUTBACK(tok, lex);
DEBUG("meta_items = " << meta_items);
// root-level macros
@@ -1658,11 +1664,11 @@ void Parse_ModRoot_Items(TokenStream& lex, AST::Module& mod, bool file_controls_ // - Silently consume ';' after the macro
// TODO: Check the tt next token before parsing to tell if this is needed
if( GET_TOK(tok, lex) != TOK_SEMICOLON )
- lex.putback(tok);
+ PUTBACK(tok, lex);
continue ;
}
else {
- lex.putback(tok);
+ PUTBACK(tok, lex);
}
// Module visibility
@@ -1671,7 +1677,7 @@ void Parse_ModRoot_Items(TokenStream& lex, AST::Module& mod, bool file_controls_ is_public = true;
}
else {
- lex.putback(tok);
+ PUTBACK(tok, lex);
}
Parse_Mod_Item(lex, file_controls_dir,path, mod, is_public, mv$(meta_items));
@@ -1692,7 +1698,7 @@ void Parse_ModRoot(TokenStream& lex, AST::Module& mod, AST::MetaItems& mod_attrs mod_attrs.push_back( mv$(item) );
}
- lex.putback(tok);
+ PUTBACK(tok, lex);
Parse_ModRoot_Items(lex, mod, file_controls_dir, path);
}
diff --git a/src/parse/token.cpp b/src/parse/token.cpp index e61afcb9..5b580fe3 100644 --- a/src/parse/token.cpp +++ b/src/parse/token.cpp @@ -5,6 +5,40 @@ #include "token.hpp" #include <common.hpp> #include <parse/parseerror.hpp> +#include "interpolated_fragment.hpp" +#include <ast/types.hpp> +#include <ast/ast.hpp> + +Token::~Token() +{ + switch(m_type) + { + case TOK_INTERPOLATED_TYPE: + delete reinterpret_cast<TypeRef*>(m_data.as_Fragment()); + break; + case TOK_INTERPOLATED_PATTERN: + delete reinterpret_cast<AST::Pattern*>(m_data.as_Fragment()); + break; + case TOK_INTERPOLATED_PATH: + delete reinterpret_cast<AST::Path*>(m_data.as_Fragment()); + break; + case TOK_INTERPOLATED_EXPR: + delete reinterpret_cast<AST::ExprNode*>(m_data.as_Fragment()); + break; + case TOK_INTERPOLATED_STMT: + delete reinterpret_cast<AST::ExprNode*>(m_data.as_Fragment()); + break; + case TOK_INTERPOLATED_BLOCK: + delete reinterpret_cast<AST::ExprNode*>(m_data.as_Fragment()); + break; + case TOK_INTERPOLATED_META: + delete reinterpret_cast<AST::MetaItem*>(m_data.as_Fragment()); + break; + default: + break; + } + +} Token::Token(): m_type(TOK_NULL) @@ -29,6 +63,101 @@ Token::Token(double val, enum eCoreType datatype): m_data( Data::make_Float({datatype, val}) ) { } +Token::Token(InterpolatedFragment& frag) +{ + switch(frag.m_type) + { + case InterpolatedFragment::TT: throw ""; + case InterpolatedFragment::TYPE: + m_type = TOK_INTERPOLATED_TYPE; + m_data = new TypeRef( *reinterpret_cast<TypeRef*>(frag.m_ptr) ); + break; + case InterpolatedFragment::PAT: + m_type = TOK_INTERPOLATED_PATTERN; + m_data = new AST::Pattern( reinterpret_cast<AST::Pattern*>(frag.m_ptr)->clone() ); + break; + case InterpolatedFragment::PATH: + m_type = TOK_INTERPOLATED_PATH; + m_data = new AST::Path( *reinterpret_cast<AST::Path*>(frag.m_ptr) ); + break; + case InterpolatedFragment::EXPR: + m_type = TOK_INTERPOLATED_EXPR; + m_data = reinterpret_cast<AST::ExprNode*>(frag.m_ptr)->clone().release(); + break; + case InterpolatedFragment::STMT: + m_type = TOK_INTERPOLATED_STMT; + m_data = reinterpret_cast<AST::ExprNode*>(frag.m_ptr)->clone().release(); + break; + case InterpolatedFragment::BLOCK: + m_type = TOK_INTERPOLATED_BLOCK; + m_data = reinterpret_cast<AST::ExprNode*>(frag.m_ptr)->clone().release(); + break; + case InterpolatedFragment::META: + m_type = TOK_INTERPOLATED_META; + m_data = new AST::MetaItem( reinterpret_cast<AST::MetaItem*>(frag.m_ptr)->clone() ); + break; + } +} + +Token Token::clone() const +{ + Token rv(m_type); + rv.m_pos = m_pos; + + assert( m_data.tag() != Data::TAGDEAD ); + TU_MATCH(Data, (m_data), (e), + (None, + ), + (String, + rv.m_data = Data::make_String(e); + ), + (Integer, + rv.m_data = Data::make_Integer(e); + ), + (Float, + rv.m_data = Data::make_Float(e); + ), + (Fragment, + assert(e); + switch(m_type) + { + case TOK_INTERPOLATED_TYPE: + rv.m_data = new TypeRef( *reinterpret_cast<TypeRef*>(e) ); + break; + case TOK_INTERPOLATED_PATTERN: + rv.m_data = new AST::Pattern( reinterpret_cast<AST::Pattern*>(e)->clone() ); + break; + case TOK_INTERPOLATED_PATH: + rv.m_data = new AST::Path( *reinterpret_cast<AST::Path*>(e) ); + break; + case TOK_INTERPOLATED_EXPR: + rv.m_data = reinterpret_cast<AST::ExprNode*>(e)->clone().release(); + break; + case TOK_INTERPOLATED_STMT: + rv.m_data = reinterpret_cast<AST::ExprNode*>(e)->clone().release(); + break; + case TOK_INTERPOLATED_BLOCK: + rv.m_data = reinterpret_cast<AST::ExprNode*>(e)->clone().release(); + break; + case TOK_INTERPOLATED_META: + rv.m_data = new AST::MetaItem( reinterpret_cast<AST::MetaItem*>(e)->clone() ); + break; + default: + assert(!"Token::clone() - fragment with invalid token type"); + break; + } + ) + ) + return rv; +} + +::std::unique_ptr<AST::ExprNode> Token::take_frag_node() +{ + assert( m_type == TOK_INTERPOLATED_EXPR || m_type == TOK_INTERPOLATED_STMT || m_type == TOK_INTERPOLATED_BLOCK ); + auto ptr = m_data.as_Fragment(); + m_data.as_Fragment() = nullptr; + return ::std::unique_ptr<AST::ExprNode>( reinterpret_cast<AST::ExprNode*>( ptr ) ); +} const char* Token::typestr(enum eTokenType type) { @@ -91,6 +220,13 @@ struct EscapedString { case TOK_NEWLINE: return "\n"; case TOK_WHITESPACE: return " "; case TOK_COMMENT: return "/*" + m_data.as_String() + "*/"; + case TOK_INTERPOLATED_TYPE: return "/*:ty*/"; + case TOK_INTERPOLATED_PATH: return "/*:path*/"; + case TOK_INTERPOLATED_PATTERN: return "/*:pat*/"; + case TOK_INTERPOLATED_EXPR: return "/*:expr*/"; + case TOK_INTERPOLATED_STMT: return "/*:stmt*/"; + case TOK_INTERPOLATED_BLOCK: return "/*:block*/"; + case TOK_INTERPOLATED_META: return "/*:meta*/"; // Value tokens case TOK_IDENT: return m_data.as_String(); case TOK_MACRO: return m_data.as_String() + "!"; @@ -251,6 +387,9 @@ SERIALISE_TYPE(Token::, "Token", { (Float, s % e.m_datatype; s.item( e.m_floatval ); + ), + (Fragment, + assert(!"Serialising interpolated macro fragment"); ) ) },{ @@ -284,6 +423,8 @@ SERIALISE_TYPE(Token::, "Token", { s.item( v ); m_data = Token::Data::make_Float({dt, v}); break; } + case Token::Data::TAG_Fragment: + assert(!"Serialising interpolated macro fragment"); } }); @@ -297,10 +438,12 @@ SERIALISE_TYPE(Token::, "Token", { case TOK_IDENT: case TOK_MACRO: case TOK_LIFETIME: - os << "\"" << EscapedString(tok.str()) << "\""; + if( tok.m_data.is_String() ) + os << "\"" << EscapedString(tok.str()) << "\""; break; case TOK_INTEGER: - os << ":" << tok.intval(); + if( tok.m_data.is_Integer() ) + os << ":" << tok.intval(); break; default: break; @@ -311,3 +454,4 @@ SERIALISE_TYPE(Token::, "Token", { { return os << p.filename << ":" << p.line; } + diff --git a/src/parse/token.hpp b/src/parse/token.hpp index 0a72cec1..25993b2f 100644 --- a/src/parse/token.hpp +++ b/src/parse/token.hpp @@ -35,6 +35,17 @@ public: }; extern ::std::ostream& operator<<(::std::ostream& os, const Position& p); +class TypeRef; +class TokenTree; +namespace AST { + class Pattern; + class Path; + class ExprNode; + class MetaItem; +}; + +class InterpolatedFragment; + class Token: public Serialisable { @@ -48,13 +59,15 @@ class Token: (Float, struct { enum eCoreType m_datatype; double m_floatval; - }) + }), + (Fragment, void*) ); enum eTokenType m_type; Data m_data; Position m_pos; public: + virtual ~Token(); Token(); Token& operator=(Token&& t) { @@ -80,20 +93,30 @@ public: (None, ), (String, m_data = Data::make_String(e); ), (Integer, m_data = Data::make_Integer(e);), - (Float, m_data = Data::make_Float(e);) + (Float, m_data = Data::make_Float(e);), + (Fragment, assert(!"Copied fragment");) ) } + Token clone() const; Token(enum eTokenType type); Token(enum eTokenType type, ::std::string str); Token(uint64_t val, enum eCoreType datatype); Token(double val, enum eCoreType datatype); + Token(InterpolatedFragment& ); enum eTokenType type() const { return m_type; } const ::std::string& str() const { return m_data.as_String(); } enum eCoreType datatype() const { TU_MATCH_DEF(Data, (m_data), (e), (assert(!"Getting datatype of invalid token type");), (Integer, return e.m_datatype;), (Float, return e.m_datatype;)) } uint64_t intval() const { return m_data.as_Integer().m_intval; } double floatval() const { return m_data.as_Float().m_floatval; } + + TypeRef& frag_type() { assert(m_type == TOK_INTERPOLATED_TYPE); return *reinterpret_cast<TypeRef*>( m_data.as_Fragment() ); } + AST::Path& frag_path() { assert(m_type == TOK_INTERPOLATED_PATH); return *reinterpret_cast<AST::Path*>( m_data.as_Fragment() ); } + AST::Pattern& frag_pattern() { assert(m_type == TOK_INTERPOLATED_PATTERN); return *reinterpret_cast<AST::Pattern*>( m_data.as_Fragment() ); } + AST::MetaItem& frag_meta() { assert(m_type == TOK_INTERPOLATED_META); return *reinterpret_cast<AST::MetaItem*>( m_data.as_Fragment() ); } + ::std::unique_ptr<AST::ExprNode> take_frag_node(); + bool operator==(const Token& r) const { if(type() != r.type()) return false; @@ -101,7 +124,8 @@ public: (None, return true;), (String, return e == re;), (Integer, return e.m_datatype == re.m_datatype && e.m_intval == re.m_intval;), - (Float, return e.m_datatype == re.m_datatype && e.m_floatval == re.m_floatval;) + (Float, return e.m_datatype == re.m_datatype && e.m_floatval == re.m_floatval;), + (Fragment, assert(!"Token equality on Fragment");) ) throw ""; } @@ -116,6 +140,8 @@ public: static eTokenType typefromstr(const ::std::string& s); SERIALISABLE_PROTOTYPES(); + + friend ::std::ostream& operator<<(::std::ostream& os, const Token& tok); }; extern ::std::ostream& operator<<(::std::ostream& os, const Token& tok); diff --git a/src/parse/tokentree.hpp b/src/parse/tokentree.hpp index 89537a64..68080689 100644 --- a/src/parse/tokentree.hpp +++ b/src/parse/tokentree.hpp @@ -10,8 +10,10 @@ class TokenTree: Token m_tok;
::std::vector<TokenTree> m_subtrees;
public:
+ virtual ~TokenTree() {}
TokenTree() {}
TokenTree(TokenTree&&) = default;
+ TokenTree& operator=(TokenTree&&) = default;
TokenTree(Token tok):
m_tok( ::std::move(tok) )
{
@@ -20,7 +22,8 @@ public: m_subtrees( ::std::move(subtrees) )
{
}
- TokenTree& operator=(TokenTree&&) = default;
+
+ TokenTree clone() const;
bool is_token() const {
return m_tok.type() != TOK_NULL;
diff --git a/src/parse/types.cpp b/src/parse/types.cpp index 370f78a3..c6e33a3d 100644 --- a/src/parse/types.cpp +++ b/src/parse/types.cpp @@ -11,21 +11,21 @@ #include <ast/ast.hpp> // === PROTOTYPES === -TypeRef Parse_Type(TokenStream& lex); -TypeRef Parse_Type_Int(TokenStream& lex); +//TypeRef Parse_Type(TokenStream& lex, bool allow_trait_list); +TypeRef Parse_Type_Int(TokenStream& lex, bool allow_trait_list); TypeRef Parse_Type_Fn(TokenStream& lex, ::std::vector<::std::string> hrls = {}); -TypeRef Parse_Type_Path(TokenStream& lex, ::std::vector<::std::string> hrls); +TypeRef Parse_Type_Path(TokenStream& lex, ::std::vector<::std::string> hrls, bool allow_trait_list); // === CODE === -TypeRef Parse_Type(TokenStream& lex) +TypeRef Parse_Type(TokenStream& lex, bool allow_trait_list) { ProtoSpan ps = lex.start_span(); - TypeRef rv = Parse_Type_Int(lex); + TypeRef rv = Parse_Type_Int(lex, allow_trait_list); //rv.set_span(lex.end_span(ps)); return rv; } -TypeRef Parse_Type_Int(TokenStream& lex) +TypeRef Parse_Type_Int(TokenStream& lex, bool allow_trait_list) { //TRACE_FUNCTION; auto ps = lex.start_span(); @@ -34,6 +34,8 @@ TypeRef Parse_Type_Int(TokenStream& lex) switch( GET_TOK(tok, lex) ) { + case TOK_INTERPOLATED_TYPE: + return mv$(tok.frag_type()); case TOK_MACRO: return TypeRef(TypeRef::TagMacro(), Parse_MacroInvocation(ps, AST::MetaItems(), mv$(tok.str()), lex)); // '!' - Only ever used as part of function prototypes, but is kinda a type... not allowed here though @@ -49,13 +51,13 @@ TypeRef Parse_Type_Int(TokenStream& lex) case TOK_RWORD_EXTERN: // 'fn' - Rust function case TOK_RWORD_FN: - lex.putback(tok); + PUTBACK(tok, lex); return Parse_Type_Fn(lex); // '<' - An associated type cast case TOK_LT: case TOK_DOUBLE_LT: { - lex.putback(tok); + PUTBACK(tok, lex); auto path = Parse_Path(lex, PATH_GENERIC_TYPE); return TypeRef(TypeRef::TagPath(), lex.end_span(ps), mv$(path)); } @@ -76,27 +78,27 @@ TypeRef Parse_Type_Int(TokenStream& lex) // TODO: Handle HRLS in fn types return Parse_Type_Fn(lex, hrls); default: - return Parse_Type_Path(lex, hrls); + return Parse_Type_Path(lex, hrls, allow_trait_list); } } // <ident> - Either a primitive, or a path case TOK_IDENT: // or a primitive - if( auto ct = coretype_fromstring(tok.str()) ) - { - return TypeRef(TypeRef::TagPrimitive(), Span(tok.get_pos()), ct); - } - lex.putback(tok); - return Parse_Type_Path(lex, {}); + //if( auto ct = coretype_fromstring(tok.str()) ) + //{ + // return TypeRef(TypeRef::TagPrimitive(), Span(tok.get_pos()), ct); + //} + PUTBACK(tok, lex); + return Parse_Type_Path(lex, {}, allow_trait_list); // - Fall through to path handling // '::' - Absolute path case TOK_DOUBLE_COLON: - lex.putback(tok); - return Parse_Type_Path(lex, {}); + PUTBACK(tok, lex); + return Parse_Type_Path(lex, {}, allow_trait_list); // 'super' - Parent relative path case TOK_RWORD_SUPER: - lex.putback(tok); - return Parse_Type_Path(lex, {}); + PUTBACK(tok, lex); + return Parse_Type_Path(lex, {}, allow_trait_list); // HACK! Convert && into & & case TOK_DOUBLE_AMP: @@ -115,7 +117,7 @@ TypeRef Parse_Type_Int(TokenStream& lex) return TypeRef(TypeRef::TagReference(), lex.end_span(ps), true, Parse_Type(lex)); } else { - lex.putback(tok); + PUTBACK(tok, lex); // Immutable reference return TypeRef(TypeRef::TagReference(), lex.end_span(ps), false, Parse_Type(lex)); } @@ -160,9 +162,9 @@ TypeRef Parse_Type_Int(TokenStream& lex) DEBUG("Tuple"); if( GET_TOK(tok, lex) == TOK_PAREN_CLOSE ) return TypeRef(TypeRef::TagTuple(), lex.end_span(ps), {}); - lex.putback(tok); + PUTBACK(tok, lex); - TypeRef inner = Parse_Type(lex); + TypeRef inner = Parse_Type(lex, true); if( GET_TOK(tok, lex) == TOK_PLUS ) { // Lifetime bounded type, NOT a tuple @@ -177,13 +179,13 @@ TypeRef Parse_Type_Int(TokenStream& lex) { ::std::vector<TypeRef> types; types.push_back( ::std::move(inner) ); - lex.putback(tok); + PUTBACK(tok, lex); while( GET_TOK(tok, lex) == TOK_COMMA ) { if( GET_TOK(tok, lex) == TOK_PAREN_CLOSE ) break; else - lex.putback(tok); + PUTBACK(tok, lex); types.push_back(Parse_Type(lex)); } CHECK_TOK(tok, TOK_PAREN_CLOSE); @@ -239,7 +241,7 @@ TypeRef Parse_Type_Fn(TokenStream& lex, ::std::vector<::std::string> hrls) } args.push_back( Parse_Type(lex) ); if( GET_TOK(tok, lex) != TOK_COMMA ) { - lex.putback(tok); + PUTBACK(tok, lex); break; } } @@ -251,36 +253,43 @@ TypeRef Parse_Type_Fn(TokenStream& lex, ::std::vector<::std::string> hrls) ret_type = Parse_Type(lex); } else { - lex.putback(tok); + PUTBACK(tok, lex); } return TypeRef(TypeRef::TagFunction(), lex.end_span(ps), ::std::move(abi), ::std::move(args), ::std::move(ret_type)); } -TypeRef Parse_Type_Path(TokenStream& lex, ::std::vector<::std::string> hrls) +TypeRef Parse_Type_Path(TokenStream& lex, ::std::vector<::std::string> hrls, bool allow_trait_list) { Token tok; auto ps = lex.start_span(); - ::std::vector<AST::Path> traits; - ::std::vector< ::std::string> lifetimes; - do { - if( LOOK_AHEAD(lex) == TOK_LIFETIME ) { - GET_TOK(tok, lex); - lifetimes.push_back( tok.str() ); - } - else - traits.push_back( Parse_Path(lex, PATH_GENERIC_TYPE) ); - } while( GET_TOK(tok, lex) == TOK_PLUS ); - lex.putback(tok); - if( hrls.size() > 0 || traits.size() > 1 || lifetimes.size() > 0 ) { - if( lifetimes.size() ) - DEBUG("TODO: Lifetime bounds on trait objects"); - return TypeRef(lex.end_span(ps), mv$(hrls), ::std::move(traits)); + if( ! allow_trait_list ) + { + return TypeRef(TypeRef::TagPath(), lex.end_span(ps), Parse_Path(lex, PATH_GENERIC_TYPE)); } - else { - return TypeRef(TypeRef::TagPath(), lex.end_span(ps), traits.at(0)); + else + { + ::std::vector<AST::Path> traits; + ::std::vector< ::std::string> lifetimes; + do { + if( LOOK_AHEAD(lex) == TOK_LIFETIME ) { + GET_TOK(tok, lex); + lifetimes.push_back( tok.str() ); + } + else + traits.push_back( Parse_Path(lex, PATH_GENERIC_TYPE) ); + } while( GET_TOK(tok, lex) == TOK_PLUS ); + PUTBACK(tok, lex); + if( hrls.size() > 0 || traits.size() > 1 || lifetimes.size() > 0 ) { + if( lifetimes.size() ) + DEBUG("TODO: Lifetime bounds on trait objects"); + return TypeRef(lex.end_span(ps), mv$(hrls), ::std::move(traits)); + } + else { + return TypeRef(TypeRef::TagPath(), lex.end_span(ps), traits.at(0)); + } } } |