From dd7680bc0d53f02e4b96b2b3896ebea16742c9bb Mon Sep 17 00:00:00 2001 From: John Hodge Date: Wed, 25 May 2016 23:16:23 +0800 Subject: Parse - Use interpolated macro fragments instead of captured TTs --- src/parse/common.hpp | 5 +- src/parse/eTokenType.enum.h | 8 ++ src/parse/expr.cpp | 131 +++++++++++++++++-------------- src/parse/interpolated_fragment.cpp | 76 ++++++++++++++++++ src/parse/interpolated_fragment.hpp | 49 ++++++++++++ src/parse/lex.cpp | 20 ++++- src/parse/parseerror.cpp | 10 +-- src/parse/parseerror.hpp | 6 +- src/parse/paths.cpp | 27 ++++--- src/parse/pattern.cpp | 24 +++--- src/parse/root.cpp | 98 +++++++++++++----------- src/parse/token.cpp | 148 +++++++++++++++++++++++++++++++++++- src/parse/token.hpp | 32 +++++++- src/parse/tokentree.hpp | 5 +- src/parse/types.cpp | 97 ++++++++++++----------- 15 files changed, 550 insertions(+), 186 deletions(-) create mode 100644 src/parse/interpolated_fragment.cpp create mode 100644 src/parse/interpolated_fragment.hpp (limited to 'src/parse') diff --git a/src/parse/common.hpp b/src/parse/common.hpp index d7f84315..c42889c3 100644 --- a/src/parse/common.hpp +++ b/src/parse/common.hpp @@ -41,7 +41,7 @@ extern ::std::vector Parse_Path_GenericList(TokenStream& lex); extern AST::MetaItem Parse_MetaItem(TokenStream& lex); extern ::AST::MacroInvocation Parse_MacroInvocation(ProtoSpan ps, ::AST::MetaItems meta_items, ::std::string name, TokenStream& lex); -extern TypeRef Parse_Type(TokenStream& lex); +extern TypeRef Parse_Type(TokenStream& lex, bool allow_trait_list = false); extern AST::Pattern Parse_Pattern(TokenStream& lex, bool is_refutable); extern void Parse_Impl_Item(TokenStream& lex, AST::Impl& impl); @@ -51,7 +51,8 @@ extern void Parse_ModRoot_Items(TokenStream& lex, AST::Module& mod, bool file_co extern AST::Expr Parse_Expr(TokenStream& lex); extern AST::Expr Parse_ExprBlock(TokenStream& lex); -extern AST::ExprNodeP Parse_Expr0(TokenStream& lex); +extern AST::ExprNodeP Parse_Expr0(TokenStream& lex); +extern AST::ExprNodeP Parse_ExprVal(TokenStream& lex); extern bool Parse_IsTokValue(eTokenType tok_type); diff --git a/src/parse/eTokenType.enum.h b/src/parse/eTokenType.enum.h index 2adfb042..f7157805 100644 --- a/src/parse/eTokenType.enum.h +++ b/src/parse/eTokenType.enum.h @@ -5,6 +5,14 @@ _(TOK_NEWLINE) _(TOK_WHITESPACE) _(TOK_COMMENT) +_(TOK_INTERPOLATED_PATH) +_(TOK_INTERPOLATED_TYPE) +_(TOK_INTERPOLATED_PATTERN) +_(TOK_INTERPOLATED_EXPR) +_(TOK_INTERPOLATED_STMT) +_(TOK_INTERPOLATED_BLOCK) +_(TOK_INTERPOLATED_META) + // Value tokens _(TOK_IDENT) _(TOK_MACRO) diff --git a/src/parse/expr.cpp b/src/parse/expr.cpp index 7094cf96..37bc03e7 100644 --- a/src/parse/expr.cpp +++ b/src/parse/expr.cpp @@ -85,7 +85,7 @@ ExprNodeP Parse_ExprBlockNode(TokenStream& lex) case TOK_RWORD_IMPL: case TOK_RWORD_FN: case TOK_RWORD_MOD: - lex.putback(tok); + PUTBACK(tok, lex); if( !local_mod ) { local_mod = lex.parse_state().get_current_mod().add_anon(); } @@ -95,7 +95,7 @@ ExprNodeP Parse_ExprBlockNode(TokenStream& lex) case TOK_RWORD_UNSAFE: if( LOOK_AHEAD(lex) != TOK_BRACE_OPEN ) { - lex.putback(tok); + PUTBACK(tok, lex); if( !local_mod ) { local_mod = lex.parse_state().get_current_mod().add_anon(); } @@ -104,7 +104,7 @@ ExprNodeP Parse_ExprBlockNode(TokenStream& lex) } // fall default: { - lex.putback(tok); + PUTBACK(tok, lex); bool expect_end = false; nodes.push_back(Parse_ExprBlockLine(lex, &expect_end)); if( nodes.back() ) { @@ -121,7 +121,7 @@ ExprNodeP Parse_ExprBlockNode(TokenStream& lex) { throw ParseError::Unexpected(lex, tok, Token(TOK_BRACE_CLOSE)); } - lex.putback(tok); + PUTBACK(tok, lex); } break; } @@ -159,7 +159,7 @@ ExprNodeP Parse_ExprBlockLine(TokenStream& lex, bool *expect_end) case TOK_RWORD_MATCH: return Parse_Expr_Match(lex); case TOK_BRACE_OPEN: - lex.putback(tok); + PUTBACK(tok, lex); return Parse_ExprBlockNode(lex); default: @@ -173,7 +173,7 @@ ExprNodeP Parse_ExprBlockLine(TokenStream& lex, bool *expect_end) case TOK_SEMICOLON: return 0; case TOK_BRACE_OPEN: - lex.putback(tok); + PUTBACK(tok, lex); return Parse_ExprBlockNode(lex); // let binding @@ -215,11 +215,11 @@ ExprNodeP Parse_ExprBlockLine(TokenStream& lex, bool *expect_end) } // Fall through to the statement code default: { - lex.putback(tok); + PUTBACK(tok, lex); auto ret = Parse_Stmt(lex); if( GET_TOK(tok, lex) != TOK_SEMICOLON ) { - lex.putback(tok); + PUTBACK(tok, lex); *expect_end = true; } return ::std::move(ret); @@ -245,7 +245,7 @@ ExprNodeP Parse_WhileStmt(TokenStream& lex, ::std::string lifetime) ::std::move(pat), ::std::move(val), Parse_ExprBlockNode(lex) ); } else { - lex.putback(tok); + PUTBACK(tok, lex); ExprNodeP cnd; { SET_PARSE_FLAG(lex, disallow_struct_literal); @@ -291,7 +291,7 @@ ExprNodeP Parse_IfStmt(TokenStream& lex) cond = Parse_Expr0(lex); } else { - lex.putback(tok); + PUTBACK(tok, lex); cond = Parse_Expr0(lex); } } @@ -309,13 +309,13 @@ ExprNodeP Parse_IfStmt(TokenStream& lex) } // - or get block else { - lex.putback(tok); + PUTBACK(tok, lex); altcode = Parse_ExprBlockNode(lex); } } // - or nothing else { - lex.putback(tok); + PUTBACK(tok, lex); } if( if_let ) @@ -343,7 +343,7 @@ ExprNodeP Parse_Expr_Match(TokenStream& lex) do { if( GET_TOK(tok, lex) == TOK_BRACE_CLOSE ) break; - lex.putback(tok); + PUTBACK(tok, lex); AST::ExprNode_Match_Arm arm; ::AST::MetaItems arm_attrs; @@ -372,7 +372,7 @@ ExprNodeP Parse_Expr_Match(TokenStream& lex) if( GET_TOK(tok, lex) == TOK_COMMA ) continue; - lex.putback(tok); + PUTBACK(tok, lex); } while( 1 ); CHECK_TOK(tok, TOK_BRACE_CLOSE); @@ -431,20 +431,20 @@ ExprNodeP Parse_Stmt(TokenStream& lex) case TOK_BRACE_CLOSE: case TOK_PAREN_CLOSE: case TOK_SQUARE_CLOSE: - lex.putback(tok); + PUTBACK(tok, lex); break; default: - lex.putback(tok); + PUTBACK(tok, lex); val = Parse_Expr1(lex); break; } return NEWNODE( AST::ExprNode_Flow, type, lifetime, ::std::move(val) ); } case TOK_BRACE_OPEN: - lex.putback(tok); + PUTBACK(tok, lex); return Parse_ExprBlockNode(lex); default: - lex.putback(tok); + PUTBACK(tok, lex); return Parse_Expr0(lex); } } @@ -460,7 +460,7 @@ ExprNodeP Parse_Stmt(TokenStream& lex) GET_CHECK_TOK(tok, lex, TOK_PAREN_OPEN); if( GET_TOK(tok, lex) != TOK_PAREN_CLOSE ) { - lex.putback(tok); + PUTBACK(tok, lex); do { if( LOOK_AHEAD(lex) == TOK_PAREN_CLOSE ) { GET_TOK(tok, lex); @@ -521,7 +521,7 @@ ExprNodeP Parse_Expr0(TokenStream& lex) return rv; default: - lex.putback(tok); + PUTBACK(tok, lex); rv->set_attrs(mv$(expr_attrs)); return rv; } @@ -542,7 +542,7 @@ ExprNodeP cur(TokenStream& lex) \ cases \ default: \ /*::std::cout << "<<" << #cur << ::std::endl; */\ - lex.putback(tok); \ + PUTBACK(tok, lex); \ return rv; \ } \ } \ @@ -585,26 +585,39 @@ ExprNodeP Parse_Expr1(TokenStream& lex) ExprNodeP (*next)(TokenStream&) = Parse_Expr1_1; ExprNodeP left, right; + // Inclusive range to a value + if( GET_TOK(tok, lex) == TOK_TRIPLE_DOT ) { + right = next(lex); + return NEWNODE( AST::ExprNode_BinOp, AST::ExprNode_BinOp::RANGE_INC, nullptr, mv$(right) ); + } + else { + PUTBACK(tok, lex); + } + + // Exclusive ranges + // - If NOT `.. `, parse a leading value if( GET_TOK(tok, lex) != TOK_DOUBLE_DOT ) { - lex.putback(tok); + PUTBACK(tok, lex); left = next(lex); + // - If NOT ` ..`, return the value if( GET_TOK(tok, lex) != TOK_DOUBLE_DOT ) { - lex.putback(tok); + PUTBACK(tok, lex); return ::std::move(left); } } - if( Parse_IsTokValue( GET_TOK(tok, lex) ) ) + assert( tok.type() == TOK_DOUBLE_DOT ); + // If the next token is part of a value, parse that value + if( Parse_IsTokValue( LOOK_AHEAD(lex) ) ) { - lex.putback(tok); right = next(lex); } else { - lex.putback(tok); + // Otherwise, leave `right` as nullptr } return NEWNODE( AST::ExprNode_BinOp, AST::ExprNode_BinOp::RANGE, ::std::move(left), ::std::move(right) ); @@ -717,7 +730,7 @@ ExprNodeP Parse_Expr12(TokenStream& lex) } else { - lex.putback(tok); + PUTBACK(tok, lex); } return rv; } @@ -752,11 +765,11 @@ ExprNodeP Parse_Expr13(TokenStream& lex) if( GET_TOK(tok, lex) == TOK_RWORD_MUT ) return NEWNODE( AST::ExprNode_UniOp, AST::ExprNode_UniOp::REFMUT, Parse_Expr12(lex) ); else { - lex.putback(tok); + PUTBACK(tok, lex); return NEWNODE( AST::ExprNode_UniOp, AST::ExprNode_UniOp::REF, Parse_Expr12(lex) ); } default: - lex.putback(tok); + PUTBACK(tok, lex); return Parse_ExprFC(lex); } } @@ -776,7 +789,7 @@ ExprNodeP Parse_ExprFC(TokenStream& lex) case TOK_PAREN_OPEN: // Expression method call - lex.putback(tok); + PUTBACK(tok, lex); val = NEWNODE( AST::ExprNode_CallObject, ::std::move(val), Parse_ParenList(lex) ); break; case TOK_SQUARE_OPEN: @@ -793,7 +806,7 @@ ExprNodeP Parse_ExprFC(TokenStream& lex) switch( GET_TOK(tok, lex) ) { case TOK_PAREN_OPEN: - lex.putback(tok); + PUTBACK(tok, lex); val = NEWNODE( AST::ExprNode_CallMethod, ::std::move(val), ::std::move(path), Parse_ParenList(lex) ); break; case TOK_DOUBLE_COLON: @@ -862,7 +875,7 @@ ExprNodeP Parse_ExprVal_Closure(TokenStream& lex, bool is_move) while( GET_TOK(tok, lex) != TOK_PIPE ) { - lex.putback(tok); + PUTBACK(tok, lex); // Irrefutable pattern AST::Pattern pat = Parse_Pattern(lex, false); @@ -870,7 +883,7 @@ ExprNodeP Parse_ExprVal_Closure(TokenStream& lex, bool is_move) if( GET_TOK(tok, lex) == TOK_COLON ) type = Parse_Type(lex); else - lex.putback(tok); + PUTBACK(tok, lex); args.push_back( ::std::make_pair( ::std::move(pat), ::std::move(type) ) ); @@ -886,12 +899,12 @@ ExprNodeP Parse_ExprVal_Closure(TokenStream& lex, bool is_move) rt = TypeRef(TypeRef::TagInvalid(), Span(tok.get_pos())); } else { - lex.putback(tok); + PUTBACK(tok, lex); rt = Parse_Type(lex); } } else - lex.putback(tok); + PUTBACK(tok, lex); auto code = Parse_Expr0(lex); @@ -932,14 +945,17 @@ ExprNodeP Parse_ExprVal(TokenStream& lex) switch( GET_TOK(tok, lex) ) { case TOK_BRACE_OPEN: - lex.putback(tok); + PUTBACK(tok, lex); return Parse_ExprBlockNode(lex); + case TOK_INTERPOLATED_EXPR: + return tok.take_frag_node(); + // TODO: Return/break/continue/... here? case TOK_RWORD_RETURN: case TOK_RWORD_CONTINUE: case TOK_RWORD_BREAK: - lex.putback(tok); + PUTBACK(tok, lex); return Parse_Stmt(lex); case TOK_RWORD_LOOP: @@ -960,7 +976,7 @@ ExprNodeP Parse_ExprVal(TokenStream& lex) // UFCS case TOK_DOUBLE_LT: - lex.putback(tok); + PUTBACK(tok, lex); case TOK_LT: { TypeRef ty = Parse_Type(lex); if( GET_TOK(tok, lex) == TOK_RWORD_AS ) { @@ -970,7 +986,7 @@ ExprNodeP Parse_ExprVal(TokenStream& lex) path = AST::Path(AST::Path::TagUfcs(), ty, trait, Parse_PathNodes(lex, PATH_GENERIC_EXPR)); } else { - lex.putback(tok); + PUTBACK(tok, lex); GET_CHECK_TOK(tok, lex, TOK_GT); // TODO: Terminating the "path" here is sometimes valid GET_CHECK_TOK(tok, lex, TOK_DOUBLE_COLON); @@ -985,31 +1001,36 @@ ExprNodeP Parse_ExprVal(TokenStream& lex) } else { - lex.putback(tok); + PUTBACK(tok, lex); path = Parse_Path(lex, PATH_GENERIC_EXPR); } } if(0) case TOK_RWORD_SUPER: { - lex.putback(tok); + PUTBACK(tok, lex); path = Parse_Path(lex, PATH_GENERIC_EXPR); } if(0) case TOK_IDENT: // Get path { - lex.putback(tok); + PUTBACK(tok, lex); path = Parse_Path(lex, false, PATH_GENERIC_EXPR); } if(0) + case TOK_INTERPOLATED_PATH: + { + path = mv$(tok.frag_path()); + } + if(0) case TOK_DOUBLE_COLON: path = Parse_Path(lex, true, PATH_GENERIC_EXPR); switch( GET_TOK(tok, lex) ) { case TOK_PAREN_OPEN: // Function call - lex.putback(tok); + PUTBACK(tok, lex); return NEWNODE( AST::ExprNode_CallPath, ::std::move(path), Parse_ParenList(lex) ); case TOK_BRACE_OPEN: if( !CHECK_PARSE_FLAG(lex, disallow_struct_literal) ) @@ -1018,7 +1039,7 @@ ExprNodeP Parse_ExprVal(TokenStream& lex) DEBUG("Not parsing struct literal"); default: // Value - lex.putback(tok); + PUTBACK(tok, lex); return NEWNODE( AST::ExprNode_NamedValue, ::std::move(path) ); } case TOK_RWORD_MOVE: @@ -1065,7 +1086,7 @@ ExprNodeP Parse_ExprVal(TokenStream& lex) else { CLEAR_PARSE_FLAG(lex, disallow_struct_literal); - lex.putback(tok); + PUTBACK(tok, lex); ExprNodeP rv = Parse_Expr0(lex); if( GET_TOK(tok, lex) == TOK_COMMA ) { @@ -1074,7 +1095,7 @@ ExprNodeP Parse_ExprVal(TokenStream& lex) do { if( GET_TOK(tok, lex) == TOK_PAREN_CLOSE ) break; - lex.putback(tok); + PUTBACK(tok, lex); ents.push_back( Parse_Expr0(lex) ); } while( GET_TOK(tok, lex) == TOK_COMMA ); rv = NEWNODE( AST::ExprNode_Tuple, ::std::move(ents) ); @@ -1090,7 +1111,7 @@ ExprNodeP Parse_ExprVal(TokenStream& lex) } else { - lex.putback(tok); + PUTBACK(tok, lex); auto first = Parse_Expr0(lex); if( GET_TOK(tok, lex) == TOK_SEMICOLON ) { @@ -1108,7 +1129,7 @@ ExprNodeP Parse_ExprVal(TokenStream& lex) if( GET_TOK(tok, lex) == TOK_SQUARE_CLOSE ) break; else - lex.putback(tok); + PUTBACK(tok, lex); items.push_back( Parse_Expr0(lex) ); GET_TOK(tok, lex); } @@ -1118,7 +1139,7 @@ ExprNodeP Parse_ExprVal(TokenStream& lex) } throw ParseError::BugCheck(lex, "Array literal fell"); case TOK_MACRO: - return Parse_ExprMacro(lex, tok); + return Parse_ExprMacro(lex, mv$(tok)); default: throw ParseError::Unexpected(lex, tok); } @@ -1131,7 +1152,7 @@ ExprNodeP Parse_ExprMacro(TokenStream& lex, Token tok) ident = mv$(tok.str()); } else { - lex.putback(tok); + PUTBACK(tok, lex); } TokenTree tt = Parse_TT(lex, true); if( tt.is_token() ) { @@ -1162,21 +1183,21 @@ TokenTree Parse_TT(TokenStream& lex, bool unwrapped) case TOK_NULL: throw ParseError::Unexpected(lex, tok); default: - return TokenTree(tok); + return TokenTree( mv$(tok) ); } ::std::vector items; if( !unwrapped ) - items.push_back(tok); + items.push_back( mv$(tok) ); while(GET_TOK(tok, lex) != closer && tok.type() != TOK_EOF) { if( tok.type() == TOK_NULL ) throw ParseError::Unexpected(lex, tok); - lex.putback(tok); + PUTBACK(tok, lex); items.push_back(Parse_TT(lex, false)); } if( !unwrapped ) - items.push_back(tok); + items.push_back( mv$(tok) ); return TokenTree(mv$(items)); } @@ -1244,7 +1265,7 @@ TokenTree Parse_TT_Path(TokenStream& lex, bool mode_expr) Parse_Path(wlex, true, (mode_expr ? PATH_GENERIC_EXPR : PATH_GENERIC_TYPE)); } else { - wlex.putback(tok); + PUTBACK(tok, lex); Parse_Path(wlex, false, (mode_expr ? PATH_GENERIC_EXPR : PATH_GENERIC_TYPE)); } diff --git a/src/parse/interpolated_fragment.cpp b/src/parse/interpolated_fragment.cpp new file mode 100644 index 00000000..dd490bf5 --- /dev/null +++ b/src/parse/interpolated_fragment.cpp @@ -0,0 +1,76 @@ +/* + */ +#include +#include "interpolated_fragment.hpp" +#include + +InterpolatedFragment::~InterpolatedFragment() +{ + if( m_ptr ) + { + switch(m_type) + { + case InterpolatedFragment::TT: delete reinterpret_cast(m_ptr); break; + case InterpolatedFragment::PAT: delete reinterpret_cast(m_ptr); break; + case InterpolatedFragment::PATH:delete reinterpret_cast(m_ptr); break; + case InterpolatedFragment::TYPE:delete reinterpret_cast(m_ptr); break; + case InterpolatedFragment::EXPR: + case InterpolatedFragment::STMT: + case InterpolatedFragment::BLOCK: + delete reinterpret_cast(m_ptr); + break; + case InterpolatedFragment::META: + delete reinterpret_cast(m_ptr); + break; + } + } +} + +InterpolatedFragment::InterpolatedFragment(InterpolatedFragment&& x): + m_type( x.m_type ) +{ + m_ptr = x.m_ptr, x.m_ptr = nullptr; +} +InterpolatedFragment& InterpolatedFragment::operator=(InterpolatedFragment&& x) +{ + m_type = x.m_type; + m_ptr = x.m_ptr, x.m_ptr = nullptr; + return *this; +} + +InterpolatedFragment::InterpolatedFragment(InterpolatedFragment::Type type, AST::ExprNode* ptr): + m_type( type ), + m_ptr( ptr ) +{ +} +InterpolatedFragment::InterpolatedFragment(AST::MetaItem v): + m_type( InterpolatedFragment::META ), + m_ptr( new AST::MetaItem(mv$(v)) ) +{ +} +InterpolatedFragment::InterpolatedFragment(TokenTree v): + m_type( InterpolatedFragment::TT ), + m_ptr( new TokenTree(mv$(v)) ) +{ +} +InterpolatedFragment::InterpolatedFragment(AST::Path v): + m_type( InterpolatedFragment::PATH ), + m_ptr( new AST::Path(mv$(v)) ) +{ +} +InterpolatedFragment::InterpolatedFragment(AST::Pattern v): + m_type( InterpolatedFragment::PAT ), + m_ptr( new AST::Pattern(mv$(v)) ) +{ +} +InterpolatedFragment::InterpolatedFragment(TypeRef v): + m_type( InterpolatedFragment::TYPE ), + m_ptr( new TypeRef(mv$(v)) ) +{ +} + +::std::ostream& operator<<(::std::ostream& os, InterpolatedFragment const& x) +{ + return os; +} + diff --git a/src/parse/interpolated_fragment.hpp b/src/parse/interpolated_fragment.hpp new file mode 100644 index 00000000..40aa764a --- /dev/null +++ b/src/parse/interpolated_fragment.hpp @@ -0,0 +1,49 @@ +/* + */ +#pragma once + +#include + +class TypeRef; +class TokenTree; +namespace AST { + class Pattern; + class Path; + class ExprNode; + class MetaItem; +}; + +class InterpolatedFragment +{ +public: + enum Type + { + TT, + PAT, + PATH, + TYPE, + + EXPR, + STMT, + BLOCK, + + META, + } m_type; + + void* m_ptr; + + InterpolatedFragment(InterpolatedFragment&& ); + InterpolatedFragment& operator=(InterpolatedFragment&& ); + //InterpolatedFragment(const InterpolatedFragment& ); + InterpolatedFragment(TokenTree ); + InterpolatedFragment(::AST::Pattern); + InterpolatedFragment(::AST::Path); + InterpolatedFragment(::TypeRef); + InterpolatedFragment(::AST::MetaItem ); + ~InterpolatedFragment(); + InterpolatedFragment(Type , ::AST::ExprNode*); + + TokenTree& as_tt() { assert(m_type == TT); return *reinterpret_cast(m_ptr); } + + friend ::std::ostream& operator<<(::std::ostream& os, const InterpolatedFragment& x); +}; diff --git a/src/parse/lex.cpp b/src/parse/lex.cpp index fd6044f4..395eee48 100644 --- a/src/parse/lex.cpp +++ b/src/parse/lex.cpp @@ -20,7 +20,8 @@ #include #include // std::count -const bool DEBUG_PRINT_TOKENS = false; +//const bool DEBUG_PRINT_TOKENS = false; +const bool DEBUG_PRINT_TOKENS = true; Lexer::Lexer(const ::std::string& filename): m_path(filename.c_str()), @@ -956,7 +957,7 @@ Token TTStream::realGetToken() const TokenTree& subtree = tree[idx]; idx ++; if( subtree.size() == 0 ) { - return subtree.tok(); + return subtree.tok().clone(); } else { m_stack.push_back( ::std::make_pair(0, &subtree ) ); @@ -1041,7 +1042,7 @@ Token TokenStream::getToken() if( m_cache_valid ) { m_cache_valid = false; - return m_cache; + return mv$(m_cache); } else if( m_lookahead.size() ) { @@ -1119,6 +1120,19 @@ Span TokenStream::end_span(ProtoSpan ps) const } +TokenTree TokenTree::clone() const +{ + if( m_subtrees.size() == 0 ) { + return TokenTree(m_tok.clone()); + } + else { + ::std::vector< TokenTree> ents; + ents.reserve( m_subtrees.size() ); + for(const auto& sub : m_subtrees) + ents.push_back( sub.clone() ); + return TokenTree( mv$(ents) ); + } +} SERIALISE_TYPE_A(TokenTree::, "TokenTree", { s.item(m_tok); s.item(m_subtrees); diff --git a/src/parse/parseerror.cpp b/src/parse/parseerror.cpp index e05ecb05..7c6f6a21 100644 --- a/src/parse/parseerror.cpp +++ b/src/parse/parseerror.cpp @@ -56,23 +56,23 @@ ParseError::BadChar::~BadChar() throw() { } -ParseError::Unexpected::Unexpected(const TokenStream& lex, Token tok): - m_tok(tok) +ParseError::Unexpected::Unexpected(const TokenStream& lex, const Token& tok)//: +// m_tok( mv$(tok) ) { auto pos = tok.get_pos(); if(pos.filename == "") pos = lex.getPosition(); ::std::cout << pos << ": Unexpected(" << tok << ")" << ::std::endl; } -ParseError::Unexpected::Unexpected(const TokenStream& lex, Token tok, Token exp): - m_tok(tok) +ParseError::Unexpected::Unexpected(const TokenStream& lex, const Token& tok, Token exp)//: +// m_tok( mv$(tok) ) { auto pos = tok.get_pos(); if(pos.filename == "") pos = lex.getPosition(); ::std::cout << pos << ": Unexpected(" << tok << ", " << exp << ")" << ::std::endl; } -ParseError::Unexpected::Unexpected(const TokenStream& lex, Token tok, ::std::vector exp) +ParseError::Unexpected::Unexpected(const TokenStream& lex, const Token& tok, ::std::vector exp) { auto pos = tok.get_pos(); if(pos.filename == "") diff --git a/src/parse/parseerror.hpp b/src/parse/parseerror.hpp index 1074bf46..42a89785 100644 --- a/src/parse/parseerror.hpp +++ b/src/parse/parseerror.hpp @@ -26,9 +26,9 @@ class Unexpected: { Token m_tok; public: - Unexpected(const TokenStream& lex, Token tok); - Unexpected(const TokenStream& lex, Token tok, Token exp); - Unexpected(const TokenStream& lex, Token tok, ::std::vector exp); + Unexpected(const TokenStream& lex, const Token& tok); + Unexpected(const TokenStream& lex, const Token& tok, Token exp); + Unexpected(const TokenStream& lex, const Token& tok, ::std::vector exp); virtual ~Unexpected() throw (); }; diff --git a/src/parse/paths.cpp b/src/parse/paths.cpp index 06545b7e..a5d10f74 100644 --- a/src/parse/paths.cpp +++ b/src/parse/paths.cpp @@ -19,6 +19,9 @@ AST::Path Parse_Path(TokenStream& lex, eParsePathGenericMode generic_mode) Token tok; switch( GET_TOK(tok, lex) ) { + case TOK_INTERPOLATED_PATH: + return mv$(tok.frag_path()); + case TOK_RWORD_SELF: GET_CHECK_TOK(tok, lex, TOK_DOUBLE_COLON); return Parse_Path(lex, false, generic_mode); @@ -47,7 +50,7 @@ AST::Path Parse_Path(TokenStream& lex, eParsePathGenericMode generic_mode) trait = Parse_Path(lex, true, PATH_GENERIC_TYPE); } else { - lex.putback(tok); + PUTBACK(tok, lex); trait = Parse_Path(lex, false, PATH_GENERIC_TYPE); } GET_CHECK_TOK(tok, lex, TOK_GT); @@ -55,7 +58,7 @@ AST::Path Parse_Path(TokenStream& lex, eParsePathGenericMode generic_mode) return AST::Path(AST::Path::TagUfcs(), ty, trait, Parse_PathNodes(lex, generic_mode)); } else { - lex.putback(tok); + PUTBACK(tok, lex); GET_CHECK_TOK(tok, lex, TOK_GT); // TODO: Terminating the "path" here is sometimes valid? GET_CHECK_TOK(tok, lex, TOK_DOUBLE_COLON); @@ -64,7 +67,7 @@ AST::Path Parse_Path(TokenStream& lex, eParsePathGenericMode generic_mode) throw ""; } default: - lex.putback(tok); + PUTBACK(tok, lex); return Parse_Path(lex, false, generic_mode); } } @@ -79,7 +82,7 @@ AST::Path Parse_Path(TokenStream& lex, bool is_abs, eParsePathGenericMode generi return AST::Path(cratename, Parse_PathNodes(lex, generic_mode)); } else { - lex.putback(tok); + PUTBACK(tok, lex); return AST::Path("", Parse_PathNodes(lex, generic_mode)); } } @@ -89,7 +92,7 @@ AST::Path Parse_Path(TokenStream& lex, bool is_abs, eParsePathGenericMode generi // return AST::Path( tok.str() ); //} //else { - // lex.putback(tok); + // PUTBACK(tok, lex); return AST::Path(AST::Path::TagRelative(), Parse_PathNodes(lex, generic_mode)); //} } @@ -133,7 +136,7 @@ AST::Path Parse_Path(TokenStream& lex, bool is_abs, eParsePathGenericMode generi } else { - lex.putback(tok); + PUTBACK(tok, lex); do { args.push_back( Parse_Type(lex) ); } while( GET_TOK(tok, lex) == TOK_COMMA ); @@ -145,7 +148,7 @@ AST::Path Parse_Path(TokenStream& lex, bool is_abs, eParsePathGenericMode generi ret_type = Parse_Type(lex); } else { - lex.putback(tok); + PUTBACK(tok, lex); } DEBUG("- Fn("<"< fcn) @@ -1190,7 +1196,7 @@ void Parse_Use(TokenStream& lex, ::std::function 0); name = path.nodes().back().name(); } @@ -1254,7 +1260,7 @@ void Parse_Use(TokenStream& lex, ::std::function #include +#include "interpolated_fragment.hpp" +#include +#include + +Token::~Token() +{ + switch(m_type) + { + case TOK_INTERPOLATED_TYPE: + delete reinterpret_cast(m_data.as_Fragment()); + break; + case TOK_INTERPOLATED_PATTERN: + delete reinterpret_cast(m_data.as_Fragment()); + break; + case TOK_INTERPOLATED_PATH: + delete reinterpret_cast(m_data.as_Fragment()); + break; + case TOK_INTERPOLATED_EXPR: + delete reinterpret_cast(m_data.as_Fragment()); + break; + case TOK_INTERPOLATED_STMT: + delete reinterpret_cast(m_data.as_Fragment()); + break; + case TOK_INTERPOLATED_BLOCK: + delete reinterpret_cast(m_data.as_Fragment()); + break; + case TOK_INTERPOLATED_META: + delete reinterpret_cast(m_data.as_Fragment()); + break; + default: + break; + } + +} Token::Token(): m_type(TOK_NULL) @@ -29,6 +63,101 @@ Token::Token(double val, enum eCoreType datatype): m_data( Data::make_Float({datatype, val}) ) { } +Token::Token(InterpolatedFragment& frag) +{ + switch(frag.m_type) + { + case InterpolatedFragment::TT: throw ""; + case InterpolatedFragment::TYPE: + m_type = TOK_INTERPOLATED_TYPE; + m_data = new TypeRef( *reinterpret_cast(frag.m_ptr) ); + break; + case InterpolatedFragment::PAT: + m_type = TOK_INTERPOLATED_PATTERN; + m_data = new AST::Pattern( reinterpret_cast(frag.m_ptr)->clone() ); + break; + case InterpolatedFragment::PATH: + m_type = TOK_INTERPOLATED_PATH; + m_data = new AST::Path( *reinterpret_cast(frag.m_ptr) ); + break; + case InterpolatedFragment::EXPR: + m_type = TOK_INTERPOLATED_EXPR; + m_data = reinterpret_cast(frag.m_ptr)->clone().release(); + break; + case InterpolatedFragment::STMT: + m_type = TOK_INTERPOLATED_STMT; + m_data = reinterpret_cast(frag.m_ptr)->clone().release(); + break; + case InterpolatedFragment::BLOCK: + m_type = TOK_INTERPOLATED_BLOCK; + m_data = reinterpret_cast(frag.m_ptr)->clone().release(); + break; + case InterpolatedFragment::META: + m_type = TOK_INTERPOLATED_META; + m_data = new AST::MetaItem( reinterpret_cast(frag.m_ptr)->clone() ); + break; + } +} + +Token Token::clone() const +{ + Token rv(m_type); + rv.m_pos = m_pos; + + assert( m_data.tag() != Data::TAGDEAD ); + TU_MATCH(Data, (m_data), (e), + (None, + ), + (String, + rv.m_data = Data::make_String(e); + ), + (Integer, + rv.m_data = Data::make_Integer(e); + ), + (Float, + rv.m_data = Data::make_Float(e); + ), + (Fragment, + assert(e); + switch(m_type) + { + case TOK_INTERPOLATED_TYPE: + rv.m_data = new TypeRef( *reinterpret_cast(e) ); + break; + case TOK_INTERPOLATED_PATTERN: + rv.m_data = new AST::Pattern( reinterpret_cast(e)->clone() ); + break; + case TOK_INTERPOLATED_PATH: + rv.m_data = new AST::Path( *reinterpret_cast(e) ); + break; + case TOK_INTERPOLATED_EXPR: + rv.m_data = reinterpret_cast(e)->clone().release(); + break; + case TOK_INTERPOLATED_STMT: + rv.m_data = reinterpret_cast(e)->clone().release(); + break; + case TOK_INTERPOLATED_BLOCK: + rv.m_data = reinterpret_cast(e)->clone().release(); + break; + case TOK_INTERPOLATED_META: + rv.m_data = new AST::MetaItem( reinterpret_cast(e)->clone() ); + break; + default: + assert(!"Token::clone() - fragment with invalid token type"); + break; + } + ) + ) + return rv; +} + +::std::unique_ptr Token::take_frag_node() +{ + assert( m_type == TOK_INTERPOLATED_EXPR || m_type == TOK_INTERPOLATED_STMT || m_type == TOK_INTERPOLATED_BLOCK ); + auto ptr = m_data.as_Fragment(); + m_data.as_Fragment() = nullptr; + return ::std::unique_ptr( reinterpret_cast( ptr ) ); +} const char* Token::typestr(enum eTokenType type) { @@ -91,6 +220,13 @@ struct EscapedString { case TOK_NEWLINE: return "\n"; case TOK_WHITESPACE: return " "; case TOK_COMMENT: return "/*" + m_data.as_String() + "*/"; + case TOK_INTERPOLATED_TYPE: return "/*:ty*/"; + case TOK_INTERPOLATED_PATH: return "/*:path*/"; + case TOK_INTERPOLATED_PATTERN: return "/*:pat*/"; + case TOK_INTERPOLATED_EXPR: return "/*:expr*/"; + case TOK_INTERPOLATED_STMT: return "/*:stmt*/"; + case TOK_INTERPOLATED_BLOCK: return "/*:block*/"; + case TOK_INTERPOLATED_META: return "/*:meta*/"; // Value tokens case TOK_IDENT: return m_data.as_String(); case TOK_MACRO: return m_data.as_String() + "!"; @@ -251,6 +387,9 @@ SERIALISE_TYPE(Token::, "Token", { (Float, s % e.m_datatype; s.item( e.m_floatval ); + ), + (Fragment, + assert(!"Serialising interpolated macro fragment"); ) ) },{ @@ -284,6 +423,8 @@ SERIALISE_TYPE(Token::, "Token", { s.item( v ); m_data = Token::Data::make_Float({dt, v}); break; } + case Token::Data::TAG_Fragment: + assert(!"Serialising interpolated macro fragment"); } }); @@ -297,10 +438,12 @@ SERIALISE_TYPE(Token::, "Token", { case TOK_IDENT: case TOK_MACRO: case TOK_LIFETIME: - os << "\"" << EscapedString(tok.str()) << "\""; + if( tok.m_data.is_String() ) + os << "\"" << EscapedString(tok.str()) << "\""; break; case TOK_INTEGER: - os << ":" << tok.intval(); + if( tok.m_data.is_Integer() ) + os << ":" << tok.intval(); break; default: break; @@ -311,3 +454,4 @@ SERIALISE_TYPE(Token::, "Token", { { return os << p.filename << ":" << p.line; } + diff --git a/src/parse/token.hpp b/src/parse/token.hpp index 0a72cec1..25993b2f 100644 --- a/src/parse/token.hpp +++ b/src/parse/token.hpp @@ -35,6 +35,17 @@ public: }; extern ::std::ostream& operator<<(::std::ostream& os, const Position& p); +class TypeRef; +class TokenTree; +namespace AST { + class Pattern; + class Path; + class ExprNode; + class MetaItem; +}; + +class InterpolatedFragment; + class Token: public Serialisable { @@ -48,13 +59,15 @@ class Token: (Float, struct { enum eCoreType m_datatype; double m_floatval; - }) + }), + (Fragment, void*) ); enum eTokenType m_type; Data m_data; Position m_pos; public: + virtual ~Token(); Token(); Token& operator=(Token&& t) { @@ -80,20 +93,30 @@ public: (None, ), (String, m_data = Data::make_String(e); ), (Integer, m_data = Data::make_Integer(e);), - (Float, m_data = Data::make_Float(e);) + (Float, m_data = Data::make_Float(e);), + (Fragment, assert(!"Copied fragment");) ) } + Token clone() const; Token(enum eTokenType type); Token(enum eTokenType type, ::std::string str); Token(uint64_t val, enum eCoreType datatype); Token(double val, enum eCoreType datatype); + Token(InterpolatedFragment& ); enum eTokenType type() const { return m_type; } const ::std::string& str() const { return m_data.as_String(); } enum eCoreType datatype() const { TU_MATCH_DEF(Data, (m_data), (e), (assert(!"Getting datatype of invalid token type");), (Integer, return e.m_datatype;), (Float, return e.m_datatype;)) } uint64_t intval() const { return m_data.as_Integer().m_intval; } double floatval() const { return m_data.as_Float().m_floatval; } + + TypeRef& frag_type() { assert(m_type == TOK_INTERPOLATED_TYPE); return *reinterpret_cast( m_data.as_Fragment() ); } + AST::Path& frag_path() { assert(m_type == TOK_INTERPOLATED_PATH); return *reinterpret_cast( m_data.as_Fragment() ); } + AST::Pattern& frag_pattern() { assert(m_type == TOK_INTERPOLATED_PATTERN); return *reinterpret_cast( m_data.as_Fragment() ); } + AST::MetaItem& frag_meta() { assert(m_type == TOK_INTERPOLATED_META); return *reinterpret_cast( m_data.as_Fragment() ); } + ::std::unique_ptr take_frag_node(); + bool operator==(const Token& r) const { if(type() != r.type()) return false; @@ -101,7 +124,8 @@ public: (None, return true;), (String, return e == re;), (Integer, return e.m_datatype == re.m_datatype && e.m_intval == re.m_intval;), - (Float, return e.m_datatype == re.m_datatype && e.m_floatval == re.m_floatval;) + (Float, return e.m_datatype == re.m_datatype && e.m_floatval == re.m_floatval;), + (Fragment, assert(!"Token equality on Fragment");) ) throw ""; } @@ -116,6 +140,8 @@ public: static eTokenType typefromstr(const ::std::string& s); SERIALISABLE_PROTOTYPES(); + + friend ::std::ostream& operator<<(::std::ostream& os, const Token& tok); }; extern ::std::ostream& operator<<(::std::ostream& os, const Token& tok); diff --git a/src/parse/tokentree.hpp b/src/parse/tokentree.hpp index 89537a64..68080689 100644 --- a/src/parse/tokentree.hpp +++ b/src/parse/tokentree.hpp @@ -10,8 +10,10 @@ class TokenTree: Token m_tok; ::std::vector m_subtrees; public: + virtual ~TokenTree() {} TokenTree() {} TokenTree(TokenTree&&) = default; + TokenTree& operator=(TokenTree&&) = default; TokenTree(Token tok): m_tok( ::std::move(tok) ) { @@ -20,7 +22,8 @@ public: m_subtrees( ::std::move(subtrees) ) { } - TokenTree& operator=(TokenTree&&) = default; + + TokenTree clone() const; bool is_token() const { return m_tok.type() != TOK_NULL; diff --git a/src/parse/types.cpp b/src/parse/types.cpp index 370f78a3..c6e33a3d 100644 --- a/src/parse/types.cpp +++ b/src/parse/types.cpp @@ -11,21 +11,21 @@ #include // === PROTOTYPES === -TypeRef Parse_Type(TokenStream& lex); -TypeRef Parse_Type_Int(TokenStream& lex); +//TypeRef Parse_Type(TokenStream& lex, bool allow_trait_list); +TypeRef Parse_Type_Int(TokenStream& lex, bool allow_trait_list); TypeRef Parse_Type_Fn(TokenStream& lex, ::std::vector<::std::string> hrls = {}); -TypeRef Parse_Type_Path(TokenStream& lex, ::std::vector<::std::string> hrls); +TypeRef Parse_Type_Path(TokenStream& lex, ::std::vector<::std::string> hrls, bool allow_trait_list); // === CODE === -TypeRef Parse_Type(TokenStream& lex) +TypeRef Parse_Type(TokenStream& lex, bool allow_trait_list) { ProtoSpan ps = lex.start_span(); - TypeRef rv = Parse_Type_Int(lex); + TypeRef rv = Parse_Type_Int(lex, allow_trait_list); //rv.set_span(lex.end_span(ps)); return rv; } -TypeRef Parse_Type_Int(TokenStream& lex) +TypeRef Parse_Type_Int(TokenStream& lex, bool allow_trait_list) { //TRACE_FUNCTION; auto ps = lex.start_span(); @@ -34,6 +34,8 @@ TypeRef Parse_Type_Int(TokenStream& lex) switch( GET_TOK(tok, lex) ) { + case TOK_INTERPOLATED_TYPE: + return mv$(tok.frag_type()); case TOK_MACRO: return TypeRef(TypeRef::TagMacro(), Parse_MacroInvocation(ps, AST::MetaItems(), mv$(tok.str()), lex)); // '!' - Only ever used as part of function prototypes, but is kinda a type... not allowed here though @@ -49,13 +51,13 @@ TypeRef Parse_Type_Int(TokenStream& lex) case TOK_RWORD_EXTERN: // 'fn' - Rust function case TOK_RWORD_FN: - lex.putback(tok); + PUTBACK(tok, lex); return Parse_Type_Fn(lex); // '<' - An associated type cast case TOK_LT: case TOK_DOUBLE_LT: { - lex.putback(tok); + PUTBACK(tok, lex); auto path = Parse_Path(lex, PATH_GENERIC_TYPE); return TypeRef(TypeRef::TagPath(), lex.end_span(ps), mv$(path)); } @@ -76,27 +78,27 @@ TypeRef Parse_Type_Int(TokenStream& lex) // TODO: Handle HRLS in fn types return Parse_Type_Fn(lex, hrls); default: - return Parse_Type_Path(lex, hrls); + return Parse_Type_Path(lex, hrls, allow_trait_list); } } // - Either a primitive, or a path case TOK_IDENT: // or a primitive - if( auto ct = coretype_fromstring(tok.str()) ) - { - return TypeRef(TypeRef::TagPrimitive(), Span(tok.get_pos()), ct); - } - lex.putback(tok); - return Parse_Type_Path(lex, {}); + //if( auto ct = coretype_fromstring(tok.str()) ) + //{ + // return TypeRef(TypeRef::TagPrimitive(), Span(tok.get_pos()), ct); + //} + PUTBACK(tok, lex); + return Parse_Type_Path(lex, {}, allow_trait_list); // - Fall through to path handling // '::' - Absolute path case TOK_DOUBLE_COLON: - lex.putback(tok); - return Parse_Type_Path(lex, {}); + PUTBACK(tok, lex); + return Parse_Type_Path(lex, {}, allow_trait_list); // 'super' - Parent relative path case TOK_RWORD_SUPER: - lex.putback(tok); - return Parse_Type_Path(lex, {}); + PUTBACK(tok, lex); + return Parse_Type_Path(lex, {}, allow_trait_list); // HACK! Convert && into & & case TOK_DOUBLE_AMP: @@ -115,7 +117,7 @@ TypeRef Parse_Type_Int(TokenStream& lex) return TypeRef(TypeRef::TagReference(), lex.end_span(ps), true, Parse_Type(lex)); } else { - lex.putback(tok); + PUTBACK(tok, lex); // Immutable reference return TypeRef(TypeRef::TagReference(), lex.end_span(ps), false, Parse_Type(lex)); } @@ -160,9 +162,9 @@ TypeRef Parse_Type_Int(TokenStream& lex) DEBUG("Tuple"); if( GET_TOK(tok, lex) == TOK_PAREN_CLOSE ) return TypeRef(TypeRef::TagTuple(), lex.end_span(ps), {}); - lex.putback(tok); + PUTBACK(tok, lex); - TypeRef inner = Parse_Type(lex); + TypeRef inner = Parse_Type(lex, true); if( GET_TOK(tok, lex) == TOK_PLUS ) { // Lifetime bounded type, NOT a tuple @@ -177,13 +179,13 @@ TypeRef Parse_Type_Int(TokenStream& lex) { ::std::vector types; types.push_back( ::std::move(inner) ); - lex.putback(tok); + PUTBACK(tok, lex); while( GET_TOK(tok, lex) == TOK_COMMA ) { if( GET_TOK(tok, lex) == TOK_PAREN_CLOSE ) break; else - lex.putback(tok); + PUTBACK(tok, lex); types.push_back(Parse_Type(lex)); } CHECK_TOK(tok, TOK_PAREN_CLOSE); @@ -239,7 +241,7 @@ TypeRef Parse_Type_Fn(TokenStream& lex, ::std::vector<::std::string> hrls) } args.push_back( Parse_Type(lex) ); if( GET_TOK(tok, lex) != TOK_COMMA ) { - lex.putback(tok); + PUTBACK(tok, lex); break; } } @@ -251,36 +253,43 @@ TypeRef Parse_Type_Fn(TokenStream& lex, ::std::vector<::std::string> hrls) ret_type = Parse_Type(lex); } else { - lex.putback(tok); + PUTBACK(tok, lex); } return TypeRef(TypeRef::TagFunction(), lex.end_span(ps), ::std::move(abi), ::std::move(args), ::std::move(ret_type)); } -TypeRef Parse_Type_Path(TokenStream& lex, ::std::vector<::std::string> hrls) +TypeRef Parse_Type_Path(TokenStream& lex, ::std::vector<::std::string> hrls, bool allow_trait_list) { Token tok; auto ps = lex.start_span(); - ::std::vector traits; - ::std::vector< ::std::string> lifetimes; - do { - if( LOOK_AHEAD(lex) == TOK_LIFETIME ) { - GET_TOK(tok, lex); - lifetimes.push_back( tok.str() ); - } - else - traits.push_back( Parse_Path(lex, PATH_GENERIC_TYPE) ); - } while( GET_TOK(tok, lex) == TOK_PLUS ); - lex.putback(tok); - if( hrls.size() > 0 || traits.size() > 1 || lifetimes.size() > 0 ) { - if( lifetimes.size() ) - DEBUG("TODO: Lifetime bounds on trait objects"); - return TypeRef(lex.end_span(ps), mv$(hrls), ::std::move(traits)); + if( ! allow_trait_list ) + { + return TypeRef(TypeRef::TagPath(), lex.end_span(ps), Parse_Path(lex, PATH_GENERIC_TYPE)); } - else { - return TypeRef(TypeRef::TagPath(), lex.end_span(ps), traits.at(0)); + else + { + ::std::vector traits; + ::std::vector< ::std::string> lifetimes; + do { + if( LOOK_AHEAD(lex) == TOK_LIFETIME ) { + GET_TOK(tok, lex); + lifetimes.push_back( tok.str() ); + } + else + traits.push_back( Parse_Path(lex, PATH_GENERIC_TYPE) ); + } while( GET_TOK(tok, lex) == TOK_PLUS ); + PUTBACK(tok, lex); + if( hrls.size() > 0 || traits.size() > 1 || lifetimes.size() > 0 ) { + if( lifetimes.size() ) + DEBUG("TODO: Lifetime bounds on trait objects"); + return TypeRef(lex.end_span(ps), mv$(hrls), ::std::move(traits)); + } + else { + return TypeRef(TypeRef::TagPath(), lex.end_span(ps), traits.at(0)); + } } } -- cgit v1.2.3