From dd7680bc0d53f02e4b96b2b3896ebea16742c9bb Mon Sep 17 00:00:00 2001 From: John Hodge Date: Wed, 25 May 2016 23:16:23 +0800 Subject: Parse - Use interpolated macro fragments instead of captured TTs --- src/ast/ast.cpp | 11 ++- src/ast/attrs.hpp | 4 + src/ast/expr.cpp | 162 ++++++++++++++++++++++++++++++------ src/ast/expr.hpp | 10 +-- src/ast/macro.hpp | 7 ++ src/ast/path.cpp | 7 +- src/ast/path.hpp | 1 + src/ast/pattern.cpp | 73 ++++++++++++++++ src/ast/pattern.hpp | 7 ++ src/ast/types.cpp | 6 +- src/ast/types.hpp | 2 + src/expand/format_args.cpp | 4 +- src/include/serialise.hpp | 6 +- src/macro_rules/eval.cpp | 92 ++++++++++---------- src/macro_rules/mod.cpp | 5 ++ src/main.cpp | 3 +- src/parse/common.hpp | 5 +- src/parse/eTokenType.enum.h | 8 ++ src/parse/expr.cpp | 131 +++++++++++++++++------------ src/parse/interpolated_fragment.cpp | 76 +++++++++++++++++ src/parse/interpolated_fragment.hpp | 49 +++++++++++ src/parse/lex.cpp | 20 ++++- src/parse/parseerror.cpp | 10 +-- src/parse/parseerror.hpp | 6 +- src/parse/paths.cpp | 27 +++--- src/parse/pattern.cpp | 24 +++--- src/parse/root.cpp | 98 ++++++++++++---------- src/parse/token.cpp | 148 +++++++++++++++++++++++++++++++- src/parse/token.hpp | 32 ++++++- src/parse/tokentree.hpp | 5 +- src/parse/types.cpp | 97 +++++++++++---------- 31 files changed, 863 insertions(+), 273 deletions(-) create mode 100644 src/parse/interpolated_fragment.cpp create mode 100644 src/parse/interpolated_fragment.hpp (limited to 'src') diff --git a/src/ast/ast.cpp b/src/ast/ast.cpp index 2521a878..802bde37 100644 --- a/src/ast/ast.cpp +++ b/src/ast/ast.cpp @@ -22,6 +22,9 @@ namespace { } } +MetaItems::~MetaItems() +{ +} MetaItems MetaItems::clone() const { return MetaItems( m_span, clone_mivec(m_items) ); @@ -45,7 +48,9 @@ SERIALISE_TYPE_A(MetaItems::, "AST_MetaItems", { s.item(m_items); }) - +MetaItem::~MetaItem() +{ +} MetaItem MetaItem::clone() const { TU_MATCH(MetaItemData, (m_data), (e), @@ -183,6 +188,10 @@ SERIALISE_TYPE_A(UseStmt::, "AST_UseStmt", { }) +MacroInvocation MacroInvocation::clone() const +{ + return MacroInvocation(m_span, m_attrs.clone(), m_macro_name, m_ident, m_input.clone()); +} SERIALISE_TYPE_A(MacroInvocation::, "AST_MacroInvocation", { s.item(m_attrs); s.item(m_macro_name); diff --git a/src/ast/attrs.hpp b/src/ast/attrs.hpp index a2b82c40..fbe63be4 100644 --- a/src/ast/attrs.hpp +++ b/src/ast/attrs.hpp @@ -14,6 +14,7 @@ public: Span m_span; ::std::vector m_items; + virtual ~MetaItems(); MetaItems() {} MetaItems(MetaItems&&) = default; MetaItems& operator=(MetaItems&&) = default; @@ -57,7 +58,10 @@ class MetaItem: ::std::string m_name; MetaItemData m_data; public: + virtual ~MetaItem(); MetaItem() {} + MetaItem(MetaItem&& ) = default; + MetaItem& operator=(MetaItem&& ) = default; MetaItem(::std::string name): m_name(name), m_data( MetaItemData::make_None({}) ) diff --git a/src/ast/expr.cpp b/src/ast/expr.cpp index 675c26b8..42010868 100644 --- a/src/ast/expr.cpp +++ b/src/ast/expr.cpp @@ -89,11 +89,20 @@ SERIALISE_TYPE(Expr::, "Expr", { ExprNode::~ExprNode() { } -#define NODE(class, serialise, _print)\ - void class::visit(NodeVisitor& nv) { nv.visit(*this); } \ - /*void class::visit(NodeVisitor& nv) const { nv.visit(*this); }*/ \ - void class::print(::std::ostream& os) const _print \ - SERIALISE_TYPE_S(class, serialise) \ +#define NODE(class, serialise, _print, _clone)\ + void class::visit(NodeVisitor& nv) { nv.visit(*this); } \ + void class::print(::std::ostream& os) const _print \ + ::std::unique_ptr class::clone() const _clone \ + SERIALISE_TYPE_S(class, serialise) +#define OPT_CLONE(node) (node.get() ? node->clone() : ::AST::ExprNodeP()) + +namespace { + static inline ExprNodeP mk_exprnodep(const Position& pos, AST::ExprNode* en) { + en->set_pos(pos); + return ExprNodeP(en); + } + #define NEWNODE(type, ...) mk_exprnodep(get_pos(), new type(__VA_ARGS__)) +} NODE(ExprNode_Block, { s.item(m_nodes); @@ -102,6 +111,13 @@ NODE(ExprNode_Block, { for(const auto& n : m_nodes) os << *n << ";"; os << "}"; +},{ + ::std::vector nodes; + for(const auto& n : m_nodes) + nodes.push_back( n->clone() ); + if( m_local_mod ) + TODO(get_pos(), "Handle cloning ExprNode_Block with a module"); + return NEWNODE(ExprNode_Block, mv$(nodes), nullptr); }) NODE(ExprNode_Macro, { @@ -115,6 +131,8 @@ NODE(ExprNode_Macro, { os << " " << m_ident << " "; } os << "(" << ")"; +},{ + return NEWNODE(ExprNode_Macro, m_name, m_ident, m_tokens.clone()); }) void operator%(::Serialiser& s, const ExprNode_Flow::Type t) { @@ -151,6 +169,8 @@ NODE(ExprNode_Flow, { case CONTINUE: os << "continue"; break; } os << " " << *m_value; +},{ + return NEWNODE(ExprNode_Flow, m_type, m_target, m_value->clone()); }) @@ -160,6 +180,8 @@ NODE(ExprNode_LetBinding, { s.item(m_value); },{ os << "let " << m_pat << ": " << m_type << " = " << *m_value; +},{ + return NEWNODE(ExprNode_LetBinding, m_pat.clone(), TypeRef(m_type), OPT_CLONE(m_value)); }) NODE(ExprNode_Assign, { @@ -167,6 +189,8 @@ NODE(ExprNode_Assign, { s.item(m_value); },{ os << *m_slot << " = " << *m_value; +},{ + return NEWNODE(ExprNode_Assign, m_op, m_slot->clone(), m_value->clone()); }) NODE(ExprNode_CallPath, { @@ -178,6 +202,12 @@ NODE(ExprNode_CallPath, { os << *a << ","; } os << ")"; +},{ + ::std::vector args; + for(const auto& a : m_args) { + args.push_back( a->clone() ); + } + return NEWNODE(ExprNode_CallPath, AST::Path(m_path), mv$(args)); }) NODE(ExprNode_CallMethod, { @@ -190,6 +220,12 @@ NODE(ExprNode_CallMethod, { os << *a << ","; } os << ")"; +},{ + ::std::vector args; + for(const auto& a : m_args) { + args.push_back( a->clone() ); + } + return NEWNODE(ExprNode_CallMethod, m_val->clone(), m_method, mv$(args)); }) NODE(ExprNode_CallObject, { @@ -201,6 +237,12 @@ NODE(ExprNode_CallObject, { os << *a << ","; } os << ")"; +},{ + ::std::vector args; + for(const auto& a : m_args) { + args.push_back( a->clone() ); + } + return NEWNODE(ExprNode_CallObject, m_val->clone(), mv$(args)); }) void operator%(::Serialiser& s, const ExprNode_Loop::Type t) { @@ -235,6 +277,8 @@ NODE(ExprNode_Loop, { s.item(m_code); },{ os << "LOOP [" << m_label << "] " << m_pattern << " in/= " << *m_cond << " " << *m_code; +},{ + return NEWNODE(ExprNode_Loop, m_label, m_type, m_pattern.clone(), OPT_CLONE(m_cond), m_code->clone()); }) SERIALISE_TYPE_A(ExprNode_Match_Arm::, "ExprNode_Match_Arm", { @@ -256,6 +300,17 @@ NODE(ExprNode_Match, { os << " => " << *arm.m_code << ","; } os << "}"; +},{ + ::std::vector< ExprNode_Match_Arm> arms; + for(const auto& arm : m_arms) { + ::std::vector< AST::Pattern> patterns; + for( const auto& pat : arm.m_patterns ) { + patterns.push_back( pat.clone() ); + } + arms.push_back( ExprNode_Match_Arm( mv$(patterns), OPT_CLONE(arm.m_cond), arm.m_code->clone() ) ); + arms.back().m_attrs = arm.m_attrs.clone(); + } + return NEWNODE(ExprNode_Match, m_val->clone(), mv$(arms)); }) NODE(ExprNode_If, { @@ -264,6 +319,8 @@ NODE(ExprNode_If, { s.item(m_false); },{ os << "if " << *m_cond << " { " << *m_true << " } else { " << *m_false << " }"; +},{ + return NEWNODE(ExprNode_If, m_cond->clone(), m_true->clone(), OPT_CLONE(m_false)); }) NODE(ExprNode_IfLet, { s.item(m_pattern); @@ -272,6 +329,8 @@ NODE(ExprNode_IfLet, { s.item(m_false); },{ os << "if let " << m_pattern << " = (" << *m_value << ") { " << *m_true << " } else { " << *m_false << " }"; +},{ + return NEWNODE(ExprNode_IfLet, m_pattern.clone(), m_value->clone(), m_true->clone(), OPT_CLONE(m_false)); }) NODE(ExprNode_Integer, { @@ -279,22 +338,30 @@ NODE(ExprNode_Integer, { s.item(m_value); },{ os << m_value; +},{ + return NEWNODE(ExprNode_Integer, m_value, m_datatype); }) NODE(ExprNode_Float, { s % m_datatype; s.item(m_value); },{ os << m_value; +},{ + return NEWNODE(ExprNode_Float, m_value, m_datatype); }) NODE(ExprNode_Bool, { s.item(m_value); },{ os << m_value; +},{ + return NEWNODE(ExprNode_Bool, m_value); }) NODE(ExprNode_String, { s.item(m_value); },{ os << "\"" << m_value << "\""; +},{ + return NEWNODE(ExprNode_String, m_value); }) NODE(ExprNode_Closure, { @@ -303,6 +370,12 @@ NODE(ExprNode_Closure, { s.item(m_code); },{ os << "/* todo: closure */"; +},{ + ExprNode_Closure::args_t args; + for(const auto& a : m_args) { + args.push_back( ::std::make_pair(a.first.clone(), TypeRef(a.second)) ); + } + return NEWNODE(ExprNode_Closure, mv$(args), TypeRef(m_return), m_code->clone()); }); NODE(ExprNode_StructLiteral, { @@ -311,6 +384,14 @@ NODE(ExprNode_StructLiteral, { s.item(m_values); },{ os << "/* todo: sl */"; +},{ + ExprNode_StructLiteral::t_values vals; + + for(const auto& v : m_values) { + vals.push_back( ::std::make_pair(v.first, v.second->clone()) ); + } + + return NEWNODE(ExprNode_StructLiteral, AST::Path(m_path), OPT_CLONE(m_base_value), mv$(vals) ); }) NODE(ExprNode_Array, { @@ -324,6 +405,18 @@ NODE(ExprNode_Array, { for(const auto& a : m_values) os << *a << ","; os << "]"; +},{ + if( m_size.get() ) + { + return NEWNODE(ExprNode_Array, m_values[0]->clone(), m_size->clone()); + } + else + { + ::std::vector nodes; + for(const auto& n : m_values) + nodes.push_back( n->clone() ); + return NEWNODE(ExprNode_Array, mv$(nodes)); + } }) NODE(ExprNode_Tuple, { @@ -334,12 +427,19 @@ NODE(ExprNode_Tuple, { os << *a << ","; } os << ")"; +},{ + ::std::vector nodes; + for(const auto& n : m_values) + nodes.push_back( n->clone() ); + return NEWNODE(ExprNode_Tuple, mv$(nodes)); }) NODE(ExprNode_NamedValue, { s.item(m_path); },{ os << m_path; +},{ + return NEWNODE(ExprNode_NamedValue, AST::Path(m_path)); }) NODE(ExprNode_Field, { @@ -347,6 +447,8 @@ NODE(ExprNode_Field, { s.item(m_name); },{ os << "(" << *m_obj << ")." << m_name; +},{ + return NEWNODE(ExprNode_Field, m_obj->clone(), m_name); }) NODE(ExprNode_Index, { @@ -354,12 +456,16 @@ NODE(ExprNode_Index, { s.item(m_idx); },{ os << "(" << *m_obj << ")[" << *m_idx << "]"; +},{ + return NEWNODE(ExprNode_Index, m_obj->clone(), m_idx->clone()); }) NODE(ExprNode_Deref, { s.item(m_value); },{ os << "*(" << *m_value << ")"; +},{ + return NEWNODE(ExprNode_Deref, m_value->clone()); }); NODE(ExprNode_Cast, { @@ -367,9 +473,11 @@ NODE(ExprNode_Cast, { s.item(m_type); },{ os << "(" << *m_value << " as " << m_type << ")"; +},{ + return NEWNODE(ExprNode_Cast, m_value->clone(), TypeRef(m_type)); }) -void operator%(::Serialiser& s, const ExprNode_BinOp::Type t) { +void operator%(::Serialiser& s, const ExprNode_BinOp::Type& t) { switch(t) { #define _(v) case ExprNode_BinOp::v: s << #v; return @@ -388,11 +496,11 @@ void operator%(::Serialiser& s, const ExprNode_BinOp::Type t) { _(BITXOR); _(SHL); _(SHR); - _(MULTIPLY); - _(DIVIDE); - _(MODULO); - _(ADD); - _(SUB); + _(MULTIPLY); + _(DIVIDE); + _(MODULO); + _(ADD); + _(SUB); _(PLACE_IN); #undef _ } @@ -417,11 +525,11 @@ void operator%(::Deserialiser& s, ExprNode_BinOp::Type& t) { _(BITXOR); _(SHL); _(SHR); - _(MULTIPLY); - _(DIVIDE); - _(MODULO); - _(ADD); - _(SUB); + _(MULTIPLY); + _(DIVIDE); + _(MODULO); + _(ADD); + _(SUB); #undef _ else throw ::std::runtime_error(""); @@ -447,16 +555,18 @@ NODE(ExprNode_BinOp, { case BITXOR: os << "^"; break; case SHR: os << ">>"; break; case SHL: os << "<<"; break; - case MULTIPLY: os << "*"; break; - case DIVIDE: os << "/"; break; - case MODULO: os << "%"; break; - case ADD: os << "+"; break; - case SUB: os << "-"; break; - case RANGE: os << ".."; break; - case RANGE_INC: os << "..."; break; + case MULTIPLY: os << "*"; break; + case DIVIDE: os << "/"; break; + case MODULO: os << "%"; break; + case ADD: os << "+"; break; + case SUB: os << "-"; break; + case RANGE: os << ".."; break; + case RANGE_INC: os << "..."; break; case PLACE_IN: os << "<-"; break; } os << " " << *m_right << ")"; +},{ + return NEWNODE(ExprNode_BinOp, m_type, OPT_CLONE(m_left), OPT_CLONE(m_right)); }) void operator%(::Serialiser& s, const ExprNode_UniOp::Type t) { @@ -501,12 +611,14 @@ NODE(ExprNode_UniOp, { case QMARK: os << "(" << *m_value << "?)"; return; } os << *m_value << ")"; +},{ + return NEWNODE(ExprNode_UniOp, m_type, m_value->clone()); }) #define NV(type, actions)\ - void NodeVisitorDef::visit(type& node) { DEBUG("DEF - "#type); actions } -// void NodeVisitorDef::visit(const type& node) { DEBUG("DEF - "#type" (const)"); actions } + void NodeVisitorDef::visit(type& node) { DEBUG("DEF - "#type); actions } +// void NodeVisitorDef::visit(const type& node) { DEBUG("DEF - "#type" (const)"); actions } NV(ExprNode_Block, { INDENT(); diff --git a/src/ast/expr.hpp b/src/ast/expr.hpp index 95f74505..eab6f260 100644 --- a/src/ast/expr.hpp +++ b/src/ast/expr.hpp @@ -28,8 +28,8 @@ public: virtual ~ExprNode() = 0; virtual void visit(NodeVisitor& nv) = 0; - //virtual void visit(NodeVisitor& nv) const = 0; virtual void print(::std::ostream& os) const = 0; + virtual ::std::unique_ptr clone() const = 0; void set_pos(Position p) { m_pos = ::std::move(p); } const Position& get_pos() const { return m_pos; } @@ -46,10 +46,10 @@ public: }; #define NODE_METHODS() \ - virtual void visit(NodeVisitor& nv) override;\ - virtual void print(::std::ostream& os) const override; \ - SERIALISABLE_PROTOTYPES();/* \ - virtual void visit(NodeVisitor& nv) const override;*/ + void visit(NodeVisitor& nv) override;\ + void print(::std::ostream& os) const override; \ + ::std::unique_ptr clone() const override; \ + SERIALISABLE_PROTOTYPES(); struct ExprNode_Block: public ExprNode diff --git a/src/ast/macro.hpp b/src/ast/macro.hpp index 85f2dea2..5cd53e1f 100644 --- a/src/ast/macro.hpp +++ b/src/ast/macro.hpp @@ -18,6 +18,11 @@ class MacroInvocation: ::std::string m_ident; TokenTree m_input; public: + MacroInvocation(MacroInvocation&&) = default; + MacroInvocation& operator=(MacroInvocation&&) = default; + MacroInvocation(const MacroInvocation&) = delete; + MacroInvocation& operator=(const MacroInvocation&) = delete; + MacroInvocation() { } @@ -30,6 +35,8 @@ public: m_input( mv$(input) ) { } + + MacroInvocation clone() const; static ::std::unique_ptr from_deserialiser(Deserialiser& s) { auto i = new MacroInvocation; diff --git a/src/ast/path.cpp b/src/ast/path.cpp index 1a1d3cc9..be8b5887 100644 --- a/src/ast/path.cpp +++ b/src/ast/path.cpp @@ -54,8 +54,8 @@ PathBinding PathBinding::clone() const // --- AST::PathNode PathNode::PathNode(::std::string name, ::std::vector args): - m_name(name), - m_params(args) + m_name( mv$(name) ), + m_params( mv$(args) ) { } Ordering PathNode::ord(const PathNode& x) const @@ -101,6 +101,9 @@ typename ::std::vector >::const_iterator find_named(const ::std::vector } // --- AST::Path +AST::Path::~Path() +{ +} AST::Path::Path(TagUfcs, TypeRef type, ::std::vector nodes): m_class( AST::Path::Class::make_UFCS({box$(type), nullptr, nodes}) ) { diff --git a/src/ast/path.hpp b/src/ast/path.hpp index 7eb17aab..4029f655 100644 --- a/src/ast/path.hpp +++ b/src/ast/path.hpp @@ -140,6 +140,7 @@ public: private: PathBinding m_binding; public: + virtual ~Path(); // INVALID Path(): m_class() diff --git a/src/ast/pattern.cpp b/src/ast/pattern.cpp index c3373206..80665008 100644 --- a/src/ast/pattern.cpp +++ b/src/ast/pattern.cpp @@ -163,6 +163,79 @@ void operator%(::Deserialiser& s, Pattern::Data::Tag& c) { s.item(n); c = Pattern::Data::tag_from_str(n); } + +Pattern::~Pattern() +{ +} + +AST::Pattern AST::Pattern::clone() const +{ + AST::Pattern rv; + rv.m_span = m_span; + rv.m_binding = m_binding; + rv.m_binding_mut = m_binding_mut; + + struct H { + static ::std::unique_ptr clone_sp(const ::std::unique_ptr& p) { + return ::std::make_unique( p->clone() ); + } + static ::std::vector clone_list(const ::std::vector& list) { + ::std::vector rv; + rv.reserve(list.size()); + for(const auto& p : list) + rv.push_back( p.clone() ); + return rv; + } + static AST::Pattern::Value clone_val(const AST::Pattern::Value& v) { + TU_MATCH(::AST::Pattern::Value, (v), (e), + (Invalid, return Value(e);), + (Integer, return Value(e);), + (String, return Value(e);), + (Named, return Value::make_Named( AST::Path(e) );) + ) + throw ""; + } + }; + + TU_MATCH(Pattern::Data, (m_data), (e), + (Any, + rv.m_data = Data::make_Any(e); + ), + (MaybeBind, + rv.m_data = Data::make_MaybeBind(e); + ), + (Macro, + rv.m_data = Data::make_Macro({ ::std::make_unique( e.inv->clone() ) }); + ), + (Box, + rv.m_data = Data::make_Box({ H::clone_sp(e.sub) }); + ), + (Ref, + rv.m_data = Data::make_Ref({ e.mut, H::clone_sp(e.sub) }); + ), + (Value, + rv.m_data = Data::make_Value({ H::clone_val(e.start), H::clone_val(e.end) }); + ), + (Tuple, + rv.m_data = Data::make_Tuple({ H::clone_list(e.sub_patterns) }); + ), + (StructTuple, + rv.m_data = Data::make_StructTuple({ ::AST::Path(e.path), H::clone_list(e.sub_patterns) }); + ), + (Struct, + ::std::vector< ::std::pair< ::std::string, Pattern> > sps; + for(const auto& sp : e.sub_patterns) + sps.push_back( ::std::make_pair(sp.first, sp.second.clone()) ); + rv.m_data = Data::make_Struct({ ::AST::Path(e.path), mv$(sps) }); + ), + (Slice, + rv.m_data = Data::make_Slice({ H::clone_list(e.leading), e.extra_bind, H::clone_list(e.trailing) }); + ) + ) + + return rv; +} + #define _D(VAR, ...) case Pattern::Data::TAG_##VAR: { m_data = Pattern::Data::make_##VAR({}); auto& ent = m_data.as_##VAR(); (void)&ent; __VA_ARGS__ } break; SERIALISE_TYPE(Pattern::, "Pattern", { s.item(m_binding); diff --git a/src/ast/pattern.hpp b/src/ast/pattern.hpp index 468bf306..a901ac89 100644 --- a/src/ast/pattern.hpp +++ b/src/ast/pattern.hpp @@ -52,8 +52,12 @@ private: Data m_data; public: + virtual ~Pattern(); + Pattern() {} + Pattern(Pattern&&) = default; + Pattern& operator=(Pattern&&) = default; struct TagMaybeBind {}; Pattern(TagMaybeBind, ::std::string name): @@ -128,6 +132,9 @@ public: const Span& span() const { return m_span; } void set_span(Span sp) { m_span = mv$(sp); } + + Pattern clone() const; + // Accessors const ::std::string& binding() const { return m_binding; } const BindType& binding_type() const { assert(m_binding != ""); return m_binding_type; } diff --git a/src/ast/types.cpp b/src/ast/types.cpp index 8a3ee638..8cc9b4ea 100644 --- a/src/ast/types.cpp +++ b/src/ast/types.cpp @@ -98,11 +98,15 @@ Ordering Type_Function::ord(const Type_Function& x) const return (*m_rettype).ord( *x.m_rettype ); } +TypeRef::~TypeRef() +{ +} + TypeRef::TypeRef(const TypeRef& other) { switch( other.m_data.tag() ) { - case TypeData::TAGDEAD: throw ""; + case TypeData::TAGDEAD: assert(!"Copying a destructed type"); #define _COPY(VAR) case TypeData::TAG_##VAR: m_data = TypeData::make_##VAR(other.m_data.as_##VAR()); break; #define _CLONE(VAR, code...) case TypeData::TAG_##VAR: { auto& old = other.m_data.as_##VAR(); m_data = TypeData::make_##VAR(code); } break; _COPY(None) diff --git a/src/ast/types.hpp b/src/ast/types.hpp index 72080723..a79a811d 100644 --- a/src/ast/types.hpp +++ b/src/ast/types.hpp @@ -108,6 +108,8 @@ class TypeRef: public: TypeData m_data; + virtual ~TypeRef(); + TypeRef(TypeRef&& other) noexcept: m_data( mv$(other.m_data) ) { diff --git a/src/expand/format_args.cpp b/src/expand/format_args.cpp index ab320e3f..1b8dfb54 100644 --- a/src/expand/format_args.cpp +++ b/src/expand/format_args.cpp @@ -19,8 +19,8 @@ class CFormatArgsExpander: if( ident != "" ) ERROR(sp, E0000, "format_args! doesn't take an ident"); - GET_CHECK_TOK(tok, lex, TOK_STRING); - auto format_string = mv$(tok.str()); + auto n = Parse_ExprVal(lex); + auto format_string = dynamic_cast(*n).m_value; // TODO: Interpolated expression "tokens" ::std::map< ::std::string, TokenTree> named_args; diff --git a/src/include/serialise.hpp b/src/include/serialise.hpp index 31abc358..664c5bac 100644 --- a/src/include/serialise.hpp +++ b/src/include/serialise.hpp @@ -13,9 +13,9 @@ class Serialiser; class Deserialiser; #define SERIALISABLE_PROTOTYPES()\ - virtual const char* serialise_tag() const override; \ - virtual void serialise(::Serialiser& s) const override; \ - virtual void deserialise(::Deserialiser& s) override + const char* serialise_tag() const override; \ + void serialise(::Serialiser& s) const override; \ + void deserialise(::Deserialiser& s) override #define SERIALISE_TYPE(method_prefix, tag_str, body, des_body) \ const char* method_prefix serialise_tag() const { return tag_str; } \ void method_prefix serialise(::Serialiser& s) const { body } \ diff --git a/src/macro_rules/eval.cpp b/src/macro_rules/eval.cpp index 44900807..ed1e1a2e 100644 --- a/src/macro_rules/eval.cpp +++ b/src/macro_rules/eval.cpp @@ -7,6 +7,10 @@ #include #include #include "pattern_checks.hpp" +#include + +extern AST::ExprNodeP Parse_ExprBlockNode(TokenStream& lex); +extern AST::ExprNodeP Parse_Stmt(TokenStream& lex); class ParameterMappings { @@ -16,9 +20,9 @@ class ParameterMappings struct Mapping { t_mapping_block block; - ::std::vector entries; + ::std::vector entries; friend ::std::ostream& operator<<(::std::ostream& os, const Mapping& x) { - os << "(" << x.block.first << ", " << x.block.second << "): '" << x.entries << "')"; + os << "(" << x.block.first << ", " << x.block.second << "): [" << x.entries << "])"; return os; } }; @@ -51,7 +55,7 @@ public: return m_layer_count+1; } - void insert(unsigned int layer, unsigned int name_index, TokenTree data) { + void insert(unsigned int layer, unsigned int name_index, InterpolatedFragment data) { if(layer > m_layer_count) m_layer_count = layer; @@ -83,14 +87,14 @@ public: } } - const TokenTree* get(unsigned int layer, unsigned int iteration, const char *name, unsigned int idx) const + InterpolatedFragment* get(unsigned int layer, unsigned int iteration, const char *name, unsigned int idx) { const auto it = m_map.find( name ); if( it == m_map.end() ) { DEBUG("m_map = " << m_map); return nullptr; } - const auto& e = *it->second; + auto& e = *it->second; if( e.block.first < layer ) { DEBUG(name<<" higher layer (" << e.block.first << ")"); return nullptr; @@ -141,7 +145,7 @@ private: const ::std::string m_crate_name; const ::std::vector& m_root_contents; - const ParameterMappings m_mappings; + ParameterMappings m_mappings; struct t_offset { @@ -157,20 +161,11 @@ private: const ::std::vector* m_cur_ents; // For faster lookup. Token m_next_token; // used for inserting a single token into the stream - ::std::unique_ptr m_ttstream; + ::std::unique_ptr m_ttstream; public: MacroExpander(const MacroExpander& x) = delete; - //MacroExpander(const MacroExpander& x): - // m_macro_name( x.m_macro_name ), - // m_crate_name(x.m_crate_name), - // m_root_contents(x.m_root_contents), - // m_mappings(x.m_mappings), - // m_offsets({ {0,0,0} }), - // m_cur_ents(&m_root_contents) - //{ - // prep_counts(); - //} + MacroExpander(const ::std::string& macro_name, const ::std::vector& contents, ParameterMappings mappings, ::std::string crate_name): m_macro_filename( FMT("Macro:" << macro_name) ), m_crate_name( mv$(crate_name) ), @@ -212,7 +207,7 @@ bool Macro_TryPattern(TTStream& lex, const MacroPatEnt& pat) return true; return Macro_TryPattern(lex, pat.subpats[0]); case MacroPatEnt::PAT_BLOCK: - return LOOK_AHEAD(lex) == TOK_BRACE_OPEN; + return LOOK_AHEAD(lex) == TOK_BRACE_OPEN || LOOK_AHEAD(lex) == TOK_INTERPOLATED_BLOCK; case MacroPatEnt::PAT_IDENT: return LOOK_AHEAD(lex) == TOK_IDENT; case MacroPatEnt::PAT_TT: @@ -228,7 +223,7 @@ bool Macro_TryPattern(TTStream& lex, const MacroPatEnt& pat) case MacroPatEnt::PAT_PAT: return is_token_pat( LOOK_AHEAD(lex) ); case MacroPatEnt::PAT_META: - return LOOK_AHEAD(lex) == TOK_IDENT; + return LOOK_AHEAD(lex) == TOK_IDENT || LOOK_AHEAD(lex) == TOK_INTERPOLATED_META; } throw ParseError::Todo(lex, FMT("Macro_TryPattern : " << pat)); } @@ -237,7 +232,6 @@ bool Macro_HandlePattern(TTStream& lex, const MacroPatEnt& pat, unsigned int lay { TRACE_FUNCTION_F("layer = " << layer); Token tok; - TokenTree val; switch(pat.type) { @@ -284,35 +278,32 @@ bool Macro_HandlePattern(TTStream& lex, const MacroPatEnt& pat, unsigned int lay throw ParseError::Unexpected(lex, TOK_EOF); else PUTBACK(tok, lex); - val = Parse_TT(lex, false); - if(0) + bound_tts.insert( layer, pat.name_index, InterpolatedFragment( Parse_TT(lex, false) ) ); + break; case MacroPatEnt::PAT_PAT: - val = Parse_TT_Pattern(lex); - if(0) + bound_tts.insert( layer, pat.name_index, InterpolatedFragment( Parse_Pattern(lex, true) ) ); + break; case MacroPatEnt::PAT_TYPE: - val = Parse_TT_Type(lex); - if(0) + bound_tts.insert( layer, pat.name_index, InterpolatedFragment( Parse_Type(lex) ) ); + break; case MacroPatEnt::PAT_EXPR: - val = Parse_TT_Expr(lex); - if(0) + bound_tts.insert( layer, pat.name_index, InterpolatedFragment( InterpolatedFragment::EXPR, Parse_Expr0(lex).release() ) ); + break; case MacroPatEnt::PAT_STMT: - val = Parse_TT_Stmt(lex); - if(0) + bound_tts.insert( layer, pat.name_index, InterpolatedFragment( InterpolatedFragment::STMT, Parse_Stmt(lex).release() ) ); + break; case MacroPatEnt::PAT_PATH: - val = Parse_TT_Path(lex, false); // non-expr mode - if(0) + bound_tts.insert( layer, pat.name_index, InterpolatedFragment( Parse_Path(lex, PATH_GENERIC_TYPE) ) ); // non-expr mode + break; case MacroPatEnt::PAT_BLOCK: - val = Parse_TT_Block(lex); - if(0) + bound_tts.insert( layer, pat.name_index, InterpolatedFragment( InterpolatedFragment::BLOCK, Parse_ExprBlockNode(lex).release() ) ); + break; case MacroPatEnt::PAT_META: - val = Parse_TT_Meta(lex); - if(0) + bound_tts.insert( layer, pat.name_index, InterpolatedFragment( Parse_MetaItem(lex) ) ); + break; case MacroPatEnt::PAT_IDENT: - { - GET_CHECK_TOK(tok, lex, TOK_IDENT); - val = TokenTree(tok); - } - bound_tts.insert( layer, pat.name_index, ::std::move(val) ); + GET_CHECK_TOK(tok, lex, TOK_IDENT); + bound_tts.insert( layer, pat.name_index, InterpolatedFragment( TokenTree(tok) ) ); break; //default: @@ -455,21 +446,28 @@ Token MacroExpander::realGetToken() else if( ent.name != "" ) { DEBUG("lookup '" << ent.name << "'"); - const TokenTree* tt; + InterpolatedFragment* frag; unsigned int search_layer = layer; do { unsigned int parent_iter = (search_layer > 0 ? m_layer_iters.at(search_layer-1) : 0); const size_t iter_idx = m_offsets.at(search_layer).loop_index; - tt = m_mappings.get(search_layer, parent_iter, ent.name.c_str(), iter_idx); - } while( !tt && search_layer-- > 0 ); - if( ! tt ) + frag = m_mappings.get(search_layer, parent_iter, ent.name.c_str(), iter_idx); + } while( !frag && search_layer-- > 0 ); + if( !frag ) { throw ParseError::Generic(*this, FMT("Cannot find '" << ent.name << "' for " << layer)); } else { - m_ttstream.reset( new TTStream(*tt) ); - return m_ttstream->getToken(); + if( frag->m_type == InterpolatedFragment::TT ) + { + m_ttstream.reset( new TTStream( frag->as_tt() ) ); + return m_ttstream->getToken(); + } + else + { + return Token( *frag ); + } } } // - Descend into a repetition diff --git a/src/macro_rules/mod.cpp b/src/macro_rules/mod.cpp index e3699a3d..24e877dc 100644 --- a/src/macro_rules/mod.cpp +++ b/src/macro_rules/mod.cpp @@ -18,6 +18,7 @@ bool is_token_path(eTokenType tt) { case TOK_DOUBLE_LT: case TOK_RWORD_SELF: case TOK_RWORD_SUPER: + case TOK_INTERPOLATED_PATH: return true; default: return false; @@ -38,6 +39,7 @@ bool is_token_pat(eTokenType tt) { case TOK_STRING: case TOK_INTEGER: case TOK_CHAR: + case TOK_INTERPOLATED_PATTERN: return true; default: return false; @@ -52,6 +54,7 @@ bool is_token_type(eTokenType tt) { case TOK_SQUARE_OPEN: case TOK_STAR: case TOK_AMP: + case TOK_INTERPOLATED_TYPE: return true; default: return false; @@ -70,6 +73,7 @@ bool is_token_expr(eTokenType tt) { case TOK_INTEGER: case TOK_STRING: + case TOK_INTERPOLATED_EXPR: return true; default: return false; @@ -82,6 +86,7 @@ bool is_token_stmt(eTokenType tt) { { case TOK_BRACE_OPEN: case TOK_RWORD_LET: + case TOK_INTERPOLATED_STMT: return true; default: return false; diff --git a/src/main.cpp b/src/main.cpp index b519f1fb..6be8b2ca 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -24,9 +24,10 @@ int g_debug_indent_level = 0; bool debug_enabled() { + //return true; //return g_cur_phase != "Parse"; //return g_cur_phase != "Parse" && g_cur_phase != "Expand"; - return g_cur_phase != "Parse" && g_cur_phase != "Expand" && g_cur_phase != "Resolve";; + return g_cur_phase != "Parse" && g_cur_phase != "Expand" && g_cur_phase != "Resolve"; } ::std::ostream& debug_output(int indent, const char* function) { diff --git a/src/parse/common.hpp b/src/parse/common.hpp index d7f84315..c42889c3 100644 --- a/src/parse/common.hpp +++ b/src/parse/common.hpp @@ -41,7 +41,7 @@ extern ::std::vector Parse_Path_GenericList(TokenStream& lex); extern AST::MetaItem Parse_MetaItem(TokenStream& lex); extern ::AST::MacroInvocation Parse_MacroInvocation(ProtoSpan ps, ::AST::MetaItems meta_items, ::std::string name, TokenStream& lex); -extern TypeRef Parse_Type(TokenStream& lex); +extern TypeRef Parse_Type(TokenStream& lex, bool allow_trait_list = false); extern AST::Pattern Parse_Pattern(TokenStream& lex, bool is_refutable); extern void Parse_Impl_Item(TokenStream& lex, AST::Impl& impl); @@ -51,7 +51,8 @@ extern void Parse_ModRoot_Items(TokenStream& lex, AST::Module& mod, bool file_co extern AST::Expr Parse_Expr(TokenStream& lex); extern AST::Expr Parse_ExprBlock(TokenStream& lex); -extern AST::ExprNodeP Parse_Expr0(TokenStream& lex); +extern AST::ExprNodeP Parse_Expr0(TokenStream& lex); +extern AST::ExprNodeP Parse_ExprVal(TokenStream& lex); extern bool Parse_IsTokValue(eTokenType tok_type); diff --git a/src/parse/eTokenType.enum.h b/src/parse/eTokenType.enum.h index 2adfb042..f7157805 100644 --- a/src/parse/eTokenType.enum.h +++ b/src/parse/eTokenType.enum.h @@ -5,6 +5,14 @@ _(TOK_NEWLINE) _(TOK_WHITESPACE) _(TOK_COMMENT) +_(TOK_INTERPOLATED_PATH) +_(TOK_INTERPOLATED_TYPE) +_(TOK_INTERPOLATED_PATTERN) +_(TOK_INTERPOLATED_EXPR) +_(TOK_INTERPOLATED_STMT) +_(TOK_INTERPOLATED_BLOCK) +_(TOK_INTERPOLATED_META) + // Value tokens _(TOK_IDENT) _(TOK_MACRO) diff --git a/src/parse/expr.cpp b/src/parse/expr.cpp index 7094cf96..37bc03e7 100644 --- a/src/parse/expr.cpp +++ b/src/parse/expr.cpp @@ -85,7 +85,7 @@ ExprNodeP Parse_ExprBlockNode(TokenStream& lex) case TOK_RWORD_IMPL: case TOK_RWORD_FN: case TOK_RWORD_MOD: - lex.putback(tok); + PUTBACK(tok, lex); if( !local_mod ) { local_mod = lex.parse_state().get_current_mod().add_anon(); } @@ -95,7 +95,7 @@ ExprNodeP Parse_ExprBlockNode(TokenStream& lex) case TOK_RWORD_UNSAFE: if( LOOK_AHEAD(lex) != TOK_BRACE_OPEN ) { - lex.putback(tok); + PUTBACK(tok, lex); if( !local_mod ) { local_mod = lex.parse_state().get_current_mod().add_anon(); } @@ -104,7 +104,7 @@ ExprNodeP Parse_ExprBlockNode(TokenStream& lex) } // fall default: { - lex.putback(tok); + PUTBACK(tok, lex); bool expect_end = false; nodes.push_back(Parse_ExprBlockLine(lex, &expect_end)); if( nodes.back() ) { @@ -121,7 +121,7 @@ ExprNodeP Parse_ExprBlockNode(TokenStream& lex) { throw ParseError::Unexpected(lex, tok, Token(TOK_BRACE_CLOSE)); } - lex.putback(tok); + PUTBACK(tok, lex); } break; } @@ -159,7 +159,7 @@ ExprNodeP Parse_ExprBlockLine(TokenStream& lex, bool *expect_end) case TOK_RWORD_MATCH: return Parse_Expr_Match(lex); case TOK_BRACE_OPEN: - lex.putback(tok); + PUTBACK(tok, lex); return Parse_ExprBlockNode(lex); default: @@ -173,7 +173,7 @@ ExprNodeP Parse_ExprBlockLine(TokenStream& lex, bool *expect_end) case TOK_SEMICOLON: return 0; case TOK_BRACE_OPEN: - lex.putback(tok); + PUTBACK(tok, lex); return Parse_ExprBlockNode(lex); // let binding @@ -215,11 +215,11 @@ ExprNodeP Parse_ExprBlockLine(TokenStream& lex, bool *expect_end) } // Fall through to the statement code default: { - lex.putback(tok); + PUTBACK(tok, lex); auto ret = Parse_Stmt(lex); if( GET_TOK(tok, lex) != TOK_SEMICOLON ) { - lex.putback(tok); + PUTBACK(tok, lex); *expect_end = true; } return ::std::move(ret); @@ -245,7 +245,7 @@ ExprNodeP Parse_WhileStmt(TokenStream& lex, ::std::string lifetime) ::std::move(pat), ::std::move(val), Parse_ExprBlockNode(lex) ); } else { - lex.putback(tok); + PUTBACK(tok, lex); ExprNodeP cnd; { SET_PARSE_FLAG(lex, disallow_struct_literal); @@ -291,7 +291,7 @@ ExprNodeP Parse_IfStmt(TokenStream& lex) cond = Parse_Expr0(lex); } else { - lex.putback(tok); + PUTBACK(tok, lex); cond = Parse_Expr0(lex); } } @@ -309,13 +309,13 @@ ExprNodeP Parse_IfStmt(TokenStream& lex) } // - or get block else { - lex.putback(tok); + PUTBACK(tok, lex); altcode = Parse_ExprBlockNode(lex); } } // - or nothing else { - lex.putback(tok); + PUTBACK(tok, lex); } if( if_let ) @@ -343,7 +343,7 @@ ExprNodeP Parse_Expr_Match(TokenStream& lex) do { if( GET_TOK(tok, lex) == TOK_BRACE_CLOSE ) break; - lex.putback(tok); + PUTBACK(tok, lex); AST::ExprNode_Match_Arm arm; ::AST::MetaItems arm_attrs; @@ -372,7 +372,7 @@ ExprNodeP Parse_Expr_Match(TokenStream& lex) if( GET_TOK(tok, lex) == TOK_COMMA ) continue; - lex.putback(tok); + PUTBACK(tok, lex); } while( 1 ); CHECK_TOK(tok, TOK_BRACE_CLOSE); @@ -431,20 +431,20 @@ ExprNodeP Parse_Stmt(TokenStream& lex) case TOK_BRACE_CLOSE: case TOK_PAREN_CLOSE: case TOK_SQUARE_CLOSE: - lex.putback(tok); + PUTBACK(tok, lex); break; default: - lex.putback(tok); + PUTBACK(tok, lex); val = Parse_Expr1(lex); break; } return NEWNODE( AST::ExprNode_Flow, type, lifetime, ::std::move(val) ); } case TOK_BRACE_OPEN: - lex.putback(tok); + PUTBACK(tok, lex); return Parse_ExprBlockNode(lex); default: - lex.putback(tok); + PUTBACK(tok, lex); return Parse_Expr0(lex); } } @@ -460,7 +460,7 @@ ExprNodeP Parse_Stmt(TokenStream& lex) GET_CHECK_TOK(tok, lex, TOK_PAREN_OPEN); if( GET_TOK(tok, lex) != TOK_PAREN_CLOSE ) { - lex.putback(tok); + PUTBACK(tok, lex); do { if( LOOK_AHEAD(lex) == TOK_PAREN_CLOSE ) { GET_TOK(tok, lex); @@ -521,7 +521,7 @@ ExprNodeP Parse_Expr0(TokenStream& lex) return rv; default: - lex.putback(tok); + PUTBACK(tok, lex); rv->set_attrs(mv$(expr_attrs)); return rv; } @@ -542,7 +542,7 @@ ExprNodeP cur(TokenStream& lex) \ cases \ default: \ /*::std::cout << "<<" << #cur << ::std::endl; */\ - lex.putback(tok); \ + PUTBACK(tok, lex); \ return rv; \ } \ } \ @@ -585,26 +585,39 @@ ExprNodeP Parse_Expr1(TokenStream& lex) ExprNodeP (*next)(TokenStream&) = Parse_Expr1_1; ExprNodeP left, right; + // Inclusive range to a value + if( GET_TOK(tok, lex) == TOK_TRIPLE_DOT ) { + right = next(lex); + return NEWNODE( AST::ExprNode_BinOp, AST::ExprNode_BinOp::RANGE_INC, nullptr, mv$(right) ); + } + else { + PUTBACK(tok, lex); + } + + // Exclusive ranges + // - If NOT `.. `, parse a leading value if( GET_TOK(tok, lex) != TOK_DOUBLE_DOT ) { - lex.putback(tok); + PUTBACK(tok, lex); left = next(lex); + // - If NOT ` ..`, return the value if( GET_TOK(tok, lex) != TOK_DOUBLE_DOT ) { - lex.putback(tok); + PUTBACK(tok, lex); return ::std::move(left); } } - if( Parse_IsTokValue( GET_TOK(tok, lex) ) ) + assert( tok.type() == TOK_DOUBLE_DOT ); + // If the next token is part of a value, parse that value + if( Parse_IsTokValue( LOOK_AHEAD(lex) ) ) { - lex.putback(tok); right = next(lex); } else { - lex.putback(tok); + // Otherwise, leave `right` as nullptr } return NEWNODE( AST::ExprNode_BinOp, AST::ExprNode_BinOp::RANGE, ::std::move(left), ::std::move(right) ); @@ -717,7 +730,7 @@ ExprNodeP Parse_Expr12(TokenStream& lex) } else { - lex.putback(tok); + PUTBACK(tok, lex); } return rv; } @@ -752,11 +765,11 @@ ExprNodeP Parse_Expr13(TokenStream& lex) if( GET_TOK(tok, lex) == TOK_RWORD_MUT ) return NEWNODE( AST::ExprNode_UniOp, AST::ExprNode_UniOp::REFMUT, Parse_Expr12(lex) ); else { - lex.putback(tok); + PUTBACK(tok, lex); return NEWNODE( AST::ExprNode_UniOp, AST::ExprNode_UniOp::REF, Parse_Expr12(lex) ); } default: - lex.putback(tok); + PUTBACK(tok, lex); return Parse_ExprFC(lex); } } @@ -776,7 +789,7 @@ ExprNodeP Parse_ExprFC(TokenStream& lex) case TOK_PAREN_OPEN: // Expression method call - lex.putback(tok); + PUTBACK(tok, lex); val = NEWNODE( AST::ExprNode_CallObject, ::std::move(val), Parse_ParenList(lex) ); break; case TOK_SQUARE_OPEN: @@ -793,7 +806,7 @@ ExprNodeP Parse_ExprFC(TokenStream& lex) switch( GET_TOK(tok, lex) ) { case TOK_PAREN_OPEN: - lex.putback(tok); + PUTBACK(tok, lex); val = NEWNODE( AST::ExprNode_CallMethod, ::std::move(val), ::std::move(path), Parse_ParenList(lex) ); break; case TOK_DOUBLE_COLON: @@ -862,7 +875,7 @@ ExprNodeP Parse_ExprVal_Closure(TokenStream& lex, bool is_move) while( GET_TOK(tok, lex) != TOK_PIPE ) { - lex.putback(tok); + PUTBACK(tok, lex); // Irrefutable pattern AST::Pattern pat = Parse_Pattern(lex, false); @@ -870,7 +883,7 @@ ExprNodeP Parse_ExprVal_Closure(TokenStream& lex, bool is_move) if( GET_TOK(tok, lex) == TOK_COLON ) type = Parse_Type(lex); else - lex.putback(tok); + PUTBACK(tok, lex); args.push_back( ::std::make_pair( ::std::move(pat), ::std::move(type) ) ); @@ -886,12 +899,12 @@ ExprNodeP Parse_ExprVal_Closure(TokenStream& lex, bool is_move) rt = TypeRef(TypeRef::TagInvalid(), Span(tok.get_pos())); } else { - lex.putback(tok); + PUTBACK(tok, lex); rt = Parse_Type(lex); } } else - lex.putback(tok); + PUTBACK(tok, lex); auto code = Parse_Expr0(lex); @@ -932,14 +945,17 @@ ExprNodeP Parse_ExprVal(TokenStream& lex) switch( GET_TOK(tok, lex) ) { case TOK_BRACE_OPEN: - lex.putback(tok); + PUTBACK(tok, lex); return Parse_ExprBlockNode(lex); + case TOK_INTERPOLATED_EXPR: + return tok.take_frag_node(); + // TODO: Return/break/continue/... here? case TOK_RWORD_RETURN: case TOK_RWORD_CONTINUE: case TOK_RWORD_BREAK: - lex.putback(tok); + PUTBACK(tok, lex); return Parse_Stmt(lex); case TOK_RWORD_LOOP: @@ -960,7 +976,7 @@ ExprNodeP Parse_ExprVal(TokenStream& lex) // UFCS case TOK_DOUBLE_LT: - lex.putback(tok); + PUTBACK(tok, lex); case TOK_LT: { TypeRef ty = Parse_Type(lex); if( GET_TOK(tok, lex) == TOK_RWORD_AS ) { @@ -970,7 +986,7 @@ ExprNodeP Parse_ExprVal(TokenStream& lex) path = AST::Path(AST::Path::TagUfcs(), ty, trait, Parse_PathNodes(lex, PATH_GENERIC_EXPR)); } else { - lex.putback(tok); + PUTBACK(tok, lex); GET_CHECK_TOK(tok, lex, TOK_GT); // TODO: Terminating the "path" here is sometimes valid GET_CHECK_TOK(tok, lex, TOK_DOUBLE_COLON); @@ -985,31 +1001,36 @@ ExprNodeP Parse_ExprVal(TokenStream& lex) } else { - lex.putback(tok); + PUTBACK(tok, lex); path = Parse_Path(lex, PATH_GENERIC_EXPR); } } if(0) case TOK_RWORD_SUPER: { - lex.putback(tok); + PUTBACK(tok, lex); path = Parse_Path(lex, PATH_GENERIC_EXPR); } if(0) case TOK_IDENT: // Get path { - lex.putback(tok); + PUTBACK(tok, lex); path = Parse_Path(lex, false, PATH_GENERIC_EXPR); } if(0) + case TOK_INTERPOLATED_PATH: + { + path = mv$(tok.frag_path()); + } + if(0) case TOK_DOUBLE_COLON: path = Parse_Path(lex, true, PATH_GENERIC_EXPR); switch( GET_TOK(tok, lex) ) { case TOK_PAREN_OPEN: // Function call - lex.putback(tok); + PUTBACK(tok, lex); return NEWNODE( AST::ExprNode_CallPath, ::std::move(path), Parse_ParenList(lex) ); case TOK_BRACE_OPEN: if( !CHECK_PARSE_FLAG(lex, disallow_struct_literal) ) @@ -1018,7 +1039,7 @@ ExprNodeP Parse_ExprVal(TokenStream& lex) DEBUG("Not parsing struct literal"); default: // Value - lex.putback(tok); + PUTBACK(tok, lex); return NEWNODE( AST::ExprNode_NamedValue, ::std::move(path) ); } case TOK_RWORD_MOVE: @@ -1065,7 +1086,7 @@ ExprNodeP Parse_ExprVal(TokenStream& lex) else { CLEAR_PARSE_FLAG(lex, disallow_struct_literal); - lex.putback(tok); + PUTBACK(tok, lex); ExprNodeP rv = Parse_Expr0(lex); if( GET_TOK(tok, lex) == TOK_COMMA ) { @@ -1074,7 +1095,7 @@ ExprNodeP Parse_ExprVal(TokenStream& lex) do { if( GET_TOK(tok, lex) == TOK_PAREN_CLOSE ) break; - lex.putback(tok); + PUTBACK(tok, lex); ents.push_back( Parse_Expr0(lex) ); } while( GET_TOK(tok, lex) == TOK_COMMA ); rv = NEWNODE( AST::ExprNode_Tuple, ::std::move(ents) ); @@ -1090,7 +1111,7 @@ ExprNodeP Parse_ExprVal(TokenStream& lex) } else { - lex.putback(tok); + PUTBACK(tok, lex); auto first = Parse_Expr0(lex); if( GET_TOK(tok, lex) == TOK_SEMICOLON ) { @@ -1108,7 +1129,7 @@ ExprNodeP Parse_ExprVal(TokenStream& lex) if( GET_TOK(tok, lex) == TOK_SQUARE_CLOSE ) break; else - lex.putback(tok); + PUTBACK(tok, lex); items.push_back( Parse_Expr0(lex) ); GET_TOK(tok, lex); } @@ -1118,7 +1139,7 @@ ExprNodeP Parse_ExprVal(TokenStream& lex) } throw ParseError::BugCheck(lex, "Array literal fell"); case TOK_MACRO: - return Parse_ExprMacro(lex, tok); + return Parse_ExprMacro(lex, mv$(tok)); default: throw ParseError::Unexpected(lex, tok); } @@ -1131,7 +1152,7 @@ ExprNodeP Parse_ExprMacro(TokenStream& lex, Token tok) ident = mv$(tok.str()); } else { - lex.putback(tok); + PUTBACK(tok, lex); } TokenTree tt = Parse_TT(lex, true); if( tt.is_token() ) { @@ -1162,21 +1183,21 @@ TokenTree Parse_TT(TokenStream& lex, bool unwrapped) case TOK_NULL: throw ParseError::Unexpected(lex, tok); default: - return TokenTree(tok); + return TokenTree( mv$(tok) ); } ::std::vector items; if( !unwrapped ) - items.push_back(tok); + items.push_back( mv$(tok) ); while(GET_TOK(tok, lex) != closer && tok.type() != TOK_EOF) { if( tok.type() == TOK_NULL ) throw ParseError::Unexpected(lex, tok); - lex.putback(tok); + PUTBACK(tok, lex); items.push_back(Parse_TT(lex, false)); } if( !unwrapped ) - items.push_back(tok); + items.push_back( mv$(tok) ); return TokenTree(mv$(items)); } @@ -1244,7 +1265,7 @@ TokenTree Parse_TT_Path(TokenStream& lex, bool mode_expr) Parse_Path(wlex, true, (mode_expr ? PATH_GENERIC_EXPR : PATH_GENERIC_TYPE)); } else { - wlex.putback(tok); + PUTBACK(tok, lex); Parse_Path(wlex, false, (mode_expr ? PATH_GENERIC_EXPR : PATH_GENERIC_TYPE)); } diff --git a/src/parse/interpolated_fragment.cpp b/src/parse/interpolated_fragment.cpp new file mode 100644 index 00000000..dd490bf5 --- /dev/null +++ b/src/parse/interpolated_fragment.cpp @@ -0,0 +1,76 @@ +/* + */ +#include +#include "interpolated_fragment.hpp" +#include + +InterpolatedFragment::~InterpolatedFragment() +{ + if( m_ptr ) + { + switch(m_type) + { + case InterpolatedFragment::TT: delete reinterpret_cast(m_ptr); break; + case InterpolatedFragment::PAT: delete reinterpret_cast(m_ptr); break; + case InterpolatedFragment::PATH:delete reinterpret_cast(m_ptr); break; + case InterpolatedFragment::TYPE:delete reinterpret_cast(m_ptr); break; + case InterpolatedFragment::EXPR: + case InterpolatedFragment::STMT: + case InterpolatedFragment::BLOCK: + delete reinterpret_cast(m_ptr); + break; + case InterpolatedFragment::META: + delete reinterpret_cast(m_ptr); + break; + } + } +} + +InterpolatedFragment::InterpolatedFragment(InterpolatedFragment&& x): + m_type( x.m_type ) +{ + m_ptr = x.m_ptr, x.m_ptr = nullptr; +} +InterpolatedFragment& InterpolatedFragment::operator=(InterpolatedFragment&& x) +{ + m_type = x.m_type; + m_ptr = x.m_ptr, x.m_ptr = nullptr; + return *this; +} + +InterpolatedFragment::InterpolatedFragment(InterpolatedFragment::Type type, AST::ExprNode* ptr): + m_type( type ), + m_ptr( ptr ) +{ +} +InterpolatedFragment::InterpolatedFragment(AST::MetaItem v): + m_type( InterpolatedFragment::META ), + m_ptr( new AST::MetaItem(mv$(v)) ) +{ +} +InterpolatedFragment::InterpolatedFragment(TokenTree v): + m_type( InterpolatedFragment::TT ), + m_ptr( new TokenTree(mv$(v)) ) +{ +} +InterpolatedFragment::InterpolatedFragment(AST::Path v): + m_type( InterpolatedFragment::PATH ), + m_ptr( new AST::Path(mv$(v)) ) +{ +} +InterpolatedFragment::InterpolatedFragment(AST::Pattern v): + m_type( InterpolatedFragment::PAT ), + m_ptr( new AST::Pattern(mv$(v)) ) +{ +} +InterpolatedFragment::InterpolatedFragment(TypeRef v): + m_type( InterpolatedFragment::TYPE ), + m_ptr( new TypeRef(mv$(v)) ) +{ +} + +::std::ostream& operator<<(::std::ostream& os, InterpolatedFragment const& x) +{ + return os; +} + diff --git a/src/parse/interpolated_fragment.hpp b/src/parse/interpolated_fragment.hpp new file mode 100644 index 00000000..40aa764a --- /dev/null +++ b/src/parse/interpolated_fragment.hpp @@ -0,0 +1,49 @@ +/* + */ +#pragma once + +#include + +class TypeRef; +class TokenTree; +namespace AST { + class Pattern; + class Path; + class ExprNode; + class MetaItem; +}; + +class InterpolatedFragment +{ +public: + enum Type + { + TT, + PAT, + PATH, + TYPE, + + EXPR, + STMT, + BLOCK, + + META, + } m_type; + + void* m_ptr; + + InterpolatedFragment(InterpolatedFragment&& ); + InterpolatedFragment& operator=(InterpolatedFragment&& ); + //InterpolatedFragment(const InterpolatedFragment& ); + InterpolatedFragment(TokenTree ); + InterpolatedFragment(::AST::Pattern); + InterpolatedFragment(::AST::Path); + InterpolatedFragment(::TypeRef); + InterpolatedFragment(::AST::MetaItem ); + ~InterpolatedFragment(); + InterpolatedFragment(Type , ::AST::ExprNode*); + + TokenTree& as_tt() { assert(m_type == TT); return *reinterpret_cast(m_ptr); } + + friend ::std::ostream& operator<<(::std::ostream& os, const InterpolatedFragment& x); +}; diff --git a/src/parse/lex.cpp b/src/parse/lex.cpp index fd6044f4..395eee48 100644 --- a/src/parse/lex.cpp +++ b/src/parse/lex.cpp @@ -20,7 +20,8 @@ #include #include // std::count -const bool DEBUG_PRINT_TOKENS = false; +//const bool DEBUG_PRINT_TOKENS = false; +const bool DEBUG_PRINT_TOKENS = true; Lexer::Lexer(const ::std::string& filename): m_path(filename.c_str()), @@ -956,7 +957,7 @@ Token TTStream::realGetToken() const TokenTree& subtree = tree[idx]; idx ++; if( subtree.size() == 0 ) { - return subtree.tok(); + return subtree.tok().clone(); } else { m_stack.push_back( ::std::make_pair(0, &subtree ) ); @@ -1041,7 +1042,7 @@ Token TokenStream::getToken() if( m_cache_valid ) { m_cache_valid = false; - return m_cache; + return mv$(m_cache); } else if( m_lookahead.size() ) { @@ -1119,6 +1120,19 @@ Span TokenStream::end_span(ProtoSpan ps) const } +TokenTree TokenTree::clone() const +{ + if( m_subtrees.size() == 0 ) { + return TokenTree(m_tok.clone()); + } + else { + ::std::vector< TokenTree> ents; + ents.reserve( m_subtrees.size() ); + for(const auto& sub : m_subtrees) + ents.push_back( sub.clone() ); + return TokenTree( mv$(ents) ); + } +} SERIALISE_TYPE_A(TokenTree::, "TokenTree", { s.item(m_tok); s.item(m_subtrees); diff --git a/src/parse/parseerror.cpp b/src/parse/parseerror.cpp index e05ecb05..7c6f6a21 100644 --- a/src/parse/parseerror.cpp +++ b/src/parse/parseerror.cpp @@ -56,23 +56,23 @@ ParseError::BadChar::~BadChar() throw() { } -ParseError::Unexpected::Unexpected(const TokenStream& lex, Token tok): - m_tok(tok) +ParseError::Unexpected::Unexpected(const TokenStream& lex, const Token& tok)//: +// m_tok( mv$(tok) ) { auto pos = tok.get_pos(); if(pos.filename == "") pos = lex.getPosition(); ::std::cout << pos << ": Unexpected(" << tok << ")" << ::std::endl; } -ParseError::Unexpected::Unexpected(const TokenStream& lex, Token tok, Token exp): - m_tok(tok) +ParseError::Unexpected::Unexpected(const TokenStream& lex, const Token& tok, Token exp)//: +// m_tok( mv$(tok) ) { auto pos = tok.get_pos(); if(pos.filename == "") pos = lex.getPosition(); ::std::cout << pos << ": Unexpected(" << tok << ", " << exp << ")" << ::std::endl; } -ParseError::Unexpected::Unexpected(const TokenStream& lex, Token tok, ::std::vector exp) +ParseError::Unexpected::Unexpected(const TokenStream& lex, const Token& tok, ::std::vector exp) { auto pos = tok.get_pos(); if(pos.filename == "") diff --git a/src/parse/parseerror.hpp b/src/parse/parseerror.hpp index 1074bf46..42a89785 100644 --- a/src/parse/parseerror.hpp +++ b/src/parse/parseerror.hpp @@ -26,9 +26,9 @@ class Unexpected: { Token m_tok; public: - Unexpected(const TokenStream& lex, Token tok); - Unexpected(const TokenStream& lex, Token tok, Token exp); - Unexpected(const TokenStream& lex, Token tok, ::std::vector exp); + Unexpected(const TokenStream& lex, const Token& tok); + Unexpected(const TokenStream& lex, const Token& tok, Token exp); + Unexpected(const TokenStream& lex, const Token& tok, ::std::vector exp); virtual ~Unexpected() throw (); }; diff --git a/src/parse/paths.cpp b/src/parse/paths.cpp index 06545b7e..a5d10f74 100644 --- a/src/parse/paths.cpp +++ b/src/parse/paths.cpp @@ -19,6 +19,9 @@ AST::Path Parse_Path(TokenStream& lex, eParsePathGenericMode generic_mode) Token tok; switch( GET_TOK(tok, lex) ) { + case TOK_INTERPOLATED_PATH: + return mv$(tok.frag_path()); + case TOK_RWORD_SELF: GET_CHECK_TOK(tok, lex, TOK_DOUBLE_COLON); return Parse_Path(lex, false, generic_mode); @@ -47,7 +50,7 @@ AST::Path Parse_Path(TokenStream& lex, eParsePathGenericMode generic_mode) trait = Parse_Path(lex, true, PATH_GENERIC_TYPE); } else { - lex.putback(tok); + PUTBACK(tok, lex); trait = Parse_Path(lex, false, PATH_GENERIC_TYPE); } GET_CHECK_TOK(tok, lex, TOK_GT); @@ -55,7 +58,7 @@ AST::Path Parse_Path(TokenStream& lex, eParsePathGenericMode generic_mode) return AST::Path(AST::Path::TagUfcs(), ty, trait, Parse_PathNodes(lex, generic_mode)); } else { - lex.putback(tok); + PUTBACK(tok, lex); GET_CHECK_TOK(tok, lex, TOK_GT); // TODO: Terminating the "path" here is sometimes valid? GET_CHECK_TOK(tok, lex, TOK_DOUBLE_COLON); @@ -64,7 +67,7 @@ AST::Path Parse_Path(TokenStream& lex, eParsePathGenericMode generic_mode) throw ""; } default: - lex.putback(tok); + PUTBACK(tok, lex); return Parse_Path(lex, false, generic_mode); } } @@ -79,7 +82,7 @@ AST::Path Parse_Path(TokenStream& lex, bool is_abs, eParsePathGenericMode generi return AST::Path(cratename, Parse_PathNodes(lex, generic_mode)); } else { - lex.putback(tok); + PUTBACK(tok, lex); return AST::Path("", Parse_PathNodes(lex, generic_mode)); } } @@ -89,7 +92,7 @@ AST::Path Parse_Path(TokenStream& lex, bool is_abs, eParsePathGenericMode generi // return AST::Path( tok.str() ); //} //else { - // lex.putback(tok); + // PUTBACK(tok, lex); return AST::Path(AST::Path::TagRelative(), Parse_PathNodes(lex, generic_mode)); //} } @@ -133,7 +136,7 @@ AST::Path Parse_Path(TokenStream& lex, bool is_abs, eParsePathGenericMode generi } else { - lex.putback(tok); + PUTBACK(tok, lex); do { args.push_back( Parse_Type(lex) ); } while( GET_TOK(tok, lex) == TOK_COMMA ); @@ -145,7 +148,7 @@ AST::Path Parse_Path(TokenStream& lex, bool is_abs, eParsePathGenericMode generi ret_type = Parse_Type(lex); } else { - lex.putback(tok); + PUTBACK(tok, lex); } DEBUG("- Fn("<"< fcn) @@ -1190,7 +1196,7 @@ void Parse_Use(TokenStream& lex, ::std::function 0); name = path.nodes().back().name(); } @@ -1254,7 +1260,7 @@ void Parse_Use(TokenStream& lex, ::std::function #include +#include "interpolated_fragment.hpp" +#include +#include + +Token::~Token() +{ + switch(m_type) + { + case TOK_INTERPOLATED_TYPE: + delete reinterpret_cast(m_data.as_Fragment()); + break; + case TOK_INTERPOLATED_PATTERN: + delete reinterpret_cast(m_data.as_Fragment()); + break; + case TOK_INTERPOLATED_PATH: + delete reinterpret_cast(m_data.as_Fragment()); + break; + case TOK_INTERPOLATED_EXPR: + delete reinterpret_cast(m_data.as_Fragment()); + break; + case TOK_INTERPOLATED_STMT: + delete reinterpret_cast(m_data.as_Fragment()); + break; + case TOK_INTERPOLATED_BLOCK: + delete reinterpret_cast(m_data.as_Fragment()); + break; + case TOK_INTERPOLATED_META: + delete reinterpret_cast(m_data.as_Fragment()); + break; + default: + break; + } + +} Token::Token(): m_type(TOK_NULL) @@ -29,6 +63,101 @@ Token::Token(double val, enum eCoreType datatype): m_data( Data::make_Float({datatype, val}) ) { } +Token::Token(InterpolatedFragment& frag) +{ + switch(frag.m_type) + { + case InterpolatedFragment::TT: throw ""; + case InterpolatedFragment::TYPE: + m_type = TOK_INTERPOLATED_TYPE; + m_data = new TypeRef( *reinterpret_cast(frag.m_ptr) ); + break; + case InterpolatedFragment::PAT: + m_type = TOK_INTERPOLATED_PATTERN; + m_data = new AST::Pattern( reinterpret_cast(frag.m_ptr)->clone() ); + break; + case InterpolatedFragment::PATH: + m_type = TOK_INTERPOLATED_PATH; + m_data = new AST::Path( *reinterpret_cast(frag.m_ptr) ); + break; + case InterpolatedFragment::EXPR: + m_type = TOK_INTERPOLATED_EXPR; + m_data = reinterpret_cast(frag.m_ptr)->clone().release(); + break; + case InterpolatedFragment::STMT: + m_type = TOK_INTERPOLATED_STMT; + m_data = reinterpret_cast(frag.m_ptr)->clone().release(); + break; + case InterpolatedFragment::BLOCK: + m_type = TOK_INTERPOLATED_BLOCK; + m_data = reinterpret_cast(frag.m_ptr)->clone().release(); + break; + case InterpolatedFragment::META: + m_type = TOK_INTERPOLATED_META; + m_data = new AST::MetaItem( reinterpret_cast(frag.m_ptr)->clone() ); + break; + } +} + +Token Token::clone() const +{ + Token rv(m_type); + rv.m_pos = m_pos; + + assert( m_data.tag() != Data::TAGDEAD ); + TU_MATCH(Data, (m_data), (e), + (None, + ), + (String, + rv.m_data = Data::make_String(e); + ), + (Integer, + rv.m_data = Data::make_Integer(e); + ), + (Float, + rv.m_data = Data::make_Float(e); + ), + (Fragment, + assert(e); + switch(m_type) + { + case TOK_INTERPOLATED_TYPE: + rv.m_data = new TypeRef( *reinterpret_cast(e) ); + break; + case TOK_INTERPOLATED_PATTERN: + rv.m_data = new AST::Pattern( reinterpret_cast(e)->clone() ); + break; + case TOK_INTERPOLATED_PATH: + rv.m_data = new AST::Path( *reinterpret_cast(e) ); + break; + case TOK_INTERPOLATED_EXPR: + rv.m_data = reinterpret_cast(e)->clone().release(); + break; + case TOK_INTERPOLATED_STMT: + rv.m_data = reinterpret_cast(e)->clone().release(); + break; + case TOK_INTERPOLATED_BLOCK: + rv.m_data = reinterpret_cast(e)->clone().release(); + break; + case TOK_INTERPOLATED_META: + rv.m_data = new AST::MetaItem( reinterpret_cast(e)->clone() ); + break; + default: + assert(!"Token::clone() - fragment with invalid token type"); + break; + } + ) + ) + return rv; +} + +::std::unique_ptr Token::take_frag_node() +{ + assert( m_type == TOK_INTERPOLATED_EXPR || m_type == TOK_INTERPOLATED_STMT || m_type == TOK_INTERPOLATED_BLOCK ); + auto ptr = m_data.as_Fragment(); + m_data.as_Fragment() = nullptr; + return ::std::unique_ptr( reinterpret_cast( ptr ) ); +} const char* Token::typestr(enum eTokenType type) { @@ -91,6 +220,13 @@ struct EscapedString { case TOK_NEWLINE: return "\n"; case TOK_WHITESPACE: return " "; case TOK_COMMENT: return "/*" + m_data.as_String() + "*/"; + case TOK_INTERPOLATED_TYPE: return "/*:ty*/"; + case TOK_INTERPOLATED_PATH: return "/*:path*/"; + case TOK_INTERPOLATED_PATTERN: return "/*:pat*/"; + case TOK_INTERPOLATED_EXPR: return "/*:expr*/"; + case TOK_INTERPOLATED_STMT: return "/*:stmt*/"; + case TOK_INTERPOLATED_BLOCK: return "/*:block*/"; + case TOK_INTERPOLATED_META: return "/*:meta*/"; // Value tokens case TOK_IDENT: return m_data.as_String(); case TOK_MACRO: return m_data.as_String() + "!"; @@ -251,6 +387,9 @@ SERIALISE_TYPE(Token::, "Token", { (Float, s % e.m_datatype; s.item( e.m_floatval ); + ), + (Fragment, + assert(!"Serialising interpolated macro fragment"); ) ) },{ @@ -284,6 +423,8 @@ SERIALISE_TYPE(Token::, "Token", { s.item( v ); m_data = Token::Data::make_Float({dt, v}); break; } + case Token::Data::TAG_Fragment: + assert(!"Serialising interpolated macro fragment"); } }); @@ -297,10 +438,12 @@ SERIALISE_TYPE(Token::, "Token", { case TOK_IDENT: case TOK_MACRO: case TOK_LIFETIME: - os << "\"" << EscapedString(tok.str()) << "\""; + if( tok.m_data.is_String() ) + os << "\"" << EscapedString(tok.str()) << "\""; break; case TOK_INTEGER: - os << ":" << tok.intval(); + if( tok.m_data.is_Integer() ) + os << ":" << tok.intval(); break; default: break; @@ -311,3 +454,4 @@ SERIALISE_TYPE(Token::, "Token", { { return os << p.filename << ":" << p.line; } + diff --git a/src/parse/token.hpp b/src/parse/token.hpp index 0a72cec1..25993b2f 100644 --- a/src/parse/token.hpp +++ b/src/parse/token.hpp @@ -35,6 +35,17 @@ public: }; extern ::std::ostream& operator<<(::std::ostream& os, const Position& p); +class TypeRef; +class TokenTree; +namespace AST { + class Pattern; + class Path; + class ExprNode; + class MetaItem; +}; + +class InterpolatedFragment; + class Token: public Serialisable { @@ -48,13 +59,15 @@ class Token: (Float, struct { enum eCoreType m_datatype; double m_floatval; - }) + }), + (Fragment, void*) ); enum eTokenType m_type; Data m_data; Position m_pos; public: + virtual ~Token(); Token(); Token& operator=(Token&& t) { @@ -80,20 +93,30 @@ public: (None, ), (String, m_data = Data::make_String(e); ), (Integer, m_data = Data::make_Integer(e);), - (Float, m_data = Data::make_Float(e);) + (Float, m_data = Data::make_Float(e);), + (Fragment, assert(!"Copied fragment");) ) } + Token clone() const; Token(enum eTokenType type); Token(enum eTokenType type, ::std::string str); Token(uint64_t val, enum eCoreType datatype); Token(double val, enum eCoreType datatype); + Token(InterpolatedFragment& ); enum eTokenType type() const { return m_type; } const ::std::string& str() const { return m_data.as_String(); } enum eCoreType datatype() const { TU_MATCH_DEF(Data, (m_data), (e), (assert(!"Getting datatype of invalid token type");), (Integer, return e.m_datatype;), (Float, return e.m_datatype;)) } uint64_t intval() const { return m_data.as_Integer().m_intval; } double floatval() const { return m_data.as_Float().m_floatval; } + + TypeRef& frag_type() { assert(m_type == TOK_INTERPOLATED_TYPE); return *reinterpret_cast( m_data.as_Fragment() ); } + AST::Path& frag_path() { assert(m_type == TOK_INTERPOLATED_PATH); return *reinterpret_cast( m_data.as_Fragment() ); } + AST::Pattern& frag_pattern() { assert(m_type == TOK_INTERPOLATED_PATTERN); return *reinterpret_cast( m_data.as_Fragment() ); } + AST::MetaItem& frag_meta() { assert(m_type == TOK_INTERPOLATED_META); return *reinterpret_cast( m_data.as_Fragment() ); } + ::std::unique_ptr take_frag_node(); + bool operator==(const Token& r) const { if(type() != r.type()) return false; @@ -101,7 +124,8 @@ public: (None, return true;), (String, return e == re;), (Integer, return e.m_datatype == re.m_datatype && e.m_intval == re.m_intval;), - (Float, return e.m_datatype == re.m_datatype && e.m_floatval == re.m_floatval;) + (Float, return e.m_datatype == re.m_datatype && e.m_floatval == re.m_floatval;), + (Fragment, assert(!"Token equality on Fragment");) ) throw ""; } @@ -116,6 +140,8 @@ public: static eTokenType typefromstr(const ::std::string& s); SERIALISABLE_PROTOTYPES(); + + friend ::std::ostream& operator<<(::std::ostream& os, const Token& tok); }; extern ::std::ostream& operator<<(::std::ostream& os, const Token& tok); diff --git a/src/parse/tokentree.hpp b/src/parse/tokentree.hpp index 89537a64..68080689 100644 --- a/src/parse/tokentree.hpp +++ b/src/parse/tokentree.hpp @@ -10,8 +10,10 @@ class TokenTree: Token m_tok; ::std::vector m_subtrees; public: + virtual ~TokenTree() {} TokenTree() {} TokenTree(TokenTree&&) = default; + TokenTree& operator=(TokenTree&&) = default; TokenTree(Token tok): m_tok( ::std::move(tok) ) { @@ -20,7 +22,8 @@ public: m_subtrees( ::std::move(subtrees) ) { } - TokenTree& operator=(TokenTree&&) = default; + + TokenTree clone() const; bool is_token() const { return m_tok.type() != TOK_NULL; diff --git a/src/parse/types.cpp b/src/parse/types.cpp index 370f78a3..c6e33a3d 100644 --- a/src/parse/types.cpp +++ b/src/parse/types.cpp @@ -11,21 +11,21 @@ #include // === PROTOTYPES === -TypeRef Parse_Type(TokenStream& lex); -TypeRef Parse_Type_Int(TokenStream& lex); +//TypeRef Parse_Type(TokenStream& lex, bool allow_trait_list); +TypeRef Parse_Type_Int(TokenStream& lex, bool allow_trait_list); TypeRef Parse_Type_Fn(TokenStream& lex, ::std::vector<::std::string> hrls = {}); -TypeRef Parse_Type_Path(TokenStream& lex, ::std::vector<::std::string> hrls); +TypeRef Parse_Type_Path(TokenStream& lex, ::std::vector<::std::string> hrls, bool allow_trait_list); // === CODE === -TypeRef Parse_Type(TokenStream& lex) +TypeRef Parse_Type(TokenStream& lex, bool allow_trait_list) { ProtoSpan ps = lex.start_span(); - TypeRef rv = Parse_Type_Int(lex); + TypeRef rv = Parse_Type_Int(lex, allow_trait_list); //rv.set_span(lex.end_span(ps)); return rv; } -TypeRef Parse_Type_Int(TokenStream& lex) +TypeRef Parse_Type_Int(TokenStream& lex, bool allow_trait_list) { //TRACE_FUNCTION; auto ps = lex.start_span(); @@ -34,6 +34,8 @@ TypeRef Parse_Type_Int(TokenStream& lex) switch( GET_TOK(tok, lex) ) { + case TOK_INTERPOLATED_TYPE: + return mv$(tok.frag_type()); case TOK_MACRO: return TypeRef(TypeRef::TagMacro(), Parse_MacroInvocation(ps, AST::MetaItems(), mv$(tok.str()), lex)); // '!' - Only ever used as part of function prototypes, but is kinda a type... not allowed here though @@ -49,13 +51,13 @@ TypeRef Parse_Type_Int(TokenStream& lex) case TOK_RWORD_EXTERN: // 'fn' - Rust function case TOK_RWORD_FN: - lex.putback(tok); + PUTBACK(tok, lex); return Parse_Type_Fn(lex); // '<' - An associated type cast case TOK_LT: case TOK_DOUBLE_LT: { - lex.putback(tok); + PUTBACK(tok, lex); auto path = Parse_Path(lex, PATH_GENERIC_TYPE); return TypeRef(TypeRef::TagPath(), lex.end_span(ps), mv$(path)); } @@ -76,27 +78,27 @@ TypeRef Parse_Type_Int(TokenStream& lex) // TODO: Handle HRLS in fn types return Parse_Type_Fn(lex, hrls); default: - return Parse_Type_Path(lex, hrls); + return Parse_Type_Path(lex, hrls, allow_trait_list); } } // - Either a primitive, or a path case TOK_IDENT: // or a primitive - if( auto ct = coretype_fromstring(tok.str()) ) - { - return TypeRef(TypeRef::TagPrimitive(), Span(tok.get_pos()), ct); - } - lex.putback(tok); - return Parse_Type_Path(lex, {}); + //if( auto ct = coretype_fromstring(tok.str()) ) + //{ + // return TypeRef(TypeRef::TagPrimitive(), Span(tok.get_pos()), ct); + //} + PUTBACK(tok, lex); + return Parse_Type_Path(lex, {}, allow_trait_list); // - Fall through to path handling // '::' - Absolute path case TOK_DOUBLE_COLON: - lex.putback(tok); - return Parse_Type_Path(lex, {}); + PUTBACK(tok, lex); + return Parse_Type_Path(lex, {}, allow_trait_list); // 'super' - Parent relative path case TOK_RWORD_SUPER: - lex.putback(tok); - return Parse_Type_Path(lex, {}); + PUTBACK(tok, lex); + return Parse_Type_Path(lex, {}, allow_trait_list); // HACK! Convert && into & & case TOK_DOUBLE_AMP: @@ -115,7 +117,7 @@ TypeRef Parse_Type_Int(TokenStream& lex) return TypeRef(TypeRef::TagReference(), lex.end_span(ps), true, Parse_Type(lex)); } else { - lex.putback(tok); + PUTBACK(tok, lex); // Immutable reference return TypeRef(TypeRef::TagReference(), lex.end_span(ps), false, Parse_Type(lex)); } @@ -160,9 +162,9 @@ TypeRef Parse_Type_Int(TokenStream& lex) DEBUG("Tuple"); if( GET_TOK(tok, lex) == TOK_PAREN_CLOSE ) return TypeRef(TypeRef::TagTuple(), lex.end_span(ps), {}); - lex.putback(tok); + PUTBACK(tok, lex); - TypeRef inner = Parse_Type(lex); + TypeRef inner = Parse_Type(lex, true); if( GET_TOK(tok, lex) == TOK_PLUS ) { // Lifetime bounded type, NOT a tuple @@ -177,13 +179,13 @@ TypeRef Parse_Type_Int(TokenStream& lex) { ::std::vector types; types.push_back( ::std::move(inner) ); - lex.putback(tok); + PUTBACK(tok, lex); while( GET_TOK(tok, lex) == TOK_COMMA ) { if( GET_TOK(tok, lex) == TOK_PAREN_CLOSE ) break; else - lex.putback(tok); + PUTBACK(tok, lex); types.push_back(Parse_Type(lex)); } CHECK_TOK(tok, TOK_PAREN_CLOSE); @@ -239,7 +241,7 @@ TypeRef Parse_Type_Fn(TokenStream& lex, ::std::vector<::std::string> hrls) } args.push_back( Parse_Type(lex) ); if( GET_TOK(tok, lex) != TOK_COMMA ) { - lex.putback(tok); + PUTBACK(tok, lex); break; } } @@ -251,36 +253,43 @@ TypeRef Parse_Type_Fn(TokenStream& lex, ::std::vector<::std::string> hrls) ret_type = Parse_Type(lex); } else { - lex.putback(tok); + PUTBACK(tok, lex); } return TypeRef(TypeRef::TagFunction(), lex.end_span(ps), ::std::move(abi), ::std::move(args), ::std::move(ret_type)); } -TypeRef Parse_Type_Path(TokenStream& lex, ::std::vector<::std::string> hrls) +TypeRef Parse_Type_Path(TokenStream& lex, ::std::vector<::std::string> hrls, bool allow_trait_list) { Token tok; auto ps = lex.start_span(); - ::std::vector traits; - ::std::vector< ::std::string> lifetimes; - do { - if( LOOK_AHEAD(lex) == TOK_LIFETIME ) { - GET_TOK(tok, lex); - lifetimes.push_back( tok.str() ); - } - else - traits.push_back( Parse_Path(lex, PATH_GENERIC_TYPE) ); - } while( GET_TOK(tok, lex) == TOK_PLUS ); - lex.putback(tok); - if( hrls.size() > 0 || traits.size() > 1 || lifetimes.size() > 0 ) { - if( lifetimes.size() ) - DEBUG("TODO: Lifetime bounds on trait objects"); - return TypeRef(lex.end_span(ps), mv$(hrls), ::std::move(traits)); + if( ! allow_trait_list ) + { + return TypeRef(TypeRef::TagPath(), lex.end_span(ps), Parse_Path(lex, PATH_GENERIC_TYPE)); } - else { - return TypeRef(TypeRef::TagPath(), lex.end_span(ps), traits.at(0)); + else + { + ::std::vector traits; + ::std::vector< ::std::string> lifetimes; + do { + if( LOOK_AHEAD(lex) == TOK_LIFETIME ) { + GET_TOK(tok, lex); + lifetimes.push_back( tok.str() ); + } + else + traits.push_back( Parse_Path(lex, PATH_GENERIC_TYPE) ); + } while( GET_TOK(tok, lex) == TOK_PLUS ); + PUTBACK(tok, lex); + if( hrls.size() > 0 || traits.size() > 1 || lifetimes.size() > 0 ) { + if( lifetimes.size() ) + DEBUG("TODO: Lifetime bounds on trait objects"); + return TypeRef(lex.end_span(ps), mv$(hrls), ::std::move(traits)); + } + else { + return TypeRef(TypeRef::TagPath(), lex.end_span(ps), traits.at(0)); + } } } -- cgit v1.2.3