From bf14a518b591b8f43a92c13d85b1ae5db8965703 Mon Sep 17 00:00:00 2001 From: John Hodge Date: Sat, 14 Mar 2015 22:29:51 +0800 Subject: Fixed macros, added range syntax and other tweaks --- src/ast/expr.cpp | 120 +++++++++++- src/ast/expr.hpp | 129 ++++++++++++- src/dump_as_rust.cpp | 82 +++++++- src/macros.cpp | 29 ++- src/macros.hpp | 22 ++- src/parse/common.hpp | 2 + src/parse/expr.cpp | 482 +++++++++++++++++++++++++++-------------------- src/parse/lex.cpp | 13 +- src/parse/lex.hpp | 52 +++-- src/parse/parseerror.cpp | 15 +- src/parse/parseerror.hpp | 3 + src/parse/root.cpp | 31 ++- src/parse/tokentree.hpp | 7 + 13 files changed, 738 insertions(+), 249 deletions(-) (limited to 'src') diff --git a/src/ast/expr.cpp b/src/ast/expr.cpp index 9740a4d9..1246ad29 100644 --- a/src/ast/expr.cpp +++ b/src/ast/expr.cpp @@ -52,17 +52,20 @@ SERIALISE_TYPE(Expr::, "Expr", { #define _(x) if(tag == #x) ptr = new x; _(ExprNode_Block) else _(ExprNode_Macro) - else _(ExprNode_Return) + else _(ExprNode_Flow) else _(ExprNode_Const) + else _(ExprNode_Import) else _(ExprNode_LetBinding) else _(ExprNode_Assign) else _(ExprNode_CallPath) else _(ExprNode_CallMethod) else _(ExprNode_CallObject) else _(ExprNode_Match) + else _(ExprNode_Loop) else _(ExprNode_If) else _(ExprNode_IfLet) else _(ExprNode_Integer) + else _(ExprNode_Closure) else _(ExprNode_StructLiteral) else _(ExprNode_Tuple) else _(ExprNode_NamedValue) @@ -107,10 +110,40 @@ NODE(ExprNode_Macro, { os << m_name << "!(" << ")"; }) -NODE(ExprNode_Return, { +void operator%(::Serialiser& s, const ExprNode_Flow::Type t) { + switch(t) + { + #define _(v) case ExprNode_Flow::v: s << #v; return + _(RETURN); + _(BREAK); + _(CONTINUE); + #undef _ + } +} +void operator%(::Deserialiser& s, ExprNode_Flow::Type& t) { + ::std::string n; + s.item(n); + if(0) ; + #define _(v) else if(n == #v) t = ExprNode_Flow::v + _(RETURN); + _(BREAK); + _(CONTINUE); + #undef _ + else + throw ::std::runtime_error(""); +} +NODE(ExprNode_Flow, { + s % m_type; + s.item(m_target); s.item(m_value); },{ - os << "return " << *m_value; + switch(m_type) + { + case RETURN: os << "return"; break; + case BREAK: os << "break"; break; + case CONTINUE: os << "continue"; break; + } + os << " " << *m_value; }) NODE(ExprNode_Const, { @@ -121,6 +154,12 @@ NODE(ExprNode_Const, { os << "const " << m_name << ": " << m_type << " = " << *m_value; }) +NODE(ExprNode_Import, { + s.item(m_imports); +},{ + os << "/* todo: use /*"; +}) + NODE(ExprNode_LetBinding, { s.item(m_pat); s.item(m_type); @@ -170,6 +209,40 @@ NODE(ExprNode_CallObject, { os << ")"; }) +void operator%(::Serialiser& s, const ExprNode_Loop::Type t) { + switch(t) + { + #define _(v) case ExprNode_Loop::v: s << #v; return + _(LOOP); + _(WHILE); + _(WHILELET); + _(FOR); + #undef _ + } +} +void operator%(::Deserialiser& s, ExprNode_Loop::Type& t) { + ::std::string n; + s.item(n); + if(0) ; + #define _(v) else if(n == #v) t = ExprNode_Loop::v + _(LOOP); + _(WHILE); + _(WHILELET); + _(FOR); + #undef _ + else + throw ::std::runtime_error(""); +} +NODE(ExprNode_Loop, { + s % m_type; + s.item(m_label); + s.item(m_pattern); + s.item(m_cond); + s.item(m_code); +},{ + //os << "LOOP [" << m_label << "] " << m_pattern << " in/= " << m_cond << " " << m_code; +}) + SERIALISE_TYPE_A(ExprNode_Match::Arm::, "ExprNode_Match_Arm", { s.item(m_patterns); s.item(m_cond); @@ -224,6 +297,19 @@ NODE(ExprNode_Bool, { },{ os << m_value; }) +NODE(ExprNode_String, { + s.item(m_value); +},{ + os << "\"" << m_value << "\""; +}) + +NODE(ExprNode_Closure, { + s.item(m_args); + s.item(m_return); + s.item(m_code); +},{ + os << "/* todo: closure */"; +}); NODE(ExprNode_StructLiteral, { s.item(m_path); @@ -256,6 +342,13 @@ NODE(ExprNode_Field, { os << "(" << *m_obj << ")." << m_name; }) +NODE(ExprNode_Index, { + s.item(m_obj); + s.item(m_idx); +},{ + os << "(" << *m_obj << ")[" << *m_idx << "]"; +}) + NODE(ExprNode_Deref, { s.item(m_value); },{ @@ -279,6 +372,7 @@ void operator%(::Serialiser& s, const ExprNode_BinOp::Type t) { _(CMPLTE); _(CMPGT); _(CMPGTE); + _(RANGE); _(BOOLAND); _(BOOLOR); _(BITAND); @@ -305,6 +399,7 @@ void operator%(::Deserialiser& s, ExprNode_BinOp::Type& t) { _(CMPLTE); _(CMPGT); _(CMPGTE); + _(RANGE); _(BOOLAND); _(BOOLOR); _(BITAND); @@ -404,7 +499,7 @@ NV(ExprNode_Macro, { DEBUG("TODO: Macro"); }) -NV(ExprNode_Return, +NV(ExprNode_Flow, { visit(node.m_value); }) @@ -412,6 +507,7 @@ NV(ExprNode_Const, { visit(node.m_value); }) +NV(ExprNode_Import, {}) NV(ExprNode_LetBinding, { // TODO: Handle recurse into Let pattern @@ -447,6 +543,13 @@ NV(ExprNode_CallObject, visit(arg); UNINDENT(); }) +NV(ExprNode_Loop, +{ + INDENT(); + visit(node.m_cond); + visit(node.m_code); + UNINDENT(); +}) NV(ExprNode_Match, { INDENT(); @@ -478,7 +581,11 @@ NV(ExprNode_IfLet, NV(ExprNode_Integer, {}) NV(ExprNode_Float, {}) NV(ExprNode_Bool, {}) +NV(ExprNode_String, {}) +NV(ExprNode_Closure, +{ +}); NV(ExprNode_StructLiteral, { visit(node.m_base_value); @@ -499,6 +606,11 @@ NV(ExprNode_Field, { visit(node.m_obj); }) +NV(ExprNode_Index, +{ + visit(node.m_obj); + visit(node.m_idx); +}) NV(ExprNode_Deref, { visit(node.m_value); diff --git a/src/ast/expr.hpp b/src/ast/expr.hpp index 4c6d6b7c..121a1f09 100644 --- a/src/ast/expr.hpp +++ b/src/ast/expr.hpp @@ -77,13 +77,21 @@ struct ExprNode_Macro: }; // Return a value -struct ExprNode_Return: +struct ExprNode_Flow: public ExprNode { + enum Type { + RETURN, + CONTINUE, + BREAK, + } m_type; + ::std::string m_target; unique_ptr m_value; - ExprNode_Return() {} - ExprNode_Return(unique_ptr&& value): + ExprNode_Flow() {} + ExprNode_Flow(Type type, ::std::string target, unique_ptr&& value): + m_type(type), + m_target( move(target) ), m_value( move(value) ) { } @@ -107,6 +115,19 @@ struct ExprNode_Const: NODE_METHODS(); }; +struct ExprNode_Import: + public ExprNode +{ + typedef ::std::vector< ::std::pair< ::std::string, AST::Path> > imports_t; + imports_t m_imports; + + ExprNode_Import() {} + ExprNode_Import(imports_t imports): + m_imports( ::std::move(imports) ) + {} + + NODE_METHODS(); +}; struct ExprNode_LetBinding: public ExprNode { @@ -131,6 +152,9 @@ struct ExprNode_Assign: NONE, ADD, SUB, + MUL, + DIV, + MOD, } m_op; unique_ptr m_slot; unique_ptr m_value; @@ -193,6 +217,42 @@ struct ExprNode_CallObject: NODE_METHODS(); }; +struct ExprNode_Loop: + public ExprNode +{ + enum Type { + LOOP, + WHILE, + WHILELET, + FOR, + } m_type; + ::std::string m_label; + AST::Pattern m_pattern; + unique_ptr m_cond; // if NULL, loop is a 'loop' + unique_ptr m_code; + + ExprNode_Loop(): m_type(LOOP) {} + ExprNode_Loop(::std::string label, unique_ptr code): + m_type(LOOP), + m_label( ::std::move(label) ), + m_code( ::std::move(code) ) + {} + ExprNode_Loop(::std::string label, unique_ptr cond, unique_ptr code): + m_type(WHILE), + m_label( ::std::move(label) ), + m_cond( ::std::move(cond) ), + m_code( ::std::move(code) ) + {} + ExprNode_Loop(::std::string label, Type type, AST::Pattern pattern, unique_ptr val, unique_ptr code): + m_type(type), + m_label( ::std::move(label) ), + m_pattern( ::std::move(pattern) ), + m_cond( ::std::move(val) ), + m_code( ::std::move(code) ) + {} + NODE_METHODS(); +}; + struct ExprNode_Match: public ExprNode { @@ -299,6 +359,39 @@ struct ExprNode_Bool: NODE_METHODS(); }; +// Literal string +struct ExprNode_String: + public ExprNode +{ + ::std::string m_value; + + ExprNode_String() {} + ExprNode_String(::std::string value): + m_value( ::std::move(value) ) + {} + + NODE_METHODS(); +}; + +// Closure / Lambda +struct ExprNode_Closure: + public ExprNode +{ + typedef ::std::vector< ::std::pair > args_t; + + args_t m_args; + TypeRef m_return; + unique_ptr m_code; + + ExprNode_Closure() {} + ExprNode_Closure(args_t args, TypeRef rv, unique_ptr code): + m_args( ::std::move(args) ), + m_return( ::std::move(rv) ), + m_code( ::std::move(code) ) + {} + + NODE_METHODS(); +}; // Literal structure struct ExprNode_StructLiteral: public ExprNode @@ -358,6 +451,20 @@ struct ExprNode_Field: } NODE_METHODS(); }; +struct ExprNode_Index: + public ExprNode +{ + ::std::unique_ptr m_obj; + ::std::unique_ptr m_idx; + + ExprNode_Index() {} + ExprNode_Index(::std::unique_ptr obj, ::std::unique_ptr idx): + m_obj( ::std::move(obj) ), + m_idx( ::std::move(idx) ) + {} + + NODE_METHODS(); +}; // Pointer dereference struct ExprNode_Deref: @@ -401,6 +508,8 @@ struct ExprNode_BinOp: CMPLTE, CMPGT, CMPGTE, + + RANGE, BOOLAND, BOOLOR, @@ -470,13 +579,15 @@ public: virtual void visit(const nt& node) = 0*/ NT(ExprNode_Block); NT(ExprNode_Macro); - NT(ExprNode_Return); + NT(ExprNode_Flow); NT(ExprNode_Const); + NT(ExprNode_Import); NT(ExprNode_LetBinding); NT(ExprNode_Assign); NT(ExprNode_CallPath); NT(ExprNode_CallMethod); NT(ExprNode_CallObject); + NT(ExprNode_Loop); NT(ExprNode_Match); NT(ExprNode_If); NT(ExprNode_IfLet); @@ -484,11 +595,14 @@ public: NT(ExprNode_Integer); NT(ExprNode_Float); NT(ExprNode_Bool); + NT(ExprNode_String); + NT(ExprNode_Closure); NT(ExprNode_StructLiteral); NT(ExprNode_Tuple); NT(ExprNode_NamedValue); NT(ExprNode_Field); + NT(ExprNode_Index); NT(ExprNode_Deref); NT(ExprNode_Cast); NT(ExprNode_BinOp); @@ -508,13 +622,15 @@ public: virtual void visit(const nt& node) override*/ NT(ExprNode_Block); NT(ExprNode_Macro); - NT(ExprNode_Return); + NT(ExprNode_Flow); NT(ExprNode_Const); + NT(ExprNode_Import); NT(ExprNode_LetBinding); NT(ExprNode_Assign); NT(ExprNode_CallPath); NT(ExprNode_CallMethod); NT(ExprNode_CallObject); + NT(ExprNode_Loop); NT(ExprNode_Match); NT(ExprNode_If); NT(ExprNode_IfLet); @@ -522,11 +638,14 @@ public: NT(ExprNode_Integer); NT(ExprNode_Float); NT(ExprNode_Bool); + NT(ExprNode_String); + NT(ExprNode_Closure); NT(ExprNode_StructLiteral); NT(ExprNode_Tuple); NT(ExprNode_NamedValue); NT(ExprNode_Field); + NT(ExprNode_Index); NT(ExprNode_Deref); NT(ExprNode_Cast); NT(ExprNode_BinOp); diff --git a/src/dump_as_rust.cpp b/src/dump_as_rust.cpp index 76bb99b4..471a0479 100644 --- a/src/dump_as_rust.cpp +++ b/src/dump_as_rust.cpp @@ -58,9 +58,14 @@ public: m_expr_root = false; m_os << n.m_name << "!( /* TODO: Macro TT */ )"; } - virtual void visit(AST::ExprNode_Return& n) override { + virtual void visit(AST::ExprNode_Flow& n) override { m_expr_root = false; - m_os << "return "; + switch(n.m_type) + { + case AST::ExprNode_Flow::RETURN: m_os << "return "; break; + case AST::ExprNode_Flow::BREAK: m_os << "break "; break; + case AST::ExprNode_Flow::CONTINUE: m_os << "continue "; break; + } AST::NodeVisitor::visit(n.m_value); } virtual void visit(AST::ExprNode_Const& n) override { @@ -70,6 +75,11 @@ public: m_os << " = "; AST::NodeVisitor::visit(n.m_value); } + virtual void visit(AST::ExprNode_Import& n) override { + m_expr_root = false; + for( const auto& item : n.m_imports ) + m_os << "use " << item.second << " as " << item.first << ";\n" << indent(); + } virtual void visit(AST::ExprNode_LetBinding& n) override { m_expr_root = false; m_os << "let "; @@ -121,6 +131,45 @@ public: m_expr_root = false; throw ::std::runtime_error("unimplemented ExprNode_CallObject"); } + virtual void visit(AST::ExprNode_Loop& n) override { + bool expr_root = m_expr_root; + m_expr_root = false; + + switch(n.m_type) + { + case AST::ExprNode_Loop::LOOP: + m_os << "loop"; + break; + case AST::ExprNode_Loop::WHILE: + m_os << "while "; + AST::NodeVisitor::visit(n.m_cond); + break; + case AST::ExprNode_Loop::WHILELET: + m_os << "while let "; + print_pattern(n.m_pattern); + m_os << " = "; + AST::NodeVisitor::visit(n.m_cond); + break; + case AST::ExprNode_Loop::FOR: + m_os << "while for "; + print_pattern(n.m_pattern); + m_os << " in "; + AST::NodeVisitor::visit(n.m_cond); + break; + } + + if( expr_root ) + { + m_os << "\n"; + m_os << indent(); + } + else + { + m_os << " "; + } + + AST::NodeVisitor::visit(n.m_code); + } virtual void visit(AST::ExprNode_Match& n) override { bool expr_root = m_expr_root; m_expr_root = false; @@ -220,6 +269,22 @@ public: AST::NodeVisitor::visit(fv); } } + virtual void visit(AST::ExprNode_Closure& n) override { + m_expr_root = false; + m_os << "|"; + bool is_first = true; + for( const auto& arg : n.m_args ) + { + if(!is_first) m_os << ", "; + is_first = false; + print_pattern(arg.first); + m_os << ": "; + print_type(arg.second); + } + m_os << "|: "; + print_type(n.m_return); + AST::NodeVisitor::visit(n.m_code); + } virtual void visit(AST::ExprNode_Integer& n) override { m_expr_root = false; switch(n.m_datatype) @@ -268,6 +333,11 @@ public: else m_os << "false"; } + virtual void visit(AST::ExprNode_String& n) override { + m_expr_root = false; + m_os << "\"" << n.m_value << "\""; + } + virtual void visit(AST::ExprNode_StructLiteral& n) override { m_expr_root = false; m_os << n.m_path << " {\n"; @@ -307,6 +377,14 @@ public: AST::NodeVisitor::visit(n.m_obj); m_os << ")." << n.m_name; } + virtual void visit(AST::ExprNode_Index& n) override { + m_expr_root = false; + m_os << "("; + AST::NodeVisitor::visit(n.m_obj); + m_os << ")["; + AST::NodeVisitor::visit(n.m_idx); + m_os << "]"; + } virtual void visit(AST::ExprNode_Deref&) override { m_expr_root = false; throw ::std::runtime_error("unimplemented ExprNode_Deref"); diff --git a/src/macros.cpp b/src/macros.cpp index 29d17b8f..f375c0d4 100644 --- a/src/macros.cpp +++ b/src/macros.cpp @@ -33,6 +33,7 @@ public: typedef ::std::multimap t_mappings; private: + const TokenStream& m_olex; const TokenTree& m_crate_path; const ::std::vector& m_root_contents; const t_mappings m_mappings; @@ -49,6 +50,7 @@ private: public: MacroExpander(const MacroExpander& x): + m_olex(x.m_olex), m_crate_path(x.m_crate_path), m_root_contents(x.m_root_contents), m_mappings(x.m_mappings), @@ -57,7 +59,8 @@ public: { prep_counts(); } - MacroExpander(const ::std::vector& contents, t_mappings mappings, const TokenTree& crate_path): + MacroExpander(const TokenStream& olex, const ::std::vector& contents, t_mappings mappings, const TokenTree& crate_path): + m_olex(olex), m_crate_path(crate_path), m_root_contents(contents), m_mappings(mappings), @@ -241,7 +244,7 @@ void Macro_HandlePattern(TTStream& lex, const MacroPatEnt& pat, unsigned int lay } -::std::unique_ptr Macro_InvokeInt(const char *name, const MacroRules& rules, TokenTree input) +::std::unique_ptr Macro_InvokeInt(const TokenStream& olex, const char *name, const MacroRules& rules, TokenTree input) { TRACE_FUNCTION; @@ -276,7 +279,7 @@ void Macro_HandlePattern(TTStream& lex, const MacroPatEnt& pat, unsigned int lay //GET_CHECK_TOK(tok, lex, close); GET_CHECK_TOK(tok, lex, TOK_EOF); DEBUG( rule.m_contents.size() << " rule contents bound to " << bound_tts.size() << " values - " << name ); - return ::std::unique_ptr( (TokenStream*)new MacroExpander(rule.m_contents, bound_tts, g_crate_path_tt) ); + return ::std::unique_ptr( (TokenStream*)new MacroExpander(olex, rule.m_contents, bound_tts, g_crate_path_tt) ); } catch(const ParseError::Base& e) { @@ -285,11 +288,12 @@ void Macro_HandlePattern(TTStream& lex, const MacroPatEnt& pat, unsigned int lay i ++; } DEBUG(""); - throw ParseError::Todo("Error when macro fails to match"); + throw ParseError::Todo(olex, "Error when macro fails to match"); } ::std::unique_ptr Macro_Invoke(const TokenStream& olex, const ::std::string& name, TokenTree input) { + DEBUG("Invoke " << name << " from " << olex.getPosition()); // XXX: EVIL HACK! - This should be removed when std loading is implemented if( g_macro_registrations.size() == 0 ) { Macro_InitDefaults(); @@ -312,7 +316,7 @@ void Macro_HandlePattern(TTStream& lex, const MacroPatEnt& pat, unsigned int lay t_macro_regs::iterator macro_reg = g_macro_registrations.find(name); if( macro_reg != g_macro_registrations.end() ) { - return Macro_InvokeInt(macro_reg->first.c_str(), macro_reg->second, input); + return Macro_InvokeInt(olex, macro_reg->first.c_str(), macro_reg->second, input); } // Search import list @@ -324,7 +328,7 @@ void Macro_HandlePattern(TTStream& lex, const MacroPatEnt& pat, unsigned int lay DEBUG("" << m.name); if( m.name == name ) { - return Macro_InvokeInt(m.name.c_str(), m.data, input); + return Macro_InvokeInt(olex, m.name.c_str(), m.data, input); } } @@ -333,7 +337,7 @@ void Macro_HandlePattern(TTStream& lex, const MacroPatEnt& pat, unsigned int lay DEBUG("" << mi.name); if( mi.name == name ) { - return Macro_InvokeInt(mi.name.c_str(), *mi.data, input); + return Macro_InvokeInt(olex, mi.name.c_str(), *mi.data, input); } } } @@ -343,7 +347,8 @@ void Macro_HandlePattern(TTStream& lex, const MacroPatEnt& pat, unsigned int lay Position MacroExpander::getPosition() const { - return Position("Macro", 0); + DEBUG("olex.getPosition() = " << m_olex.getPosition()); + return Position(FMT("Macro:" << ""), m_offsets[0].first); } Token MacroExpander::realGetToken() { @@ -380,6 +385,7 @@ Token MacroExpander::realGetToken() // - Name // HACK: Handle $crate with a special name if( ent.name == "*crate" ) { + DEBUG("Crate name hack"); m_ttstream.reset( new TTStream(m_crate_path) ); return m_ttstream->getToken(); } @@ -394,6 +400,7 @@ Token MacroExpander::realGetToken() { if( i == iter_idx ) { + DEBUG(ent.name << " #" << i << " - Setting TT"); m_ttstream.reset( new TTStream(it->second) ); return m_ttstream->getToken(); } @@ -536,7 +543,7 @@ MacroToken::MacroToken(Token tok): } Position MacroToken::getPosition() const { - return Position("macro", 0); + return Position("MacroToken", 0); } Token MacroToken::realGetToken() { @@ -550,7 +557,7 @@ MacroStringify::MacroStringify(const TokenTree& input) } Position MacroStringify::getPosition() const { - return Position("stringify", 0); + return Position("Stringify", 0); } Token MacroStringify::realGetToken() { @@ -569,6 +576,7 @@ void operator%(Serialiser& s, MacroPatEnt::Type c) { #define _(v) case MacroPatEnt::v: s << #v; return _(PAT_TOKEN); _(PAT_TT); + _(PAT_TYPE); _(PAT_EXPR); _(PAT_LOOP); //_(PAT_OPTLOOP); @@ -586,6 +594,7 @@ void operator%(::Deserialiser& s, MacroPatEnt::Type& c) { if(0) ; _(PAT_TOKEN); _(PAT_TT); + _(PAT_TYPE); _(PAT_EXPR); _(PAT_LOOP); //_(PAT_OPTLOOP); diff --git a/src/macros.hpp b/src/macros.hpp index 2fa74698..bb25f38a 100644 --- a/src/macros.hpp +++ b/src/macros.hpp @@ -39,7 +39,15 @@ public: } friend ::std::ostream& operator<<(::std::ostream& os, const MacroRuleEnt& x) { - return os << "MacroRuleEnt( '"< Parse_Path_GenericList(TokenStream& lex); extern TypeRef Parse_Type(TokenStream& lex); +extern void Parse_Use(TokenStream& lex, ::std::function fcn); extern AST::Expr Parse_Expr(TokenStream& lex, bool const_only); extern AST::Expr Parse_ExprBlock(TokenStream& lex); diff --git a/src/parse/expr.cpp b/src/parse/expr.cpp index 10b942ac..d0d5bc7f 100644 --- a/src/parse/expr.cpp +++ b/src/parse/expr.cpp @@ -17,6 +17,7 @@ ExprNodeP Parse_Stmt(TokenStream& lex, bool& opt_semicolon); ExprNodeP Parse_Expr0(TokenStream& lex); ExprNodeP Parse_ExprBlocks(TokenStream& lex); ExprNodeP Parse_IfStmt(TokenStream& lex); +ExprNodeP Parse_Expr_Match(TokenStream& lex); ExprNodeP Parse_Expr1(TokenStream& lex); AST::Expr Parse_Expr(TokenStream& lex, bool const_only) @@ -142,14 +143,21 @@ AST::Pattern Parse_PatternReal(TokenStream& lex) case TOK_DOUBLE_COLON: // 2. Paths are enum/struct names return Parse_PatternReal_Path( lex, Parse_Path(lex, true, PATH_GENERIC_EXPR) ); + case TOK_DASH: + GET_CHECK_TOK(tok, lex, TOK_INTEGER); + return AST::Pattern( AST::Pattern::TagValue(), NEWNODE(AST::ExprNode_Integer, -tok.intval(), tok.datatype()) ); case TOK_INTEGER: return AST::Pattern( AST::Pattern::TagValue(), NEWNODE(AST::ExprNode_Integer, tok.intval(), tok.datatype()) ); + case TOK_RWORD_TRUE: + return AST::Pattern( AST::Pattern::TagValue(), NEWNODE( AST::ExprNode_Bool, true ) ); + case TOK_RWORD_FALSE: + return AST::Pattern( AST::Pattern::TagValue(), NEWNODE( AST::ExprNode_Bool, false ) ); case TOK_STRING: - throw ParseError::Todo("string patterns"); + return AST::Pattern( AST::Pattern::TagValue(), NEWNODE(AST::ExprNode_String, tok.str()) ); case TOK_PAREN_OPEN: return AST::Pattern(AST::Pattern::TagTuple(), Parse_PatternList(lex)); case TOK_SQUARE_OPEN: - throw ParseError::Todo("array patterns"); + throw ParseError::Todo(lex, "array patterns"); default: throw ParseError::Unexpected(lex, tok); } @@ -218,8 +226,18 @@ pass_value: ExprNodeP Parse_Stmt(TokenStream& lex, bool& opt_semicolon) { TRACE_FUNCTION; - Token tok; + + ::std::string lifetime; + + if( GET_TOK(tok, lex) == TOK_LIFETIME ) { + lifetime = tok.str(); + GET_CHECK_TOK(tok, lex, TOK_COLON); + } + else { + lex.putback(tok); + } + // 1. Handle 'let' // 2. Handle new blocks // 3. Handle a sequence of expressions broken by ';' @@ -229,6 +247,14 @@ ExprNodeP Parse_Stmt(TokenStream& lex, bool& opt_semicolon) lex.putback(tok); opt_semicolon = true; return Parse_ExprBlockNode(lex); + case TOK_RWORD_USE: { + opt_semicolon = false; + ::std::vector< ::std::pair< ::std::string, AST::Path> > imports; + Parse_Use(lex, [&imports](AST::Path p, std::string s) { + imports.push_back( ::std::make_pair( ::std::move(s), ::std::move(p) ) ); + }); + return NEWNODE( AST::ExprNode_Import, ::std::move(imports) ); + } case TOK_RWORD_CONST: { opt_semicolon = false; GET_CHECK_TOK(tok, lex, TOK_IDENT); @@ -253,17 +279,68 @@ ExprNodeP Parse_Stmt(TokenStream& lex, bool& opt_semicolon) ExprNodeP val = Parse_ExprBlocks(lex); return NEWNODE( AST::ExprNode_LetBinding, ::std::move(pat), ::std::move(type), ::std::move(val) ); } - case TOK_RWORD_RETURN: - return NEWNODE( AST::ExprNode_Return, Parse_Expr1(lex) ); + + case TOK_RWORD_RETURN: { + if( lifetime.size() ) + throw ParseError::Unexpected(lex, Token(TOK_LIFETIME, lifetime)); + ExprNodeP val; + if( GET_TOK(tok, lex) != TOK_SEMICOLON ) { + lex.putback(tok); + val = Parse_Expr1(lex); + } + else + lex.putback(tok); + return NEWNODE( AST::ExprNode_Flow, AST::ExprNode_Flow::RETURN, "", ::std::move(val) ); + } + case TOK_RWORD_CONTINUE: + case TOK_RWORD_BREAK: + { + if( lifetime.size() ) + throw ParseError::Unexpected(lex, Token(TOK_LIFETIME, lifetime)); + AST::ExprNode_Flow::Type type; + switch(tok.type()) + { + case TOK_RWORD_CONTINUE: type = AST::ExprNode_Flow::CONTINUE; break; + case TOK_RWORD_BREAK: type = AST::ExprNode_Flow::BREAK; break; + default: throw ParseError::BugCheck(/*lex,*/ "continue/break"); + } + if( GET_TOK(tok, lex) == TOK_LIFETIME ) + { + lifetime = tok.str(); + GET_TOK(tok, lex); + } + ExprNodeP val; + if( tok.type() != TOK_SEMICOLON ) { + lex.putback(tok); + val = Parse_Expr1(lex); + } + else + lex.putback(tok); + return NEWNODE( AST::ExprNode_Flow, type, lifetime, ::std::move(val) ); + } + case TOK_RWORD_LOOP: - throw ParseError::Todo(lex, "loop"); - case TOK_RWORD_FOR: - throw ParseError::Todo(lex, "for"); - case TOK_RWORD_WHILE: - throw ParseError::Todo(lex, "while"); + opt_semicolon = true; + return NEWNODE( AST::ExprNode_Loop, lifetime, Parse_ExprBlockNode(lex) ); + case TOK_RWORD_WHILE: { + opt_semicolon = true; + ExprNodeP cnd = Parse_Expr1(lex); + return NEWNODE( AST::ExprNode_Loop, lifetime, ::std::move(cnd), Parse_ExprBlockNode(lex) ); + } + case TOK_RWORD_FOR: { + opt_semicolon = true; + AST::Pattern pat = Parse_Pattern(lex); + GET_CHECK_TOK(tok, lex, TOK_RWORD_IN); + ExprNodeP val = Parse_Expr1(lex); + return NEWNODE( AST::ExprNode_Loop, lifetime, AST::ExprNode_Loop::FOR, + ::std::move(pat), ::std::move(val), Parse_ExprBlockNode(lex) ); + } case TOK_RWORD_IF: opt_semicolon = true; return Parse_IfStmt(lex); + case TOK_RWORD_MATCH: + opt_semicolon = true; + return Parse_Expr_Match(lex); default: lex.putback(tok); return Parse_Expr0(lex); @@ -295,14 +372,25 @@ ExprNodeP Parse_Expr0(TokenStream& lex) Token tok; ExprNodeP rv = Parse_ExprBlocks(lex); + auto op = AST::ExprNode_Assign::NONE; switch( GET_TOK(tok, lex) ) { - case TOK_EQUAL: - return NEWNODE( AST::ExprNode_Assign, AST::ExprNode_Assign::NONE, ::std::move(rv), Parse_Expr1(lex) ); case TOK_PLUS_EQUAL: - return NEWNODE( AST::ExprNode_Assign, AST::ExprNode_Assign::ADD, ::std::move(rv), Parse_Expr1(lex) ); + op = AST::ExprNode_Assign::ADD; + if(0) case TOK_DASH_EQUAL: - return NEWNODE( AST::ExprNode_Assign, AST::ExprNode_Assign::SUB, ::std::move(rv), Parse_Expr1(lex) ); + op = AST::ExprNode_Assign::SUB; + if(0) + case TOK_SLASH_EQUAL: + op = AST::ExprNode_Assign::DIV; + if(0) + case TOK_STAR_EQUAL: + op = AST::ExprNode_Assign::MUL; + if(0) + case TOK_EQUAL: + op = AST::ExprNode_Assign::NONE; + return NEWNODE( AST::ExprNode_Assign, op, ::std::move(rv), Parse_ExprBlocks(lex) ); + default: lex.putback(tok); return rv; @@ -314,21 +402,24 @@ ExprNodeP Parse_Expr0(TokenStream& lex) ExprNodeP Parse_IfStmt(TokenStream& lex) { TRACE_FUNCTION; - SET_PARSE_FLAG(lex, disallow_struct_literal); Token tok; ExprNodeP cond; AST::Pattern pat; bool if_let = false; - if( GET_TOK(tok, lex) == TOK_RWORD_LET ) { - if_let = true; - pat = Parse_Pattern(lex); - GET_CHECK_TOK(tok, lex, TOK_EQUAL); - cond = Parse_Expr0(lex); - } - else { - lex.putback(tok); - cond = Parse_Expr0(lex); + + { + SET_PARSE_FLAG(lex, disallow_struct_literal); + if( GET_TOK(tok, lex) == TOK_RWORD_LET ) { + if_let = true; + pat = Parse_Pattern(lex); + GET_CHECK_TOK(tok, lex, TOK_EQUAL); + cond = Parse_Expr0(lex); + } + else { + lex.putback(tok); + cond = Parse_Expr0(lex); + } } // Contents @@ -370,6 +461,7 @@ ExprNodeP Parse_Expr_Match(TokenStream& lex) SET_PARSE_FLAG(lex, disallow_struct_literal); switch_val = Parse_Expr1(lex); } + ASSERT(lex, !CHECK_PARSE_FLAG(lex, disallow_struct_literal) ); GET_CHECK_TOK(tok, lex, TOK_BRACE_OPEN); ::std::vector< AST::ExprNode_Match::Arm > arms; @@ -449,8 +541,59 @@ ExprNodeP cur(TokenStream& lex) \ } \ } \ } +bool Parse_IsTokValue(eTokenType tok_type) +{ + switch( tok_type ) + { + case TOK_DOUBLE_COLON: + case TOK_IDENT: + case TOK_INTEGER: + case TOK_FLOAT: + case TOK_STRING: + case TOK_RWORD_TRUE: + case TOK_RWORD_FALSE: + case TOK_RWORD_BOX: + case TOK_PAREN_OPEN: + return true; + default: + return false; + } + +} +ExprNodeP Parse_Expr1_5(TokenStream& lex); +// Very evil handling for '..' +ExprNodeP Parse_Expr1(TokenStream& lex) +{ + Token tok; + ExprNodeP (*next)(TokenStream&) = Parse_Expr1_5; + ExprNodeP left, right; + + if( GET_TOK(tok, lex) != TOK_DOUBLE_DOT ) + { + lex.putback(tok); + + left = next(lex); + + if( GET_TOK(tok, lex) != TOK_DOUBLE_DOT ) + { + lex.putback(tok); + return ::std::move(left); + } + } + if( Parse_IsTokValue( GET_TOK(tok, lex) ) ) + { + lex.putback(tok); + right = next(lex); + } + else + { + lex.putback(tok); + } + + return NEWNODE( AST::ExprNode_BinOp, AST::ExprNode_BinOp::RANGE, ::std::move(left), ::std::move(right) ); +} // 1: Bool OR -LEFTASSOC(Parse_Expr1, Parse_Expr2, +LEFTASSOC(Parse_Expr1_5, Parse_Expr2, case TOK_DOUBLE_PIPE: rv = NEWNODE( AST::ExprNode_BinOp, AST::ExprNode_BinOp::BOOLOR, ::std::move(rv), next(lex)); break; @@ -536,7 +679,8 @@ LEFTASSOC(Parse_Expr11, Parse_Expr12, rv = NEWNODE( AST::ExprNode_BinOp, AST::ExprNode_BinOp::DIVIDE, ::std::move(rv), next(lex)); break; case TOK_PERCENT: - throw ParseError::Todo("expr - modulo"); + rv = NEWNODE( AST::ExprNode_BinOp, AST::ExprNode_BinOp::MODULO, ::std::move(rv), next(lex)); + break; ) // 12: Unaries ExprNodeP Parse_ExprFC(TokenStream& lex); @@ -575,23 +719,32 @@ ExprNodeP Parse_ExprFC(TokenStream& lex) lex.putback(tok); val = NEWNODE( AST::ExprNode_CallObject, ::std::move(val), Parse_ParenList(lex) ); break; + case TOK_SQUARE_OPEN: { + ExprNodeP idx = Parse_Expr0(lex); + val = NEWNODE( AST::ExprNode_Index, ::std::move(val), ::std::move(idx) ); + GET_CHECK_TOK(tok, lex, TOK_SQUARE_CLOSE); + } + break; case TOK_DOT: // Field access / method call // TODO: What about tuple indexing? switch(GET_TOK(tok, lex)) { case TOK_IDENT: { - ::std::string name = tok.str(); + AST::PathNode path(tok.str(), {}); switch( GET_TOK(tok, lex) ) { case TOK_PAREN_OPEN: lex.putback(tok); - val = NEWNODE( AST::ExprNode_CallMethod, ::std::move(val), AST::PathNode(name, {}), Parse_ParenList(lex) ); + val = NEWNODE( AST::ExprNode_CallMethod, ::std::move(val), ::std::move(path), Parse_ParenList(lex) ); break; case TOK_DOUBLE_COLON: - throw ParseError::Todo("method calls - generic"); + GET_CHECK_TOK(tok, lex, TOK_LT); + path.args() = Parse_Path_GenericList(lex); + val = NEWNODE( AST::ExprNode_CallMethod, ::std::move(val), ::std::move(path), Parse_ParenList(lex) ); + break; default: - val = NEWNODE( AST::ExprNode_Field, ::std::move(val), ::std::string(name) ); + val = NEWNODE( AST::ExprNode_Field, ::std::move(val), ::std::string(path.name()) ); lex.putback(tok); break; } @@ -641,6 +794,42 @@ ExprNodeP Parse_ExprVal_StructLiteral(TokenStream& lex, AST::Path path) return NEWNODE( AST::ExprNode_StructLiteral, path, ::std::move(base_val), ::std::move(items) ); } +ExprNodeP Parse_ExprVal_Closure(TokenStream& lex) +{ + TRACE_FUNCTION; + Token tok; + + ::std::vector< ::std::pair > args; + + while( GET_TOK(tok, lex) != TOK_PIPE ) + { + lex.putback(tok); + AST::Pattern pat = Parse_Pattern(lex); + + TypeRef type; + if( GET_TOK(tok, lex) == TOK_COLON ) + type = Parse_Type(lex); + else + lex.putback(tok); + + args.push_back( ::std::make_pair( ::std::move(pat), ::std::move(type) ) ); + + if( GET_TOK(tok, lex) != TOK_COMMA ) + break; + } + CHECK_TOK(tok, TOK_PIPE); + + TypeRef rt; + if( GET_TOK(tok, lex) == TOK_COLON ) + rt = Parse_Type(lex); + else + lex.putback(tok); + + auto code = Parse_Expr0(lex); + + return NEWNODE( AST::ExprNode_Closure, ::std::move(args), ::std::move(rt), ::std::move(code) ); +} + ExprNodeP Parse_FormatArgs(TokenStream& lex) { TRACE_FUNCTION; @@ -690,15 +879,23 @@ ExprNodeP Parse_ExprVal(TokenStream& lex) case TOK_BRACE_OPEN: if( !CHECK_PARSE_FLAG(lex, disallow_struct_literal) ) return Parse_ExprVal_StructLiteral(lex, ::std::move(path)); + else + DEBUG("Not parsing struct literal"); default: // Value lex.putback(tok); return NEWNODE( AST::ExprNode_NamedValue, ::std::move(path) ); } + case TOK_DOUBLE_PIPE: + lex.putback(Token(TOK_PIPE)); + case TOK_PIPE: + return Parse_ExprVal_Closure(lex); case TOK_INTEGER: return NEWNODE( AST::ExprNode_Integer, tok.intval(), tok.datatype() ); case TOK_FLOAT: return NEWNODE( AST::ExprNode_Float, tok.floatval(), tok.datatype() ); + case TOK_STRING: + return NEWNODE( AST::ExprNode_String, tok.str() ); case TOK_RWORD_TRUE: return NEWNODE( AST::ExprNode_Bool, true ); case TOK_RWORD_FALSE: @@ -795,197 +992,82 @@ TokenTree Parse_TT(TokenStream& lex, bool unwrapped) return TokenTree(items); } +/// A wrapping lexer that +class TTLexer: + public TokenStream +{ + TokenStream& m_input; + Token m_last_token; + ::std::vector m_output; +public: + TTLexer(TokenStream& input): + m_input(input) + { + } + + virtual Position getPosition() const override { return m_input.getPosition(); } + virtual Token realGetToken() override { + Token tok = m_input.getToken(); + m_output.push_back( TokenTree(tok) ); + return tok; + } + + TokenTree get_output() { + unsigned int eat = (TokenStream::m_cache_valid ? 1 : 0) + TokenStream::m_lookahead.size(); + DEBUG(eat << " tokens were not consumed"); + assert( m_output.size() >= eat ); + assert( m_input.m_lookahead.size() == 0 ); + assert( m_input.m_cache_valid == false ); + for( unsigned int i = 0; i < eat; i ++ ) + { + Token tok = m_output[ m_output.size() - eat + i ].tok(); + DEBUG("Unconsume " << tok); + m_input.m_lookahead.push_back( tok ); + } + DEBUG("- output was [" << m_output << "]"); + m_output.erase( m_output.end() - eat, m_output.end() ); + DEBUG("Returning [" << m_output << "]"); + return ::std::move(m_output); + } +}; + TokenTree Parse_TT_Type(TokenStream& lex) { TRACE_FUNCTION; - Token tok; + TTLexer wlex(lex); - ::std::vector ret; - switch(GET_TOK(tok, lex)) - { - case TOK_AMP: - throw ParseError::Todo("TokenTree type &-ptr"); - case TOK_STAR: - throw ParseError::Todo("TokenTree type *-ptr"); - case TOK_DOUBLE_COLON: - case TOK_IDENT: - lex.putback(tok); - return Parse_TT_Path(lex, false); - default: - throw ParseError::Unexpected(lex, tok); - } - return TokenTree(ret); + // discard result + Parse_Type(wlex); + + return wlex.get_output(); } +/// Parse a token tree path TokenTree Parse_TT_Path(TokenStream& lex, bool mode_expr) { TRACE_FUNCTION; - + TTLexer wlex(lex); Token tok; - ::std::vector ret; - if( GET_TOK(tok, lex) == TOK_DOUBLE_COLON ) { - ret.push_back(TokenTree(tok)); + if( GET_TOK(tok, wlex) == TOK_DOUBLE_COLON ) { + Parse_Path(wlex, true, (mode_expr ? PATH_GENERIC_EXPR : PATH_GENERIC_TYPE)); } else { - lex.putback(tok); + wlex.putback(tok); + Parse_Path(wlex, false, (mode_expr ? PATH_GENERIC_EXPR : PATH_GENERIC_TYPE)); } - for(;;) - { - // Expect an ident - GET_CHECK_TOK(tok, lex, TOK_IDENT); - ret.push_back(TokenTree(tok)); - // If mode is expr, check for a double colon here - if( mode_expr ) - { - if( GET_TOK(tok, lex) != TOK_DOUBLE_COLON ) - break; - ret.push_back( TokenTree(tok) ); - } - - if( GET_TOK(tok, lex) == TOK_LT ) - { - do { - ret.push_back( TokenTree(tok) ); - ret.push_back(Parse_TT_Type(lex)); - } while(GET_TOK(tok, lex) == TOK_COMMA); - if( tok.type() != TOK_GT ) - { - if(tok.type() == TOK_DOUBLE_GT) { - ret.push_back(TokenTree(Token(TOK_GT))); - lex.putback(Token(TOK_GT)); - } - else { - CHECK_TOK(tok, TOK_GT); - } - } - else { - ret.push_back(TokenTree(tok)); - } - - if( GET_TOK(tok, lex) != TOK_DOUBLE_COLON ) - break; - ret.push_back(TokenTree(tok)); - } - else - { - lex.putback(tok); - - if( !mode_expr ) - { - if( GET_TOK(tok, lex) != TOK_DOUBLE_COLON ) - break; - ret.push_back(TokenTree(tok)); - } - } - } - lex.putback(tok); - return TokenTree(ret); -} -/// Parse a token tree path -TokenTree Parse_TT_Val(TokenStream& lex) -{ - Token tok; - ::std::vector ret; - switch(GET_TOK(tok, lex)) - { - case TOK_PAREN_OPEN: - lex.putback(tok); - return Parse_TT(lex, false); - - case TOK_IDENT: - case TOK_DOUBLE_COLON: { - lex.putback(tok); - TokenTree inner = Parse_TT_Path(lex, true); - if(GET_TOK(tok, lex) == TOK_BRACE_OPEN) { - lex.putback(tok); - ret.push_back(inner); - ret.push_back(Parse_TT(lex, false)); - } - else { - lex.putback(tok); - return inner; - } - break; } - case TOK_RWORD_SELF: - case TOK_INTEGER: - case TOK_FLOAT: - case TOK_STRING: - return TokenTree(tok); - case TOK_RWORD_MATCH: - ret.push_back(TokenTree(tok)); - ret.push_back(Parse_TT(lex, false)); - break; - case TOK_RWORD_IF: - ret.push_back(TokenTree(tok)); - ret.push_back(Parse_TT_Expr(lex)); - if( GET_TOK(tok, lex) == TOK_RWORD_ELSE ) { - ret.push_back(TokenTree(tok)); - ret.push_back(Parse_TT(lex, false)); - } - else { - lex.putback(tok); - } - break; - default: - // Oh, fail :( - throw ParseError::Unexpected(lex, tok); - } - return TokenTree(ret); + return wlex.get_output(); } /// Parse a token tree expression TokenTree Parse_TT_Expr(TokenStream& lex) { TRACE_FUNCTION; + TTLexer wlex(lex); - Token tok; - ::std::vector ret; - - ret.push_back(Parse_TT_Val(lex)); - // 1. Get left associative blocks until nothing matches - bool cont = true; - while(cont) - { - switch(GET_TOK(tok, lex)) - { - case TOK_PLUS: - case TOK_DASH: - case TOK_SLASH: - case TOK_STAR: - case TOK_PERCENT: - ret.push_back(tok); - ret.push_back(Parse_TT_Val(lex)); - break; - case TOK_PAREN_OPEN: - lex.putback(tok); - ret.push_back(Parse_TT(lex, false)); - break; - case TOK_DOT: - ret.push_back(tok); - GET_CHECK_TOK(tok, lex, TOK_IDENT); - ret.push_back(tok); - switch(GET_TOK(tok, lex)) - { - case TOK_DOUBLE_COLON: - throw ParseError::Todo("Generic type params in TT expr"); - case TOK_PAREN_OPEN: - lex.putback(tok); - ret.push_back(Parse_TT(lex, false)); - break; - default: - lex.putback(tok); - break; - } - break; - default: - lex.putback(tok); - cont = false; - break; - } - } - return TokenTree(ret); - + Parse_Expr1(wlex); + + return wlex.get_output(); } TokenTree Parse_TT_Stmt(TokenStream& lex) { diff --git a/src/parse/lex.cpp b/src/parse/lex.cpp index b290b3f1..d469d46d 100644 --- a/src/parse/lex.cpp +++ b/src/parse/lex.cpp @@ -623,6 +623,7 @@ SERIALISE_TYPE_S(Token, { TTStream::TTStream(const TokenTree& input_tt): m_input_tt(input_tt) { + DEBUG("input_tt = [" << input_tt << "]"); m_stack.push_back( ::std::make_pair(0, &input_tt) ); } TTStream::~TTStream() @@ -671,6 +672,14 @@ TokenStream::~TokenStream() { } +Token TokenStream::innerGetToken() +{ + Token ret = this->realGetToken(); + if( ret.get_pos().filename.size() == 0 ) + ret.set_pos( this->getPosition() ); + //DEBUG("ret.get_pos() = " << ret.get_pos()); + return ret; +} Token TokenStream::getToken() { if( m_cache_valid ) @@ -687,7 +696,7 @@ Token TokenStream::getToken() } else { - Token ret = this->realGetToken(); + Token ret = this->innerGetToken(); ::std::cout << "getToken[" << typeid(*this).name() << "] - " << ret << ::std::endl; return ret; } @@ -723,7 +732,7 @@ eTokenType TokenStream::lookahead(unsigned int i) while( i >= m_lookahead.size() ) { DEBUG("lookahead - read #" << m_lookahead.size()); - m_lookahead.push_back( this->realGetToken() ); + m_lookahead.push_back( this->innerGetToken() ); } return m_lookahead[i].type(); diff --git a/src/parse/lex.hpp b/src/parse/lex.hpp index c754c61d..da56c6ba 100644 --- a/src/parse/lex.hpp +++ b/src/parse/lex.hpp @@ -12,6 +12,23 @@ enum eTokenType #undef _ }; +struct Position +{ + ::std::string filename; + unsigned int line; + + Position(): + filename(""), + line(0) + {} + Position(::std::string filename, unsigned int line): + filename(filename), + line(line) + { + } +}; +extern ::std::ostream& operator<<(::std::ostream& os, const Position& p); + class Token: public Serialisable { @@ -22,6 +39,7 @@ class Token: uint64_t m_intval; double m_floatval; }; + Position m_pos; public: Token(); Token(const Token& t) = default; @@ -30,7 +48,8 @@ public: m_type(t.m_type), m_str( ::std::move(t.m_str) ), m_datatype( t.m_datatype ), - m_intval( t.m_intval ) + m_intval( t.m_intval ), + m_pos( ::std::move(t.m_pos) ) { t.m_type = TOK_NULL; } @@ -45,6 +64,9 @@ public: uint64_t intval() const { return m_intval; } double floatval() const { return m_floatval; } + void set_pos(Position pos) { m_pos = pos; } + const Position& get_pos() const { return m_pos; } + static const char* typestr(enum eTokenType type); static eTokenType typefromstr(const ::std::string& s); @@ -53,27 +75,23 @@ public: extern ::std::ostream& operator<<(::std::ostream& os, const Token& tok); -struct Position -{ - ::std::string filename; - unsigned int line; - - Position(::std::string filename, unsigned int line): - filename(filename), - line(line) - { - } -}; -extern ::std::ostream& operator<<(::std::ostream& os, const Position& p); - /// State the parser needs to pass down via a second channel. struct ParseState { bool disallow_struct_literal = false; + + friend ::std::ostream& operator<<(::std::ostream& os, const ParseState& ps) { + os << "ParseState {"; + if(ps.disallow_struct_literal) os << " disallow_struct_literal"; + os << " }"; + return os; + } }; class TokenStream { + friend class TTLexer; // needs access to internals to know what was consumed + bool m_cache_valid; Token m_cache; ::std::vector m_lookahead; @@ -90,6 +108,8 @@ public: protected: virtual Token realGetToken() = 0; +private: + Token innerGetToken(); }; class SavedParseState @@ -100,9 +120,11 @@ public: SavedParseState(TokenStream& lex, ParseState state): m_lex(lex), m_state(state) - {} + { + } ~SavedParseState() { + DEBUG("Restoring " << m_state); m_lex.parse_state() = m_state; } }; diff --git a/src/parse/parseerror.cpp b/src/parse/parseerror.cpp index e5385257..3905ae33 100644 --- a/src/parse/parseerror.cpp +++ b/src/parse/parseerror.cpp @@ -17,6 +17,11 @@ ParseError::Generic::Generic(const TokenStream& lex, ::std::string message) ::std::cout << lex.getPosition() << ": Generic(" << message << ")" << ::std::endl; } +ParseError::BugCheck::BugCheck(const TokenStream& lex, ::std::string message): + m_message(message) +{ + ::std::cout << lex.getPosition() << "BugCheck(" << message << ")" << ::std::endl; +} ParseError::BugCheck::BugCheck(::std::string message): m_message(message) { @@ -49,12 +54,18 @@ ParseError::BadChar::~BadChar() throw() ParseError::Unexpected::Unexpected(const TokenStream& lex, Token tok): m_tok(tok) { - ::std::cout << lex.getPosition() << ": Unexpected(" << tok << ")" << ::std::endl; + auto pos = tok.get_pos(); + if(pos.filename == "") + pos = lex.getPosition(); + ::std::cout << pos << ": Unexpected(" << tok << ")" << ::std::endl; } ParseError::Unexpected::Unexpected(const TokenStream& lex, Token tok, Token exp): m_tok(tok) { - ::std::cout << lex.getPosition() << ": Unexpected(" << tok << ", " << exp << ")" << ::std::endl; + auto pos = tok.get_pos(); + if(pos.filename == "") + pos = lex.getPosition(); + ::std::cout << pos << ": Unexpected(" << tok << ", " << exp << ")" << ::std::endl; } ParseError::Unexpected::~Unexpected() throw() { diff --git a/src/parse/parseerror.hpp b/src/parse/parseerror.hpp index bfea40b3..911312c5 100644 --- a/src/parse/parseerror.hpp +++ b/src/parse/parseerror.hpp @@ -29,6 +29,7 @@ class BugCheck: ::std::string m_message; public: BugCheck(::std::string message); + BugCheck(const TokenStream& lex, ::std::string message); virtual ~BugCheck() throw () {} }; @@ -64,6 +65,8 @@ public: }; +#define ASSERT(lex, cnd) do { if( !(cnd) ) throw ParseError::BugCheck(lex, "Assertion failed: "#cnd); } while(0) + } #endif // PARSEERROR_HPP_INCLUDED diff --git a/src/parse/root.cpp b/src/parse/root.cpp index 5b14ff02..c64dbb17 100644 --- a/src/parse/root.cpp +++ b/src/parse/root.cpp @@ -149,9 +149,16 @@ TypeRef Parse_Type(TokenStream& lex) { TRACE_FUNCTION; - Token tok = lex.getToken(); - switch(tok.type()) + Token tok; + + switch( GET_TOK(tok, lex) ) { + case TOK_RWORD_EXTERN: + GET_CHECK_TOK(tok, lex, TOK_STRING); + // abi = tok.str(); + GET_CHECK_TOK(tok, lex, TOK_RWORD_FN); + case TOK_RWORD_FN: + throw ParseError::Todo(lex, "Function types"); case TOK_LT: { DEBUG("Associated type"); // ::Inner @@ -306,7 +313,7 @@ void Parse_TypeConds(TokenStream& lex, AST::TypeParams& params) } /// Parse a function definition (after the 'fn') -AST::Function Parse_FunctionDef(TokenStream& lex, bool allow_no_code=false) +AST::Function Parse_FunctionDef(TokenStream& lex, AST::MetaItems attrs, bool allow_no_code=false) { TRACE_FUNCTION; @@ -618,7 +625,7 @@ AST::Trait Parse_TraitDef(TokenStream& lex, const AST::MetaItems& meta_items) case TOK_RWORD_FN: { GET_CHECK_TOK(tok, lex, TOK_IDENT); ::std::string name = tok.str(); - trait.add_function( ::std::move(name), Parse_FunctionDef(lex, true) ); + trait.add_function( ::std::move(name), Parse_FunctionDef(lex, item_attrs, true) ); break; } default: throw ParseError::Generic("Unexpected token, expected 'type' or 'fn'"); @@ -776,7 +783,7 @@ AST::Impl Parse_Impl(TokenStream& lex) case TOK_RWORD_FN: { GET_CHECK_TOK(tok, lex, TOK_IDENT); ::std::string name = tok.str(); - impl.add_function(is_public, name, Parse_FunctionDef(lex)); + impl.add_function(is_public, name, Parse_FunctionDef(lex, item_attrs)); break; } default: @@ -1255,10 +1262,22 @@ void Parse_ModRoot_Items(TokenStream& lex, AST::Crate& crate, AST::Module& mod, mod.add_global(is_public, is_mut, name, type, val); break; } + case TOK_RWORD_UNSAFE: + meta_items.push_back( AST::MetaItem("#UNSAFE") ); + switch(GET_TOK(tok, lex)) + { + case TOK_RWORD_FN: + GET_CHECK_TOK(tok, lex, TOK_IDENT); + mod.add_function(is_public, tok.str(), Parse_FunctionDef(lex, ::std::move(meta_items))); + break; + default: + throw ParseError::Unexpected(lex, tok); + } + break; case TOK_RWORD_FN: { GET_CHECK_TOK(tok, lex, TOK_IDENT); ::std::string name = tok.str(); - mod.add_function(is_public, name, Parse_FunctionDef(lex)); + mod.add_function(is_public, name, Parse_FunctionDef(lex, ::std::move(meta_items))); break; } case TOK_RWORD_TYPE: { GET_CHECK_TOK(tok, lex, TOK_IDENT); diff --git a/src/parse/tokentree.hpp b/src/parse/tokentree.hpp index b3156f66..eaaf6623 100644 --- a/src/parse/tokentree.hpp +++ b/src/parse/tokentree.hpp @@ -28,6 +28,13 @@ public: const Token& tok() const { return m_tok; } + + friend ::std::ostream& operator<<(::std::ostream& os, const TokenTree& tt) { + if( tt.m_subtrees.size() == 0 ) + return os << "TokenTree(" << tt.m_tok << ")"; + else + return os << "TokenTree([" << tt.m_subtrees << "])"; + } }; class TTStream: -- cgit v1.2.3