From e71fccd80610bc05b0a90338b2b3beb9a0b94c22 Mon Sep 17 00:00:00 2001 From: John Hodge Date: Tue, 10 Mar 2015 15:37:02 +0800 Subject: Rework macro handling to support correct repetitions --- Makefile | 4 +- samples/env_logger.rs | 0 samples/getopts.rs | 0 samples/log.rs | 3 +- samples/std.rs | 9 +- src/ast/ast.hpp | 5 +- src/ast/expr.cpp | 18 +++- src/ast/expr.hpp | 16 +++- src/convert/resolve.cpp | 6 +- src/convert/typecheck_expr.cpp | 6 +- src/dump_as_rust.cpp | 15 +++- src/macros.cpp | 192 +++++++++++++++++++++++++++++++-------- src/macros.hpp | 66 +++++++++++--- src/parse/expr.cpp | 29 ++++-- src/parse/lex.hpp | 4 +- src/parse/root.cpp | 200 +++++++++++++++++++++++++++++++++++------ 16 files changed, 467 insertions(+), 106 deletions(-) create mode 100644 samples/env_logger.rs create mode 100644 samples/getopts.rs diff --git a/Makefile b/Makefile index 41d68943..56c7485f 100644 --- a/Makefile +++ b/Makefile @@ -29,11 +29,11 @@ all: $(BIN) clean: $(RM) -r $(BIN) $(OBJ) -output/%.ast: samples/%.rs +output/%.ast: samples/%.rs $(BIN) @mkdir -p output/ $(DBG) $(BIN) $< --emit ast -o $@ 2>&1 | tee $@_dbg.txt ; test $${PIPESTATUS[0]} -eq 0 -test: $(BIN) samples/1.rs output/std.ast output/log.ast +test: $(BIN) samples/1.rs output/std.ast output/log.ast output/env_logger.ast output/getopts.ast mkdir -p output/ # $(DBG) $(BIN) samples/1.rs --crate-path output/std.ast -o output/test.c 2>&1 | tee output/1_dbg.txt $(DBG) $(BIN) ../../BinaryView2/src/main.rs --crate-path output/ -o output/test.c 2>&1 | tee output/1_dbg.txt ; test $${PIPESTATUS[0]} -eq 0 diff --git a/samples/env_logger.rs b/samples/env_logger.rs new file mode 100644 index 00000000..e69de29b diff --git a/samples/getopts.rs b/samples/getopts.rs new file mode 100644 index 00000000..e69de29b diff --git a/samples/log.rs b/samples/log.rs index eb9f544e..7b9f693e 100644 --- a/samples/log.rs +++ b/samples/log.rs @@ -1,5 +1,6 @@ +#[macro_export] macro_rules! error{ - ($( $v:tt )*) => {}; + ($( $v:tt )*) => {()}; } diff --git a/samples/std.rs b/samples/std.rs index 100fcb5e..11913768 100644 --- a/samples/std.rs +++ b/samples/std.rs @@ -6,7 +6,7 @@ macro_rules! try { } #[macro_export] macro_rules! assert{ - ($e:expr) => ({ if !($e) { panic!("Assertion {} failed", stringify!($e)); } }); + ($e:expr) => ({ if !($e) { panic!("assertion failed: {}", stringify!($e)); } }); } #[macro_export] macro_rules! assert_eq{ @@ -22,6 +22,13 @@ macro_rules! assert_eq{ }); } +#[macro_export] +macro_rules! write{ + ($f:expr, $($fmt:tt)*) => { + $crate::fmt::write($f, format_args!($($fmt)*)) + }; +} + pub mod option { pub enum Option diff --git a/src/ast/ast.hpp b/src/ast/ast.hpp index c1c9c283..1da79696 100644 --- a/src/ast/ast.hpp +++ b/src/ast/ast.hpp @@ -159,8 +159,9 @@ class MetaItem; class MetaItems: public Serialisable { - ::std::vector m_items; public: + ::std::vector m_items; + MetaItems() {} MetaItems(::std::vector items): m_items(items) @@ -197,6 +198,8 @@ public: void mark_used() {} const ::std::string& name() const { return m_name; } + bool has_sub_items() const { return m_sub_items.m_items.size() > 0; } + MetaItems& items() { return m_sub_items; } SERIALISABLE_PROTOTYPES(); }; diff --git a/src/ast/expr.cpp b/src/ast/expr.cpp index 9ddd75a9..0ed99e3f 100644 --- a/src/ast/expr.cpp +++ b/src/ast/expr.cpp @@ -160,6 +160,11 @@ NODE(ExprNode_CallObject, { os << ")"; }) +SERIALISE_TYPE_A(ExprNode_Match::Arm::, "ExprNode_Match_Arm", { + s.item(m_patterns); + s.item(m_cond); + s.item(m_code); +}); NODE(ExprNode_Match, { s.item(m_val); s.item(m_arms); @@ -167,7 +172,11 @@ NODE(ExprNode_Match, { os << "match ("<<*m_val<<") {"; for(const auto& arm : m_arms) { - os << " " << arm.first << " => " << *arm.second << ","; + for( const auto& pat : arm.m_patterns ) + os << " " << pat; + os << " if " << *arm.m_cond; + + os << " => " << *arm.m_code << ","; } os << "}"; }) @@ -326,6 +335,8 @@ NODE(ExprNode_BinOp, { case MULTIPLY: os << "*"; break; case DIVIDE: os << "/"; break; case MODULO: os << "%"; break; + case ADD: os << "+"; break; + case SUB: os << "-"; break; } os << " " << *m_right << ")"; }) @@ -427,7 +438,10 @@ NV(ExprNode_Match, INDENT(); visit(node.m_val); for( auto& arm : node.m_arms ) - visit(arm.second); + { + visit(arm.m_cond); + visit(arm.m_code); + } UNINDENT(); }) NV(ExprNode_If, diff --git a/src/ast/expr.hpp b/src/ast/expr.hpp index 55da44d6..e01606d7 100644 --- a/src/ast/expr.hpp +++ b/src/ast/expr.hpp @@ -173,12 +173,22 @@ struct ExprNode_CallObject: struct ExprNode_Match: public ExprNode { - typedef ::std::vector< ::std::pair > > arm_t; + struct Arm: + public Serialisable + { + ::std::vector m_patterns; + unique_ptr m_cond; + + unique_ptr m_code; + + SERIALISABLE_PROTOTYPES(); + }; + unique_ptr m_val; - arm_t m_arms; + ::std::vector m_arms; ExprNode_Match() {} - ExprNode_Match(unique_ptr&& val, arm_t&& arms): + ExprNode_Match(unique_ptr val, ::std::vector arms): m_val( ::std::move(val) ), m_arms( ::std::move(arms) ) { diff --git a/src/convert/resolve.cpp b/src/convert/resolve.cpp index 9a1033c9..9aae9847 100644 --- a/src/convert/resolve.cpp +++ b/src/convert/resolve.cpp @@ -105,8 +105,10 @@ public: for( auto& arm : node.m_arms ) { m_res.start_scope(); - m_res.handle_pattern(arm.first, TypeRef()); - AST::NodeVisitor::visit(arm.second); + for( auto& pat : arm.m_patterns ) + m_res.handle_pattern(pat, TypeRef()); + AST::NodeVisitor::visit(arm.m_cond); + AST::NodeVisitor::visit(arm.m_code); m_res.end_scope(); } } diff --git a/src/convert/typecheck_expr.cpp b/src/convert/typecheck_expr.cpp index 49a17243..d7f8afb1 100644 --- a/src/convert/typecheck_expr.cpp +++ b/src/convert/typecheck_expr.cpp @@ -371,8 +371,10 @@ void CTC_NodeVisitor::visit(AST::ExprNode_Match& node) for( auto& arm : node.m_arms ) { m_tc.start_scope(); - m_tc.handle_pattern(arm.first, node.m_val->get_res_type()); - AST::NodeVisitor::visit(arm.second); + for( auto& pat : arm.m_patterns ) + m_tc.handle_pattern(pat, node.m_val->get_res_type()); + AST::NodeVisitor::visit(arm.m_cond); + AST::NodeVisitor::visit(arm.m_code); m_tc.end_scope(); } } diff --git a/src/dump_as_rust.cpp b/src/dump_as_rust.cpp index c9a64371..d28ab084 100644 --- a/src/dump_as_rust.cpp +++ b/src/dump_as_rust.cpp @@ -134,9 +134,20 @@ public: for( auto& arm : n.m_arms ) { m_os << indent(); - print_pattern( arm.first ); + bool is_first = true; + for( const auto& pat : arm.m_patterns ) { + if(!is_first) + m_os << "|"; + is_first = false; + print_pattern(pat); + } + if( arm.m_cond ) + { + m_os << " if "; + AST::NodeVisitor::visit(arm.m_cond); + } m_os << " => "; - AST::NodeVisitor::visit(arm.second); + AST::NodeVisitor::visit(arm.m_code); m_os << ",\n"; } diff --git a/src/macros.cpp b/src/macros.cpp index 71e0b85e..b20f9fad 100644 --- a/src/macros.cpp +++ b/src/macros.cpp @@ -12,6 +12,11 @@ typedef ::std::map< ::std::string, MacroRules> t_macro_regs; t_macro_regs g_macro_registrations; const LList* g_macro_module; +TokenTree g_crate_path_tt = TokenTree({ + TokenTree(Token(TOK_DOUBLE_COLON)), + TokenTree(Token(TOK_STRING, "--CRATE--")), + }); + void Macro_SetModule(const LList& mod) { g_macro_module = &mod; @@ -70,10 +75,7 @@ void Macro_InitDefaults() } } -typedef ::std::map single_tts_t; -typedef ::std::multimap repeated_tts_t; - -void Macro_HandlePattern(TTStream& lex, const MacroPatEnt& pat, bool rep, single_tts_t& bound_tts, repeated_tts_t& rep_bound_tts) +void Macro_HandlePattern(TTStream& lex, const MacroPatEnt& pat, unsigned int layer, MacroExpander::t_mappings& bound_tts) { Token tok; TokenTree val; @@ -85,7 +87,7 @@ void Macro_HandlePattern(TTStream& lex, const MacroPatEnt& pat, bool rep, single break; case MacroPatEnt::PAT_LOOP: //case MacroPatEnt::PAT_OPTLOOP: - if( rep ) + if( layer ) { throw ParseError::BugCheck("Nested macro loop"); } @@ -97,7 +99,7 @@ void Macro_HandlePattern(TTStream& lex, const MacroPatEnt& pat, bool rep, single DEBUG("Try"); TTStream saved = lex; try { - Macro_HandlePattern(lex, pat.subpats[0], true, bound_tts, rep_bound_tts); + Macro_HandlePattern(lex, pat.subpats[0], layer+1, bound_tts); } catch(const ParseError::Base& e) { DEBUG("Breakout"); @@ -106,7 +108,7 @@ void Macro_HandlePattern(TTStream& lex, const MacroPatEnt& pat, bool rep, single } for( unsigned int i = 1; i < pat.subpats.size(); i ++ ) { - Macro_HandlePattern(lex, pat.subpats[i], true, bound_tts, rep_bound_tts); + Macro_HandlePattern(lex, pat.subpats[i], layer+1, bound_tts); } DEBUG("succ"); if( pat.tok.type() != TOK_NULL ) @@ -147,10 +149,7 @@ void Macro_HandlePattern(TTStream& lex, const MacroPatEnt& pat, bool rep, single GET_CHECK_TOK(tok, lex, TOK_IDENT); val = TokenTree(tok); } - if(rep) - rep_bound_tts.insert( std::make_pair(pat.name.c_str(), val) ); - else - bound_tts.insert( std::make_pair(pat.name.c_str(), val) ); + bound_tts.insert( ::std::make_pair( ::std::make_pair(layer, pat.name.c_str()), ::std::move(val) ) ); break; //default: @@ -182,20 +181,19 @@ MacroExpander Macro_InvokeInt(const char *name, const MacroRules& rules, TokenTr throw ParseError::Unexpected(lex, tok); } */ - ::std::map bound_tts; - ::std::multimap rep_bound_tts; + MacroExpander::t_mappings bound_tts; // Parse according to rules try { for(const auto& pat : rule.m_pattern) { - Macro_HandlePattern(lex, pat, false, bound_tts, rep_bound_tts); + Macro_HandlePattern(lex, pat, 0, bound_tts); } //GET_CHECK_TOK(tok, lex, close); GET_CHECK_TOK(tok, lex, TOK_EOF); DEBUG( rule.m_contents.size() << " rule contents bound to " << bound_tts.size() << " values - " << name ); - return MacroExpander(rule.m_contents, bound_tts); + return MacroExpander(rule.m_contents, bound_tts, g_crate_path_tt); } catch(const ParseError::Base& e) { @@ -213,7 +211,17 @@ MacroExpander Macro_Invoke(const char* name, TokenTree input) if( g_macro_registrations.size() == 0 ) { Macro_InitDefaults(); } - // 1. Locate macro with that name + + // 1. Builtin syntax extensions + { + const ::std::string sname = name; + if( sname == "format_args" ) + { + throw ParseError::Todo("format_args"); + } + } + + // 2. Locate macro with that name t_macro_regs::iterator macro_reg = g_macro_registrations.find(name); if( macro_reg != g_macro_registrations.end() ) { @@ -259,26 +267,132 @@ Token MacroExpander::realGetToken() return rv; m_ttstream.reset(); } - DEBUG("ofs " << m_ofs << " < " << m_contents.size()); - if( m_ofs < m_contents.size() ) + //DEBUG("ofs " << m_offsets << " < " << m_root_contents.size()); + + // Check offset of lowest layer + while(m_offsets.size() > 0) { - const MacroRuleEnt& ent = m_contents[m_ofs]; - m_ofs ++; - if( ent.name.size() != 0 ) { - // Binding! - m_ttstream.reset( new TTStream(m_mappings.at(ent.name.c_str())) ); - return m_ttstream->getToken(); + assert(m_offsets.size() > 0); + unsigned int layer = m_offsets.size() - 1; + // - If that layer has hit its limit + const auto& ents = *m_cur_ents; + size_t idx = m_offsets.back().first; + m_offsets.back().first ++; + + //DEBUG("ents = " << ents); + + // Check if limit has been reached + if( idx < ents.size() ) + { + const auto& ent = ents[idx]; + // - If not, just handle the next entry + // Check type of entry + if( ent.name != "" ) + { + // - Name + // HACK: Handle $crate with a special name + if( ent.name == "*crate" ) { + m_ttstream.reset( new TTStream(m_crate_path) ); + return m_ttstream->getToken(); + } + else { + const auto tt_i = m_mappings.find( ::std::make_pair(layer, ent.name.c_str()) ); + if( tt_i == m_mappings.end() ) + throw ParseError::Generic( FMT("Cannot find mapping name: " << ent.name << " for layer " << layer) ); + m_ttstream.reset( new TTStream(tt_i->second) ); + return m_ttstream->getToken(); + } + } + else if( ent.subpats.size() != 0 ) + { + // New layer + // - Push an offset + m_offsets.push_back( ::std::make_pair(0, 0) ); + // - Save the current layer + m_cur_ents = getCurLayer(); + // - Restart loop for new layer + } + else + { + // Raw token + return ent.tok; + } + // Fall through for loop } - else { - return ent.tok; + else + { + // - Otherwise, restart/end loop and fall through + unsigned int layer_max = m_layer_counts.at(layer); + if( m_offsets.back().second + 1 < layer_max ) + { + DEBUG("Restart layer"); + m_offsets.back().first = 0; + m_offsets.back().second ++; + // Fall through and restart layer + } + else + { + DEBUG("Terminate layer"); + // Terminate loop, fall through to lower layers + m_offsets.pop_back(); + // - Special case: End of macro, avoid issues + if( m_offsets.size() == 0 ) + break; + m_cur_ents = getCurLayer(); + } } - throw ParseError::Todo("MacroExpander - realGetToken"); - } + } // while( m_offsets NONEMPTY ) DEBUG("EOF"); return Token(TOK_EOF); } +/// Count the number of names at each layer +void MacroExpander::prep_counts() +{ + struct TMP { + size_t count; + const char *first_name; + }; + ::std::vector counts; + + for( const auto& ent : m_mappings ) + { + unsigned int layer = ent.first.first; + const char *name = ent.first.second; + + if( layer >= counts.size() ) + { + counts.resize(layer+1); + } + auto& l = counts[layer]; + if( l.first_name == NULL || ::std::strcmp(l.first_name, name) == 0 ) + { + l.first_name = name; + l.count += 1; + } + } + + for( const auto& l : counts ) + { + m_layer_counts.push_back(l.count); + } +} +const ::std::vector* MacroExpander::getCurLayer() const +{ + assert( m_offsets.size() > 0 ); + const ::std::vector* ents = &m_root_contents; + for( unsigned int i = 0; i < m_offsets.size()-1; i ++ ) + { + unsigned int ofs = m_offsets[i].first; + //DEBUG(i << " ofs=" << ofs << " / " << ents->size()); + assert( ofs > 0 && ofs <= ents->size() ); + ents = &(*ents)[ofs-1].subpats; + //DEBUG("ents = " << ents); + } + return ents; +} + SERIALISE_TYPE_S(MacroRule, { s.item(m_pattern); s.item(m_contents); @@ -287,19 +401,18 @@ SERIALISE_TYPE_S(MacroRule, { void operator%(Serialiser& s, MacroPatEnt::Type c) { switch(c) { - #define _(ns,v) case ns v: s << #v; return; - _(MacroPatEnt::,PAT_TOKEN); - _(MacroPatEnt::,PAT_TT); - _(MacroPatEnt::,PAT_EXPR); - _(MacroPatEnt::,PAT_LOOP); - //_(MacroPatEnt::,PAT_OPTLOOP); - _(MacroPatEnt::,PAT_STMT); - _(MacroPatEnt::,PAT_PATH); - _(MacroPatEnt::,PAT_BLOCK); - _(MacroPatEnt::,PAT_IDENT); + #define _(v) case MacroPatEnt::v: s << #v; return + _(PAT_TOKEN); + _(PAT_TT); + _(PAT_EXPR); + _(PAT_LOOP); + //_(PAT_OPTLOOP); + _(PAT_STMT); + _(PAT_PATH); + _(PAT_BLOCK); + _(PAT_IDENT); #undef _ } - s << "--TODO--"; } void operator%(::Deserialiser& s, MacroPatEnt::Type& c) { ::std::string n; @@ -329,4 +442,5 @@ SERIALISE_TYPE_S(MacroPatEnt, { SERIALISE_TYPE_S(MacroRuleEnt, { s.item(name); s.item(tok); + s.item(subpats); }); diff --git a/src/macros.hpp b/src/macros.hpp index 4de41782..74001cae 100644 --- a/src/macros.hpp +++ b/src/macros.hpp @@ -16,6 +16,7 @@ class MacroRuleEnt: Token tok; ::std::string name; + ::std::vector subpats; public: MacroRuleEnt(): tok(TOK_NULL), @@ -31,6 +32,15 @@ public: name(name) { } + MacroRuleEnt(Token tok, ::std::vector subpats): + tok(tok), + subpats(subpats) + { + } + + friend ::std::ostream& operator<<(::std::ostream& os, const MacroRuleEnt& x) { + return os << "MacroRuleEnt( '"< MacroRules; -struct cmp_str { - bool operator()(const char* a, const char* b) const { - return ::std::strcmp(a, b) < 0; - } -}; - class MacroExpander: public TokenStream { - typedef ::std::map t_mappings; +public: + // MultiMap (layer, name) -> TokenTree + // - Multiple values are only allowed for layer>0 + typedef ::std::pair t_mapping_key; + + struct cmp_mk { + bool operator()(const t_mapping_key& a, const t_mapping_key& b) const { + return a.first < b.first || ::std::strcmp(a.second, b.second) < 0; + } + }; + typedef ::std::multimap t_mappings; + +private: + const TokenTree& m_crate_path; + const ::std::vector& m_root_contents; const t_mappings m_mappings; - const ::std::vector& m_contents; - size_t m_ofs; + + /// Layer states : Index and Iteration + ::std::vector< ::std::pair > m_offsets; + + /// Cached pointer to the current layer + const ::std::vector* m_cur_ents; // For faster lookup. + /// Iteration counts for each layer + ::std::vector m_layer_counts; ::std::auto_ptr m_ttstream; + public: MacroExpander(const MacroExpander& x): + m_crate_path(x.m_crate_path), + m_root_contents(x.m_root_contents), m_mappings(x.m_mappings), - m_contents(x.m_contents), - m_ofs(0) + m_offsets({ {0,0} }), + m_cur_ents(&m_root_contents) { + prep_counts(); } - MacroExpander(const ::std::vector& contents, t_mappings mappings): + MacroExpander(const ::std::vector& contents, t_mappings mappings, const TokenTree& crate_path): + m_crate_path(crate_path), + m_root_contents(contents), m_mappings(mappings), - m_contents(contents), - m_ofs(0) + m_offsets({ {0,0} }), + m_cur_ents(&m_root_contents) { + prep_counts(); } virtual Position getPosition() const override; virtual Token realGetToken() override; +private: + const ::std::vector* getCurLayer() const; + void prep_counts(); }; extern void Macro_SetModule(const LList& mod); diff --git a/src/parse/expr.cpp b/src/parse/expr.cpp index 7fb41b0f..be4d67a6 100644 --- a/src/parse/expr.cpp +++ b/src/parse/expr.cpp @@ -356,22 +356,34 @@ ExprNodeP Parse_Expr_Match(TokenStream& lex) Token tok; // 1. Get expression - ExprNodeP switch_val = Parse_Expr1(lex); + ExprNodeP switch_val; + { + SET_PARSE_FLAG(lex, disallow_struct_literal); + switch_val = Parse_Expr1(lex); + } GET_CHECK_TOK(tok, lex, TOK_BRACE_OPEN); - ::std::vector< ::std::pair > arms; + ::std::vector< AST::ExprNode_Match::Arm > arms; do { if( GET_TOK(tok, lex) == TOK_BRACE_CLOSE ) break; lex.putback(tok); - AST::Pattern pat = Parse_Pattern(lex); + AST::ExprNode_Match::Arm arm; + do { + arm.m_patterns.push_back( Parse_Pattern(lex) ); + } while( GET_TOK(tok, lex) == TOK_PIPE ); - GET_CHECK_TOK(tok, lex, TOK_FATARROW); + if( tok.type() == TOK_RWORD_IF ) + { + arm.m_cond = Parse_Expr1(lex); + GET_TOK(tok, lex); + } + CHECK_TOK(tok, TOK_FATARROW); bool opt_semicolon = false; - ExprNodeP val = Parse_Stmt(lex, opt_semicolon); + arm.m_code = Parse_Stmt(lex, opt_semicolon); - arms.push_back( ::std::make_pair( ::std::move(pat), ::std::move(val) ) ); + arms.push_back( ::std::move(arm) ); if( GET_TOK(tok, lex) == TOK_COMMA ) continue; @@ -708,6 +720,9 @@ TokenTree Parse_TT(TokenStream& lex, bool unwrapped) case TOK_BRACE_OPEN: closer = TOK_BRACE_CLOSE; break; + case TOK_EOF: + case TOK_NULL: + throw ParseError::Unexpected(lex, tok); default: return TokenTree(tok); } @@ -717,6 +732,8 @@ TokenTree Parse_TT(TokenStream& lex, bool unwrapped) items.push_back(tok); while(GET_TOK(tok, lex) != closer && tok.type() != TOK_EOF) { + if( tok.type() == TOK_NULL ) + throw ParseError::Unexpected(lex, tok); lex.putback(tok); items.push_back(Parse_TT(lex, false)); } diff --git a/src/parse/lex.hpp b/src/parse/lex.hpp index 41bac795..25c2d530 100644 --- a/src/parse/lex.hpp +++ b/src/parse/lex.hpp @@ -95,8 +95,8 @@ public: } }; -#define SET_PARSE_FLAG(lex, flag) SavedParseState(lex, lex.parse_state()); lex.parse_state().flag = true -#define CLEAR_PARSE_FLAG(lex, flag) SavedParseState(lex, lex.parse_state()); lex.parse_state().flag = false +#define SET_PARSE_FLAG(lex, flag) SavedParseState _sps(lex, lex.parse_state()); lex.parse_state().flag = true +#define CLEAR_PARSE_FLAG(lex, flag) SavedParseState _sps(lex, lex.parse_state()); lex.parse_state().flag = false #define CHECK_PARSE_FLAG(lex, flag) (lex.parse_state().flag == true) class Lexer diff --git a/src/parse/root.cpp b/src/parse/root.cpp index e3d0ac0a..8ae7bc1f 100644 --- a/src/parse/root.cpp +++ b/src/parse/root.cpp @@ -95,7 +95,18 @@ AST::Path Parse_PathFrom(TokenStream& lex, AST::Path path, eParsePathGenericMode AST::Path Parse_Path(TokenStream& lex, bool is_abs, eParsePathGenericMode generic_mode) { if( is_abs ) - return Parse_PathFrom(lex, AST::Path(AST::Path::TagAbsolute()), generic_mode); + { + Token tok; + if( GET_TOK(tok, lex) == TOK_STRING ) { + ::std::string cratename = tok.str(); + GET_CHECK_TOK(tok, lex, TOK_DOUBLE_COLON); + return Parse_PathFrom(lex, AST::Path(cratename, {}), generic_mode); + } + else { + lex.putback(tok); + return Parse_PathFrom(lex, AST::Path(AST::Path::TagAbsolute()), generic_mode); + } + } else return Parse_PathFrom(lex, AST::Path(), generic_mode); } @@ -794,25 +805,27 @@ void Parse_Use(Preproc& lex, ::std::function fcn fcn(path, name); } -MacroRule Parse_MacroRules_Var(Preproc& lex) +::std::vector Parse_MacroRules_Pat(TokenStream& lex, bool allow_sub, enum eTokenType open, enum eTokenType close) { TRACE_FUNCTION; Token tok; - MacroRule rule; + ::std::vector ret; - // Pattern - enum eTokenType close; - switch(GET_TOK(tok, lex)) - { - case TOK_BRACE_OPEN: close = TOK_BRACE_CLOSE; break; - case TOK_PAREN_OPEN: close = TOK_PAREN_CLOSE; break; - default: - throw ParseError::Unexpected(lex, tok); - } - // - Pattern entries - while( GET_TOK(tok, lex) != close ) + int depth = 0; + while( GET_TOK(tok, lex) != close || depth > 0 ) { + if( tok.type() == open ) + { + depth ++; + } + else if( tok.type() == close ) + { + if(depth == 0) + throw ParseError::Generic(FMT("Unmatched " << Token(close) << " in macro pattern")); + depth --; + } + switch(tok.type()) { case TOK_DOLLAR: @@ -823,43 +836,122 @@ MacroRule Parse_MacroRules_Var(Preproc& lex) GET_CHECK_TOK(tok, lex, TOK_COLON); GET_CHECK_TOK(tok, lex, TOK_IDENT); ::std::string type = tok.str(); - if( type == "expr" ) - rule.m_pattern.push_back( MacroPatEnt(name, MacroPatEnt::PAT_EXPR) ); + if(0) + ; + else if( type == "expr" ) + ret.push_back( MacroPatEnt(name, MacroPatEnt::PAT_EXPR) ); + else if( type == "tt" ) + ret.push_back( MacroPatEnt(name, MacroPatEnt::PAT_TT) ); else throw ParseError::Generic(FMT("Unknown fragment type " << type)); break; } case TOK_PAREN_OPEN: - throw ParseError::Todo("Repetitions in macro_rules"); + if( allow_sub ) + { + auto subpat = Parse_MacroRules_Pat(lex, false, TOK_PAREN_OPEN, TOK_PAREN_CLOSE); + enum eTokenType joiner = TOK_NULL; + GET_TOK(tok, lex); + if( tok.type() != TOK_PLUS && tok.type() != TOK_STAR ) + { + DEBUG("Joiner = " << tok); + joiner = tok.type(); + GET_TOK(tok, lex); + } + DEBUG("tok = " << tok); + switch(tok.type()) + { + case TOK_PLUS: + throw ParseError::Todo(lex, "+ repetitions"); + break; + case TOK_STAR: + DEBUG("$()* " << subpat); + ret.push_back( MacroPatEnt(Token(joiner), ::std::move(subpat)) ); + break; + default: + throw ParseError::Unexpected(lex, tok); + } + } + else + { + throw ParseError::Generic(FMT("Nested repetitions in macro")); + } + break; default: throw ParseError::Unexpected(lex, tok); } break; + case TOK_EOF: + throw ParseError::Unexpected(lex, tok); default: - rule.m_pattern.push_back( MacroPatEnt(tok) ); + ret.push_back( MacroPatEnt(tok) ); break; } } - GET_CHECK_TOK(tok, lex, TOK_FATARROW); + return ret; +} - // Replacement - auto rep = Parse_TT(lex, true); // - don't care about the enclosing brackets - TTStream slex(rep); +::std::vector Parse_MacroRules_Cont(Preproc& lex, bool allow_sub, enum eTokenType open, enum eTokenType close) +{ + TRACE_FUNCTION; - // parse token tree into a flattend replacement, handling expansions - while(GET_TOK(tok, slex) != TOK_EOF) + Token tok; + ::std::vector ret; + + int depth = 0; + while( GET_TOK(tok, lex) != close || depth > 0 ) { + if( tok.type() == TOK_EOF ) { + throw ParseError::Unexpected(lex, tok); + } + if( tok.type() == TOK_NULL ) continue ; + + if( tok.type() == open ) + { + DEBUG("depth++"); + depth ++; + } + else if( tok.type() == close ) + { + DEBUG("depth--"); + if(depth == 0) + throw ParseError::Generic(FMT("Unmatched " << Token(close) << " in macro content")); + depth --; + } + if( tok.type() == TOK_DOLLAR ) { - GET_TOK(tok, slex); + GET_TOK(tok, lex); - if( tok.type() == TOK_PAREN_OPEN ) + if( allow_sub && tok.type() == TOK_PAREN_OPEN ) { - throw ParseError::Todo("Repetitions in macro_rules content"); + auto content = Parse_MacroRules_Cont(lex, false, TOK_PAREN_OPEN, TOK_PAREN_CLOSE); + + GET_TOK(tok, lex); + enum eTokenType joiner = TOK_NULL; + if( tok.type() != TOK_PLUS && tok.type() != TOK_STAR ) + { + joiner = tok.type(); + GET_TOK(tok, lex); + } + DEBUG("joiner = " << Token(joiner) << ", content = " << content); + switch(tok.type()) + { + case TOK_STAR: + ret.push_back( MacroRuleEnt(joiner, ::std::move(content)) ); + break; + default: + throw ParseError::Unexpected(lex, tok); + } + } else if( tok.type() == TOK_IDENT ) { - rule.m_contents.push_back( MacroRuleEnt(tok.str()) ); + ret.push_back( MacroRuleEnt(tok.str()) ); + } + else if( tok.type() == TOK_RWORD_CRATE ) + { + ret.push_back( MacroRuleEnt("*crate") ); } else { @@ -868,10 +960,46 @@ MacroRule Parse_MacroRules_Var(Preproc& lex) } else { - rule.m_contents.push_back( MacroRuleEnt(tok) ); + ret.push_back( MacroRuleEnt(tok) ); } } + + return ret; +} + +MacroRule Parse_MacroRules_Var(Preproc& lex) +{ + TRACE_FUNCTION; + Token tok; + + MacroRule rule; + + // Pattern + enum eTokenType close; + switch(GET_TOK(tok, lex)) + { + case TOK_BRACE_OPEN: close = TOK_BRACE_CLOSE; break; + case TOK_PAREN_OPEN: close = TOK_PAREN_CLOSE; break; + default: + throw ParseError::Unexpected(lex, tok); + } + // - Pattern entries + rule.m_pattern = Parse_MacroRules_Pat(lex, true, tok.type(), close); + + GET_CHECK_TOK(tok, lex, TOK_FATARROW); + // Replacement + switch(GET_TOK(tok, lex)) + { + case TOK_BRACE_OPEN: close = TOK_BRACE_CLOSE; break; + case TOK_PAREN_OPEN: close = TOK_PAREN_CLOSE; break; + default: + throw ParseError::Unexpected(lex, tok); + } + rule.m_contents = Parse_MacroRules_Cont(lex, true, tok.type(), close); + + DEBUG("Rule - ["< "<has_sub_items() ) + { + throw ParseError::Todo("selective macro_use"); + } + else + { + mod.add_macro_import(crate, name, ""); + } + } + GET_CHECK_TOK(tok, lex, TOK_SEMICOLON); break; } default: -- cgit v1.2.3