diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/ast/crate.cpp | 1 | ||||
| -rw-r--r-- | src/ast/dump.cpp | 5 | ||||
| -rw-r--r-- | src/expand/asm.cpp | 1 | ||||
| -rw-r--r-- | src/expand/cfg.cpp | 9 | ||||
| -rw-r--r-- | src/expand/concat.cpp | 3 | ||||
| -rw-r--r-- | src/expand/env.cpp | 1 | ||||
| -rw-r--r-- | src/expand/file_line.cpp | 6 | ||||
| -rw-r--r-- | src/expand/format_args.cpp | 2 | ||||
| -rw-r--r-- | src/expand/include.cpp | 4 | ||||
| -rw-r--r-- | src/expand/macro_rules.cpp | 1 | ||||
| -rw-r--r-- | src/expand/rustc_diagnostics.cpp | 1 | ||||
| -rw-r--r-- | src/expand/stringify.cpp | 6 | ||||
| -rw-r--r-- | src/include/span.hpp | 1 | ||||
| -rw-r--r-- | src/macro_rules/eval.cpp | 2 | ||||
| -rw-r--r-- | src/parse/common.hpp | 4 | ||||
| -rw-r--r-- | src/parse/lex.cpp | 238 | ||||
| -rw-r--r-- | src/parse/lex.hpp | 93 | ||||
| -rw-r--r-- | src/parse/parseerror.hpp | 2 | ||||
| -rw-r--r-- | src/parse/root.cpp | 6 | ||||
| -rw-r--r-- | src/parse/tokenstream.cpp | 124 | ||||
| -rw-r--r-- | src/parse/tokenstream.hpp | 99 | ||||
| -rw-r--r-- | src/parse/tokentree.cpp | 23 | ||||
| -rw-r--r-- | src/parse/tokentree.hpp | 53 | ||||
| -rw-r--r-- | src/parse/ttstream.cpp | 109 | ||||
| -rw-r--r-- | src/parse/ttstream.hpp | 52 |
25 files changed, 466 insertions, 380 deletions
diff --git a/src/ast/crate.cpp b/src/ast/crate.cpp index f99389d0..b18ca662 100644 --- a/src/ast/crate.cpp +++ b/src/ast/crate.cpp @@ -6,6 +6,7 @@ #include "../expand/cfg.hpp" #include <hir/hir.hpp> // HIR::Crate #include <hir/main_bindings.hpp> // HIR_Deserialise +#include <fstream> namespace { bool check_item_cfg(const ::AST::MetaItems& attrs) diff --git a/src/ast/dump.cpp b/src/ast/dump.cpp index 33ef5911..15fb6c4a 100644 --- a/src/ast/dump.cpp +++ b/src/ast/dump.cpp @@ -9,6 +9,7 @@ #include <ast/ast.hpp> #include <ast/expr.hpp> #include <main_bindings.hpp> +#include <fstream> #include <cpp_unpack.h> @@ -564,9 +565,9 @@ private: void dec_indent(); }; -void Dump_Rust(const char *Filename, const AST::Crate& crate) +void Dump_Rust(const char *filename, const AST::Crate& crate) { - ::std::ofstream os(Filename); + ::std::ofstream os(filename); RustPrinter printer(os); printer.handle_module(crate.root_module()); } diff --git a/src/expand/asm.cpp b/src/expand/asm.cpp index 783e6a20..3ca7bb72 100644 --- a/src/expand/asm.cpp +++ b/src/expand/asm.cpp @@ -8,6 +8,7 @@ #include <common.hpp> #include <synext_macro.hpp> #include <parse/tokentree.hpp> +#include <parse/ttstream.hpp> class CAsmExpander: public ExpandProcMacro diff --git a/src/expand/cfg.cpp b/src/expand/cfg.cpp index 0a4617cf..00e3921c 100644 --- a/src/expand/cfg.cpp +++ b/src/expand/cfg.cpp @@ -1,9 +1,14 @@ /* + * MRustC - Rust Compiler + * - By John Hodge (Mutabah/thePowersGang) + * + * expand/cfg.hpp + * - cfg! and #[cfg] handling */ #include <synext.hpp> -#include <parse/tokentree.hpp> -#include <parse/lex.hpp> #include <parse/common.hpp> +#include <parse/tokentree.hpp> +#include <parse/ttstream.hpp> #include "cfg.hpp" #include <ast/expr.hpp> // Needed to clear a ExprNodeP #include <ast/crate.hpp> diff --git a/src/expand/concat.cpp b/src/expand/concat.cpp index c15be18b..7475325a 100644 --- a/src/expand/concat.cpp +++ b/src/expand/concat.cpp @@ -9,7 +9,8 @@ #include "../parse/common.hpp" #include "../parse/parseerror.hpp" #include "../parse/tokentree.hpp" -#include "../parse/lex.hpp" +#include "../parse/ttstream.hpp" +#include "../parse/lex.hpp" // For Codepoint #include <ast/expr.hpp> class CConcatExpander: diff --git a/src/expand/env.cpp b/src/expand/env.cpp index 85742549..db51fa67 100644 --- a/src/expand/env.cpp +++ b/src/expand/env.cpp @@ -7,6 +7,7 @@ */ #include <synext_macro.hpp> #include <parse/common.hpp> +#include <parse/ttstream.hpp> #include <ast/expr.hpp> // ExprNode_* #include <synext.hpp> // for Expand_BareExpr diff --git a/src/expand/file_line.cpp b/src/expand/file_line.cpp index 8ce56436..a485fe61 100644 --- a/src/expand/file_line.cpp +++ b/src/expand/file_line.cpp @@ -1,7 +1,13 @@ /* + * MRustC - Rust Compiler + * - By John Hodge (Mutabah/thePowersGang) + * + * expand/file_line.cpp + * - file! line! and macro_path! macros */ #include <synext.hpp> #include "../parse/common.hpp" +#include "../parse/ttstream.hpp" class CExpanderFile: public ExpandProcMacro diff --git a/src/expand/format_args.cpp b/src/expand/format_args.cpp index 4857a6c2..6881e19a 100644 --- a/src/expand/format_args.cpp +++ b/src/expand/format_args.cpp @@ -10,7 +10,7 @@ #include "../parse/common.hpp" #include "../parse/parseerror.hpp" #include "../parse/tokentree.hpp" -#include "../parse/lex.hpp" +#include "../parse/ttstream.hpp" #include "../parse/interpolated_fragment.hpp" #include <ast/crate.hpp> // for m_load_std #include <ast/expr.hpp> // for ExprNode_* diff --git a/src/expand/include.cpp b/src/expand/include.cpp index b7fe0f53..83a2aa03 100644 --- a/src/expand/include.cpp +++ b/src/expand/include.cpp @@ -8,8 +8,8 @@ #include <synext_macro.hpp> #include <parse/common.hpp> #include <parse/parseerror.hpp> // for GET_CHECK_TOK -#include <parse/tokentree.hpp> // TTStream -#include <parse/lex.hpp> +#include <parse/ttstream.hpp> +#include <parse/lex.hpp> // Lexer (new files) #include <ast/expr.hpp> namespace { diff --git a/src/expand/macro_rules.cpp b/src/expand/macro_rules.cpp index c460e35d..163504db 100644 --- a/src/expand/macro_rules.cpp +++ b/src/expand/macro_rules.cpp @@ -13,6 +13,7 @@ #include "../ast/expr.hpp" #include "../ast/ast.hpp" #include "../parse/common.hpp" +#include "../parse/ttstream.hpp" #include <ast/crate.hpp> #include "macro_rules.hpp" #include <macro_rules/macro_rules.hpp> diff --git a/src/expand/rustc_diagnostics.cpp b/src/expand/rustc_diagnostics.cpp index e9990a93..b5337d26 100644 --- a/src/expand/rustc_diagnostics.cpp +++ b/src/expand/rustc_diagnostics.cpp @@ -7,6 +7,7 @@ */ #include <synext.hpp> #include <parse/common.hpp> // TokenTree etc +#include <parse/ttstream.hpp> class CExpanderRegisterDiagnostic: public ExpandProcMacro diff --git a/src/expand/stringify.cpp b/src/expand/stringify.cpp index 2d70ca27..ce16847b 100644 --- a/src/expand/stringify.cpp +++ b/src/expand/stringify.cpp @@ -1,7 +1,13 @@ /* + * MRustC - Rust Compiler + * - By John Hodge (Mutabah/thePowersGang) + * + * expand/stringify.cpp + * - stringify! macro */ #include <synext.hpp> #include "../parse/common.hpp" +#include "../parse/ttstream.hpp" class CExpander: public ExpandProcMacro diff --git a/src/include/span.hpp b/src/include/span.hpp index 81ccd179..eafebb0a 100644 --- a/src/include/span.hpp +++ b/src/include/span.hpp @@ -8,6 +8,7 @@ #pragma once #include <rc_string.hpp> +#include <functional> enum ErrorType { diff --git a/src/macro_rules/eval.cpp b/src/macro_rules/eval.cpp index 110e8525..f7a5aab1 100644 --- a/src/macro_rules/eval.cpp +++ b/src/macro_rules/eval.cpp @@ -8,7 +8,7 @@ #include <common.hpp> #include "macro_rules.hpp" #include <parse/parseerror.hpp> -#include <parse/tokentree.hpp> +#include <parse/ttstream.hpp> #include <parse/common.hpp> #include <limits.h> #include "pattern_checks.hpp" diff --git a/src/parse/common.hpp b/src/parse/common.hpp index 7bf41d22..57d6f24d 100644 --- a/src/parse/common.hpp +++ b/src/parse/common.hpp @@ -8,6 +8,7 @@ #ifndef PARSE_COMMON_HPP_INCLUDED
#define PARSE_COMMON_HPP_INCLUDED
#include <iostream>
+#include "tokenstream.hpp"
#include "../ast/ast.hpp"
#define GET_TOK(tok, lex) ((tok = lex.getToken()).type())
@@ -60,6 +61,9 @@ extern AST::ExprNodeP Parse_ExprBlockLine(TokenStream& lex, bool *add_silence); extern AST::ExprNodeP Parse_ExprBlockLine_WithItems(TokenStream& lex, ::std::shared_ptr<AST::Module>& local_mod, bool& add_silence_if_end);
extern AST::ExprNodeP Parse_Stmt(TokenStream& lex);
+// unwrapped = Exclude the enclosing brackets (used by macro parse code)
+extern TokenTree Parse_TT(TokenStream& lex, bool unwrapped);
+
extern bool Parse_IsTokValue(eTokenType tok_type);
diff --git a/src/parse/lex.cpp b/src/parse/lex.cpp index 9acb56e0..d6cfcddb 100644 --- a/src/parse/lex.cpp +++ b/src/parse/lex.cpp @@ -4,11 +4,6 @@ * * parse/lex.cpp * - Lexer (converts input file to token stream) - * - * Provides: - * - Lexer : The file->token lexer - * - TTStream : A stream of tokens from a TokenTree - * - TokenStream : Common interface for all token streams */ #include "lex.hpp" #include "tokentree.hpp" @@ -18,11 +13,7 @@ #include <iostream> #include <cstdlib> // strtol #include <typeinfo> -#include <algorithm> // std::count - -const bool DEBUG_PRINT_TOKENS = false; -//const bool DEBUG_PRINT_TOKENS = true; -//#define DEBUG_PRINT_TOKENS debug_enabled("Lexer Tokens") +#include <algorithm> // std::count Lexer::Lexer(const ::std::string& filename): m_path(filename.c_str()), @@ -978,230 +969,9 @@ void Lexer::ungetc() m_last_char_valid = true; } -TTStream::TTStream(const TokenTree& input_tt) -{ - DEBUG("input_tt = [" << input_tt << "]"); - m_stack.push_back( ::std::make_pair(0, &input_tt) ); -} -TTStream::~TTStream() -{ -} -Token TTStream::realGetToken() -{ - while(m_stack.size() > 0) - { - // If current index is above TT size, go up - unsigned int& idx = m_stack.back().first; - const TokenTree& tree = *m_stack.back().second; - - if(idx == 0 && tree.is_token()) { - idx ++; - return tree.tok(); - } - - if(idx < tree.size()) - { - const TokenTree& subtree = tree[idx]; - idx ++; - if( subtree.size() == 0 ) { - m_hygiene_ptr = &m_stack.back().second->hygiene(); - return subtree.tok().clone(); - } - else { - m_stack.push_back( ::std::make_pair(0, &subtree) ); - } - } - else { - m_stack.pop_back(); - } - } - //m_hygiene = nullptr; - return Token(TOK_EOF); -} -Position TTStream::getPosition() const -{ - return Position("TTStream", 0,0); -} -Ident::Hygiene TTStream::getHygiene() const -{ - //assert( m_hygiene ); - return *m_hygiene_ptr; -} - - -TTStreamO::TTStreamO(TokenTree input_tt): - m_input_tt( mv$(input_tt) ) -{ - m_stack.push_back( ::std::make_pair(0, nullptr) ); -} -TTStreamO::~TTStreamO() -{ -} -Token TTStreamO::realGetToken() -{ - while(m_stack.size() > 0) - { - // If current index is above TT size, go up - unsigned int& idx = m_stack.back().first; - TokenTree& tree = *( m_stack.back().second ? m_stack.back().second : &m_input_tt ); - - if(idx == 0 && tree.is_token()) { - idx ++; - m_last_pos = tree.tok().get_pos(); - return mv$(tree.tok()); - } - - if(idx < tree.size()) - { - TokenTree& subtree = tree[idx]; - idx ++; - if( subtree.size() == 0 ) { - m_last_pos = subtree.tok().get_pos(); - return mv$( subtree.tok() ); - } - else { - m_stack.push_back( ::std::make_pair(0, &subtree) ); - } - } - else { - m_stack.pop_back(); - } - } - return Token(TOK_EOF); -} -Position TTStreamO::getPosition() const -{ - return m_last_pos; -} -Ident::Hygiene TTStreamO::getHygiene() const -{ - return (m_stack.back().second ? m_stack.back().second->hygiene() : m_input_tt.hygiene()); -} - - -TokenStream::TokenStream(): - m_cache_valid(false) -{ -} -TokenStream::~TokenStream() -{ -} - -Token TokenStream::innerGetToken() -{ - Token ret = this->realGetToken(); - if( ret.get_pos().filename == "" ) - ret.set_pos( this->getPosition() ); - //DEBUG("ret.get_pos() = " << ret.get_pos()); - return ret; -} -Token TokenStream::getToken() -{ - if( m_cache_valid ) - { - m_cache_valid = false; - return mv$(m_cache); - } - else if( m_lookahead.size() ) - { - Token ret = mv$( m_lookahead.front() ); - m_lookahead.erase(m_lookahead.begin()); - if( DEBUG_PRINT_TOKENS ) { - ::std::cout << "getToken[" << typeid(*this).name() << "] - " << ret.get_pos() << "-" << ret << ::std::endl; - } - return ret; - } - else - { - Token ret = this->innerGetToken(); - if( DEBUG_PRINT_TOKENS ) { - ::std::cout << "getToken[" << typeid(*this).name() << "] - " << ret.get_pos() << "-" << ret << ::std::endl; - } - return ret; - } -} -void TokenStream::putback(Token tok) -{ - if( m_cache_valid ) - { - DEBUG("" << getPosition() << " - Double putback: " << tok << " but " << m_cache); - throw ParseError::BugCheck("Double putback"); - } - else - { - m_cache_valid = true; - m_cache = mv$(tok); - } -} - -eTokenType TokenStream::lookahead(unsigned int i) -{ - const unsigned int MAX_LOOKAHEAD = 3; - - if( m_cache_valid ) - { - if( i == 0 ) - return m_cache.type(); - i --; - } - - if( i >= MAX_LOOKAHEAD ) - throw ParseError::BugCheck("Excessive lookahead"); - - while( i >= m_lookahead.size() ) - { - DEBUG("lookahead - read #" << m_lookahead.size()); - m_lookahead.push_back( this->innerGetToken() ); - } - - DEBUG("lookahead(" << i << ") = " << m_lookahead[i]); - return m_lookahead[i].type(); -} - -ProtoSpan TokenStream::start_span() const -{ - auto p = this->getPosition(); - return ProtoSpan { - .filename = p.filename, - .start_line = p.line, - .start_ofs = p.ofs, - }; -} -Span TokenStream::end_span(ProtoSpan ps) const -{ - auto p = this->getPosition(); - return Span( - ps.filename, - ps.start_line, ps.start_ofs, - p.line, p.ofs - ); -} -Ident TokenStream::get_ident(Token tok) const -{ - if(tok.type() == TOK_IDENT) { - return Ident(getHygiene(), tok.str()); - } - else if( tok.type() == TOK_INTERPOLATED_IDENT ) { - TODO(getPosition(), ""); - } - else { - throw ParseError::Unexpected(*this, tok); - } -} - -TokenTree TokenTree::clone() const -{ - if( m_subtrees.size() == 0 ) { - return TokenTree(m_tok.clone()); - } - else { - ::std::vector< TokenTree> ents; - ents.reserve( m_subtrees.size() ); - for(const auto& sub : m_subtrees) - ents.push_back( sub.clone() ); - return TokenTree( m_hygiene, mv$(ents) ); - } -} +// -------------------------------------------------------------------- +// Codepoint - Unicode codepoint. +// -------------------------------------------------------------------- bool Codepoint::isspace() const { switch(this->v) diff --git a/src/parse/lex.hpp b/src/parse/lex.hpp index d147d79d..3d41c3fc 100644 --- a/src/parse/lex.hpp +++ b/src/parse/lex.hpp @@ -8,99 +8,9 @@ #ifndef LEX_HPP_INCLUDED #define LEX_HPP_INCLUDED -#include <debug.hpp> -#include <serialise.hpp> -#include "../coretypes.hpp" #include <string> #include <fstream> - -#include "../include/span.hpp" - -#include "token.hpp" - -namespace AST { - class Module; - class MetaItems; -} - -/// State the parser needs to pass down via a second channel. -struct ParseState -{ - // Used for "for/if/while" to handle ambiguity - bool disallow_struct_literal = false; - // A debugging hook that disables expansion of macros - bool no_expand_macros = false; - - ::AST::Module* module = nullptr; - ::AST::MetaItems* parent_attrs = nullptr; - - ::AST::Module& get_current_mod() { - assert(this->module); - return *this->module; - } - - friend ::std::ostream& operator<<(::std::ostream& os, const ParseState& ps) { - os << "ParseState {"; - if(ps.disallow_struct_literal) os << " disallow_struct_literal"; - if(ps.no_expand_macros) os << " no_expand_macros"; - os << " }"; - return os; - } -}; - -class TokenStream -{ - friend class TTLexer; // needs access to internals to know what was consumed - - bool m_cache_valid; - Token m_cache; - ::std::vector<Token> m_lookahead; - ParseState m_parse_state; -public: - TokenStream(); - virtual ~TokenStream(); - Token getToken(); - void putback(Token tok); - eTokenType lookahead(unsigned int count); - - virtual Position getPosition() const = 0; - virtual Ident::Hygiene getHygiene() const = 0; - - ParseState& parse_state() { return m_parse_state; } - - ProtoSpan start_span() const; - Span end_span(ProtoSpan ps) const; - - Ident get_ident(Token tok) const; - -protected: - virtual Token realGetToken() = 0; -private: - Token innerGetToken(); -}; - -class SavedParseState -{ - TokenStream& m_lex; - ParseState m_state; -public: - SavedParseState(TokenStream& lex, ParseState state): - m_lex(lex), - m_state(state) - { - } - ~SavedParseState() - { - DEBUG("Restoring " << m_state); - m_lex.parse_state() = m_state; - } -}; - -#define SET_MODULE(lex, mod) SavedParseState _sps(lex, lex.parse_state()); lex.parse_state().module = &(mod) -#define SET_ATTRS(lex, attrs) SavedParseState _sps(lex, lex.parse_state()); lex.parse_state().parent_attrs = &(attrs) -#define SET_PARSE_FLAG(lex, flag) SavedParseState _sps(lex, lex.parse_state()); lex.parse_state().flag = true -#define CLEAR_PARSE_FLAG(lex, flag) SavedParseState _sps(lex, lex.parse_state()); lex.parse_state().flag = false -#define CHECK_PARSE_FLAG(lex, flag) (lex.parse_state().flag == true) +#include "tokenstream.hpp" struct Codepoint { uint32_t v; @@ -116,6 +26,7 @@ struct Codepoint { }; extern ::std::string& operator+=(::std::string& s, const Codepoint& cp); extern ::std::ostream& operator<<(::std::ostream& s, const Codepoint& cp); + typedef Codepoint uchar; class Lexer: diff --git a/src/parse/parseerror.hpp b/src/parse/parseerror.hpp index b847ccc4..0a00f60a 100644 --- a/src/parse/parseerror.hpp +++ b/src/parse/parseerror.hpp @@ -2,7 +2,7 @@ #define PARSEERROR_HPP_INCLUDED
#include <stdexcept>
-#include "lex.hpp"
+#include "tokenstream.hpp"
#include <compile_error.hpp>
namespace ParseError {
diff --git a/src/parse/root.cpp b/src/parse/root.cpp index 2f5114ed..a6683925 100644 --- a/src/parse/root.cpp +++ b/src/parse/root.cpp @@ -9,13 +9,15 @@ * - Parse_Crate : Handles crate attrbutes, and passes on to Parse_ModRoot
* - Parse_ModRoot
*/
-#include "../ast/ast.hpp"
-#include "../ast/crate.hpp"
+#include <ast/ast.hpp>
+#include <ast/crate.hpp>
#include "parseerror.hpp"
#include "common.hpp"
#include <cassert>
#include <hir/hir.hpp> // ABI_RUST - TODO: Move elsewhere?
#include <expand/cfg.hpp> // check_cfg - for `mod nonexistant;`
+#include <fstream> // Used by directory path
+#include "lex.hpp" // New file lexer
template<typename T>
Spanned<T> get_spanned(TokenStream& lex, ::std::function<T()> f) {
diff --git a/src/parse/tokenstream.cpp b/src/parse/tokenstream.cpp new file mode 100644 index 00000000..5d9919f6 --- /dev/null +++ b/src/parse/tokenstream.cpp @@ -0,0 +1,124 @@ +/* + * MRustC - Rust Compiler + * - By John Hodge (Mutabah/thePowersGang) + * + * parse/tokenstream.cpp + * - TokenStream - Parser token source interface + */ +#include "tokenstream.hpp" +#include <common.hpp> +#include "parseerror.hpp" + +const bool DEBUG_PRINT_TOKENS = false; +//const bool DEBUG_PRINT_TOKENS = true; +//#define DEBUG_PRINT_TOKENS debug_enabled("Lexer Tokens") + +TokenStream::TokenStream(): + m_cache_valid(false) +{ +} +TokenStream::~TokenStream() +{ +} + +Token TokenStream::innerGetToken() +{ + Token ret = this->realGetToken(); + if( ret.get_pos().filename == "" ) + ret.set_pos( this->getPosition() ); + //DEBUG("ret.get_pos() = " << ret.get_pos()); + return ret; +} +Token TokenStream::getToken() +{ + if( m_cache_valid ) + { + m_cache_valid = false; + return mv$(m_cache); + } + else if( m_lookahead.size() ) + { + Token ret = mv$( m_lookahead.front() ); + m_lookahead.erase(m_lookahead.begin()); + if( DEBUG_PRINT_TOKENS ) { + ::std::cout << "getToken[" << typeid(*this).name() << "] - " << ret.get_pos() << "-" << ret << ::std::endl; + } + return ret; + } + else + { + Token ret = this->innerGetToken(); + if( DEBUG_PRINT_TOKENS ) { + ::std::cout << "getToken[" << typeid(*this).name() << "] - " << ret.get_pos() << "-" << ret << ::std::endl; + } + return ret; + } +} +void TokenStream::putback(Token tok) +{ + if( m_cache_valid ) + { + DEBUG("" << getPosition() << " - Double putback: " << tok << " but " << m_cache); + throw ParseError::BugCheck("Double putback"); + } + else + { + m_cache_valid = true; + m_cache = mv$(tok); + } +} + +eTokenType TokenStream::lookahead(unsigned int i) +{ + const unsigned int MAX_LOOKAHEAD = 3; + + if( m_cache_valid ) + { + if( i == 0 ) + return m_cache.type(); + i --; + } + + if( i >= MAX_LOOKAHEAD ) + throw ParseError::BugCheck("Excessive lookahead"); + + while( i >= m_lookahead.size() ) + { + DEBUG("lookahead - read #" << m_lookahead.size()); + m_lookahead.push_back( this->innerGetToken() ); + } + + DEBUG("lookahead(" << i << ") = " << m_lookahead[i]); + return m_lookahead[i].type(); +} + +ProtoSpan TokenStream::start_span() const +{ + auto p = this->getPosition(); + return ProtoSpan { + .filename = p.filename, + .start_line = p.line, + .start_ofs = p.ofs, + }; +} +Span TokenStream::end_span(ProtoSpan ps) const +{ + auto p = this->getPosition(); + return Span( + ps.filename, + ps.start_line, ps.start_ofs, + p.line, p.ofs + ); +} +Ident TokenStream::get_ident(Token tok) const +{ + if(tok.type() == TOK_IDENT) { + return Ident(getHygiene(), tok.str()); + } + else if( tok.type() == TOK_INTERPOLATED_IDENT ) { + TODO(getPosition(), ""); + } + else { + throw ParseError::Unexpected(*this, tok); + } +} diff --git a/src/parse/tokenstream.hpp b/src/parse/tokenstream.hpp new file mode 100644 index 00000000..c6e2de37 --- /dev/null +++ b/src/parse/tokenstream.hpp @@ -0,0 +1,99 @@ +/* + * MRustC - Rust Compiler + * - By John Hodge (Mutabah/thePowersGang) + * + * parse/tokenstream.hpp + * - Parser stream (TokenStream) header + */ +#pragma once + +#include <iostream> +#include <vector> +#include <span.hpp> +#include <debug.hpp> +#include <ident.hpp> +#include "token.hpp" + +namespace AST { + class Module; + class MetaItems; +} + +/// State the parser needs to pass down via a second channel. +struct ParseState +{ + // Used for "for/if/while" to handle ambiguity + bool disallow_struct_literal = false; + // A debugging hook that disables expansion of macros + bool no_expand_macros = false; + + ::AST::Module* module = nullptr; + ::AST::MetaItems* parent_attrs = nullptr; + + ::AST::Module& get_current_mod() { + assert(this->module); + return *this->module; + } + + friend ::std::ostream& operator<<(::std::ostream& os, const ParseState& ps) { + os << "ParseState {"; + if(ps.disallow_struct_literal) os << " disallow_struct_literal"; + if(ps.no_expand_macros) os << " no_expand_macros"; + os << " }"; + return os; + } +}; + +class TokenStream +{ + friend class TTLexer; // needs access to internals to know what was consumed + + bool m_cache_valid; + Token m_cache; + ::std::vector<Token> m_lookahead; + ParseState m_parse_state; +public: + TokenStream(); + virtual ~TokenStream(); + Token getToken(); + void putback(Token tok); + eTokenType lookahead(unsigned int count); + + virtual Position getPosition() const = 0; + virtual Ident::Hygiene getHygiene() const = 0; + + ParseState& parse_state() { return m_parse_state; } + + ProtoSpan start_span() const; + Span end_span(ProtoSpan ps) const; + + Ident get_ident(Token tok) const; + +protected: + virtual Token realGetToken() = 0; +private: + Token innerGetToken(); +}; + +class SavedParseState +{ + TokenStream& m_lex; + ParseState m_state; +public: + SavedParseState(TokenStream& lex, ParseState state): + m_lex(lex), + m_state(state) + { + } + ~SavedParseState() + { + DEBUG("Restoring " << m_state); + m_lex.parse_state() = m_state; + } +}; + +#define SET_MODULE(lex, mod) SavedParseState _sps(lex, lex.parse_state()); lex.parse_state().module = &(mod) +#define SET_ATTRS(lex, attrs) SavedParseState _sps(lex, lex.parse_state()); lex.parse_state().parent_attrs = &(attrs) +#define SET_PARSE_FLAG(lex, flag) SavedParseState _sps(lex, lex.parse_state()); lex.parse_state().flag = true +#define CLEAR_PARSE_FLAG(lex, flag) SavedParseState _sps(lex, lex.parse_state()); lex.parse_state().flag = false +#define CHECK_PARSE_FLAG(lex, flag) (lex.parse_state().flag == true) diff --git a/src/parse/tokentree.cpp b/src/parse/tokentree.cpp new file mode 100644 index 00000000..8fad6386 --- /dev/null +++ b/src/parse/tokentree.cpp @@ -0,0 +1,23 @@ +/* + * MRustC - Rust Compiler + * - By John Hodge (Mutabah/thePowersGang) + * + * parse/tokentree.cpp + * - Token Tree (collection of tokens) + */ +#include "tokentree.hpp" +#include <common.hpp> + +TokenTree TokenTree::clone() const +{ + if( m_subtrees.size() == 0 ) { + return TokenTree(m_tok.clone()); + } + else { + ::std::vector< TokenTree> ents; + ents.reserve( m_subtrees.size() ); + for(const auto& sub : m_subtrees) + ents.push_back( sub.clone() ); + return TokenTree( m_hygiene, mv$(ents) ); + } +} diff --git a/src/parse/tokentree.hpp b/src/parse/tokentree.hpp index 627d11e3..50ca72a5 100644 --- a/src/parse/tokentree.hpp +++ b/src/parse/tokentree.hpp @@ -1,7 +1,16 @@ +/*
+ * MRustC - Rust Compiler
+ * - By John Hodge (Mutabah/thePowersGang)
+ *
+ * parse/tokentree.hpp
+ * - Token Trees (groups of tokens
+ */
#ifndef TOKENTREE_HPP_INCLUDED
#define TOKENTREE_HPP_INCLUDED
-#include "lex.hpp"
+#include "token.hpp"
+#include <ident.hpp>
+#include <vector>
class TokenTree
{
@@ -55,46 +64,4 @@ public: }
};
-class TTStream:
- public TokenStream
-{
- ::std::vector< ::std::pair<unsigned int, const TokenTree*> > m_stack;
- const Ident::Hygiene* m_hygiene_ptr = nullptr;
-public:
- TTStream(const TokenTree& input_tt);
- ~TTStream();
-
- TTStream& operator=(const TTStream& x) { m_stack = x.m_stack; return *this; }
-
- Position getPosition() const override;
- Ident::Hygiene getHygiene() const override;
-
-protected:
- Token realGetToken() override;
-};
-
-class TTStreamO:
- public TokenStream
-{
- Position m_last_pos;
- TokenTree m_input_tt;
- ::std::vector< ::std::pair<unsigned int, TokenTree*> > m_stack;
-public:
- TTStreamO(TokenTree input_tt);
- TTStreamO(TTStreamO&& x) = default;
- ~TTStreamO();
-
- TTStreamO& operator=(const TTStreamO& x) { m_stack = x.m_stack; return *this; }
- TTStreamO& operator=(TTStreamO&& x) = default;
-
- Position getPosition() const override;
- Ident::Hygiene getHygiene() const override;
-
-protected:
- Token realGetToken() override;
-};
-
-// unwrapped = Exclude the enclosing brackets (used by macro parse code)
-extern TokenTree Parse_TT(TokenStream& lex, bool unwrapped);
-
#endif // TOKENTREE_HPP_INCLUDED
diff --git a/src/parse/ttstream.cpp b/src/parse/ttstream.cpp new file mode 100644 index 00000000..d969f514 --- /dev/null +++ b/src/parse/ttstream.cpp @@ -0,0 +1,109 @@ +/* + * MRustC - Rust Compiler + * - By John Hodge (Mutabah/thePowersGang) + * + * parse/ttstream.cpp + * - Token-Tree backed token streams + */ +#include "ttstream.hpp" +#include <common.hpp> + +TTStream::TTStream(const TokenTree& input_tt) +{ + DEBUG("input_tt = [" << input_tt << "]"); + m_stack.push_back( ::std::make_pair(0, &input_tt) ); +} +TTStream::~TTStream() +{ +} +Token TTStream::realGetToken() +{ + while(m_stack.size() > 0) + { + // If current index is above TT size, go up + unsigned int& idx = m_stack.back().first; + const TokenTree& tree = *m_stack.back().second; + + if(idx == 0 && tree.is_token()) { + idx ++; + return tree.tok(); + } + + if(idx < tree.size()) + { + const TokenTree& subtree = tree[idx]; + idx ++; + if( subtree.size() == 0 ) { + m_hygiene_ptr = &m_stack.back().second->hygiene(); + return subtree.tok().clone(); + } + else { + m_stack.push_back( ::std::make_pair(0, &subtree) ); + } + } + else { + m_stack.pop_back(); + } + } + //m_hygiene = nullptr; + return Token(TOK_EOF); +} +Position TTStream::getPosition() const +{ + return Position("TTStream", 0,0); +} +Ident::Hygiene TTStream::getHygiene() const +{ + assert( m_hygiene_ptr ); + return *m_hygiene_ptr; +} + + +TTStreamO::TTStreamO(TokenTree input_tt): + m_input_tt( mv$(input_tt) ) +{ + m_stack.push_back( ::std::make_pair(0, nullptr) ); +} +TTStreamO::~TTStreamO() +{ +} +Token TTStreamO::realGetToken() +{ + while(m_stack.size() > 0) + { + // If current index is above TT size, go up + unsigned int& idx = m_stack.back().first; + TokenTree& tree = *( m_stack.back().second ? m_stack.back().second : &m_input_tt ); + + if(idx == 0 && tree.is_token()) { + idx ++; + m_last_pos = tree.tok().get_pos(); + return mv$(tree.tok()); + } + + if(idx < tree.size()) + { + TokenTree& subtree = tree[idx]; + idx ++; + if( subtree.size() == 0 ) { + m_last_pos = subtree.tok().get_pos(); + return mv$( subtree.tok() ); + } + else { + m_stack.push_back( ::std::make_pair(0, &subtree) ); + } + } + else { + m_stack.pop_back(); + } + } + return Token(TOK_EOF); +} +Position TTStreamO::getPosition() const +{ + return m_last_pos; +} +Ident::Hygiene TTStreamO::getHygiene() const +{ + return (m_stack.back().second ? m_stack.back().second->hygiene() : m_input_tt.hygiene()); +} diff --git a/src/parse/ttstream.hpp b/src/parse/ttstream.hpp new file mode 100644 index 00000000..7b7fec4a --- /dev/null +++ b/src/parse/ttstream.hpp @@ -0,0 +1,52 @@ +/* + * MRustC - Rust Compiler + * - By John Hodge (Mutabah/thePowersGang) + * + * parse/ttstrea.hpp + * - Token tree streams (for post-lex parsing) + */ +#pragma once + +#include "tokentree.hpp" +#include "tokenstream.hpp" + +/// Borrowed TTStream +class TTStream: + public TokenStream +{ + ::std::vector< ::std::pair<unsigned int, const TokenTree*> > m_stack; + const Ident::Hygiene* m_hygiene_ptr = nullptr; +public: + TTStream(const TokenTree& input_tt); + ~TTStream(); + + TTStream& operator=(const TTStream& x) { m_stack = x.m_stack; return *this; } + + Position getPosition() const override; + Ident::Hygiene getHygiene() const override; + +protected: + Token realGetToken() override; +}; + +/// Owned TTStream +class TTStreamO: + public TokenStream +{ + Position m_last_pos; + TokenTree m_input_tt; + ::std::vector< ::std::pair<unsigned int, TokenTree*> > m_stack; +public: + TTStreamO(TokenTree input_tt); + TTStreamO(TTStreamO&& x) = default; + ~TTStreamO(); + + TTStreamO& operator=(const TTStreamO& x) { m_stack = x.m_stack; return *this; } + TTStreamO& operator=(TTStreamO&& x) = default; + + Position getPosition() const override; + Ident::Hygiene getHygiene() const override; + +protected: + Token realGetToken() override; +}; |