diff options
Diffstat (limited to 'src/parse/lex.cpp')
| -rw-r--r-- | src/parse/lex.cpp | 238 |
1 files changed, 4 insertions, 234 deletions
diff --git a/src/parse/lex.cpp b/src/parse/lex.cpp index 9acb56e0..d6cfcddb 100644 --- a/src/parse/lex.cpp +++ b/src/parse/lex.cpp @@ -4,11 +4,6 @@ * * parse/lex.cpp * - Lexer (converts input file to token stream) - * - * Provides: - * - Lexer : The file->token lexer - * - TTStream : A stream of tokens from a TokenTree - * - TokenStream : Common interface for all token streams */ #include "lex.hpp" #include "tokentree.hpp" @@ -18,11 +13,7 @@ #include <iostream> #include <cstdlib> // strtol #include <typeinfo> -#include <algorithm> // std::count - -const bool DEBUG_PRINT_TOKENS = false; -//const bool DEBUG_PRINT_TOKENS = true; -//#define DEBUG_PRINT_TOKENS debug_enabled("Lexer Tokens") +#include <algorithm> // std::count Lexer::Lexer(const ::std::string& filename): m_path(filename.c_str()), @@ -978,230 +969,9 @@ void Lexer::ungetc() m_last_char_valid = true; } -TTStream::TTStream(const TokenTree& input_tt) -{ - DEBUG("input_tt = [" << input_tt << "]"); - m_stack.push_back( ::std::make_pair(0, &input_tt) ); -} -TTStream::~TTStream() -{ -} -Token TTStream::realGetToken() -{ - while(m_stack.size() > 0) - { - // If current index is above TT size, go up - unsigned int& idx = m_stack.back().first; - const TokenTree& tree = *m_stack.back().second; - - if(idx == 0 && tree.is_token()) { - idx ++; - return tree.tok(); - } - - if(idx < tree.size()) - { - const TokenTree& subtree = tree[idx]; - idx ++; - if( subtree.size() == 0 ) { - m_hygiene_ptr = &m_stack.back().second->hygiene(); - return subtree.tok().clone(); - } - else { - m_stack.push_back( ::std::make_pair(0, &subtree) ); - } - } - else { - m_stack.pop_back(); - } - } - //m_hygiene = nullptr; - return Token(TOK_EOF); -} -Position TTStream::getPosition() const -{ - return Position("TTStream", 0,0); -} -Ident::Hygiene TTStream::getHygiene() const -{ - //assert( m_hygiene ); - return *m_hygiene_ptr; -} - - -TTStreamO::TTStreamO(TokenTree input_tt): - m_input_tt( mv$(input_tt) ) -{ - m_stack.push_back( ::std::make_pair(0, nullptr) ); -} -TTStreamO::~TTStreamO() -{ -} -Token TTStreamO::realGetToken() -{ - while(m_stack.size() > 0) - { - // If current index is above TT size, go up - unsigned int& idx = m_stack.back().first; - TokenTree& tree = *( m_stack.back().second ? m_stack.back().second : &m_input_tt ); - - if(idx == 0 && tree.is_token()) { - idx ++; - m_last_pos = tree.tok().get_pos(); - return mv$(tree.tok()); - } - - if(idx < tree.size()) - { - TokenTree& subtree = tree[idx]; - idx ++; - if( subtree.size() == 0 ) { - m_last_pos = subtree.tok().get_pos(); - return mv$( subtree.tok() ); - } - else { - m_stack.push_back( ::std::make_pair(0, &subtree) ); - } - } - else { - m_stack.pop_back(); - } - } - return Token(TOK_EOF); -} -Position TTStreamO::getPosition() const -{ - return m_last_pos; -} -Ident::Hygiene TTStreamO::getHygiene() const -{ - return (m_stack.back().second ? m_stack.back().second->hygiene() : m_input_tt.hygiene()); -} - - -TokenStream::TokenStream(): - m_cache_valid(false) -{ -} -TokenStream::~TokenStream() -{ -} - -Token TokenStream::innerGetToken() -{ - Token ret = this->realGetToken(); - if( ret.get_pos().filename == "" ) - ret.set_pos( this->getPosition() ); - //DEBUG("ret.get_pos() = " << ret.get_pos()); - return ret; -} -Token TokenStream::getToken() -{ - if( m_cache_valid ) - { - m_cache_valid = false; - return mv$(m_cache); - } - else if( m_lookahead.size() ) - { - Token ret = mv$( m_lookahead.front() ); - m_lookahead.erase(m_lookahead.begin()); - if( DEBUG_PRINT_TOKENS ) { - ::std::cout << "getToken[" << typeid(*this).name() << "] - " << ret.get_pos() << "-" << ret << ::std::endl; - } - return ret; - } - else - { - Token ret = this->innerGetToken(); - if( DEBUG_PRINT_TOKENS ) { - ::std::cout << "getToken[" << typeid(*this).name() << "] - " << ret.get_pos() << "-" << ret << ::std::endl; - } - return ret; - } -} -void TokenStream::putback(Token tok) -{ - if( m_cache_valid ) - { - DEBUG("" << getPosition() << " - Double putback: " << tok << " but " << m_cache); - throw ParseError::BugCheck("Double putback"); - } - else - { - m_cache_valid = true; - m_cache = mv$(tok); - } -} - -eTokenType TokenStream::lookahead(unsigned int i) -{ - const unsigned int MAX_LOOKAHEAD = 3; - - if( m_cache_valid ) - { - if( i == 0 ) - return m_cache.type(); - i --; - } - - if( i >= MAX_LOOKAHEAD ) - throw ParseError::BugCheck("Excessive lookahead"); - - while( i >= m_lookahead.size() ) - { - DEBUG("lookahead - read #" << m_lookahead.size()); - m_lookahead.push_back( this->innerGetToken() ); - } - - DEBUG("lookahead(" << i << ") = " << m_lookahead[i]); - return m_lookahead[i].type(); -} - -ProtoSpan TokenStream::start_span() const -{ - auto p = this->getPosition(); - return ProtoSpan { - .filename = p.filename, - .start_line = p.line, - .start_ofs = p.ofs, - }; -} -Span TokenStream::end_span(ProtoSpan ps) const -{ - auto p = this->getPosition(); - return Span( - ps.filename, - ps.start_line, ps.start_ofs, - p.line, p.ofs - ); -} -Ident TokenStream::get_ident(Token tok) const -{ - if(tok.type() == TOK_IDENT) { - return Ident(getHygiene(), tok.str()); - } - else if( tok.type() == TOK_INTERPOLATED_IDENT ) { - TODO(getPosition(), ""); - } - else { - throw ParseError::Unexpected(*this, tok); - } -} - -TokenTree TokenTree::clone() const -{ - if( m_subtrees.size() == 0 ) { - return TokenTree(m_tok.clone()); - } - else { - ::std::vector< TokenTree> ents; - ents.reserve( m_subtrees.size() ); - for(const auto& sub : m_subtrees) - ents.push_back( sub.clone() ); - return TokenTree( m_hygiene, mv$(ents) ); - } -} +// -------------------------------------------------------------------- +// Codepoint - Unicode codepoint. +// -------------------------------------------------------------------- bool Codepoint::isspace() const { switch(this->v) |