1 files changed, 2 insertions, 91 deletions
diff --git a/src/parse/lex.hpp b/src/parse/lex.hpp
index d147d79d..3d41c3fc 100644
--- a/src/parse/lex.hpp
+++ b/src/parse/lex.hpp
@@ -8,99 +8,9 @@
 #ifndef LEX_HPP_INCLUDED
 #define LEX_HPP_INCLUDED
 
-#include <debug.hpp>
-#include <serialise.hpp>
-#include "../coretypes.hpp"
 #include <string>
 #include <fstream>
-
-#include "../include/span.hpp"
-
-#include "token.hpp"
-
-namespace AST {
-    class Module;
-    class MetaItems;
-}
-
-/// State the parser needs to pass down via a second channel.
-struct ParseState
-{
-    // Used for "for/if/while" to handle ambiguity
-    bool disallow_struct_literal = false;
-    // A debugging hook that disables expansion of macros
-    bool no_expand_macros = false;
-    
-    ::AST::Module*  module = nullptr;
-    ::AST::MetaItems*   parent_attrs = nullptr;
-    
-    ::AST::Module& get_current_mod() {
-        assert(this->module);
-        return *this->module;
-    }
-    
-    friend ::std::ostream& operator<<(::std::ostream& os, const ParseState& ps) {
-        os << "ParseState {";
-        if(ps.disallow_struct_literal)  os << " disallow_struct_literal";
-        if(ps.no_expand_macros)  os << " no_expand_macros";
-        os << " }";
-        return os;
-    }
-};
-
-class TokenStream
-{
-    friend class TTLexer;   // needs access to internals to know what was consumed
-    
-    bool    m_cache_valid;
-    Token   m_cache;
-    ::std::vector<Token>    m_lookahead;
-    ParseState  m_parse_state;
-public:
-    TokenStream();
-    virtual ~TokenStream();
-    Token   getToken();
-    void    putback(Token tok);
-    eTokenType  lookahead(unsigned int count);
-    
-    virtual Position getPosition() const = 0;
-    virtual Ident::Hygiene getHygiene() const = 0;
-    
-    ParseState& parse_state() { return m_parse_state; }
-    
-    ProtoSpan   start_span() const;
-    Span    end_span(ProtoSpan ps) const;
-    
-    Ident get_ident(Token tok) const;
-    
-protected:
-    virtual Token   realGetToken() = 0;
-private:
-    Token innerGetToken();
-};
-
-class SavedParseState
-{
-    TokenStream&    m_lex;
-    ParseState  m_state;
-public:
-    SavedParseState(TokenStream& lex, ParseState state):
-        m_lex(lex),
-        m_state(state)
-    {
-    }
-    ~SavedParseState()
-    {
-        DEBUG("Restoring " << m_state);
-        m_lex.parse_state() = m_state;
-    }
-};
-
-#define SET_MODULE(lex, mod)    SavedParseState _sps(lex, lex.parse_state()); lex.parse_state().module = &(mod)
-#define SET_ATTRS(lex, attrs)    SavedParseState _sps(lex, lex.parse_state()); lex.parse_state().parent_attrs = &(attrs)
-#define SET_PARSE_FLAG(lex, flag)    SavedParseState _sps(lex, lex.parse_state()); lex.parse_state().flag = true
-#define CLEAR_PARSE_FLAG(lex, flag)    SavedParseState _sps(lex, lex.parse_state()); lex.parse_state().flag = false
-#define CHECK_PARSE_FLAG(lex, flag) (lex.parse_state().flag == true)
+#include "tokenstream.hpp"
 
 struct Codepoint {
     uint32_t    v;
@@ -116,6 +26,7 @@ struct Codepoint {
 };
 extern ::std::string& operator+=(::std::string& s, const Codepoint& cp);
 extern ::std::ostream& operator<<(::std::ostream& s, const Codepoint& cp);
+
 typedef Codepoint   uchar;
 
 class Lexer: