1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
|
/*
* MRustC - Rust Compiler
* - By John Hodge (Mutabah/thePowersGang)
*
* parse/tokenstream.hpp
* - Parser stream (TokenStream) header
*/
#pragma once
#include <iostream>
#include <vector>
#include <span.hpp>
#include <debug.hpp>
#include <ident.hpp>
#include "token.hpp"
namespace AST {
class Module;
class Crate;
class AttributeList;
}
/// State the parser needs to pass down via a second channel.
struct ParseState
{
// Used for "for/if/while" to handle ambiguity
bool disallow_struct_literal = false;
// A debugging hook that disables expansion of macros
bool no_expand_macros = false;
const ::AST::Crate* crate = nullptr;
::AST::Module* module = nullptr;
::AST::AttributeList* parent_attrs = nullptr;
::AST::Module& get_current_mod() {
assert(this->module);
return *this->module;
}
friend ::std::ostream& operator<<(::std::ostream& os, const ParseState& ps) {
os << "ParseState {";
if(ps.disallow_struct_literal) os << " disallow_struct_literal";
if(ps.no_expand_macros) os << " no_expand_macros";
os << " }";
return os;
}
};
class TokenStream
{
friend class TTLexer; // needs access to internals to know what was consumed
bool m_cache_valid;
Token m_cache;
Ident::Hygiene m_hygiene;
::std::vector< ::std::pair<Token, Ident::Hygiene> > m_lookahead;
ParseState m_parse_state;
public:
TokenStream();
virtual ~TokenStream();
Token getToken();
void putback(Token tok);
eTokenType lookahead(unsigned int count);
Ident::Hygiene getHygiene() const;
virtual void push_hygine() {}
virtual void pop_hygine() {}
ParseState& parse_state() { return m_parse_state; }
ProtoSpan start_span() const;
Span end_span(ProtoSpan ps) const;
Span point_span() const;
Ident get_ident(Token tok) const;
protected:
virtual Position getPosition() const = 0;
virtual ::std::shared_ptr<Span> outerSpan() const { return ::std::shared_ptr<Span>(0); }
virtual Token realGetToken() = 0;
virtual Ident::Hygiene realGetHygiene() const = 0;
private:
Token innerGetToken();
};
class SavedParseState
{
TokenStream& m_lex;
ParseState m_state;
public:
SavedParseState(TokenStream& lex, ParseState state):
m_lex(lex),
m_state(state)
{
}
~SavedParseState()
{
DEBUG("Restoring " << m_state);
m_lex.parse_state() = m_state;
}
};
#define SET_MODULE(lex, mod) SavedParseState _sps(lex, lex.parse_state()); lex.parse_state().module = &(mod)
#define SET_ATTRS(lex, attrs) SavedParseState _sps(lex, lex.parse_state()); lex.parse_state().parent_attrs = &(attrs)
#define SET_PARSE_FLAG(lex, flag) SavedParseState _sps(lex, lex.parse_state()); lex.parse_state().flag = true
#define CLEAR_PARSE_FLAG(lex, flag) SavedParseState _sps(lex, lex.parse_state()); lex.parse_state().flag = false
#define CHECK_PARSE_FLAG(lex, flag) (lex.parse_state().flag == true)
|
