1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
|
/*
* MRustC - Rust Compiler
* - By John Hodge (Mutabah/thePowersGang)
*
* parse/token.hpp
* - Lexer Tokens
*/
#pragma once
#include <rc_string.hpp>
#include <tagged_union.hpp>
#include "../coretypes.hpp"
#include <ident.hpp>
#include <memory>
enum eTokenType
{
#define _(t) t,
#include "eTokenType.enum.h"
#undef _
};
class Position
{
public:
RcString filename;
unsigned int line;
unsigned int ofs;
Position():
filename(""),
line(0),
ofs(0)
{}
Position(RcString filename, unsigned int line, unsigned int ofs):
filename(filename),
line(line),
ofs(ofs)
{
}
};
extern ::std::ostream& operator<<(::std::ostream& os, const Position& p);
class TypeRef;
class TokenTree;
namespace AST {
typedef bool Visibility;
class Pattern;
class Path;
class ExprNode;
class Attribute;
class Item;
template<typename T>
struct Named;
};
class InterpolatedFragment;
class Token
{
friend class HirSerialiser;
friend class HirDeserialiser;
TAGGED_UNION(Data, None,
(None, struct {}),
(IString, RcString),
(String, ::std::string),
(Integer, struct {
enum eCoreType m_datatype;
uint64_t m_intval;
}),
(Float, struct {
enum eCoreType m_datatype;
double m_floatval;
}),
(Fragment, void*)
);
enum eTokenType m_type;
Data m_data;
Position m_pos;
Token(enum eTokenType t, Data d, Position p):
m_type(t),
m_data( ::std::move(d) ),
m_pos( ::std::move(p) )
{
}
public:
virtual ~Token();
Token();
Token& operator=(Token&& t)
{
m_type = t.m_type; t.m_type = TOK_NULL;
m_data = ::std::move(t.m_data);
m_pos = ::std::move(t.m_pos);
return *this;
}
Token(Token&& t):
m_type(t.m_type),
m_data( ::std::move(t.m_data) ),
m_pos( ::std::move(t.m_pos) )
{
t.m_type = TOK_NULL;
}
Token(const Token& t);
Token clone() const;
Token(enum eTokenType type);
Token(enum eTokenType type, ::std::string str);
Token(enum eTokenType type, RcString str);
Token(uint64_t val, enum eCoreType datatype);
Token(double val, enum eCoreType datatype);
Token(const InterpolatedFragment& );
struct TagTakeIP {};
Token(TagTakeIP, InterpolatedFragment );
enum eTokenType type() const { return m_type; }
const RcString& istr() const { return m_data.as_IString(); }
::std::string& str() { return m_data.as_String(); }
const ::std::string& str() const { return m_data.as_String(); }
enum eCoreType datatype() const { TU_MATCH_DEF(Data, (m_data), (e), (assert(!"Getting datatype of invalid token type");), (Integer, return e.m_datatype;), (Float, return e.m_datatype;)) throw ""; }
uint64_t intval() const { return m_data.as_Integer().m_intval; }
double floatval() const { return m_data.as_Float().m_floatval; }
// TODO: Replace these with a way of getting a InterpolatedFragment&
TypeRef& frag_type() { assert(m_type == TOK_INTERPOLATED_TYPE); return *reinterpret_cast<TypeRef*>( m_data.as_Fragment() ); }
AST::Path& frag_path() { assert(m_type == TOK_INTERPOLATED_PATH); return *reinterpret_cast<AST::Path*>( m_data.as_Fragment() ); }
AST::Pattern& frag_pattern() { assert(m_type == TOK_INTERPOLATED_PATTERN); return *reinterpret_cast<AST::Pattern*>( m_data.as_Fragment() ); }
AST::Attribute& frag_meta() { assert(m_type == TOK_INTERPOLATED_META); return *reinterpret_cast<AST::Attribute*>( m_data.as_Fragment() ); }
::std::unique_ptr<AST::ExprNode> take_frag_node();
::AST::Named<AST::Item> take_frag_item();
::AST::Visibility take_frag_vis();
bool operator==(const Token& r) const {
if(type() != r.type())
return false;
TU_MATCH(Data, (m_data, r.m_data), (e, re),
(None, return true;),
(IString, return e == re; ),
(String, return e == re; ),
(Integer, return e.m_datatype == re.m_datatype && e.m_intval == re.m_intval;),
(Float, return e.m_datatype == re.m_datatype && e.m_floatval == re.m_floatval;),
(Fragment, assert(!"Token equality on Fragment");)
)
throw "";
}
bool operator!=(const Token& r) const { return !(*this == r); }
::std::string to_str() const;
void set_pos(Position pos) { m_pos = pos; }
const Position& get_pos() const { return m_pos; }
static const char* typestr(enum eTokenType type);
static eTokenType typefromstr(const ::std::string& s);
friend ::std::ostream& operator<<(::std::ostream& os, const Token& tok);
};
extern ::std::ostream& operator<<(::std::ostream& os, const Token& tok);
|
