summaryrefslogtreecommitdiff
path: root/src/parse
diff options
context:
space:
mode:
Diffstat (limited to 'src/parse')
-rw-r--r--src/parse/expr.cpp13
-rw-r--r--src/parse/lex.cpp2
-rw-r--r--src/parse/lex.hpp2
-rw-r--r--src/parse/tokenstream.cpp15
-rw-r--r--src/parse/tokenstream.hpp6
-rw-r--r--src/parse/tokentree.cpp21
-rw-r--r--src/parse/tokentree.hpp22
-rw-r--r--src/parse/ttstream.cpp17
-rw-r--r--src/parse/ttstream.hpp5
9 files changed, 67 insertions, 36 deletions
diff --git a/src/parse/expr.cpp b/src/parse/expr.cpp
index 832e6a4a..f2ec1857 100644
--- a/src/parse/expr.cpp
+++ b/src/parse/expr.cpp
@@ -1239,7 +1239,8 @@ ExprNodeP Parse_ExprMacro(TokenStream& lex, Token tok)
// Token Tree Parsing
TokenTree Parse_TT(TokenStream& lex, bool unwrapped)
{
- TRACE_FUNCTION;
+ TokenTree rv;
+ TRACE_FUNCTION_FR("", rv);
Token tok = lex.getToken();
eTokenType closer = TOK_PAREN_CLOSE;
@@ -1270,12 +1271,13 @@ TokenTree Parse_TT(TokenStream& lex, bool unwrapped)
case TOK_BRACE_CLOSE:
throw ParseError::Unexpected(lex, tok);
default:
- return TokenTree( mv$(tok) );
+ rv = TokenTree(lex.getHygiene(), mv$(tok) );
+ return rv;
}
::std::vector<TokenTree> items;
if( !unwrapped )
- items.push_back( mv$(tok) );
+ items.push_back( TokenTree(lex.getHygiene(), mv$(tok)) );
while(GET_TOK(tok, lex) != closer && tok.type() != TOK_EOF)
{
if( tok.type() == TOK_NULL )
@@ -1284,6 +1286,7 @@ TokenTree Parse_TT(TokenStream& lex, bool unwrapped)
items.push_back(Parse_TT(lex, false));
}
if( !unwrapped )
- items.push_back( mv$(tok) );
- return TokenTree(lex.getHygiene(), mv$(items));
+ items.push_back( TokenTree(lex.getHygiene(), mv$(tok)) );
+ rv = TokenTree(lex.getHygiene(), mv$(items));
+ return rv;
}
diff --git a/src/parse/lex.cpp b/src/parse/lex.cpp
index d6cfcddb..39842545 100644
--- a/src/parse/lex.cpp
+++ b/src/parse/lex.cpp
@@ -235,7 +235,7 @@ Position Lexer::getPosition() const
{
return Position(m_path, m_line, m_line_ofs);
}
-Ident::Hygiene Lexer::getHygiene() const
+Ident::Hygiene Lexer::realGetHygiene() const
{
return m_hygiene;
}
diff --git a/src/parse/lex.hpp b/src/parse/lex.hpp
index 3d41c3fc..572846da 100644
--- a/src/parse/lex.hpp
+++ b/src/parse/lex.hpp
@@ -46,7 +46,7 @@ public:
Lexer(const ::std::string& filename);
Position getPosition() const override;
- Ident::Hygiene getHygiene() const override;
+ Ident::Hygiene realGetHygiene() const override;
Token realGetToken() override;
private:
diff --git a/src/parse/tokenstream.cpp b/src/parse/tokenstream.cpp
index 5d9919f6..0182e548 100644
--- a/src/parse/tokenstream.cpp
+++ b/src/parse/tokenstream.cpp
@@ -38,7 +38,8 @@ Token TokenStream::getToken()
}
else if( m_lookahead.size() )
{
- Token ret = mv$( m_lookahead.front() );
+ Token ret = mv$( m_lookahead.front().first );
+ m_hygiene = m_lookahead.front().second;
m_lookahead.erase(m_lookahead.begin());
if( DEBUG_PRINT_TOKENS ) {
::std::cout << "getToken[" << typeid(*this).name() << "] - " << ret.get_pos() << "-" << ret << ::std::endl;
@@ -48,6 +49,7 @@ Token TokenStream::getToken()
else
{
Token ret = this->innerGetToken();
+ m_hygiene = this->realGetHygiene();
if( DEBUG_PRINT_TOKENS ) {
::std::cout << "getToken[" << typeid(*this).name() << "] - " << ret.get_pos() << "-" << ret << ::std::endl;
}
@@ -85,11 +87,18 @@ eTokenType TokenStream::lookahead(unsigned int i)
while( i >= m_lookahead.size() )
{
DEBUG("lookahead - read #" << m_lookahead.size());
- m_lookahead.push_back( this->innerGetToken() );
+ auto tok = this->innerGetToken();
+ auto hygiene = this->realGetHygiene();
+ m_lookahead.push_back( ::std::make_pair(mv$(tok), mv$(hygiene)) );
}
DEBUG("lookahead(" << i << ") = " << m_lookahead[i]);
- return m_lookahead[i].type();
+ return m_lookahead[i].first.type();
+}
+
+Ident::Hygiene TokenStream::getHygiene() const
+{
+ return m_hygiene;
}
ProtoSpan TokenStream::start_span() const
diff --git a/src/parse/tokenstream.hpp b/src/parse/tokenstream.hpp
index c6e2de37..ae737d0d 100644
--- a/src/parse/tokenstream.hpp
+++ b/src/parse/tokenstream.hpp
@@ -50,7 +50,8 @@ class TokenStream
bool m_cache_valid;
Token m_cache;
- ::std::vector<Token> m_lookahead;
+ Ident::Hygiene m_hygiene;
+ ::std::vector< ::std::pair<Token, Ident::Hygiene> > m_lookahead;
ParseState m_parse_state;
public:
TokenStream();
@@ -60,7 +61,7 @@ public:
eTokenType lookahead(unsigned int count);
virtual Position getPosition() const = 0;
- virtual Ident::Hygiene getHygiene() const = 0;
+ Ident::Hygiene getHygiene() const;
ParseState& parse_state() { return m_parse_state; }
@@ -71,6 +72,7 @@ public:
protected:
virtual Token realGetToken() = 0;
+ virtual Ident::Hygiene realGetHygiene() const = 0;
private:
Token innerGetToken();
};
diff --git a/src/parse/tokentree.cpp b/src/parse/tokentree.cpp
index 8fad6386..25e8ad44 100644
--- a/src/parse/tokentree.cpp
+++ b/src/parse/tokentree.cpp
@@ -11,7 +11,7 @@
TokenTree TokenTree::clone() const
{
if( m_subtrees.size() == 0 ) {
- return TokenTree(m_tok.clone());
+ return TokenTree(m_hygiene, m_tok.clone());
}
else {
::std::vector< TokenTree> ents;
@@ -21,3 +21,22 @@ TokenTree TokenTree::clone() const
return TokenTree( m_hygiene, mv$(ents) );
}
}
+
+::std::ostream& operator<<(::std::ostream& os, const TokenTree& tt)
+{
+ os << tt.m_hygiene;
+ if( tt.m_subtrees.size() == 0 )
+ return os << tt.m_tok;
+ else {
+ os << "TT([";
+ bool first = true;
+ for(const auto& i : tt.m_subtrees) {
+ if(!first)
+ os << ", ";
+ os << i;
+ first = false;
+ }
+ os << "])";
+ return os;
+ }
+}
diff --git a/src/parse/tokentree.hpp b/src/parse/tokentree.hpp
index 50ca72a5..27b85b45 100644
--- a/src/parse/tokentree.hpp
+++ b/src/parse/tokentree.hpp
@@ -26,6 +26,11 @@ public:
m_tok( ::std::move(tok) )
{
}
+ TokenTree(Ident::Hygiene hygiene, Token tok):
+ m_hygiene( ::std::move(hygiene) ),
+ m_tok( ::std::move(tok) )
+ {
+ }
TokenTree(Ident::Hygiene hygiene, ::std::vector<TokenTree> subtrees):
m_hygiene( ::std::move(hygiene) ),
m_subtrees( ::std::move(subtrees) )
@@ -46,22 +51,7 @@ public:
Token& tok() { return m_tok; }
const Ident::Hygiene& hygiene() const { return m_hygiene; }
- friend ::std::ostream& operator<<(::std::ostream& os, const TokenTree& tt) {
- if( tt.m_subtrees.size() == 0 )
- return os << tt.m_tok;
- else {
- os << "TT([";
- bool first = true;
- for(const auto& i : tt.m_subtrees) {
- if(!first)
- os << ", ";
- os << i;
- first = false;
- }
- os << "])";
- return os;
- }
- }
+ friend ::std::ostream& operator<<(::std::ostream& os, const TokenTree& tt);
};
#endif // TOKENTREE_HPP_INCLUDED
diff --git a/src/parse/ttstream.cpp b/src/parse/ttstream.cpp
index d969f514..cf2dfb63 100644
--- a/src/parse/ttstream.cpp
+++ b/src/parse/ttstream.cpp
@@ -22,10 +22,12 @@ Token TTStream::realGetToken()
{
// If current index is above TT size, go up
unsigned int& idx = m_stack.back().first;
+ assert( m_stack.back().second );
const TokenTree& tree = *m_stack.back().second;
if(idx == 0 && tree.is_token()) {
idx ++;
+ m_hygiene_ptr = &tree.hygiene();
return tree.tok();
}
@@ -34,7 +36,7 @@ Token TTStream::realGetToken()
const TokenTree& subtree = tree[idx];
idx ++;
if( subtree.size() == 0 ) {
- m_hygiene_ptr = &m_stack.back().second->hygiene();
+ m_hygiene_ptr = &subtree.hygiene();
return subtree.tok().clone();
}
else {
@@ -52,7 +54,7 @@ Position TTStream::getPosition() const
{
return Position("TTStream", 0,0);
}
-Ident::Hygiene TTStream::getHygiene() const
+Ident::Hygiene TTStream::realGetHygiene() const
{
assert( m_hygiene_ptr );
return *m_hygiene_ptr;
@@ -73,11 +75,12 @@ Token TTStreamO::realGetToken()
{
// If current index is above TT size, go up
unsigned int& idx = m_stack.back().first;
- TokenTree& tree = *( m_stack.back().second ? m_stack.back().second : &m_input_tt );
+ TokenTree& tree = (m_stack.back().second ? *m_stack.back().second : m_input_tt);
if(idx == 0 && tree.is_token()) {
idx ++;
m_last_pos = tree.tok().get_pos();
+ m_hygiene_ptr = &tree.hygiene();
return mv$(tree.tok());
}
@@ -87,6 +90,7 @@ Token TTStreamO::realGetToken()
idx ++;
if( subtree.size() == 0 ) {
m_last_pos = subtree.tok().get_pos();
+ m_hygiene_ptr = &subtree.hygiene();
return mv$( subtree.tok() );
}
else {
@@ -103,7 +107,10 @@ Position TTStreamO::getPosition() const
{
return m_last_pos;
}
-Ident::Hygiene TTStreamO::getHygiene() const
+Ident::Hygiene TTStreamO::realGetHygiene() const
{
- return (m_stack.back().second ? m_stack.back().second->hygiene() : m_input_tt.hygiene());
+ // Empty.
+ if(!m_hygiene_ptr)
+ return Ident::Hygiene();
+ return *m_hygiene_ptr;
}
diff --git a/src/parse/ttstream.hpp b/src/parse/ttstream.hpp
index 7b7fec4a..575178ac 100644
--- a/src/parse/ttstream.hpp
+++ b/src/parse/ttstream.hpp
@@ -23,9 +23,9 @@ public:
TTStream& operator=(const TTStream& x) { m_stack = x.m_stack; return *this; }
Position getPosition() const override;
- Ident::Hygiene getHygiene() const override;
protected:
+ Ident::Hygiene realGetHygiene() const override;
Token realGetToken() override;
};
@@ -36,6 +36,7 @@ class TTStreamO:
Position m_last_pos;
TokenTree m_input_tt;
::std::vector< ::std::pair<unsigned int, TokenTree*> > m_stack;
+ const Ident::Hygiene* m_hygiene_ptr = nullptr;
public:
TTStreamO(TokenTree input_tt);
TTStreamO(TTStreamO&& x) = default;
@@ -45,8 +46,8 @@ public:
TTStreamO& operator=(TTStreamO&& x) = default;
Position getPosition() const override;
- Ident::Hygiene getHygiene() const override;
protected:
+ Ident::Hygiene realGetHygiene() const override;
Token realGetToken() override;
};
generated by cgit v1.2.3 (git 2.39.1) at 2025-05-09 20:08:59 +0000