summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorJohn Hodge <tpg@mutabah.net>2016-11-02 19:38:45 +0800
committerJohn Hodge <tpg@mutabah.net>2016-11-02 19:38:45 +0800
commit983ff1c26e15a13e6a1e0e4bb1e964ec496a4777 (patch)
treebc39e1e44e63b9440dd186c2781e4ad02f82c2c5
parent739c2f5a2124e9aa4fcc01e8fc1dbfe147a37254 (diff)
downloadmrust-983ff1c26e15a13e6a1e0e4bb1e964ec496a4777.tar.gz
Parse - Rework hygiene (with spelling fix) for take 2
-rw-r--r--src/ast/path.cpp2
-rw-r--r--src/ast/path.hpp6
-rw-r--r--src/ast/pattern.hpp2
-rw-r--r--src/expand/env.cpp2
-rw-r--r--src/expand/format_args.cpp22
-rw-r--r--src/ident.cpp45
-rw-r--r--src/include/ident.hpp48
-rw-r--r--src/macro_rules/eval.cpp20
-rw-r--r--src/macro_rules/parse.cpp6
-rw-r--r--src/parse/eTokenType.enum.h1
-rw-r--r--src/parse/expr.cpp8
-rw-r--r--src/parse/lex.cpp62
-rw-r--r--src/parse/lex.hpp13
-rw-r--r--src/parse/paths.cpp7
-rw-r--r--src/parse/pattern.cpp19
-rw-r--r--src/parse/root.cpp88
-rw-r--r--src/parse/token.cpp20
-rw-r--r--src/parse/token.hpp8
-rw-r--r--src/parse/tokentree.hpp16
-rw-r--r--src/resolve/absolute.cpp12
20 files changed, 196 insertions, 211 deletions
diff --git a/src/ast/path.cpp b/src/ast/path.cpp
index 25c0c208..ac023c8f 100644
--- a/src/ast/path.cpp
+++ b/src/ast/path.cpp
@@ -162,7 +162,7 @@ AST::Path::Path(const Path& x):
m_class = Class::make_Local({ent.name});
),
(Relative,
- m_class = Class::make_Relative({ent.hygine, ent.nodes});
+ m_class = Class::make_Relative({ent.hygiene, ent.nodes});
),
(Self,
m_class = Class::make_Self({ent.nodes});
diff --git a/src/ast/path.hpp b/src/ast/path.hpp
index 5c2481a0..b45767a2 100644
--- a/src/ast/path.hpp
+++ b/src/ast/path.hpp
@@ -162,7 +162,7 @@ public:
::std::string name;
} ),
(Relative, struct { // General relative
- Ident::Hygine hygine; // Taken from the first ident
+ Ident::Hygiene hygiene;
::std::vector<PathNode> nodes;
} ),
(Self, struct { // Module-relative
@@ -226,8 +226,8 @@ public:
// RELATIVE
struct TagRelative {};
- Path(TagRelative, Ident::Hygine hygine, ::std::vector<PathNode> nodes):
- m_class( Class::make_Relative({ mv$(hygine), mv$(nodes) }) )
+ Path(TagRelative, Ident::Hygiene hygiene, ::std::vector<PathNode> nodes):
+ m_class( Class::make_Relative({ mv$(hygiene), mv$(nodes) }) )
{}
// SELF
struct TagSelf {};
diff --git a/src/ast/pattern.hpp b/src/ast/pattern.hpp
index 0dff4071..a2df7b59 100644
--- a/src/ast/pattern.hpp
+++ b/src/ast/pattern.hpp
@@ -35,7 +35,7 @@ public:
unsigned int m_slot;
PatternBinding():
- m_name({0,{}}, ""),
+ m_name({}, ""),
m_type(Type::MOVE),
m_mutable(false),
m_slot( ~0u )
diff --git a/src/expand/env.cpp b/src/expand/env.cpp
index 989fbb7a..85742549 100644
--- a/src/expand/env.cpp
+++ b/src/expand/env.cpp
@@ -67,7 +67,7 @@ class CExpanderOptionEnv:
rv.push_back( Token(TOK_PAREN_OPEN) );
rv.push_back( Token(TOK_STRING, var_val_cstr) );
rv.push_back( Token(TOK_PAREN_CLOSE) );
- return box$( TTStreamO(TokenTree( mv$(rv) )) );
+ return box$( TTStreamO(TokenTree( {}, mv$(rv) )) );
}
}
};
diff --git a/src/expand/format_args.cpp b/src/expand/format_args.cpp
index 977d7d69..4857a6c2 100644
--- a/src/expand/format_args.cpp
+++ b/src/expand/format_args.cpp
@@ -365,17 +365,17 @@ namespace {
break;
case ::AST::Crate::LOAD_CORE:
toks.push_back( TokenTree(TOK_DOUBLE_COLON) );
- toks.push_back( Token(Ident("core")) );
+ toks.push_back( Token(TOK_IDENT, "core") );
break;
case ::AST::Crate::LOAD_STD:
toks.push_back( TokenTree(TOK_DOUBLE_COLON) );
- toks.push_back( Token(Ident("std")) );
+ toks.push_back( Token(TOK_IDENT, "std") );
break;
}
for(auto ent : il)
{
toks.push_back( TokenTree(TOK_DOUBLE_COLON) );
- toks.push_back( Token(Ident(ent)) );
+ toks.push_back( Token(TOK_IDENT, ent) );
}
}
}
@@ -417,7 +417,7 @@ class CFormatArgsExpander:
if( lex.lookahead(0) == TOK_IDENT && lex.lookahead(1) == TOK_EQUAL )
{
GET_CHECK_TOK(tok, lex, TOK_IDENT);
- auto name = mv$(tok.ident_str());
+ auto name = mv$(tok.str());
GET_CHECK_TOK(tok, lex, TOK_EQUAL);
@@ -478,7 +478,7 @@ class CFormatArgsExpander:
toks.push_back( TokenTree(TOK_PAREN_OPEN) );
for(unsigned int i = 0; i < free_args.size() + named_args.size(); i ++ )
{
- toks.push_back( Token(Ident(FMT("a" << i))) );
+ toks.push_back( Token(TOK_IDENT, FMT("a" << i)) );
toks.push_back( TokenTree(TOK_COMMA) );
}
toks.push_back( TokenTree(TOK_PAREN_CLOSE) );
@@ -490,13 +490,13 @@ class CFormatArgsExpander:
// - Contains N+1 entries, where N is the number of fragments
{
toks.push_back( TokenTree(TOK_RWORD_STATIC) );
- toks.push_back( Token(Ident("FRAGMENTS")) );
+ toks.push_back( Token(TOK_IDENT, "FRAGMENTS") );
toks.push_back( TokenTree(TOK_COLON) );
toks.push_back( TokenTree(TOK_SQUARE_OPEN) );
toks.push_back( Token(TOK_AMP) );
toks.push_back( Token(TOK_LIFETIME, "static") );
- toks.push_back( Token(Ident("str")) );
+ toks.push_back( Token(TOK_IDENT, "str") );
toks.push_back( Token(TOK_SEMICOLON) );
toks.push_back( Token(fragments.size() + 1, CORETYPE_UINT) );
toks.push_back( TokenTree(TOK_SQUARE_CLOSE) );
@@ -522,7 +522,7 @@ class CFormatArgsExpander:
toks.push_back( TokenTree(TOK_PAREN_OPEN) );
{
toks.push_back( TokenTree(TOK_AMP) );
- toks.push_back( Token(Ident("FRAGMENTS")) );
+ toks.push_back( Token(TOK_IDENT, "FRAGMENTS") );
toks.push_back( TokenTree(TOK_COMMA) );
toks.push_back( TokenTree(TOK_AMP) );
@@ -531,7 +531,7 @@ class CFormatArgsExpander:
{
push_path(toks, crate, {"fmt", "ArgumentV1", "new"});
toks.push_back( Token(TOK_PAREN_OPEN) );
- toks.push_back( Token(Ident(FMT("a" << frag.arg_index))) );
+ toks.push_back( Token(TOK_IDENT, FMT("a" << frag.arg_index)) );
toks.push_back( TokenTree(TOK_COMMA) );
@@ -554,7 +554,7 @@ class CFormatArgsExpander:
toks.push_back( TokenTree(TOK_PAREN_OPEN) );
{
toks.push_back( TokenTree(TOK_AMP) );
- toks.push_back( Token(Ident("FRAGMENTS")) );
+ toks.push_back( Token(TOK_IDENT, "FRAGMENTS") );
toks.push_back( TokenTree(TOK_COMMA) );
// 1. Generate a set of arguments+formatters
@@ -574,7 +574,7 @@ class CFormatArgsExpander:
toks.push_back( TokenTree(TOK_BRACE_CLOSE) );
toks.push_back( TokenTree(TOK_BRACE_CLOSE) );
- return box$( TTStreamO(TokenTree(mv$(toks))) );
+ return box$( TTStreamO(TokenTree(Ident::Hygiene::new_scope(), mv$(toks))) );
}
};
diff --git a/src/ident.cpp b/src/ident.cpp
index 5d0fe00c..a5a6bdbb 100644
--- a/src/ident.cpp
+++ b/src/ident.cpp
@@ -3,49 +3,20 @@
* - By John Hodge (Mutabah/thePowersGang)
*
* include/ident.cpp
- * - Identifiers with hygine
+ * - Identifiers with hygiene
*/
#include <iostream>
#include <ident.hpp>
#include <debug.hpp>
#include <common.hpp> // vector print
-bool Ident::Hygine::is_visible(const Hygine& src) const
+unsigned int Ident::Hygiene::g_next_scope = 0;
+
+bool Ident::Hygiene::is_visible(const Hygiene& src) const
{
- // HACK: Disable hygine for now
+ // HACK: Disable hygiene for now
return true;
-
- DEBUG("*this = " << *this << ", src=" << src);
- if( this->file_num != src.file_num ) {
- DEBUG("- Different file");
- return false;
- }
-
- // `this` is the item, `src` is the ident used to access it
-
- // If this is from a deeper point than the source, it won't work.
- if( this->indexes.size() > src.indexes.size() ) {
- DEBUG("- Not subset: len");
- return false;
- }
-
- // If this hygine list (barring the last) is a subset of the source
- for(unsigned int i = 0; i < this->indexes.size()-1 - 1; i ++)
- {
- if( this->indexes[i] != src.indexes[i] ) {
- DEBUG("- Not subset: " << i);
- return false;
- }
- }
-
- unsigned int end = this->indexes.size()-1-1;
- // Allow match if this ident is from before the addressing ident
- if( this->indexes[end] < src.indexes[end] ) {
- return true;
- }
-
- DEBUG("- Not before");
- return false;
+ //return x.scope_index == src.scope_index;
}
::std::ostream& operator<<(::std::ostream& os, const Ident& x) {
@@ -53,8 +24,8 @@ bool Ident::Hygine::is_visible(const Hygine& src) const
return os;
}
-::std::ostream& operator<<(::std::ostream& os, const Ident::Hygine& x) {
- os << "{" << x.file_num << ": [" << x.indexes << "]}";
+::std::ostream& operator<<(::std::ostream& os, const Ident::Hygiene& x) {
+ os << "{" << x.scope_index << "}";
return os;
}
diff --git a/src/include/ident.hpp b/src/include/ident.hpp
index d5ce2e77..cff918ca 100644
--- a/src/include/ident.hpp
+++ b/src/include/ident.hpp
@@ -3,7 +3,7 @@
* - By John Hodge (Mutabah/thePowersGang)
*
* include/ident.hpp
- * - Identifiers with hygine
+ * - Identifiers with hygiene
*/
#pragma once
#include <vector>
@@ -11,42 +11,50 @@
struct Ident
{
- struct Hygine
+ class Hygiene
{
- unsigned int file_num;
- ::std::vector<unsigned int> indexes;
+ static unsigned g_next_scope;
+ unsigned int scope_index;
- Hygine(unsigned int file, ::std::vector<unsigned int> indexes):
- file_num(file),
- indexes(::std::move(indexes))
+ Hygiene(unsigned int index):
+ scope_index(index)
{}
+ public:
+ Hygiene():
+ scope_index(0)
+ {}
+
+ static Hygiene new_scope()
+ {
+ return Hygiene(++g_next_scope);
+ }
- Hygine(Hygine&& x) = default;
- Hygine(const Hygine& x) = default;
- Hygine& operator=(Hygine&& x) = default;
- Hygine& operator=(const Hygine& x) = default;
+ Hygiene(Hygiene&& x) = default;
+ Hygiene(const Hygiene& x) = default;
+ Hygiene& operator=(Hygiene&& x) = default;
+ Hygiene& operator=(const Hygiene& x) = default;
// Returns true if an ident with hygine `souce` can see an ident with this hygine
- bool is_visible(const Hygine& source) const;
- bool operator==(const Hygine& x) const { return file_num == x.file_num && indexes == x.indexes; }
- bool operator!=(const Hygine& x) const { return file_num != x.file_num || indexes != x.indexes; }
+ bool is_visible(const Hygiene& source) const;
+ bool operator==(const Hygiene& x) const { return scope_index == x.scope_index; }
+ bool operator!=(const Hygiene& x) const { return scope_index != x.scope_index; }
- friend ::std::ostream& operator<<(::std::ostream& os, const Hygine& v);
+ friend ::std::ostream& operator<<(::std::ostream& os, const Hygiene& v);
};
- Hygine hygine;
+ Hygiene hygiene;
::std::string name;
Ident(const char* name):
- hygine(~0u, {}),
+ hygiene(),
name(name)
{ }
Ident(::std::string name):
- hygine(~0u, {}),
+ hygiene(),
name(::std::move(name))
{ }
- Ident(Hygine hygine, ::std::string name):
- hygine(::std::move(hygine)), name(::std::move(name))
+ Ident(Hygiene hygiene, ::std::string name):
+ hygiene(::std::move(hygiene)), name(::std::move(name))
{ }
Ident(Ident&& x) = default;
diff --git a/src/macro_rules/eval.cpp b/src/macro_rules/eval.cpp
index 31665a25..110e8525 100644
--- a/src/macro_rules/eval.cpp
+++ b/src/macro_rules/eval.cpp
@@ -566,6 +566,7 @@ class MacroExpander:
Token m_next_token; // used for inserting a single token into the stream
::std::unique_ptr<TTStreamO> m_ttstream;
+ Ident::Hygiene m_hygiene;
public:
MacroExpander(const MacroExpander& x) = delete;
@@ -574,12 +575,14 @@ public:
m_macro_filename( FMT("Macro:" << macro_name) ),
m_crate_name( mv$(crate_name) ),
m_mappings( mv$(mappings) ),
- m_state( contents, m_mappings )
+ m_state( contents, m_mappings ),
+ m_hygiene( Ident::Hygiene::new_scope() )
{
}
- virtual Position getPosition() const override;
- virtual Token realGetToken() override;
+ Position getPosition() const override;
+ Ident::Hygiene getHygiene() const override;
+ Token realGetToken() override;
};
void Macro_InitDefaults()
@@ -1044,6 +1047,17 @@ Position MacroExpander::getPosition() const
// TODO: Return a far better span - invocaion location?
return Position(m_macro_filename, 0, m_state.top_pos());
}
+Ident::Hygiene MacroExpander::getHygiene() const
+{
+ if( m_ttstream )
+ {
+ return m_ttstream->getHygiene();
+ }
+ else
+ {
+ return m_hygiene;
+ }
+}
Token MacroExpander::realGetToken()
{
// Use m_next_token first
diff --git a/src/macro_rules/parse.cpp b/src/macro_rules/parse.cpp
index bd54c67d..c7a6ef89 100644
--- a/src/macro_rules/parse.cpp
+++ b/src/macro_rules/parse.cpp
@@ -49,10 +49,10 @@ public:
switch( GET_TOK(tok, lex) )
{
case TOK_IDENT: {
- ::std::string name = mv$(tok.ident_str());
+ ::std::string name = mv$(tok.str());
GET_CHECK_TOK(tok, lex, TOK_COLON);
GET_CHECK_TOK(tok, lex, TOK_IDENT);
- ::std::string type = mv$(tok.ident_str());
+ ::std::string type = mv$(tok.str());
unsigned int idx = ::std::find( names.begin(), names.end(), name ) - names.begin();
if( idx == names.size() )
@@ -202,7 +202,7 @@ public:
else if( tok.type() == TOK_IDENT )
{
// Look up the named parameter in the list of param names for this arm
- const auto& name = tok.ident_str();
+ const auto& name = tok.str();
unsigned int idx = ::std::find(var_names.begin(), var_names.end(), name) - var_names.begin();
if( idx == var_names.size() )
ERROR(lex.getPosition(), E0000, "Macro variable $" << name << " not found");
diff --git a/src/parse/eTokenType.enum.h b/src/parse/eTokenType.enum.h
index 9caa3aba..8d410b0c 100644
--- a/src/parse/eTokenType.enum.h
+++ b/src/parse/eTokenType.enum.h
@@ -5,6 +5,7 @@ _(TOK_NEWLINE)
_(TOK_WHITESPACE)
_(TOK_COMMENT)
+_(TOK_INTERPOLATED_IDENT)
_(TOK_INTERPOLATED_PATH)
_(TOK_INTERPOLATED_TYPE)
_(TOK_INTERPOLATED_PATTERN)
diff --git a/src/parse/expr.cpp b/src/parse/expr.cpp
index f6ebca1f..832e6a4a 100644
--- a/src/parse/expr.cpp
+++ b/src/parse/expr.cpp
@@ -889,7 +889,7 @@ ExprNodeP Parse_ExprFC(TokenStream& lex)
switch(GET_TOK(tok, lex))
{
case TOK_IDENT: {
- AST::PathNode path( mv$(tok.ident_str()) , {});
+ AST::PathNode path( mv$(tok.str()) , {});
switch( GET_TOK(tok, lex) )
{
case TOK_PAREN_OPEN:
@@ -965,7 +965,7 @@ ExprNodeP Parse_ExprVal_StructLiteral(TokenStream& lex, AST::Path path)
::std::vector< ::std::pair< ::std::string, ::std::unique_ptr<AST::ExprNode>> > items;
while( GET_TOK(tok, lex) == TOK_IDENT )
{
- auto name = mv$(tok.ident_str());
+ auto name = mv$(tok.str());
GET_CHECK_TOK(tok, lex, TOK_COLON);
ExprNodeP val = Parse_Stmt(lex);
items.push_back( ::std::make_pair(::std::move(name), ::std::move(val)) );
@@ -1224,7 +1224,7 @@ ExprNodeP Parse_ExprMacro(TokenStream& lex, Token tok)
::std::string name = tok.str();
::std::string ident;
if( GET_TOK(tok, lex) == TOK_IDENT ) {
- ident = mv$(tok.ident_str());
+ ident = mv$(tok.str());
}
else {
PUTBACK(tok, lex);
@@ -1285,5 +1285,5 @@ TokenTree Parse_TT(TokenStream& lex, bool unwrapped)
}
if( !unwrapped )
items.push_back( mv$(tok) );
- return TokenTree(mv$(items));
+ return TokenTree(lex.getHygiene(), mv$(items));
}
diff --git a/src/parse/lex.cpp b/src/parse/lex.cpp
index a5c4e372..9acb56e0 100644
--- a/src/parse/lex.cpp
+++ b/src/parse/lex.cpp
@@ -29,7 +29,8 @@ Lexer::Lexer(const ::std::string& filename):
m_line(1),
m_line_ofs(0),
m_istream(filename.c_str()),
- m_last_char_valid(false)
+ m_last_char_valid(false),
+ m_hygiene( Ident::Hygiene::new_scope() )
{
if( !m_istream.is_open() )
{
@@ -243,6 +244,10 @@ Position Lexer::getPosition() const
{
return Position(m_path, m_line, m_line_ofs);
}
+Ident::Hygiene Lexer::getHygiene() const
+{
+ return m_hygiene;
+}
Token Lexer::realGetToken()
{
while(true)
@@ -251,29 +256,6 @@ Token Lexer::realGetToken()
//::std::cout << "getTokenInt: tok = " << tok << ::std::endl;
switch(tok.type())
{
- case TOK_IDENT:
- if( m_hygine_stack.size() > 0 )
- {
- m_hygine_stack.back() ++;
- }
- return tok;
-
- case TOK_PAREN_OPEN:
- case TOK_SQUARE_OPEN:
- case TOK_BRACE_OPEN:
- case TOK_ATTR_OPEN:
- case TOK_CATTR_OPEN:
- m_hygine_stack.push_back(0);
- return tok;
- case TOK_PAREN_CLOSE:
- case TOK_SQUARE_CLOSE:
- case TOK_BRACE_CLOSE:
- m_hygine_stack.pop_back();
- if( m_hygine_stack.size() > 0 ) {
- m_hygine_stack.back() ++;
- }
- return tok;
-
case TOK_NEWLINE:
m_line ++;
m_line_ofs = 0;
@@ -790,7 +772,7 @@ Token Lexer::getTokenInt_Identifier(Codepoint leader, Codepoint leader2)
if( str < RWORDS[i].chars ) break;
if( str == RWORDS[i].chars ) return Token((enum eTokenType)RWORDS[i].type);
}
- return Token(Ident( Ident::Hygine(m_file_index, m_hygine_stack), mv$(str) ));
+ return Token(TOK_IDENT, mv$(str));
}
}
@@ -1022,22 +1004,29 @@ Token TTStream::realGetToken()
const TokenTree& subtree = tree[idx];
idx ++;
if( subtree.size() == 0 ) {
+ m_hygiene_ptr = &m_stack.back().second->hygiene();
return subtree.tok().clone();
}
else {
- m_stack.push_back( ::std::make_pair(0, &subtree ) );
+ m_stack.push_back( ::std::make_pair(0, &subtree) );
}
}
else {
m_stack.pop_back();
}
}
+ //m_hygiene = nullptr;
return Token(TOK_EOF);
}
Position TTStream::getPosition() const
{
return Position("TTStream", 0,0);
}
+Ident::Hygiene TTStream::getHygiene() const
+{
+ //assert( m_hygiene );
+ return *m_hygiene_ptr;
+}
TTStreamO::TTStreamO(TokenTree input_tt):
@@ -1071,7 +1060,7 @@ Token TTStreamO::realGetToken()
return mv$( subtree.tok() );
}
else {
- m_stack.push_back( ::std::make_pair(0, &subtree ) );
+ m_stack.push_back( ::std::make_pair(0, &subtree) );
}
}
else {
@@ -1084,6 +1073,10 @@ Position TTStreamO::getPosition() const
{
return m_last_pos;
}
+Ident::Hygiene TTStreamO::getHygiene() const
+{
+ return (m_stack.back().second ? m_stack.back().second->hygiene() : m_input_tt.hygiene());
+}
TokenStream::TokenStream():
@@ -1183,7 +1176,18 @@ Span TokenStream::end_span(ProtoSpan ps) const
p.line, p.ofs
);
}
-
+Ident TokenStream::get_ident(Token tok) const
+{
+ if(tok.type() == TOK_IDENT) {
+ return Ident(getHygiene(), tok.str());
+ }
+ else if( tok.type() == TOK_INTERPOLATED_IDENT ) {
+ TODO(getPosition(), "");
+ }
+ else {
+ throw ParseError::Unexpected(*this, tok);
+ }
+}
TokenTree TokenTree::clone() const
{
@@ -1195,7 +1199,7 @@ TokenTree TokenTree::clone() const
ents.reserve( m_subtrees.size() );
for(const auto& sub : m_subtrees)
ents.push_back( sub.clone() );
- return TokenTree( mv$(ents) );
+ return TokenTree( m_hygiene, mv$(ents) );
}
}
diff --git a/src/parse/lex.hpp b/src/parse/lex.hpp
index 208fa670..d147d79d 100644
--- a/src/parse/lex.hpp
+++ b/src/parse/lex.hpp
@@ -62,13 +62,17 @@ public:
Token getToken();
void putback(Token tok);
eTokenType lookahead(unsigned int count);
+
virtual Position getPosition() const = 0;
+ virtual Ident::Hygiene getHygiene() const = 0;
ParseState& parse_state() { return m_parse_state; }
ProtoSpan start_span() const;
Span end_span(ProtoSpan ps) const;
+ Ident get_ident(Token tok) const;
+
protected:
virtual Token realGetToken() = 0;
private:
@@ -126,14 +130,13 @@ class Lexer:
Codepoint m_last_char;
Token m_next_token; // Used when lexing generated two tokens
- unsigned int m_file_index;
- // Incremented on every ident and TT close, pushed on TT open
- ::std::vector<unsigned int> m_hygine_stack;
+ Ident::Hygiene m_hygiene;
public:
Lexer(const ::std::string& filename);
- virtual Position getPosition() const override;
- virtual Token realGetToken() override;
+ Position getPosition() const override;
+ Ident::Hygiene getHygiene() const override;
+ Token realGetToken() override;
private:
Token getTokenInt();
diff --git a/src/parse/paths.cpp b/src/parse/paths.cpp
index 5e0faf90..e600e51e 100644
--- a/src/parse/paths.cpp
+++ b/src/parse/paths.cpp
@@ -91,8 +91,9 @@ AST::Path Parse_Path(TokenStream& lex, bool is_abs, eParsePathGenericMode generi
}
}
else {
+ // TODO: TOK_INTERPOLATED_IDENT?
GET_CHECK_TOK(tok, lex, TOK_IDENT);
- auto hygine = mv$( tok.ident().hygine );
+ auto hygine = lex.getHygiene();
PUTBACK(tok, lex);
return AST::Path(AST::Path::TagRelative(), mv$(hygine), Parse_PathNodes(lex, generic_mode));
}
@@ -111,7 +112,7 @@ AST::Path Parse_Path(TokenStream& lex, bool is_abs, eParsePathGenericMode generi
::AST::PathParams params;
CHECK_TOK(tok, TOK_IDENT);
- auto component = mv$( tok.ident_str() );
+ auto component = mv$( tok.str() );
GET_TOK(tok, lex);
if( generic_mode == PATH_GENERIC_TYPE )
@@ -218,7 +219,7 @@ AST::Path Parse_Path(TokenStream& lex, bool is_abs, eParsePathGenericMode generi
case TOK_IDENT:
if( LOOK_AHEAD(lex) == TOK_EQUAL )
{
- ::std::string name = mv$(tok.ident_str());
+ ::std::string name = mv$(tok.str());
GET_CHECK_TOK(tok, lex, TOK_EQUAL);
assoc_bounds.push_back( ::std::make_pair( mv$(name), Parse_Type(lex,false) ) );
break;
diff --git a/src/parse/pattern.cpp b/src/parse/pattern.cpp
index df251bd3..f43a1205 100644
--- a/src/parse/pattern.cpp
+++ b/src/parse/pattern.cpp
@@ -85,7 +85,7 @@ AST::Pattern Parse_Pattern(TokenStream& lex, bool is_refutable)
if( expect_bind )
{
CHECK_TOK(tok, TOK_IDENT);
- auto bind_name = tok.take_ident();
+ auto bind_name = Ident(lex.getHygiene(), mv$(tok.str()));
// If there's no '@' after it, it's a name binding only (_ pattern)
if( GET_TOK(tok, lex) != TOK_AT )
{
@@ -114,21 +114,23 @@ AST::Pattern Parse_Pattern(TokenStream& lex, bool is_refutable)
break;
// Known binding `ident @`
case TOK_AT:
- binding = AST::PatternBinding( tok.take_ident(), bind_type/*MOVE*/, is_mut/*false*/ );
+ binding = AST::PatternBinding( Ident(lex.getHygiene(), mv$(tok.str())), bind_type/*MOVE*/, is_mut/*false*/ );
GET_TOK(tok, lex); // '@'
GET_TOK(tok, lex); // Match lex.putback() below
break;
- default: // Maybe bind
+ default: { // Maybe bind
+ Ident name = Ident(lex.getHygiene(), mv$(tok.str()));
// if the pattern can be refuted (i.e this could be an enum variant), return MaybeBind
if( is_refutable ) {
assert(bind_type == ::AST::PatternBinding::Type::MOVE);
assert(is_mut == false);
- return AST::Pattern(AST::Pattern::TagMaybeBind(), tok.take_ident());
+ return AST::Pattern(AST::Pattern::TagMaybeBind(), mv$(name));
}
// Otherwise, it IS a binding
else {
- return AST::Pattern(AST::Pattern::TagBind(), tok.take_ident(), bind_type, is_mut);
+ return AST::Pattern(AST::Pattern::TagBind(), mv$(name), bind_type, is_mut);
}
+ break;}
}
}
else
@@ -138,7 +140,6 @@ AST::Pattern Parse_Pattern(TokenStream& lex, bool is_refutable)
PUTBACK(tok, lex);
auto pat = Parse_PatternReal(lex, is_refutable);
- //pat.set_bind( mv$(binding) );
pat.binding() = mv$(binding);
return pat;
}
@@ -276,10 +277,10 @@ AST::Pattern Parse_PatternReal_Slice(TokenStream& lex, bool is_refutable)
::AST::PatternBinding binding;
if( tok.type() == TOK_RWORD_REF && lex.lookahead(0) == TOK_IDENT && lex.lookahead(1) == TOK_DOUBLE_DOT ) {
GET_TOK(tok, lex);
- binding = ::AST::PatternBinding( tok.take_ident(), ::AST::PatternBinding::Type::REF, false );
+ binding = ::AST::PatternBinding( lex.get_ident(mv$(tok)), ::AST::PatternBinding::Type::REF, false );
}
else if( tok.type() == TOK_IDENT && lex.lookahead(0) == TOK_DOUBLE_DOT) {
- binding = ::AST::PatternBinding( tok.take_ident(), ::AST::PatternBinding::Type::MOVE, false );
+ binding = ::AST::PatternBinding( lex.get_ident(mv$(tok)), ::AST::PatternBinding::Type::MOVE, false );
}
else if( tok.type() == TOK_UNDERSCORE && lex.lookahead(0) == TOK_DOUBLE_DOT) {
// No binding, but switching to trailing
@@ -466,7 +467,7 @@ AST::Pattern Parse_PatternStruct(TokenStream& lex, AST::Path path, bool is_refut
}
CHECK_TOK(tok, TOK_IDENT);
- auto field_ident = tok.take_ident();
+ auto field_ident = lex.get_ident(mv$(tok));
::std::string field_name;
GET_TOK(tok, lex);
diff --git a/src/parse/root.cpp b/src/parse/root.cpp
index 6c9acc14..2f5114ed 100644
--- a/src/parse/root.cpp
+++ b/src/parse/root.cpp
@@ -133,7 +133,7 @@ AST::GenericParams Parse_GenericParams(TokenStream& lex)
if( tok.type() == TOK_IDENT )
{
// TODO: Hygine
- ::std::string param_name = mv$(tok.ident_str());
+ ::std::string param_name = mv$(tok.str());
ret.add_ty_param( AST::TypeParam( param_name ) );
auto param_ty = TypeRef(lex.getPosition(), param_name);
@@ -529,7 +529,7 @@ AST::Struct Parse_Struct(TokenStream& lex, const AST::MetaItems& meta_items)
}
CHECK_TOK(tok, TOK_IDENT);
- auto name = mv$(tok.ident_str());
+ auto name = mv$(tok.str());
GET_CHECK_TOK(tok, lex, TOK_COLON);
TypeRef type = Parse_Type(lex);
@@ -615,7 +615,7 @@ AST::Trait Parse_TraitDef(TokenStream& lex, const AST::MetaItems& meta_items)
}
bool is_specialisable = false;
- if( tok.type() == TOK_IDENT && tok.ident_str() == "default" ) {
+ if( tok.type() == TOK_IDENT && tok.str() == "default" ) {
is_specialisable = true;
GET_TOK(tok, lex);
}
@@ -629,7 +629,7 @@ AST::Trait Parse_TraitDef(TokenStream& lex, const AST::MetaItems& meta_items)
{
case TOK_RWORD_STATIC: {
GET_CHECK_TOK(tok, lex, TOK_IDENT);
- auto name = mv$(tok.ident_str());
+ auto name = mv$(tok.str());
GET_CHECK_TOK(tok, lex, TOK_COLON);
auto ty = Parse_Type(lex);
GET_CHECK_TOK(tok, lex, TOK_SEMICOLON);
@@ -646,7 +646,7 @@ AST::Trait Parse_TraitDef(TokenStream& lex, const AST::MetaItems& meta_items)
break; }
case TOK_RWORD_CONST: {
GET_CHECK_TOK(tok, lex, TOK_IDENT);
- auto name = mv$(tok.ident_str());
+ auto name = mv$(tok.str());
GET_CHECK_TOK(tok, lex, TOK_COLON);
auto ty = Parse_Type(lex);
@@ -664,7 +664,7 @@ AST::Trait Parse_TraitDef(TokenStream& lex, const AST::MetaItems& meta_items)
case TOK_RWORD_TYPE: {
auto atype_params = ::AST::GenericParams { };
GET_CHECK_TOK(tok, lex, TOK_IDENT);
- auto name = mv$(tok.ident_str());
+ auto name = mv$(tok.str());
if( GET_TOK(tok, lex) == TOK_COLON )
{
// Bounded associated type
@@ -704,7 +704,7 @@ AST::Trait Parse_TraitDef(TokenStream& lex, const AST::MetaItems& meta_items)
CHECK_TOK(tok, TOK_RWORD_FN);
case TOK_RWORD_FN: {
GET_CHECK_TOK(tok, lex, TOK_IDENT);
- ::std::string name = tok.take_ident().into_string();
+ ::std::string name = mv$(tok.str());
// Self allowed, prototype-form allowed (optional names and no code)
auto fcn = Parse_FunctionDef(lex, abi, true, true, fn_is_unsafe, fn_is_const);
if( GET_TOK(tok, lex) == TOK_BRACE_OPEN )
@@ -768,7 +768,7 @@ AST::Enum Parse_EnumDef(TokenStream& lex, const AST::MetaItems& meta_items)
SET_ATTRS(lex, item_attrs);
CHECK_TOK(tok, TOK_IDENT);
- ::std::string name = tok.take_ident().into_string();
+ ::std::string name = mv$(tok.str());
if( GET_TOK(tok, lex) == TOK_PAREN_OPEN )
{
::std::vector<TypeRef> types;
@@ -815,7 +815,7 @@ AST::Enum Parse_EnumDef(TokenStream& lex, const AST::MetaItems& meta_items)
}
GET_CHECK_TOK(tok, lex, TOK_IDENT);
- auto name = tok.take_ident().into_string();
+ auto name = mv$(tok.str());
GET_CHECK_TOK(tok, lex, TOK_COLON);
auto ty = Parse_Type(lex);
fields.push_back( ::AST::StructItem(mv$(field_attrs), true, mv$(name), mv$(ty)) );
@@ -888,7 +888,7 @@ AST::Enum Parse_EnumDef(TokenStream& lex, const AST::MetaItems& meta_items)
}
GET_CHECK_TOK(tok, lex, TOK_IDENT);
- auto name = tok.take_ident().into_string();
+ auto name = mv$(tok.str());
GET_CHECK_TOK(tok, lex, TOK_COLON);
auto ty = Parse_Type(lex);
@@ -913,7 +913,7 @@ AST::MetaItem Parse_MetaItem(TokenStream& lex)
}
CHECK_TOK(tok, TOK_IDENT);
- ::std::string name = tok.take_ident().into_string();
+ ::std::string name = mv$(tok.str());
switch(GET_TOK(tok, lex))
{
case TOK_EQUAL:
@@ -1071,7 +1071,7 @@ void Parse_Impl_Item(TokenStream& lex, AST::Impl& impl)
}
bool is_specialisable = false;
- if( tok.type() == TOK_IDENT && tok.ident() == "default" ) {
+ if( tok.type() == TOK_IDENT && tok.str() == "default" ) {
is_specialisable = true;
GET_TOK(tok, lex);
}
@@ -1083,7 +1083,7 @@ void Parse_Impl_Item(TokenStream& lex, AST::Impl& impl)
{
case TOK_RWORD_TYPE: {
GET_CHECK_TOK(tok, lex, TOK_IDENT);
- auto name = tok.take_ident().into_string();
+ auto name = mv$(tok.str());
GET_CHECK_TOK(tok, lex, TOK_EQUAL);
impl.add_type(is_public, is_specialisable, name, Parse_Type(lex));
GET_CHECK_TOK(tok, lex, TOK_SEMICOLON);
@@ -1098,7 +1098,7 @@ void Parse_Impl_Item(TokenStream& lex, AST::Impl& impl)
if( tok.type() != TOK_RWORD_FN && tok.type() != TOK_RWORD_UNSAFE && !fn_is_unsafe )
{
CHECK_TOK(tok, TOK_IDENT);
- auto name = tok.take_ident().into_string();
+ auto name = mv$(tok.str());
GET_CHECK_TOK(tok, lex, TOK_COLON);
auto ty = Parse_Type(lex);
GET_CHECK_TOK(tok, lex, TOK_EQUAL);
@@ -1134,7 +1134,7 @@ void Parse_Impl_Item(TokenStream& lex, AST::Impl& impl)
case TOK_RWORD_FN: {
GET_CHECK_TOK(tok, lex, TOK_IDENT);
// TODO: Hygine on function names? - Not in impl blocks?
- ::std::string name = tok.take_ident().into_string();
+ ::std::string name = mv$(tok.str());
DEBUG("Function " << name);
// - Self allowed, can't be prototype-form
auto fcn = Parse_FunctionDefWithCode(lex, abi, true, fn_is_unsafe, fn_is_const);
@@ -1185,7 +1185,7 @@ AST::ExternBlock Parse_ExternBlock(TokenStream& lex, ::std::string abi, ::AST::M
{
case TOK_RWORD_FN: {
GET_CHECK_TOK(tok, lex, TOK_IDENT);
- auto name = tok.take_ident().into_string();
+ auto name = mv$(tok.str());
// parse function as prototype
// - no self, "safe" and not const
auto i = ::AST::Item( Parse_FunctionDef(lex, abi, false, true, false,false) );
@@ -1203,7 +1203,7 @@ AST::ExternBlock Parse_ExternBlock(TokenStream& lex, ::std::string abi, ::AST::M
else
PUTBACK(tok, lex);
GET_CHECK_TOK(tok, lex, TOK_IDENT);
- auto name = tok.take_ident().into_string();
+ auto name = mv$(tok.str());
GET_CHECK_TOK(tok, lex, TOK_COLON);
auto type = Parse_Type(lex);
GET_CHECK_TOK(tok, lex, TOK_SEMICOLON);
@@ -1242,13 +1242,13 @@ void Parse_Use_Set(TokenStream& lex, const ProtoSpan& ps, const AST::Path& base_
}
else {
CHECK_TOK(tok, TOK_IDENT);
- path = base_path + AST::PathNode(tok.ident_str(), {});
- name = mv$(tok.ident_str());
+ path = base_path + AST::PathNode(tok.str(), {});
+ name = mv$(tok.str());
}
if( GET_TOK(tok, lex) == TOK_RWORD_AS ) {
GET_CHECK_TOK(tok, lex, TOK_IDENT);
- name = mv$(tok.ident_str());
+ name = mv$(tok.str());
}
else {
PUTBACK(tok, lex);
@@ -1282,7 +1282,7 @@ void Parse_Use(TokenStream& lex, ::std::function<void(AST::UseStmt, ::std::strin
path = AST::Path( AST::Path::TagSuper(), count, {} );
break; }
case TOK_IDENT:
- path.append( AST::PathNode(tok.take_ident().into_string(), {}) );
+ path.append( AST::PathNode(mv$(tok.str()), {}) );
break;
// Leading :: is allowed and ignored for the $crate feature
case TOK_DOUBLE_COLON:
@@ -1309,7 +1309,7 @@ void Parse_Use(TokenStream& lex, ::std::function<void(AST::UseStmt, ::std::strin
{
if( GET_TOK(tok, lex) == TOK_IDENT )
{
- path.append( AST::PathNode(tok.take_ident().into_string(), {}) );
+ path.append( AST::PathNode( mv$(tok.str()), {}) );
}
else
{
@@ -1338,7 +1338,7 @@ void Parse_Use(TokenStream& lex, ::std::function<void(AST::UseStmt, ::std::strin
if( tok.type() == TOK_RWORD_AS )
{
GET_CHECK_TOK(tok, lex, TOK_IDENT);
- name = tok.take_ident().into_string();
+ name = mv$(tok.str());
}
else
{
@@ -1356,7 +1356,7 @@ void Parse_Use(TokenStream& lex, ::std::function<void(AST::UseStmt, ::std::strin
Token tok;
::std::string ident;
if( GET_TOK(tok, lex) == TOK_IDENT ) {
- ident = tok.take_ident().into_string();
+ ident = mv$(tok.str());
}
else {
PUTBACK(tok, lex);
@@ -1447,7 +1447,7 @@ void Parse_Use(TokenStream& lex, ::std::function<void(AST::UseStmt, ::std::strin
// `extern "<ABI>" fn ...`
case TOK_RWORD_FN: {
GET_CHECK_TOK(tok, lex, TOK_IDENT);
- item_name = mv$(tok.ident_str());
+ item_name = mv$(tok.str());
item_data = ::AST::Item( Parse_FunctionDefWithCode(lex, abi, false, false,false) );
break; }
// `extern "ABI" {`
@@ -1462,7 +1462,7 @@ void Parse_Use(TokenStream& lex, ::std::function<void(AST::UseStmt, ::std::strin
// `extern fn ...`
case TOK_RWORD_FN:
GET_CHECK_TOK(tok, lex, TOK_IDENT);
- item_name = mv$(tok.ident_str());
+ item_name = mv$(tok.str());
item_data = ::AST::Item( Parse_FunctionDefWithCode(lex, "C", false, false,false) );
break;
@@ -1484,17 +1484,17 @@ void Parse_Use(TokenStream& lex, ::std::function<void(AST::UseStmt, ::std::strin
item_data = ::AST::Item::make_Crate({ tok.str() });
GET_CHECK_TOK(tok, lex, TOK_RWORD_AS);
GET_CHECK_TOK(tok, lex, TOK_IDENT);
- item_name = mv$(tok.ident_str());
+ item_name = mv$(tok.str());
break;
// `extern crate crate_name;`
// `extern crate crate_name as other_name;`
case TOK_IDENT:
- item_name = mv$(tok.ident_str());
+ item_name = mv$(tok.str());
if(GET_TOK(tok, lex) == TOK_RWORD_AS) {
item_data = ::AST::Item::make_Crate({ mv$(item_name) });
GET_CHECK_TOK(tok, lex, TOK_IDENT);
- item_name = mv$(tok.ident_str());
+ item_name = mv$(tok.str());
}
else {
PUTBACK(tok, lex);
@@ -1517,7 +1517,7 @@ void Parse_Use(TokenStream& lex, ::std::function<void(AST::UseStmt, ::std::strin
switch( GET_TOK(tok, lex) )
{
case TOK_IDENT: {
- item_name = mv$(tok.ident_str());
+ item_name = mv$(tok.str());
GET_CHECK_TOK(tok, lex, TOK_COLON);
TypeRef type = Parse_Type(lex);
@@ -1529,12 +1529,12 @@ void Parse_Use(TokenStream& lex, ::std::function<void(AST::UseStmt, ::std::strin
case TOK_RWORD_UNSAFE:
GET_CHECK_TOK(tok, lex, TOK_RWORD_FN);
GET_CHECK_TOK(tok, lex, TOK_IDENT);
- item_name = mv$(tok.ident_str());
+ item_name = mv$(tok.str());
item_data = ::AST::Item( Parse_FunctionDefWithCode(lex, ABI_RUST, false, true,true/*unsafe,const*/) );
break;
case TOK_RWORD_FN:
GET_CHECK_TOK(tok, lex, TOK_IDENT);
- item_name = mv$(tok.ident_str());
+ item_name = mv$(tok.str());
// - self not allowed, not prototype
item_data = ::AST::Item( Parse_FunctionDefWithCode(lex, ABI_RUST, false, false,true/*unsafe,const*/) );
break;
@@ -1551,7 +1551,7 @@ void Parse_Use(TokenStream& lex, ::std::function<void(AST::UseStmt, ::std::strin
GET_TOK(tok, lex);
}
CHECK_TOK(tok, TOK_IDENT);
- item_name = mv$(tok.ident_str());
+ item_name = mv$(tok.str());
GET_CHECK_TOK(tok, lex, TOK_COLON);
TypeRef type = Parse_Type(lex);
@@ -1582,20 +1582,20 @@ void Parse_Use(TokenStream& lex, ::std::function<void(AST::UseStmt, ::std::strin
}
GET_CHECK_TOK(tok, lex, TOK_RWORD_FN);
GET_CHECK_TOK(tok, lex, TOK_IDENT);
- item_name = mv$(tok.ident_str());
+ item_name = mv$(tok.str());
item_data = ::AST::Item( Parse_FunctionDefWithCode(lex, abi, false, true,false/*unsafe,const*/) );
break; }
// `unsafe fn`
case TOK_RWORD_FN:
GET_CHECK_TOK(tok, lex, TOK_IDENT);
- item_name = mv$(tok.ident_str());
+ item_name = mv$(tok.str());
// - self not allowed, not prototype
item_data = ::AST::Item( Parse_FunctionDefWithCode(lex, ABI_RUST, false, true,false/*unsafe,const*/) );
break;
// `unsafe trait`
case TOK_RWORD_TRAIT:
GET_CHECK_TOK(tok, lex, TOK_IDENT);
- item_name = mv$(tok.ident_str());
+ item_name = mv$(tok.str());
// TODO: Mark as unsafe
meta_items.push_back( AST::MetaItem("#UNSAFE") );
item_data = ::AST::Item( Parse_TraitDef(lex, meta_items) );
@@ -1610,33 +1610,33 @@ void Parse_Use(TokenStream& lex, ::std::function<void(AST::UseStmt, ::std::strin
// `fn`
case TOK_RWORD_FN:
GET_CHECK_TOK(tok, lex, TOK_IDENT);
- item_name = mv$(tok.ident_str());
+ item_name = mv$(tok.str());
// - self not allowed, not prototype
item_data = ::AST::Item( Parse_FunctionDefWithCode(lex, ABI_RUST, false, false,false/*unsafe,const*/) );
break;
// `type`
case TOK_RWORD_TYPE:
GET_CHECK_TOK(tok, lex, TOK_IDENT);
- item_name = mv$(tok.ident_str());
+ item_name = mv$(tok.str());
item_data = ::AST::Item( Parse_TypeAlias(lex) );
break;
// `struct`
case TOK_RWORD_STRUCT:
GET_CHECK_TOK(tok, lex, TOK_IDENT);
- item_name = mv$(tok.ident_str());
+ item_name = mv$(tok.str());
item_data = ::AST::Item( Parse_Struct(lex, meta_items) );
break;
// `enum`
case TOK_RWORD_ENUM:
GET_CHECK_TOK(tok, lex, TOK_IDENT);
- item_name = mv$(tok.ident_str());
+ item_name = mv$(tok.str());
item_data = ::AST::Item( Parse_EnumDef(lex, meta_items) );
break;
// Contextual keywords
case TOK_IDENT:
- if( tok.ident_str() == "union" ) {
+ if( tok.str() == "union" ) {
GET_CHECK_TOK(tok, lex, TOK_IDENT);
- item_name = mv$(tok.ident_str());
+ item_name = mv$(tok.str());
item_data = ::AST::Item( Parse_Union(lex, meta_items) );
}
else {
@@ -1649,13 +1649,13 @@ void Parse_Use(TokenStream& lex, ::std::function<void(AST::UseStmt, ::std::strin
// `trait`
case TOK_RWORD_TRAIT:
GET_CHECK_TOK(tok, lex, TOK_IDENT);
- item_name = mv$(tok.ident_str());
+ item_name = mv$(tok.str());
item_data = ::AST::Item( Parse_TraitDef(lex, meta_items) );
break;
case TOK_RWORD_MOD: {
GET_CHECK_TOK(tok, lex, TOK_IDENT);
- auto name = mv$(tok.ident_str());
+ auto name = mv$(tok.str());
DEBUG("Sub module '" << name << "'");
AST::Module submod( mod_path + name );
diff --git a/src/parse/token.cpp b/src/parse/token.cpp
index 9f84df79..95adf594 100644
--- a/src/parse/token.cpp
+++ b/src/parse/token.cpp
@@ -53,12 +53,7 @@ Token::Token(enum eTokenType type, ::std::string str):
m_type(type),
m_data(Data::make_String(mv$(str)))
{
- assert( m_type != TOK_IDENT );
}
-Token::Token(Ident ident):
- m_type(TOK_IDENT),
- m_data(Data( mv$(ident) ))
-{}
Token::Token(uint64_t val, enum eCoreType datatype):
m_type(TOK_INTEGER),
m_data( Data::make_Integer({datatype, val}) )
@@ -157,7 +152,6 @@ Token::Token(const Token& t):
(None, ),
(String, m_data = Data::make_String(e); ),
(Integer, m_data = Data::make_Integer(e);),
- (Ident , m_data = Data(e);),
(Float, m_data = Data::make_Float(e);),
(Fragment, BUG(t.m_pos, "Attempted to copy a fragment - " << t);)
)
@@ -177,7 +171,6 @@ Token Token::clone() const
(Integer,
rv.m_data = Data::make_Integer(e);
),
- (Ident, rv.m_data = Data(e); ),
(Float,
rv.m_data = Data::make_Float(e);
),
@@ -305,8 +298,9 @@ struct EscapedString {
case TOK_INTERPOLATED_BLOCK: return "/*:block*/";
case TOK_INTERPOLATED_META: return "/*:meta*/";
case TOK_INTERPOLATED_ITEM: return "/*:item*/";
+ case TOK_INTERPOLATED_IDENT: return "/*:ident*/";
// Value tokens
- case TOK_IDENT: return m_data.as_Ident().name;
+ case TOK_IDENT: return m_data.as_String();
case TOK_MACRO: return m_data.as_String() + "!";
case TOK_LIFETIME: return "'" + m_data.as_String();
case TOK_INTEGER: return FMT(m_data.as_Integer().m_intval); // TODO: suffix for type
@@ -471,10 +465,6 @@ SERIALISE_TYPE(Token::, "Token", {
s % e.m_datatype;
s.item( e.m_intval );
),
- (Ident,
- // Hygine contex for serialised idents is left for the loader
- s << e.name;
- ),
(Float,
s % e.m_datatype;
s.item( e.m_floatval );
@@ -507,12 +497,6 @@ SERIALISE_TYPE(Token::, "Token", {
s.item( v );
m_data = Token::Data::make_Integer({dt, v});
break; }
- case Token::Data::TAG_Ident: {
- ::std::string str;
- s.item( str );
- // - Hygine is populated by caller, fill with dummy values for now
- m_data = Token::Data::make_Ident(Ident { {~0u,{}}, mv$(str) });
- break; }
case Token::Data::TAG_Float: {
enum eCoreType dt;
double v;
diff --git a/src/parse/token.hpp b/src/parse/token.hpp
index 94458f1f..3e7f7c24 100644
--- a/src/parse/token.hpp
+++ b/src/parse/token.hpp
@@ -61,7 +61,6 @@ class Token:
TAGGED_UNION(Data, None,
(None, struct {}),
(String, ::std::string),
- (Ident, Ident),
(Integer, struct {
enum eCoreType m_datatype;
uint64_t m_intval;
@@ -99,7 +98,6 @@ public:
Token(enum eTokenType type);
Token(enum eTokenType type, ::std::string str);
- Token(Ident ident);
Token(uint64_t val, enum eCoreType datatype);
Token(double val, enum eCoreType datatype);
Token(const InterpolatedFragment& );
@@ -113,11 +111,6 @@ public:
uint64_t intval() const { return m_data.as_Integer().m_intval; }
double floatval() const { return m_data.as_Float().m_floatval; }
- const Ident& ident() const { return m_data.as_Ident(); }
- Ident take_ident() { return ::std::move(m_data.as_Ident()); }
- ::std::string& ident_str() { return m_data.as_Ident().name; }
- const ::std::string& ident_str() const { return m_data.as_Ident().name; }
-
TypeRef& frag_type() { assert(m_type == TOK_INTERPOLATED_TYPE); return *reinterpret_cast<TypeRef*>( m_data.as_Fragment() ); }
AST::Path& frag_path() { assert(m_type == TOK_INTERPOLATED_PATH); return *reinterpret_cast<AST::Path*>( m_data.as_Fragment() ); }
AST::Pattern& frag_pattern() { assert(m_type == TOK_INTERPOLATED_PATTERN); return *reinterpret_cast<AST::Pattern*>( m_data.as_Fragment() ); }
@@ -131,7 +124,6 @@ public:
TU_MATCH(Data, (m_data, r.m_data), (e, re),
(None, return true;),
(String, return e == re; ),
- (Ident, return e == re; ),
(Integer, return e.m_datatype == re.m_datatype && e.m_intval == re.m_intval;),
(Float, return e.m_datatype == re.m_datatype && e.m_floatval == re.m_floatval;),
(Fragment, assert(!"Token equality on Fragment");)
diff --git a/src/parse/tokentree.hpp b/src/parse/tokentree.hpp
index 82c4f97d..627d11e3 100644
--- a/src/parse/tokentree.hpp
+++ b/src/parse/tokentree.hpp
@@ -5,6 +5,7 @@
class TokenTree
{
+ Ident::Hygiene m_hygiene;
Token m_tok;
::std::vector<TokenTree> m_subtrees;
public:
@@ -16,7 +17,8 @@ public:
m_tok( ::std::move(tok) )
{
}
- TokenTree(::std::vector<TokenTree> subtrees):
+ TokenTree(Ident::Hygiene hygiene, ::std::vector<TokenTree> subtrees):
+ m_hygiene( ::std::move(hygiene) ),
m_subtrees( ::std::move(subtrees) )
{
}
@@ -33,6 +35,7 @@ public:
TokenTree& operator[](unsigned int idx) { return m_subtrees[idx]; }
const Token& tok() const { return m_tok; }
Token& tok() { return m_tok; }
+ const Ident::Hygiene& hygiene() const { return m_hygiene; }
friend ::std::ostream& operator<<(::std::ostream& os, const TokenTree& tt) {
if( tt.m_subtrees.size() == 0 )
@@ -56,16 +59,18 @@ class TTStream:
public TokenStream
{
::std::vector< ::std::pair<unsigned int, const TokenTree*> > m_stack;
+ const Ident::Hygiene* m_hygiene_ptr = nullptr;
public:
TTStream(const TokenTree& input_tt);
~TTStream();
TTStream& operator=(const TTStream& x) { m_stack = x.m_stack; return *this; }
- virtual Position getPosition() const override;
+ Position getPosition() const override;
+ Ident::Hygiene getHygiene() const override;
protected:
- virtual Token realGetToken() override;
+ Token realGetToken() override;
};
class TTStreamO:
@@ -82,10 +87,11 @@ public:
TTStreamO& operator=(const TTStreamO& x) { m_stack = x.m_stack; return *this; }
TTStreamO& operator=(TTStreamO&& x) = default;
- virtual Position getPosition() const override;
+ Position getPosition() const override;
+ Ident::Hygiene getHygiene() const override;
protected:
- virtual Token realGetToken() override;
+ Token realGetToken() override;
};
// unwrapped = Exclude the enclosing brackets (used by macro parse code)
diff --git a/src/resolve/absolute.cpp b/src/resolve/absolute.cpp
index a37e0fce..38f7b27b 100644
--- a/src/resolve/absolute.cpp
+++ b/src/resolve/absolute.cpp
@@ -259,7 +259,7 @@ struct Context
}
return "";
}
- AST::Path lookup(const Span& sp, const ::std::string& name, const Ident::Hygine& src_context, LookupMode mode) const {
+ AST::Path lookup(const Span& sp, const ::std::string& name, const Ident::Hygiene& src_context, LookupMode mode) const {
auto rv = this->lookup_opt(name, src_context, mode);
if( !rv.is_valid() ) {
switch(mode)
@@ -352,7 +352,7 @@ struct Context
}
return false;
}
- AST::Path lookup_opt(const ::std::string& name, const Ident::Hygine& src_context, LookupMode mode) const {
+ AST::Path lookup_opt(const ::std::string& name, const Ident::Hygiene& src_context, LookupMode mode) const {
for(auto it = m_name_context.rbegin(); it != m_name_context.rend(); ++ it)
{
@@ -379,7 +379,7 @@ struct Context
else {
for( auto it2 = e.variables.rbegin(); it2 != e.variables.rend(); ++ it2 )
{
- if( it2->first.name == name && it2->first.hygine.is_visible(src_context) ) {
+ if( it2->first.name == name && it2->first.hygiene.is_visible(src_context) ) {
::AST::Path rv(name);
rv.bind_variable( it2->second );
return rv;
@@ -1244,7 +1244,7 @@ void Resolve_Absolute_Path(/*const*/ Context& context, const Span& sp, Context::
if(e.nodes.size() > 1)
{
// Look up type/module name
- auto p = context.lookup(sp, e.nodes[0].name(), e.hygine, Context::LookupMode::Namespace);
+ auto p = context.lookup(sp, e.nodes[0].name(), e.hygiene, Context::LookupMode::Namespace);
DEBUG("Found type/mod - " << p);
// HACK: If this is a primitive name, and resolved to a module.
// - If the next component isn't found in the located module
@@ -1333,7 +1333,7 @@ void Resolve_Absolute_Path(/*const*/ Context& context, const Span& sp, Context::
}
else {
// Look up value
- auto p = context.lookup(sp, e.nodes[0].name(), e.hygine, mode);
+ auto p = context.lookup(sp, e.nodes[0].name(), e.hygiene, mode);
//DEBUG("Found path " << p << " for " << path);
if( p.is_absolute() ) {
assert( !p.nodes().empty() );
@@ -1715,7 +1715,7 @@ void Resolve_Absolute_Pattern(Context& context, bool allow_refutable, ::AST::Pa
if( allow_refutable ) {
auto name = mv$( e.name );
// Attempt to resolve the name in the current namespace, and if it fails, it's a binding
- auto p = context.lookup_opt( name.name, name.hygine, Context::LookupMode::Pattern );
+ auto p = context.lookup_opt( name.name, name.hygiene, Context::LookupMode::Pattern );
if( p.is_valid() ) {
Resolve_Absolute_Path(context, pat.span(), Context::LookupMode::Pattern, p);
pat = ::AST::Pattern(::AST::Pattern::TagValue(), ::AST::Pattern::Value::make_Named(mv$(p)));