Fixed macros, added range syntax and other tweaks

13 files changed, 738 insertions, 249 deletions

diff --git a/src/ast/expr.cpp b/src/ast/expr.cpp
index 9740a4d9..1246ad29 100644
--- a/src/ast/expr.cpp
+++ b/src/ast/expr.cpp
@@ -52,17 +52,20 @@ SERIALISE_TYPE(Expr::, "Expr", {
     #define _(x)    if(tag == #x) ptr = new x;
          _(ExprNode_Block)
     else _(ExprNode_Macro)
-    else _(ExprNode_Return)
+    else _(ExprNode_Flow)
     else _(ExprNode_Const)
+    else _(ExprNode_Import)
     else _(ExprNode_LetBinding)
     else _(ExprNode_Assign)
     else _(ExprNode_CallPath)
     else _(ExprNode_CallMethod)
     else _(ExprNode_CallObject)
     else _(ExprNode_Match)
+    else _(ExprNode_Loop)
     else _(ExprNode_If)
     else _(ExprNode_IfLet)
     else _(ExprNode_Integer)
+    else _(ExprNode_Closure)
     else _(ExprNode_StructLiteral)
     else _(ExprNode_Tuple)
     else _(ExprNode_NamedValue)
@@ -107,10 +110,40 @@ NODE(ExprNode_Macro, {
     os << m_name << "!(" << ")";
 })
 
-NODE(ExprNode_Return, {
+void operator%(::Serialiser& s, const ExprNode_Flow::Type t) {
+    switch(t)
+    {
+    #define _(v)    case ExprNode_Flow::v: s << #v; return
+    _(RETURN);
+    _(BREAK);
+    _(CONTINUE);
+    #undef _
+    }
+}
+void operator%(::Deserialiser& s, ExprNode_Flow::Type& t) {
+    ::std::string   n;
+    s.item(n);
+    if(0)   ;
+    #define _(v)    else if(n == #v) t = ExprNode_Flow::v
+    _(RETURN);
+    _(BREAK);
+    _(CONTINUE);
+    #undef _
+    else
+        throw ::std::runtime_error("");
+}
+NODE(ExprNode_Flow, {
+    s % m_type;
+    s.item(m_target);
     s.item(m_value);
 },{
-    os << "return " << *m_value;
+    switch(m_type)
+    {
+    case RETURN:    os << "return"; break;
+    case BREAK:     os << "break"; break;
+    case CONTINUE:  os << "continue"; break;
+    }
+    os << " " << *m_value;
 })
 
 NODE(ExprNode_Const, {
@@ -121,6 +154,12 @@ NODE(ExprNode_Const, {
     os << "const " << m_name << ": " << m_type << " = " << *m_value;
 })
 
+NODE(ExprNode_Import, {
+    s.item(m_imports);
+},{
+    os << "/* todo: use /*";
+})
+
 NODE(ExprNode_LetBinding, {
     s.item(m_pat);
     s.item(m_type);
@@ -170,6 +209,40 @@ NODE(ExprNode_CallObject, {
     os << ")";
 })
 
+void operator%(::Serialiser& s, const ExprNode_Loop::Type t) {
+    switch(t)
+    {
+    #define _(v)    case ExprNode_Loop::v: s << #v; return
+    _(LOOP);
+    _(WHILE);
+    _(WHILELET);
+    _(FOR);
+    #undef _
+    }
+}
+void operator%(::Deserialiser& s, ExprNode_Loop::Type& t) {
+    ::std::string   n;
+    s.item(n);
+    if(0)   ;
+    #define _(v)    else if(n == #v) t = ExprNode_Loop::v
+    _(LOOP);
+    _(WHILE);
+    _(WHILELET);
+    _(FOR);
+    #undef _
+    else
+        throw ::std::runtime_error("");
+}
+NODE(ExprNode_Loop, {
+    s % m_type;
+    s.item(m_label);
+    s.item(m_pattern);
+    s.item(m_cond);
+    s.item(m_code);
+},{
+    //os << "LOOP [" << m_label << "] " << m_pattern << " in/= " << m_cond << " " << m_code;
+})
+
 SERIALISE_TYPE_A(ExprNode_Match::Arm::, "ExprNode_Match_Arm", {
     s.item(m_patterns);
     s.item(m_cond);
@@ -224,6 +297,19 @@ NODE(ExprNode_Bool, {
 },{
     os << m_value;
 })
+NODE(ExprNode_String, {
+    s.item(m_value);
+},{
+    os << "\"" << m_value << "\"";
+})
+
+NODE(ExprNode_Closure, {
+    s.item(m_args);
+    s.item(m_return);
+    s.item(m_code);
+},{
+    os << "/* todo: closure */";
+});
 
 NODE(ExprNode_StructLiteral, {
     s.item(m_path);
@@ -256,6 +342,13 @@ NODE(ExprNode_Field, {
     os << "(" << *m_obj << ")." << m_name;
 })
 
+NODE(ExprNode_Index, {
+    s.item(m_obj);
+    s.item(m_idx);
+},{
+    os << "(" << *m_obj << ")[" << *m_idx << "]";
+})
+
 NODE(ExprNode_Deref, {
     s.item(m_value);
 },{
@@ -279,6 +372,7 @@ void operator%(::Serialiser& s, const ExprNode_BinOp::Type t) {
     _(CMPLTE);
     _(CMPGT);
     _(CMPGTE);
+    _(RANGE);
     _(BOOLAND);
     _(BOOLOR);
     _(BITAND);
@@ -305,6 +399,7 @@ void operator%(::Deserialiser& s, ExprNode_BinOp::Type& t) {
     _(CMPLTE);
     _(CMPGT);
     _(CMPGTE);
+    _(RANGE);
     _(BOOLAND);
     _(BOOLOR);
     _(BITAND);
@@ -404,7 +499,7 @@ NV(ExprNode_Macro,
 {
     DEBUG("TODO: Macro");
 })
-NV(ExprNode_Return,
+NV(ExprNode_Flow,
 {
     visit(node.m_value);
 })
@@ -412,6 +507,7 @@ NV(ExprNode_Const,
 {
     visit(node.m_value);
 })
+NV(ExprNode_Import, {})
 NV(ExprNode_LetBinding,
 {
     // TODO: Handle recurse into Let pattern
@@ -447,6 +543,13 @@ NV(ExprNode_CallObject,
         visit(arg);
     UNINDENT();
 })
+NV(ExprNode_Loop,
+{
+    INDENT();
+    visit(node.m_cond);
+    visit(node.m_code);
+    UNINDENT();
+})
 NV(ExprNode_Match,
 {
     INDENT();
@@ -478,7 +581,11 @@ NV(ExprNode_IfLet,
 NV(ExprNode_Integer, {})
 NV(ExprNode_Float, {})
 NV(ExprNode_Bool, {})
+NV(ExprNode_String, {})
 
+NV(ExprNode_Closure,
+{
+});
 NV(ExprNode_StructLiteral,
 {
     visit(node.m_base_value);
@@ -499,6 +606,11 @@ NV(ExprNode_Field,
 {
     visit(node.m_obj);
 })
+NV(ExprNode_Index,
+{
+    visit(node.m_obj);
+    visit(node.m_idx);
+})
 NV(ExprNode_Deref,
 {
     visit(node.m_value);
diff --git a/src/ast/expr.hpp b/src/ast/expr.hpp
index 4c6d6b7c..121a1f09 100644
--- a/src/ast/expr.hpp
+++ b/src/ast/expr.hpp
@@ -77,13 +77,21 @@ struct ExprNode_Macro:
 };
 
 // Return a value
-struct ExprNode_Return:
+struct ExprNode_Flow:
     public ExprNode
 {
+    enum Type {
+        RETURN,
+        CONTINUE,
+        BREAK,
+    } m_type;
+    ::std::string   m_target;
     unique_ptr<ExprNode>    m_value;
 
-    ExprNode_Return() {}
-    ExprNode_Return(unique_ptr<ExprNode>&& value):
+    ExprNode_Flow() {}
+    ExprNode_Flow(Type type, ::std::string target, unique_ptr<ExprNode>&& value):
+        m_type(type),
+        m_target( move(target) ),
         m_value( move(value) )
     {
     }
@@ -107,6 +115,19 @@ struct ExprNode_Const:
     
     NODE_METHODS();
 };
+struct ExprNode_Import:
+    public ExprNode
+{
+    typedef ::std::vector< ::std::pair< ::std::string, AST::Path> > imports_t;
+    imports_t   m_imports;
+    
+    ExprNode_Import() {}
+    ExprNode_Import(imports_t imports):
+        m_imports( ::std::move(imports) )
+    {}
+    
+    NODE_METHODS();
+};
 struct ExprNode_LetBinding:
     public ExprNode
 {
@@ -131,6 +152,9 @@ struct ExprNode_Assign:
         NONE,
         ADD,
         SUB,
+        MUL,
+        DIV,
+        MOD,
     } m_op;
     unique_ptr<ExprNode>    m_slot;
     unique_ptr<ExprNode>    m_value;
@@ -193,6 +217,42 @@ struct ExprNode_CallObject:
     NODE_METHODS();
 };
 
+struct ExprNode_Loop:
+    public ExprNode
+{
+    enum Type {
+        LOOP,
+        WHILE,
+        WHILELET,
+        FOR,
+    } m_type;
+    ::std::string   m_label;
+    AST::Pattern    m_pattern;
+    unique_ptr<ExprNode>    m_cond; // if NULL, loop is a 'loop'
+    unique_ptr<ExprNode>    m_code;
+
+    ExprNode_Loop(): m_type(LOOP) {}
+    ExprNode_Loop(::std::string label, unique_ptr<ExprNode> code):
+        m_type(LOOP),
+        m_label( ::std::move(label) ),
+        m_code( ::std::move(code) )
+    {}
+    ExprNode_Loop(::std::string label, unique_ptr<ExprNode> cond, unique_ptr<ExprNode> code):
+        m_type(WHILE),
+        m_label( ::std::move(label) ),
+        m_cond( ::std::move(cond) ),
+        m_code( ::std::move(code) )
+    {}
+    ExprNode_Loop(::std::string label, Type type, AST::Pattern pattern, unique_ptr<ExprNode> val, unique_ptr<ExprNode> code):
+        m_type(type),
+        m_label( ::std::move(label) ),
+        m_pattern( ::std::move(pattern) ),
+        m_cond( ::std::move(val) ),
+        m_code( ::std::move(code) )
+    {}
+    NODE_METHODS();
+};
+
 struct ExprNode_Match:
     public ExprNode
 {
@@ -299,6 +359,39 @@ struct ExprNode_Bool:
     
     NODE_METHODS();
 };
+// Literal string
+struct ExprNode_String:
+    public ExprNode
+{
+    ::std::string   m_value;
+    
+    ExprNode_String() {}
+    ExprNode_String(::std::string value):
+        m_value( ::std::move(value) )
+    {}
+    
+    NODE_METHODS();
+};
+
+// Closure / Lambda
+struct ExprNode_Closure:
+    public ExprNode
+{
+    typedef ::std::vector< ::std::pair<AST::Pattern, TypeRef> > args_t;
+    
+    args_t  m_args;
+    TypeRef m_return;
+    unique_ptr<ExprNode>    m_code;
+    
+    ExprNode_Closure() {}
+    ExprNode_Closure(args_t args, TypeRef rv, unique_ptr<ExprNode> code):
+        m_args( ::std::move(args) ),
+        m_return( ::std::move(rv) ),
+        m_code( ::std::move(code) )
+    {}
+    
+    NODE_METHODS();
+};
 // Literal structure
 struct ExprNode_StructLiteral:
     public ExprNode
@@ -358,6 +451,20 @@ struct ExprNode_Field:
     }
     NODE_METHODS();
 };
+struct ExprNode_Index:
+    public ExprNode
+{
+    ::std::unique_ptr<ExprNode> m_obj;
+    ::std::unique_ptr<ExprNode> m_idx;
+    
+    ExprNode_Index() {}
+    ExprNode_Index(::std::unique_ptr<ExprNode> obj, ::std::unique_ptr<ExprNode> idx):
+        m_obj( ::std::move(obj) ),
+        m_idx( ::std::move(idx) )
+    {}
+    
+    NODE_METHODS();
+};
 
 // Pointer dereference
 struct ExprNode_Deref:
@@ -401,6 +508,8 @@ struct ExprNode_BinOp:
 		CMPLTE,
 		CMPGT,
 		CMPGTE,
+        
+        RANGE,
         BOOLAND,
         BOOLOR,
 
@@ -470,13 +579,15 @@ public:
 		virtual void visit(const nt& node) = 0*/
 	NT(ExprNode_Block);  
     NT(ExprNode_Macro);
-    NT(ExprNode_Return);
+    NT(ExprNode_Flow);
     NT(ExprNode_Const);
+    NT(ExprNode_Import);
     NT(ExprNode_LetBinding);
     NT(ExprNode_Assign);
     NT(ExprNode_CallPath);
     NT(ExprNode_CallMethod);
     NT(ExprNode_CallObject);
+    NT(ExprNode_Loop);
     NT(ExprNode_Match);
     NT(ExprNode_If);
     NT(ExprNode_IfLet);
@@ -484,11 +595,14 @@ public:
     NT(ExprNode_Integer);
     NT(ExprNode_Float);
     NT(ExprNode_Bool);
+    NT(ExprNode_String);
+    NT(ExprNode_Closure);
     NT(ExprNode_StructLiteral);
     NT(ExprNode_Tuple);
     NT(ExprNode_NamedValue);
     
     NT(ExprNode_Field);
+    NT(ExprNode_Index);
     NT(ExprNode_Deref);
     NT(ExprNode_Cast);
     NT(ExprNode_BinOp);
@@ -508,13 +622,15 @@ public:
 		virtual void visit(const nt& node) override*/
 	NT(ExprNode_Block);  
     NT(ExprNode_Macro);
-    NT(ExprNode_Return);
+    NT(ExprNode_Flow);
     NT(ExprNode_Const);
+    NT(ExprNode_Import);
     NT(ExprNode_LetBinding);
     NT(ExprNode_Assign);
     NT(ExprNode_CallPath);
     NT(ExprNode_CallMethod);
     NT(ExprNode_CallObject);
+    NT(ExprNode_Loop);
     NT(ExprNode_Match);
     NT(ExprNode_If);
     NT(ExprNode_IfLet);
@@ -522,11 +638,14 @@ public:
     NT(ExprNode_Integer);
     NT(ExprNode_Float);
     NT(ExprNode_Bool);
+    NT(ExprNode_String);
+    NT(ExprNode_Closure);
     NT(ExprNode_StructLiteral);
     NT(ExprNode_Tuple);
     NT(ExprNode_NamedValue);
     
     NT(ExprNode_Field);
+    NT(ExprNode_Index);
     NT(ExprNode_Deref);
     NT(ExprNode_Cast);
     NT(ExprNode_BinOp);
diff --git a/src/dump_as_rust.cpp b/src/dump_as_rust.cpp
index 76bb99b4..471a0479 100644
--- a/src/dump_as_rust.cpp
+++ b/src/dump_as_rust.cpp
@@ -58,9 +58,14 @@ public:
         m_expr_root = false;
         m_os << n.m_name << "!( /* TODO: Macro TT */ )";
     }
-    virtual void visit(AST::ExprNode_Return& n) override {
+    virtual void visit(AST::ExprNode_Flow& n) override {
         m_expr_root = false;
-        m_os << "return ";
+        switch(n.m_type)
+        {
+        case AST::ExprNode_Flow::RETURN:    m_os << "return ";  break;
+        case AST::ExprNode_Flow::BREAK:     m_os << "break ";  break;
+        case AST::ExprNode_Flow::CONTINUE:  m_os << "continue ";  break;
+        }
         AST::NodeVisitor::visit(n.m_value);
     }
     virtual void visit(AST::ExprNode_Const& n) override {
@@ -70,6 +75,11 @@ public:
         m_os << " = ";
         AST::NodeVisitor::visit(n.m_value);
     }
+    virtual void visit(AST::ExprNode_Import& n) override {
+        m_expr_root = false;
+        for( const auto& item : n.m_imports )
+            m_os << "use " << item.second << " as " << item.first << ";\n" << indent();
+    }
     virtual void visit(AST::ExprNode_LetBinding& n) override {
         m_expr_root = false;
         m_os << "let ";
@@ -121,6 +131,45 @@ public:
         m_expr_root = false;
         throw ::std::runtime_error("unimplemented ExprNode_CallObject");
     }
+    virtual void visit(AST::ExprNode_Loop& n) override {
+        bool expr_root = m_expr_root;
+        m_expr_root = false;
+        
+        switch(n.m_type)
+        {
+        case AST::ExprNode_Loop::LOOP:
+            m_os << "loop";
+            break;
+        case AST::ExprNode_Loop::WHILE:
+            m_os << "while ";
+            AST::NodeVisitor::visit(n.m_cond);
+            break;
+        case AST::ExprNode_Loop::WHILELET:
+            m_os << "while let ";
+            print_pattern(n.m_pattern);
+            m_os << " = ";
+            AST::NodeVisitor::visit(n.m_cond);
+            break;
+        case AST::ExprNode_Loop::FOR:
+            m_os << "while for ";
+            print_pattern(n.m_pattern);
+            m_os << " in ";
+            AST::NodeVisitor::visit(n.m_cond);
+            break;
+        }
+        
+        if( expr_root )
+        {
+            m_os << "\n";
+            m_os << indent();
+        }
+        else
+        {
+            m_os << " ";
+        }
+
+        AST::NodeVisitor::visit(n.m_code);
+    }
     virtual void visit(AST::ExprNode_Match& n) override {
         bool expr_root = m_expr_root;
         m_expr_root = false;
@@ -220,6 +269,22 @@ public:
             AST::NodeVisitor::visit(fv);
         }
     }
+    virtual void visit(AST::ExprNode_Closure& n) override {
+        m_expr_root = false;
+        m_os << "|";
+        bool is_first = true;
+        for( const auto& arg : n.m_args )
+        {
+            if(!is_first)   m_os << ", ";
+            is_first = false;
+            print_pattern(arg.first);
+            m_os << ": ";
+            print_type(arg.second);
+        }
+        m_os << "|: ";
+        print_type(n.m_return);
+        AST::NodeVisitor::visit(n.m_code);
+    }
     virtual void visit(AST::ExprNode_Integer& n) override {
         m_expr_root = false;
         switch(n.m_datatype)
@@ -268,6 +333,11 @@ public:
         else
             m_os << "false";
     }
+    virtual void visit(AST::ExprNode_String& n) override {
+        m_expr_root = false;
+        m_os << "\"" << n.m_value << "\"";
+    }
+    
     virtual void visit(AST::ExprNode_StructLiteral& n) override {
         m_expr_root = false;
         m_os << n.m_path << " {\n";
@@ -307,6 +377,14 @@ public:
         AST::NodeVisitor::visit(n.m_obj);
         m_os << ")." << n.m_name;
     }
+    virtual void visit(AST::ExprNode_Index& n) override {
+        m_expr_root = false;
+        m_os << "(";
+        AST::NodeVisitor::visit(n.m_obj);
+        m_os << ")[";
+        AST::NodeVisitor::visit(n.m_idx);
+        m_os << "]";
+    }
     virtual void visit(AST::ExprNode_Deref&) override {
         m_expr_root = false;
         throw ::std::runtime_error("unimplemented ExprNode_Deref");
diff --git a/src/macros.cpp b/src/macros.cpp
index 29d17b8f..f375c0d4 100644
--- a/src/macros.cpp
+++ b/src/macros.cpp
@@ -33,6 +33,7 @@ public:
     typedef ::std::multimap<t_mapping_key, TokenTree, cmp_mk>    t_mappings;

 

 private:

+    const TokenStream&  m_olex;

     const TokenTree&    m_crate_path;

     const ::std::vector<MacroRuleEnt>&  m_root_contents;

     const t_mappings    m_mappings;

@@ -49,6 +50,7 @@ private:
     

 public:

     MacroExpander(const MacroExpander& x):

+        m_olex(x.m_olex),

         m_crate_path(x.m_crate_path),

         m_root_contents(x.m_root_contents),

         m_mappings(x.m_mappings),

@@ -57,7 +59,8 @@ public:
     {

         prep_counts();

     }

-    MacroExpander(const ::std::vector<MacroRuleEnt>& contents, t_mappings mappings, const TokenTree& crate_path):

+    MacroExpander(const TokenStream& olex, const ::std::vector<MacroRuleEnt>& contents, t_mappings mappings, const TokenTree& crate_path):

+        m_olex(olex),

         m_crate_path(crate_path),

         m_root_contents(contents),

         m_mappings(mappings),

@@ -241,7 +244,7 @@ void Macro_HandlePattern(TTStream& lex, const MacroPatEnt& pat, unsigned int lay
 

 }

 

-::std::unique_ptr<TokenStream> Macro_InvokeInt(const char *name, const MacroRules& rules, TokenTree input)

+::std::unique_ptr<TokenStream> Macro_InvokeInt(const TokenStream& olex, const char *name, const MacroRules& rules, TokenTree input)

 {

     TRACE_FUNCTION;

     

@@ -276,7 +279,7 @@ void Macro_HandlePattern(TTStream& lex, const MacroPatEnt& pat, unsigned int lay
             //GET_CHECK_TOK(tok, lex, close);

             GET_CHECK_TOK(tok, lex, TOK_EOF);

             DEBUG( rule.m_contents.size() << " rule contents bound to " << bound_tts.size() << " values - " << name );

-            return ::std::unique_ptr<TokenStream>( (TokenStream*)new MacroExpander(rule.m_contents, bound_tts, g_crate_path_tt) );

+            return ::std::unique_ptr<TokenStream>( (TokenStream*)new MacroExpander(olex, rule.m_contents, bound_tts, g_crate_path_tt) );

         }

         catch(const ParseError::Base& e)

         {

@@ -285,11 +288,12 @@ void Macro_HandlePattern(TTStream& lex, const MacroPatEnt& pat, unsigned int lay
         i ++;

     }

     DEBUG("");

-    throw ParseError::Todo("Error when macro fails to match");

+    throw ParseError::Todo(olex, "Error when macro fails to match");

 }

 

 ::std::unique_ptr<TokenStream> Macro_Invoke(const TokenStream& olex, const ::std::string& name, TokenTree input)

 {

+    DEBUG("Invoke " << name << " from " << olex.getPosition());

     // XXX: EVIL HACK! - This should be removed when std loading is implemented

     if( g_macro_registrations.size() == 0 ) {

         Macro_InitDefaults();

@@ -312,7 +316,7 @@ void Macro_HandlePattern(TTStream& lex, const MacroPatEnt& pat, unsigned int lay
     t_macro_regs::iterator macro_reg = g_macro_registrations.find(name);

     if( macro_reg != g_macro_registrations.end() )

     {

-        return Macro_InvokeInt(macro_reg->first.c_str(), macro_reg->second, input);

+        return Macro_InvokeInt(olex, macro_reg->first.c_str(), macro_reg->second, input);

     }

     

     // Search import list

@@ -324,7 +328,7 @@ void Macro_HandlePattern(TTStream& lex, const MacroPatEnt& pat, unsigned int lay
             DEBUG("" << m.name);

             if( m.name == name )

             {

-                return Macro_InvokeInt(m.name.c_str(), m.data, input);

+                return Macro_InvokeInt(olex, m.name.c_str(), m.data, input);

             }

         }

         

@@ -333,7 +337,7 @@ void Macro_HandlePattern(TTStream& lex, const MacroPatEnt& pat, unsigned int lay
             DEBUG("" << mi.name);

             if( mi.name == name )

             {

-                return Macro_InvokeInt(mi.name.c_str(), *mi.data, input);

+                return Macro_InvokeInt(olex, mi.name.c_str(), *mi.data, input);

             }

         }

     }

@@ -343,7 +347,8 @@ void Macro_HandlePattern(TTStream& lex, const MacroPatEnt& pat, unsigned int lay
 

 Position MacroExpander::getPosition() const

 {

-    return Position("Macro", 0);

+    DEBUG("olex.getPosition() = " << m_olex.getPosition());

+    return Position(FMT("Macro:" << ""), m_offsets[0].first);

 }

 Token MacroExpander::realGetToken()

 {

@@ -380,6 +385,7 @@ Token MacroExpander::realGetToken()
                 // - Name

                 // HACK: Handle $crate with a special name

                 if( ent.name == "*crate" ) {

+                    DEBUG("Crate name hack");

                     m_ttstream.reset( new TTStream(m_crate_path) );

                     return m_ttstream->getToken();

                 }

@@ -394,6 +400,7 @@ Token MacroExpander::realGetToken()
                     {

                         if( i == iter_idx )

                         {

+                            DEBUG(ent.name << " #" << i << " - Setting TT");

                             m_ttstream.reset( new TTStream(it->second) );

                             return m_ttstream->getToken();

                         }

@@ -536,7 +543,7 @@ MacroToken::MacroToken(Token tok):
 }

 Position MacroToken::getPosition() const

 {

-    return Position("macro", 0);

+    return Position("MacroToken", 0);

 }

 Token MacroToken::realGetToken()

 {

@@ -550,7 +557,7 @@ MacroStringify::MacroStringify(const TokenTree& input)
 }

 Position MacroStringify::getPosition() const

 {

-    return Position("stringify", 0);

+    return Position("Stringify", 0);

 }

 Token MacroStringify::realGetToken()

 {

@@ -569,6 +576,7 @@ void operator%(Serialiser& s, MacroPatEnt::Type c) {
     #define _(v) case MacroPatEnt::v: s << #v; return

     _(PAT_TOKEN);

     _(PAT_TT);

+    _(PAT_TYPE);

     _(PAT_EXPR);

     _(PAT_LOOP);

     //_(PAT_OPTLOOP);

@@ -586,6 +594,7 @@ void operator%(::Deserialiser& s, MacroPatEnt::Type& c) {
     if(0) ;

     _(PAT_TOKEN);

     _(PAT_TT);

+    _(PAT_TYPE);

     _(PAT_EXPR);

     _(PAT_LOOP);

     //_(PAT_OPTLOOP);

diff --git a/src/macros.hpp b/src/macros.hpp
index 2fa74698..bb25f38a 100644
--- a/src/macros.hpp
+++ b/src/macros.hpp
@@ -39,7 +39,15 @@ public:
     }

 

     friend ::std::ostream& operator<<(::std::ostream& os, const MacroRuleEnt& x) {

-        return os << "MacroRuleEnt( '"<<x.name<<"'" << x.tok << ", " << x.subpats << ")";

+        os << "MacroRuleEnt(";

+        if(x.name.size())

+            os << "'"<<x.name<<"'";

+        else if( x.subpats.size() )

+            os << x.tok << " [" << x.subpats << "]";

+        else

+            os << x.tok;

+        os << ")";

+        return os;

     }

 

     SERIALISABLE_PROTOTYPES();

@@ -90,8 +98,16 @@ struct MacroPatEnt:
     {

     }

 

-    friend ::std::ostream& operator<<(::std::ostream& os, const MacroPatEnt& mpe) {

-        return os << "MacroPatEnt( '"<<mpe.name<<"'" << mpe.tok << ", " << mpe.subpats << ")";

+    friend ::std::ostream& operator<<(::std::ostream& os, const MacroPatEnt& x) {

+        os << "MacroPatEnt(";

+        if(x.name.size())

+            os << "'"<<x.name<<"'";

+        else if( x.subpats.size() )

+            os << x.tok << " [" << x.subpats << "]";

+        else

+            os << x.tok;

+        os << ")";

+        return os;

     }

     

     SERIALISABLE_PROTOTYPES();

diff --git a/src/parse/common.hpp b/src/parse/common.hpp
index 3bc599a9..4af48174 100644
--- a/src/parse/common.hpp
+++ b/src/parse/common.hpp
@@ -25,7 +25,9 @@ enum eParsePathGenericMode
 };

 

 extern AST::Path   Parse_Path(TokenStream& lex, bool is_abs, eParsePathGenericMode generic_mode);

+extern ::std::vector<TypeRef>   Parse_Path_GenericList(TokenStream& lex);

 extern TypeRef     Parse_Type(TokenStream& lex);

+extern void Parse_Use(TokenStream& lex, ::std::function<void(AST::Path, ::std::string)> fcn);

 extern AST::Expr   Parse_Expr(TokenStream& lex, bool const_only);

 extern AST::Expr   Parse_ExprBlock(TokenStream& lex);

 

diff --git a/src/parse/expr.cpp b/src/parse/expr.cpp
index 10b942ac..d0d5bc7f 100644
--- a/src/parse/expr.cpp
+++ b/src/parse/expr.cpp
@@ -17,6 +17,7 @@ ExprNodeP Parse_Stmt(TokenStream& lex, bool& opt_semicolon);
 ExprNodeP Parse_Expr0(TokenStream& lex);

 ExprNodeP Parse_ExprBlocks(TokenStream& lex);

 ExprNodeP Parse_IfStmt(TokenStream& lex);

+ExprNodeP Parse_Expr_Match(TokenStream& lex);

 ExprNodeP Parse_Expr1(TokenStream& lex);

 

 AST::Expr Parse_Expr(TokenStream& lex, bool const_only)

@@ -142,14 +143,21 @@ AST::Pattern Parse_PatternReal(TokenStream& lex)
     case TOK_DOUBLE_COLON:

         // 2. Paths are enum/struct names

         return Parse_PatternReal_Path( lex, Parse_Path(lex, true, PATH_GENERIC_EXPR) );

+    case TOK_DASH:

+        GET_CHECK_TOK(tok, lex, TOK_INTEGER);

+        return AST::Pattern( AST::Pattern::TagValue(), NEWNODE(AST::ExprNode_Integer, -tok.intval(), tok.datatype()) );

     case TOK_INTEGER:

         return AST::Pattern( AST::Pattern::TagValue(), NEWNODE(AST::ExprNode_Integer, tok.intval(), tok.datatype()) );

+    case TOK_RWORD_TRUE:

+        return AST::Pattern( AST::Pattern::TagValue(), NEWNODE( AST::ExprNode_Bool, true ) );

+    case TOK_RWORD_FALSE:

+        return AST::Pattern( AST::Pattern::TagValue(), NEWNODE( AST::ExprNode_Bool, false ) );

     case TOK_STRING:

-        throw ParseError::Todo("string patterns");

+        return AST::Pattern( AST::Pattern::TagValue(), NEWNODE(AST::ExprNode_String, tok.str()) );

     case TOK_PAREN_OPEN:

         return AST::Pattern(AST::Pattern::TagTuple(), Parse_PatternList(lex));

     case TOK_SQUARE_OPEN:

-        throw ParseError::Todo("array patterns");

+        throw ParseError::Todo(lex, "array patterns");

     default:

         throw ParseError::Unexpected(lex, tok);

     }

@@ -218,8 +226,18 @@ pass_value:
 ExprNodeP Parse_Stmt(TokenStream& lex, bool& opt_semicolon)

 {

     TRACE_FUNCTION;

-

     Token   tok;

+

+    ::std::string   lifetime;

+    

+    if( GET_TOK(tok, lex) == TOK_LIFETIME ) {

+        lifetime = tok.str();

+        GET_CHECK_TOK(tok, lex, TOK_COLON);

+    }

+    else {

+        lex.putback(tok);

+    }

+    

     // 1. Handle 'let'

     // 2. Handle new blocks

     // 3. Handle a sequence of expressions broken by ';'

@@ -229,6 +247,14 @@ ExprNodeP Parse_Stmt(TokenStream& lex, bool& opt_semicolon)
         lex.putback(tok);

         opt_semicolon = true;

         return Parse_ExprBlockNode(lex);

+    case TOK_RWORD_USE: {

+        opt_semicolon = false;

+        ::std::vector< ::std::pair< ::std::string, AST::Path> > imports;

+        Parse_Use(lex, [&imports](AST::Path p, std::string s) {

+            imports.push_back( ::std::make_pair( ::std::move(s), ::std::move(p) ) );

+            });

+        return NEWNODE( AST::ExprNode_Import, ::std::move(imports) );

+        }

     case TOK_RWORD_CONST: {

         opt_semicolon = false;

         GET_CHECK_TOK(tok, lex, TOK_IDENT);

@@ -253,17 +279,68 @@ ExprNodeP Parse_Stmt(TokenStream& lex, bool& opt_semicolon)
         ExprNodeP val = Parse_ExprBlocks(lex);

         return NEWNODE( AST::ExprNode_LetBinding, ::std::move(pat), ::std::move(type), ::std::move(val) );

         }

-    case TOK_RWORD_RETURN:

-        return NEWNODE( AST::ExprNode_Return, Parse_Expr1(lex) );

+    

+    case TOK_RWORD_RETURN: {

+        if( lifetime.size() )

+            throw ParseError::Unexpected(lex, Token(TOK_LIFETIME, lifetime));

+        ExprNodeP   val;

+        if( GET_TOK(tok, lex) != TOK_SEMICOLON ) {

+            lex.putback(tok);

+            val = Parse_Expr1(lex);

+        }

+        else

+            lex.putback(tok);

+        return NEWNODE( AST::ExprNode_Flow, AST::ExprNode_Flow::RETURN, "", ::std::move(val) );

+        }

+    case TOK_RWORD_CONTINUE:

+    case TOK_RWORD_BREAK:

+        {

+        if( lifetime.size() )

+            throw ParseError::Unexpected(lex, Token(TOK_LIFETIME, lifetime));

+        AST::ExprNode_Flow::Type    type;

+        switch(tok.type())

+        {

+        case TOK_RWORD_CONTINUE: type = AST::ExprNode_Flow::CONTINUE; break;

+        case TOK_RWORD_BREAK:    type = AST::ExprNode_Flow::BREAK;    break;

+        default:    throw ParseError::BugCheck(/*lex,*/ "continue/break");

+        }

+        if( GET_TOK(tok, lex) == TOK_LIFETIME )

+        {

+            lifetime = tok.str();

+            GET_TOK(tok, lex);

+        }

+        ExprNodeP   val;

+        if( tok.type() != TOK_SEMICOLON ) {

+            lex.putback(tok);

+            val = Parse_Expr1(lex);

+        }

+        else

+            lex.putback(tok);

+        return NEWNODE( AST::ExprNode_Flow, type, lifetime, ::std::move(val) );

+        } 

+    

     case TOK_RWORD_LOOP:

-        throw ParseError::Todo(lex, "loop");

-    case TOK_RWORD_FOR:

-        throw ParseError::Todo(lex, "for");

-    case TOK_RWORD_WHILE:

-        throw ParseError::Todo(lex, "while");

+        opt_semicolon = true;

+        return NEWNODE( AST::ExprNode_Loop, lifetime, Parse_ExprBlockNode(lex) );

+    case TOK_RWORD_WHILE: {

+        opt_semicolon = true;

+        ExprNodeP cnd = Parse_Expr1(lex);

+        return NEWNODE( AST::ExprNode_Loop, lifetime, ::std::move(cnd), Parse_ExprBlockNode(lex) );

+        }

+    case TOK_RWORD_FOR: {

+        opt_semicolon = true;

+        AST::Pattern    pat = Parse_Pattern(lex);

+        GET_CHECK_TOK(tok, lex, TOK_RWORD_IN);

+        ExprNodeP val = Parse_Expr1(lex);

+        return NEWNODE( AST::ExprNode_Loop, lifetime, AST::ExprNode_Loop::FOR,

+                ::std::move(pat), ::std::move(val), Parse_ExprBlockNode(lex) );

+        }

     case TOK_RWORD_IF:

         opt_semicolon = true;

         return Parse_IfStmt(lex);

+    case TOK_RWORD_MATCH:

+        opt_semicolon = true;

+        return Parse_Expr_Match(lex);

     default:

         lex.putback(tok);

         return Parse_Expr0(lex);

@@ -295,14 +372,25 @@ ExprNodeP Parse_Expr0(TokenStream& lex)
     Token tok;

 

     ExprNodeP rv = Parse_ExprBlocks(lex);

+    auto op = AST::ExprNode_Assign::NONE;

     switch( GET_TOK(tok, lex) )

     {

-    case TOK_EQUAL:

-        return NEWNODE( AST::ExprNode_Assign, AST::ExprNode_Assign::NONE, ::std::move(rv), Parse_Expr1(lex) );

     case TOK_PLUS_EQUAL:

-        return NEWNODE( AST::ExprNode_Assign, AST::ExprNode_Assign::ADD, ::std::move(rv), Parse_Expr1(lex) );

+        op = AST::ExprNode_Assign::ADD;

+        if(0)

     case TOK_DASH_EQUAL:

-        return NEWNODE( AST::ExprNode_Assign, AST::ExprNode_Assign::SUB, ::std::move(rv), Parse_Expr1(lex) );

+        op = AST::ExprNode_Assign::SUB;

+        if(0)

+    case TOK_SLASH_EQUAL:

+        op = AST::ExprNode_Assign::DIV;

+        if(0)

+    case TOK_STAR_EQUAL:

+        op = AST::ExprNode_Assign::MUL;

+        if(0)

+    case TOK_EQUAL:

+        op = AST::ExprNode_Assign::NONE;

+        return NEWNODE( AST::ExprNode_Assign, op, ::std::move(rv), Parse_ExprBlocks(lex) );

+    

     default:

         lex.putback(tok);

         return rv;

@@ -314,21 +402,24 @@ ExprNodeP Parse_Expr0(TokenStream& lex)
 ExprNodeP Parse_IfStmt(TokenStream& lex)

 {

     TRACE_FUNCTION;

-    SET_PARSE_FLAG(lex, disallow_struct_literal);

 

     Token   tok;

     ExprNodeP cond;

     AST::Pattern    pat;

     bool if_let = false;

-    if( GET_TOK(tok, lex) == TOK_RWORD_LET ) {

-        if_let = true;

-        pat = Parse_Pattern(lex);

-        GET_CHECK_TOK(tok, lex, TOK_EQUAL);

-        cond = Parse_Expr0(lex);

-    }

-    else {

-        lex.putback(tok);

-        cond = Parse_Expr0(lex);

+    

+    {

+        SET_PARSE_FLAG(lex, disallow_struct_literal);

+        if( GET_TOK(tok, lex) == TOK_RWORD_LET ) {

+            if_let = true;

+            pat = Parse_Pattern(lex);

+            GET_CHECK_TOK(tok, lex, TOK_EQUAL);

+            cond = Parse_Expr0(lex);

+        }

+        else {

+            lex.putback(tok);

+            cond = Parse_Expr0(lex);

+        }

     }

 

     // Contents

@@ -370,6 +461,7 @@ ExprNodeP Parse_Expr_Match(TokenStream& lex)
         SET_PARSE_FLAG(lex, disallow_struct_literal);

         switch_val = Parse_Expr1(lex);

     }

+    ASSERT(lex, !CHECK_PARSE_FLAG(lex, disallow_struct_literal) );

     GET_CHECK_TOK(tok, lex, TOK_BRACE_OPEN);

     

     ::std::vector< AST::ExprNode_Match::Arm >    arms;

@@ -449,8 +541,59 @@ ExprNodeP cur(TokenStream& lex) \
         } \

     } \

 }

+bool Parse_IsTokValue(eTokenType tok_type)

+{

+    switch( tok_type )

+    {

+    case TOK_DOUBLE_COLON:

+    case TOK_IDENT:

+    case TOK_INTEGER:

+    case TOK_FLOAT:

+    case TOK_STRING:

+    case TOK_RWORD_TRUE:

+    case TOK_RWORD_FALSE:

+    case TOK_RWORD_BOX:

+    case TOK_PAREN_OPEN:

+        return true;

+    default:

+        return false;

+    }

+    

+}

+ExprNodeP Parse_Expr1_5(TokenStream& lex);

+// Very evil handling for '..'

+ExprNodeP Parse_Expr1(TokenStream& lex)

+{

+    Token   tok;

+    ExprNodeP (*next)(TokenStream&) = Parse_Expr1_5;

+    ExprNodeP   left, right;

+    

+    if( GET_TOK(tok, lex) != TOK_DOUBLE_DOT )

+    {

+        lex.putback(tok);

+        

+        left = next(lex);

+        

+        if( GET_TOK(tok, lex) != TOK_DOUBLE_DOT )

+        {

+            lex.putback(tok);

+            return ::std::move(left);

+        }

+    }

+    if( Parse_IsTokValue( GET_TOK(tok, lex) ) )

+    {

+        lex.putback(tok);

+        right = next(lex);

+    }

+    else

+    {

+        lex.putback(tok);

+    }

+    

+    return NEWNODE( AST::ExprNode_BinOp, AST::ExprNode_BinOp::RANGE, ::std::move(left), ::std::move(right) );

+}

 // 1: Bool OR

-LEFTASSOC(Parse_Expr1, Parse_Expr2,

+LEFTASSOC(Parse_Expr1_5, Parse_Expr2,

     case TOK_DOUBLE_PIPE:

         rv = NEWNODE( AST::ExprNode_BinOp, AST::ExprNode_BinOp::BOOLOR, ::std::move(rv), next(lex));

         break;

@@ -536,7 +679,8 @@ LEFTASSOC(Parse_Expr11, Parse_Expr12,
         rv = NEWNODE( AST::ExprNode_BinOp, AST::ExprNode_BinOp::DIVIDE, ::std::move(rv), next(lex));

         break;

     case TOK_PERCENT:

-        throw ParseError::Todo("expr - modulo");

+        rv = NEWNODE( AST::ExprNode_BinOp, AST::ExprNode_BinOp::MODULO, ::std::move(rv), next(lex));

+        break;

 )

 // 12: Unaries

 ExprNodeP Parse_ExprFC(TokenStream& lex);

@@ -575,23 +719,32 @@ ExprNodeP Parse_ExprFC(TokenStream& lex)
             lex.putback(tok);

             val = NEWNODE( AST::ExprNode_CallObject, ::std::move(val), Parse_ParenList(lex) );

             break;

+        case TOK_SQUARE_OPEN: {

+            ExprNodeP   idx = Parse_Expr0(lex);

+            val = NEWNODE( AST::ExprNode_Index, ::std::move(val), ::std::move(idx) );

+            GET_CHECK_TOK(tok, lex, TOK_SQUARE_CLOSE);

+            }

+            break;

         case TOK_DOT:

             // Field access / method call

             // TODO: What about tuple indexing?

             switch(GET_TOK(tok, lex))

             {

             case TOK_IDENT: {

-                ::std::string name = tok.str();

+                AST::PathNode   path(tok.str(), {});

                 switch( GET_TOK(tok, lex) )

                 {

                 case TOK_PAREN_OPEN:

                     lex.putback(tok);

-                    val = NEWNODE( AST::ExprNode_CallMethod, ::std::move(val), AST::PathNode(name, {}), Parse_ParenList(lex) );

+                    val = NEWNODE( AST::ExprNode_CallMethod, ::std::move(val), ::std::move(path), Parse_ParenList(lex) );

                     break;

                 case TOK_DOUBLE_COLON:

-                    throw ParseError::Todo("method calls - generic");

+                    GET_CHECK_TOK(tok, lex, TOK_LT);

+                    path.args() = Parse_Path_GenericList(lex);

+                    val = NEWNODE( AST::ExprNode_CallMethod, ::std::move(val), ::std::move(path), Parse_ParenList(lex) );

+                    break;

                 default:

-                    val = NEWNODE( AST::ExprNode_Field, ::std::move(val), ::std::string(name) );

+                    val = NEWNODE( AST::ExprNode_Field, ::std::move(val), ::std::string(path.name()) );

                     lex.putback(tok);

                     break;

                 }

@@ -641,6 +794,42 @@ ExprNodeP Parse_ExprVal_StructLiteral(TokenStream& lex, AST::Path path)
     return NEWNODE( AST::ExprNode_StructLiteral, path, ::std::move(base_val), ::std::move(items) );

 }

 

+ExprNodeP Parse_ExprVal_Closure(TokenStream& lex)

+{

+    TRACE_FUNCTION;

+    Token   tok;

+    

+    ::std::vector< ::std::pair<AST::Pattern, TypeRef> > args;

+    

+    while( GET_TOK(tok, lex) != TOK_PIPE )

+    {

+        lex.putback(tok);

+        AST::Pattern    pat = Parse_Pattern(lex);

+    

+        TypeRef type;

+        if( GET_TOK(tok, lex) == TOK_COLON )

+            type = Parse_Type(lex);

+        else

+            lex.putback(tok);

+        

+        args.push_back( ::std::make_pair( ::std::move(pat), ::std::move(type) ) );

+        

+        if( GET_TOK(tok, lex) != TOK_COMMA )

+            break;

+    }

+    CHECK_TOK(tok, TOK_PIPE);

+    

+    TypeRef rt;

+    if( GET_TOK(tok, lex) == TOK_COLON )

+        rt = Parse_Type(lex);

+    else

+        lex.putback(tok);

+    

+    auto code = Parse_Expr0(lex);

+    

+    return NEWNODE( AST::ExprNode_Closure, ::std::move(args), ::std::move(rt), ::std::move(code) );

+}

+

 ExprNodeP Parse_FormatArgs(TokenStream& lex)

 {

     TRACE_FUNCTION;

@@ -690,15 +879,23 @@ ExprNodeP Parse_ExprVal(TokenStream& lex)
         case TOK_BRACE_OPEN:

             if( !CHECK_PARSE_FLAG(lex, disallow_struct_literal) )

                 return Parse_ExprVal_StructLiteral(lex, ::std::move(path));

+            else

+                DEBUG("Not parsing struct literal");

         default:

             // Value

             lex.putback(tok);

             return NEWNODE( AST::ExprNode_NamedValue, ::std::move(path) );

         }

+    case TOK_DOUBLE_PIPE:

+        lex.putback(Token(TOK_PIPE));

+    case TOK_PIPE:

+        return Parse_ExprVal_Closure(lex);

     case TOK_INTEGER:

         return NEWNODE( AST::ExprNode_Integer, tok.intval(), tok.datatype() );

     case TOK_FLOAT:

         return NEWNODE( AST::ExprNode_Float, tok.floatval(), tok.datatype() );

+    case TOK_STRING:

+        return NEWNODE( AST::ExprNode_String, tok.str() );

     case TOK_RWORD_TRUE:

         return NEWNODE( AST::ExprNode_Bool, true );

     case TOK_RWORD_FALSE:

@@ -795,197 +992,82 @@ TokenTree Parse_TT(TokenStream& lex, bool unwrapped)
     return TokenTree(items);

 }

 

+/// A wrapping lexer that 

+class TTLexer:

+    public TokenStream

+{

+    TokenStream&    m_input;

+    Token   m_last_token;

+    ::std::vector<TokenTree>    m_output;

+public:

+    TTLexer(TokenStream& input):

+        m_input(input)

+    {

+    }

+    

+    virtual Position getPosition() const override { return m_input.getPosition(); }

+    virtual Token realGetToken() override {

+        Token tok = m_input.getToken();

+        m_output.push_back( TokenTree(tok) );

+        return tok;

+    }

+

+    TokenTree   get_output() {

+        unsigned int eat = (TokenStream::m_cache_valid ? 1 : 0) + TokenStream::m_lookahead.size();

+        DEBUG(eat << " tokens were not consumed");

+        assert( m_output.size() >= eat );

+        assert( m_input.m_lookahead.size() == 0 );

+        assert( m_input.m_cache_valid == false );

+        for( unsigned int i = 0; i < eat; i ++ )

+        {

+            Token tok = m_output[ m_output.size() - eat + i ].tok();

+            DEBUG("Unconsume " << tok);

+            m_input.m_lookahead.push_back( tok );

+        }

+        DEBUG("- output was [" << m_output << "]");

+        m_output.erase( m_output.end() - eat, m_output.end() );

+        DEBUG("Returning [" << m_output << "]");

+        return ::std::move(m_output);

+    }

+};

+

 TokenTree Parse_TT_Type(TokenStream& lex)

 {

     TRACE_FUNCTION;

-    Token   tok;

+    TTLexer wlex(lex);

     

-    ::std::vector<TokenTree>    ret;

-    switch(GET_TOK(tok, lex))

-    {

-    case TOK_AMP:

-        throw ParseError::Todo("TokenTree type &-ptr");

-    case TOK_STAR:

-        throw ParseError::Todo("TokenTree type *-ptr");

-    case TOK_DOUBLE_COLON:

-    case TOK_IDENT:

-        lex.putback(tok);

-        return Parse_TT_Path(lex, false);

-    default:

-        throw ParseError::Unexpected(lex, tok);

-    }

-    return TokenTree(ret);

+    // discard result

+    Parse_Type(wlex);

+    

+    return wlex.get_output();

 }

 

+/// Parse a token tree path

 TokenTree Parse_TT_Path(TokenStream& lex, bool mode_expr)

 {

     TRACE_FUNCTION;

-    

+    TTLexer wlex(lex);

     Token   tok;

     

-    ::std::vector<TokenTree>    ret;

-    if( GET_TOK(tok, lex) == TOK_DOUBLE_COLON ) {

-        ret.push_back(TokenTree(tok));

+    if( GET_TOK(tok, wlex) == TOK_DOUBLE_COLON ) {

+        Parse_Path(wlex, true, (mode_expr ? PATH_GENERIC_EXPR : PATH_GENERIC_TYPE));

     }

     else {

-        lex.putback(tok);

+        wlex.putback(tok);

+        Parse_Path(wlex, false, (mode_expr ? PATH_GENERIC_EXPR : PATH_GENERIC_TYPE));

     }

     

-    for(;;)

-    {

-        // Expect an ident

-        GET_CHECK_TOK(tok, lex, TOK_IDENT);

-        ret.push_back(TokenTree(tok));

-        // If mode is expr, check for a double colon here

-        if( mode_expr )

-        {

-            if( GET_TOK(tok, lex) != TOK_DOUBLE_COLON )

-                break;

-            ret.push_back( TokenTree(tok) );

-        }

-        

-        if( GET_TOK(tok, lex) == TOK_LT )

-        {

-            do {

-                ret.push_back( TokenTree(tok) );

-                ret.push_back(Parse_TT_Type(lex));

-            } while(GET_TOK(tok, lex) == TOK_COMMA);

-            if( tok.type() != TOK_GT )

-            {

-                if(tok.type() == TOK_DOUBLE_GT) {

-                    ret.push_back(TokenTree(Token(TOK_GT)));

-                    lex.putback(Token(TOK_GT));

-                }

-                else {

-                    CHECK_TOK(tok, TOK_GT);

-                }

-            }

-            else {

-                ret.push_back(TokenTree(tok));

-            }

-            

-            if( GET_TOK(tok, lex) != TOK_DOUBLE_COLON )

-                break;

-            ret.push_back(TokenTree(tok));

-        }

-        else

-        {

-            lex.putback(tok);

-            

-            if( !mode_expr )

-            {

-                if( GET_TOK(tok, lex) != TOK_DOUBLE_COLON )

-                    break;

-                ret.push_back(TokenTree(tok));

-            }

-        }

-    }

-    lex.putback(tok);

-    return TokenTree(ret);

-}

-/// Parse a token tree path

-TokenTree Parse_TT_Val(TokenStream& lex)

-{

-    Token   tok;

-    ::std::vector<TokenTree>    ret;

-    switch(GET_TOK(tok, lex))

-    {

-    case TOK_PAREN_OPEN:

-        lex.putback(tok);

-        return Parse_TT(lex, false);

-

-    case TOK_IDENT:

-    case TOK_DOUBLE_COLON: {

-        lex.putback(tok);

-        TokenTree inner = Parse_TT_Path(lex, true);

-        if(GET_TOK(tok, lex) == TOK_BRACE_OPEN) {

-            lex.putback(tok);

-            ret.push_back(inner);

-            ret.push_back(Parse_TT(lex, false));

-        }

-        else {

-            lex.putback(tok);

-            return inner;

-        }

-        break; }

-    case TOK_RWORD_SELF:

-    case TOK_INTEGER:

-    case TOK_FLOAT:

-    case TOK_STRING:

-        return TokenTree(tok);

-    case TOK_RWORD_MATCH:

-        ret.push_back(TokenTree(tok));

-        ret.push_back(Parse_TT(lex, false));

-        break;

-    case TOK_RWORD_IF:

-        ret.push_back(TokenTree(tok));

-        ret.push_back(Parse_TT_Expr(lex));

-        if( GET_TOK(tok, lex) == TOK_RWORD_ELSE ) {

-            ret.push_back(TokenTree(tok));

-            ret.push_back(Parse_TT(lex, false));

-        }

-        else {

-            lex.putback(tok);

-        }

-        break;

-    default:

-        // Oh, fail :(

-        throw ParseError::Unexpected(lex, tok);

-    }

-    return TokenTree(ret);

+    return wlex.get_output();

 }

 /// Parse a token tree expression

 TokenTree Parse_TT_Expr(TokenStream& lex)

 {

     TRACE_FUNCTION;

+    TTLexer wlex(lex);

     

-    Token   tok;

-    ::std::vector<TokenTree>    ret;

-

-    ret.push_back(Parse_TT_Val(lex));

-    // 1. Get left associative blocks until nothing matches

-    bool cont = true;

-    while(cont)

-    {

-        switch(GET_TOK(tok, lex))

-        {

-        case TOK_PLUS:

-        case TOK_DASH:

-        case TOK_SLASH:

-        case TOK_STAR:

-        case TOK_PERCENT:

-            ret.push_back(tok);

-            ret.push_back(Parse_TT_Val(lex));

-            break;

-        case TOK_PAREN_OPEN:

-            lex.putback(tok);

-            ret.push_back(Parse_TT(lex, false));

-            break;

-        case TOK_DOT:

-            ret.push_back(tok);

-            GET_CHECK_TOK(tok, lex, TOK_IDENT);

-            ret.push_back(tok);

-            switch(GET_TOK(tok, lex))

-            {

-            case TOK_DOUBLE_COLON:

-                throw ParseError::Todo("Generic type params in TT expr");

-            case TOK_PAREN_OPEN:

-                lex.putback(tok);

-                ret.push_back(Parse_TT(lex, false));

-                break;

-            default:

-                lex.putback(tok);

-                break;

-            }

-            break;

-        default:

-            lex.putback(tok);

-            cont = false;

-            break;

-        }

-    }

-    return TokenTree(ret);

-

+    Parse_Expr1(wlex);

+    

+    return wlex.get_output();

 }

 TokenTree Parse_TT_Stmt(TokenStream& lex)

 {

diff --git a/src/parse/lex.cpp b/src/parse/lex.cpp
index b290b3f1..d469d46d 100644
--- a/src/parse/lex.cpp
+++ b/src/parse/lex.cpp
@@ -623,6 +623,7 @@ SERIALISE_TYPE_S(Token, {
 TTStream::TTStream(const TokenTree& input_tt):
     m_input_tt(input_tt)
 {
+    DEBUG("input_tt = [" << input_tt << "]");
     m_stack.push_back( ::std::make_pair(0, &input_tt) );
 }
 TTStream::~TTStream()
@@ -671,6 +672,14 @@ TokenStream::~TokenStream()
 {
 }
 
+Token TokenStream::innerGetToken()
+{
+    Token ret = this->realGetToken();
+    if( ret.get_pos().filename.size() == 0 )
+        ret.set_pos( this->getPosition() );
+    //DEBUG("ret.get_pos() = " << ret.get_pos());
+    return ret;
+}
 Token TokenStream::getToken()
 {
     if( m_cache_valid )
@@ -687,7 +696,7 @@ Token TokenStream::getToken()
     }
     else
     {
-        Token ret = this->realGetToken();
+        Token ret = this->innerGetToken();
         ::std::cout << "getToken[" << typeid(*this).name() << "] - " << ret << ::std::endl;
         return ret;
     }
@@ -723,7 +732,7 @@ eTokenType TokenStream::lookahead(unsigned int i)
     while( i >= m_lookahead.size() )
     {
         DEBUG("lookahead - read #" << m_lookahead.size());
-        m_lookahead.push_back( this->realGetToken() );
+        m_lookahead.push_back( this->innerGetToken() );
     }
     
     return m_lookahead[i].type();
diff --git a/src/parse/lex.hpp b/src/parse/lex.hpp
index c754c61d..da56c6ba 100644
--- a/src/parse/lex.hpp
+++ b/src/parse/lex.hpp
@@ -12,6 +12,23 @@ enum eTokenType
     #undef _
 };
 
+struct Position
+{
+    ::std::string   filename;
+    unsigned int    line;
+    
+    Position():
+        filename(""),
+        line(0)
+    {}
+    Position(::std::string filename, unsigned int line):
+        filename(filename),
+        line(line)
+    {
+    }
+};
+extern ::std::ostream& operator<<(::std::ostream& os, const Position& p);
+
 class Token:
     public Serialisable
 {
@@ -22,6 +39,7 @@ class Token:
         uint64_t    m_intval;
         double  m_floatval;
     };
+    Position    m_pos;
 public:
     Token();
     Token(const Token& t) = default;
@@ -30,7 +48,8 @@ public:
         m_type(t.m_type),
         m_str( ::std::move(t.m_str) ),
         m_datatype( t.m_datatype ),
-        m_intval( t.m_intval )
+        m_intval( t.m_intval ),
+        m_pos( ::std::move(t.m_pos) )
     {
         t.m_type = TOK_NULL;
     }
@@ -45,6 +64,9 @@ public:
     uint64_t intval() const { return m_intval; }
     double floatval() const { return m_floatval; }
 
+    void set_pos(Position pos) { m_pos = pos; }
+    const Position& get_pos() const { return m_pos; }
+    
     static const char* typestr(enum eTokenType type);
     static eTokenType typefromstr(const ::std::string& s);
     
@@ -53,27 +75,23 @@ public:
 
 extern ::std::ostream&  operator<<(::std::ostream& os, const Token& tok);
 
-struct Position
-{
-    ::std::string   filename;
-    unsigned int    line;
-    
-    Position(::std::string filename, unsigned int line):
-        filename(filename),
-        line(line)
-    {
-    }
-};
-extern ::std::ostream& operator<<(::std::ostream& os, const Position& p);
-
 /// State the parser needs to pass down via a second channel.
 struct ParseState
 {
     bool disallow_struct_literal = false;
+    
+    friend ::std::ostream& operator<<(::std::ostream& os, const ParseState& ps) {
+        os << "ParseState {";
+        if(ps.disallow_struct_literal)  os << " disallow_struct_literal";
+        os << " }";
+        return os;
+    }
 };
 
 class TokenStream
 {
+    friend class TTLexer;   // needs access to internals to know what was consumed
+    
     bool    m_cache_valid;
     Token   m_cache;
     ::std::vector<Token>    m_lookahead;
@@ -90,6 +108,8 @@ public:
     
 protected:
     virtual Token   realGetToken() = 0;
+private:
+    Token innerGetToken();
 };
 
 class SavedParseState
@@ -100,9 +120,11 @@ public:
     SavedParseState(TokenStream& lex, ParseState state):
         m_lex(lex),
         m_state(state)
-    {}
+    {
+    }
     ~SavedParseState()
     {
+        DEBUG("Restoring " << m_state);
         m_lex.parse_state() = m_state;
     }
 };
diff --git a/src/parse/parseerror.cpp b/src/parse/parseerror.cpp
index e5385257..3905ae33 100644
--- a/src/parse/parseerror.cpp
+++ b/src/parse/parseerror.cpp
@@ -17,6 +17,11 @@ ParseError::Generic::Generic(const TokenStream& lex, ::std::string message)
     ::std::cout << lex.getPosition() << ": Generic(" << message << ")" << ::std::endl;

 }

 

+ParseError::BugCheck::BugCheck(const TokenStream& lex, ::std::string message):

+    m_message(message)

+{

+    ::std::cout << lex.getPosition() << "BugCheck(" << message << ")" << ::std::endl;

+}

 ParseError::BugCheck::BugCheck(::std::string message):

     m_message(message)

 {

@@ -49,12 +54,18 @@ ParseError::BadChar::~BadChar() throw()
 ParseError::Unexpected::Unexpected(const TokenStream& lex, Token tok):

     m_tok(tok)

 {

-    ::std::cout << lex.getPosition() << ": Unexpected(" << tok << ")" << ::std::endl;

+    auto pos = tok.get_pos();

+    if(pos.filename == "")

+        pos = lex.getPosition();

+    ::std::cout << pos << ": Unexpected(" << tok << ")" << ::std::endl;

 }

 ParseError::Unexpected::Unexpected(const TokenStream& lex, Token tok, Token exp):

     m_tok(tok)

 {

-    ::std::cout << lex.getPosition() << ": Unexpected(" << tok << ", " << exp << ")" << ::std::endl;

+    auto pos = tok.get_pos();

+    if(pos.filename == "")

+        pos = lex.getPosition();

+    ::std::cout << pos << ": Unexpected(" << tok << ", " << exp << ")" << ::std::endl;

 }

 ParseError::Unexpected::~Unexpected() throw()

 {

diff --git a/src/parse/parseerror.hpp b/src/parse/parseerror.hpp
index bfea40b3..911312c5 100644
--- a/src/parse/parseerror.hpp
+++ b/src/parse/parseerror.hpp
@@ -29,6 +29,7 @@ class BugCheck:
     ::std::string   m_message;

 public:

     BugCheck(::std::string message);

+    BugCheck(const TokenStream& lex, ::std::string message);

     virtual ~BugCheck() throw () {}

 };

 

@@ -64,6 +65,8 @@ public:
 

 };

 

+#define ASSERT(lex, cnd)    do { if( !(cnd) ) throw ParseError::BugCheck(lex, "Assertion failed: "#cnd); } while(0)

+

 }

 

 #endif // PARSEERROR_HPP_INCLUDED

diff --git a/src/parse/root.cpp b/src/parse/root.cpp
index 5b14ff02..c64dbb17 100644
--- a/src/parse/root.cpp
+++ b/src/parse/root.cpp
@@ -149,9 +149,16 @@ TypeRef Parse_Type(TokenStream& lex)
 {

     TRACE_FUNCTION;

 

-    Token tok = lex.getToken();

-    switch(tok.type())

+    Token tok;

+    

+    switch( GET_TOK(tok, lex) )

     {

+    case TOK_RWORD_EXTERN:

+        GET_CHECK_TOK(tok, lex, TOK_STRING);

+        // abi = tok.str();

+        GET_CHECK_TOK(tok, lex, TOK_RWORD_FN);

+    case TOK_RWORD_FN:

+        throw ParseError::Todo(lex, "Function types");

     case TOK_LT: {

         DEBUG("Associated type");

         // <Type as Trait>::Inner

@@ -306,7 +313,7 @@ void Parse_TypeConds(TokenStream& lex, AST::TypeParams& params)
 }

 

 /// Parse a function definition (after the 'fn')

-AST::Function Parse_FunctionDef(TokenStream& lex, bool allow_no_code=false)

+AST::Function Parse_FunctionDef(TokenStream& lex, AST::MetaItems attrs, bool allow_no_code=false)

 {

     TRACE_FUNCTION;

 

@@ -618,7 +625,7 @@ AST::Trait Parse_TraitDef(TokenStream& lex, const AST::MetaItems& meta_items)
         case TOK_RWORD_FN: {

             GET_CHECK_TOK(tok, lex, TOK_IDENT);

             ::std::string name = tok.str();

-            trait.add_function( ::std::move(name), Parse_FunctionDef(lex, true) );

+            trait.add_function( ::std::move(name), Parse_FunctionDef(lex, item_attrs, true) );

             break; }

         default:

             throw ParseError::Generic("Unexpected token, expected 'type' or 'fn'");

@@ -776,7 +783,7 @@ AST::Impl Parse_Impl(TokenStream& lex)
         case TOK_RWORD_FN: {

             GET_CHECK_TOK(tok, lex, TOK_IDENT);

             ::std::string name = tok.str();

-            impl.add_function(is_public, name, Parse_FunctionDef(lex));

+            impl.add_function(is_public, name, Parse_FunctionDef(lex, item_attrs));

             break; }

 

         default:

@@ -1255,10 +1262,22 @@ void Parse_ModRoot_Items(TokenStream& lex, AST::Crate& crate, AST::Module& mod,
             mod.add_global(is_public, is_mut, name, type, val);

             break; }

 

+        case TOK_RWORD_UNSAFE:

+            meta_items.push_back( AST::MetaItem("#UNSAFE") );

+            switch(GET_TOK(tok, lex))

+            {

+            case TOK_RWORD_FN:

+                GET_CHECK_TOK(tok, lex, TOK_IDENT);

+                mod.add_function(is_public, tok.str(), Parse_FunctionDef(lex, ::std::move(meta_items)));

+                break;

+            default:

+                throw ParseError::Unexpected(lex, tok);

+            }

+            break;

         case TOK_RWORD_FN: {

             GET_CHECK_TOK(tok, lex, TOK_IDENT);

             ::std::string name = tok.str();

-            mod.add_function(is_public, name, Parse_FunctionDef(lex));

+            mod.add_function(is_public, name, Parse_FunctionDef(lex, ::std::move(meta_items)));

             break; }

         case TOK_RWORD_TYPE: {

             GET_CHECK_TOK(tok, lex, TOK_IDENT);

diff --git a/src/parse/tokentree.hpp b/src/parse/tokentree.hpp
index b3156f66..eaaf6623 100644
--- a/src/parse/tokentree.hpp
+++ b/src/parse/tokentree.hpp
@@ -28,6 +28,13 @@ public:
     const Token& tok() const {

         return m_tok;

     }

+    

+    friend ::std::ostream& operator<<(::std::ostream& os, const TokenTree& tt) {

+        if( tt.m_subtrees.size() == 0 )

+            return os << "TokenTree(" << tt.m_tok << ")";

+        else

+            return os << "TokenTree([" << tt.m_subtrees << "])";

+    }

 };

 

 class TTStream: