proc_macro - Working invocations (some work needed on AST->TT)

7 files changed, 854 insertions, 33 deletions

diff --git a/lib/libproc_macro/src/lib.rs b/lib/libproc_macro/src/lib.rs
index 5db439b2..6d094893 100644
--- a/lib/libproc_macro/src/lib.rs
+++ b/lib/libproc_macro/src/lib.rs
@@ -9,6 +9,7 @@ pub struct TokenStream {
     inner: Vec<Token>,
 }
 
+#[derive(Debug)]
 enum Token {
     Symbol(String),
     Ident(String),
@@ -22,7 +23,7 @@ enum Token {
     Fragment(FragmentType, u64),    // Type and a key
 }
 #[repr(u8)]
-#[derive(Copy,Clone)] // TODO: Is this just a mrustc thing?
+#[derive(Copy,Clone,Debug)] // TODO: Is this just a mrustc thing?
 enum FragmentType {
     Ident = 0,
     Tt = 1,
@@ -73,21 +74,33 @@ impl<T: Iterator<Item=char>> CharStream<T> {
 }
 
 pub struct LexError {
-    _inner: (),
+    inner: &'static str,
 }
 impl ::std::fmt::Debug for LexError {
     fn fmt(&self, f: &mut ::std::fmt::Formatter) -> ::std::fmt::Result {
-        f.write_str("LexError")
+        write!(f, "LexError({})", self.inner)
     }
 }
 impl FromStr for TokenStream {
     type Err = LexError;
     fn from_str(src: &str) -> Result<TokenStream, LexError> {
+        //eprintln!("TokenStream::from_str({:?})\r", src);
         let rv = Vec::new();
         let mut it = CharStream::new(src.chars());
 
-        fn err() -> Result<TokenStream,LexError> {
-            Err(LexError { _inner: () })
+        fn err(s: &'static str) -> Result<TokenStream,LexError> {
+            Err(LexError { inner: s })
+        }
+
+        fn get_ident<T: Iterator<Item=char>>(it: &mut CharStream<T>, mut s: String) -> String
+        {
+            let mut c = it.cur();
+            while c.is_xid_continue() || c.is_digit(10)
+            {
+                s.push(c);
+                c = some_else!(it.consume() => break);
+            }
+            s
         }
 
         'outer: while ! it.is_complete()
@@ -95,6 +108,7 @@ impl FromStr for TokenStream {
             let c = it.cur();
 
             if c.is_whitespace() {
+                it.consume();
                 continue ;
             }
 
@@ -132,8 +146,17 @@ impl FromStr for TokenStream {
                 ];
             if c == '\''
             {
-                // Lifetime or char lit
-                panic!("TODO: Lifetime / char literal");
+                it.consume();
+                c = it.cur();
+                if c.is_xid_start() && it.next().map(|x| x != '\'').unwrap_or(true) {
+                    // Lifetime
+                    let ident = get_ident(&mut it, String::new());
+                    rv.push(Token::Lifetime(ident))
+                }
+                else {
+                    // Char lit
+                    panic!("TODO: char literal");
+                }
             }
             else if c == '/' && it.next() == Some('/')
             {
@@ -160,27 +183,39 @@ impl FromStr for TokenStream {
 
                     if c == 'r'
                     {
+                        // TODO: If this isn't a string, start parsing an ident instead.
                         let ident_str = if is_byte { "br" } else { "r" };
                         c = some_else!(it.consume() => { rv.push(Token::Ident(ident_str.into())); break });
                         let mut hashes = 0;
                         while c == '#' {
                             hashes += 1;
-                            c = some_else!(it.consume() => return err());
+                            c = some_else!(it.consume() => return err("rawstr eof"));
                         }
 
                         if c != '"' {
-                            return err();
+                            if hashes == 0 {
+                                let s = get_ident(&mut it, ident_str.to_string());
+                                rv.push(Token::Ident(s));
+                            }
+                            else {
+                                rv.push(Token::Ident(ident_str.into()));
+                            }
+                            while hashes > 0 {
+                                rv.push(Token::Symbol("#".into()));
+                                hashes -= 1;
+                            }
+                            continue 'outer;
                         }
 
                         let req_hashes = hashes;
                         let mut rawstr = String::new();
                         loop
                         {
-                            c = some_else!(it.consume() => return err());
+                            c = some_else!(it.consume() => return err("Rawstr eof"));
                             if c == '"' {
                                 let mut hashes = 0;
                                 while hashes < req_hashes {
-                                    c = some_else!(it.consume() => return err());
+                                    c = some_else!(it.consume() => return err("rawstr eof"));
                                     if c != '#' { break ; }
                                     hashes += 1;
                                 }
@@ -214,7 +249,26 @@ impl FromStr for TokenStream {
                     else if c == '\"'
                     {
                         // String literal
-                        panic!("TODO: Escaped string literal");
+                        let mut s = String::new();
+                        loop
+                        {
+                            c = some_else!(it.consume() => return err("str eof"));
+                            if c == '"' {
+                                it.consume();
+                                break ;
+                            }
+                            else if c == '\\' {
+                                match some_else!(it.consume() => return err("str eof"))
+                                {
+                                c @ _ => panic!("Unknown escape in string {}", c),
+                                }
+                            }
+                            else {
+                                s.push(c);
+                            }
+                        }
+                        rv.push(Token::String(s));
+                        continue 'outer;
                     }
                     else
                     {
@@ -224,23 +278,77 @@ impl FromStr for TokenStream {
                 }
 
                 // Identifier.
-                if c.is_xid_start()
+                if c.is_xid_start() || c == '_'
                 {
-                    let mut ident = String::new();
-                    while c.is_xid_continue()
+                    let ident = get_ident(&mut it, String::new());
+                    if ident == "_" {
+                        rv.push(Token::Symbol(ident));
+                    }
+                    else {
+                        rv.push(Token::Ident(ident));
+                    }
+                }
+                else if c.is_digit(10)
+                {
+                    let base =
+                        if c == '0' {
+                            match it.consume()
+                            {
+                            Some('x') => { it.consume(); 16 },
+                            Some('o') => { it.consume(); 8 },
+                            Some('b') => { it.consume(); 2 },
+                            _ => 10,
+                            }
+                        }
+                        else {
+                            10
+                        };
+                    let mut v = 0;
+                    let mut c = it.cur();
+                    'int: loop
                     {
-                        ident.push(c);
-                        c = some_else!(it.consume() => break);
+                        while c == '_' {
+                            c = some_else!( it.consume() => { break 'int; } );
+                        }
+                        if c == 'u' || c == 'i' {
+                            let s = get_ident(&mut it, String::new());
+                            match &*s
+                            {
+                            "u8"    => rv.push(Token::UnsignedInt(v,   8)), "i8"    => rv.push(Token::SignedInt(v as i128,   8)),
+                            "u16"   => rv.push(Token::UnsignedInt(v,  16)), "i16"   => rv.push(Token::SignedInt(v as i128,  16)),
+                            "u32"   => rv.push(Token::UnsignedInt(v,  32)), "i32"   => rv.push(Token::SignedInt(v as i128,  32)),
+                            "u64"   => rv.push(Token::UnsignedInt(v,  64)), "i64"   => rv.push(Token::SignedInt(v as i128,  64)),
+                            "u128"  => rv.push(Token::UnsignedInt(v, 128)), "i128"  => rv.push(Token::SignedInt(v as i128, 128)),
+                            "usize" => rv.push(Token::UnsignedInt(v,   1)), "isize" => rv.push(Token::SignedInt(v as i128,   1)),
+                            _ => return err("Unexpected integer suffix"),
+                            }
+                            continue 'outer;
+                        }
+                        else if let Some(d) = c.to_digit(base) {
+                            v *= base as u128;
+                            v += d as u128;
+                            c = some_else!( it.consume() => { break 'int; } );
+                        }
+                        else if c == '.' {
+                            panic!("TODO: Floating point");
+                        }
+                        else {
+                            break;
+                        }
                     }
-                    rv.push(Token::Ident(ident));
+                    rv.push(Token::UnsignedInt(v, 0));
+                    continue 'outer;
                 }
                 // Punctuation?
                 else if c as u32 <= 0xFF
                 {
-                    let mut start = match syms.binary_search_by(|v| Ord::cmp(&v[0], &(c as u8)))
+                    let mut start = match syms.iter().position(|v| v[0] == (c as u8))
                         {
-                        Ok(start) => start,
-                        Err(_) => return err(),
+                        Some(start) => start,
+                        None => {
+                            eprint!("Unknown operator character '{}'\r\n", c);
+                            return err("Unknown operator")
+                            },
                         };
                     let mut end = start+1;
                     while end < syms.len() && syms[end][0] == c as u8 {
@@ -251,12 +359,12 @@ impl FromStr for TokenStream {
                     loop
                     {
                         let syms = &syms[start..end];
-                        assert!(ofs == syms[0].len());
+                        assert_eq!(ofs, syms[0].len(), "{:?}", syms[0]);
                         c = some_else!(it.consume() => break);
-                        let step = match syms[1..].binary_search_by(|v| Ord::cmp(&v[ofs], &(c as u8)))
+                        let step = match syms[1..].iter().position(|v| v[ofs] == (c as u8))
                             {
-                            Ok(s) => s+1,
-                            Err(_) => break,
+                            Some(s) => s+1,
+                            None => break,
                             };
                         start += step;
                         end = start+1;
@@ -265,12 +373,12 @@ impl FromStr for TokenStream {
                         }
                         ofs += 1;
                     }
-                    assert!(syms[start].len() == ofs);
+                    assert_eq!(syms[start].len(), ofs);
                     rv.push(Token::Symbol(::std::str::from_utf8(syms[start]).unwrap().into()));
                 }
                 else
                 {
-                    return err();
+                    return err("Unexpectec character");
                 }
             }
         }
@@ -369,7 +477,7 @@ pub fn recv_token_stream() -> TokenStream
             match self.inner.read(&mut b)
             {
             Ok(1) => Some(b[0]),
-            Ok(0) => None,
+            Ok(0) => panic!("Unexpected EOF reading from stdin"),
             Ok(_) => panic!("Bad byte count"),
             Err(e) => panic!("Error reading from stdin - {}", e),
             }
@@ -407,7 +515,7 @@ pub fn recv_token_stream() -> TokenStream
             match self.inner.read_exact(&mut buf)
             {
             Ok(_) => {},
-            Err(e) => panic!("Error reading from stdin - {}", e),
+            Err(e) => panic!("Error reading from stdin get_byte_vec({}) - {}", size, e),
             }
 
             buf
@@ -436,7 +544,11 @@ pub fn recv_token_stream() -> TokenStream
         let hdr_b = some_else!( s.getb() => break );
         toks.push(match hdr_b
             {
-            0 => Token::Symbol( s.get_string() ),
+            0 => {
+                let sym = s.get_string();
+                if sym == "" { break ; }
+                Token::Symbol( sym )
+                },
             1 => Token::Ident( s.get_string() ),
             2 => Token::Lifetime( s.get_string() ),
             3 => Token::String( s.get_string() ),
@@ -455,6 +567,7 @@ pub fn recv_token_stream() -> TokenStream
                 Token::Float(s.get_f64(), ty)
                 }
             });
+        eprintln!("> {:?}\r", toks.last().unwrap());
     }
     TokenStream {
         inner: toks,
@@ -500,6 +613,7 @@ pub fn send_token_stream(ts: TokenStream)
 
     for t in &ts.inner
     {
+        //eprintln!("{:?}\r", t);
         match t
         {
         &Token::Symbol(ref v)   => { s.putb(0); s.put_bytes(v.as_bytes()); },
@@ -514,6 +628,11 @@ pub fn send_token_stream(ts: TokenStream)
         &Token::Fragment(ty, key)  => { s.putb(9); s.putb(ty as u8); s.put_u128v(key as u128); },
         }
     }
+
+    // Empty symbol indicates EOF
+    s.putb(0); s.putb(0);
+    drop(s);
+    ::std::io::Write::flush(&mut ::std::io::stdout());
 }
 
 pub struct MacroDesc
@@ -524,5 +643,24 @@ pub struct MacroDesc
 
 pub fn main(macros: &[MacroDesc])
 {
+    let mac_name = ::std::env::args().nth(1).expect("Was not passed a macro name");
+    eprintln!("Searching for macro {}\r", mac_name);
+    for m in macros
+    {
+        if m.name == mac_name {
+            use std::io::Write;
+            ::std::io::stdout().write(&[0]);
+            ::std::io::stdout().flush();
+            eprintln!("Waiting for input\r");
+            let input = recv_token_stream();
+            eprintln!("INPUT = `{}`\r", input);
+            let output = (m.handler)( input );
+            eprintln!("OUTPUT = `{}`\r", output);
+            send_token_stream(output);
+            eprintln!("Done");
+            return ;
+        }
+    }
+    panic!("Unknown macro name '{}'", mac_name);
 }
 
diff --git a/src/expand/derive.cpp b/src/expand/derive.cpp
index 25cc6742..3b25831c 100644
--- a/src/expand/derive.cpp
+++ b/src/expand/derive.cpp
@@ -2227,8 +2227,14 @@ static void derive_item(const Span& sp, const AST::Crate& crate, AST::Module& mo
                 else {
                     // proc_macro - Invoke the handler.
                     auto lex = ProcMacro_Invoke(sp, crate, mac_path.first, path.nodes().back().name(), item);
-                    Parse_ModRoot_Items(*lex, mod);
-                    found = true;
+                    if( lex )
+                    {
+                        Parse_ModRoot_Items(*lex, mod);
+                        found = true;
+                    }
+                    else {
+                        ERROR(sp, E0000, "proc_macro derive failed");
+                    }
                     break;
                 }
             }
diff --git a/src/expand/proc_macro.cpp b/src/expand/proc_macro.cpp
new file mode 100644
index 00000000..ea8625de
--- /dev/null
+++ b/src/expand/proc_macro.cpp
@@ -0,0 +1,625 @@
+/*
+ * MRustC - Rust Compiler
+ * - By John Hodge (Mutabah/thePowersGang)
+ *
+ * expand/proc_macro.cpp
+ * - Support for the `#[proc_macro_derive]` attribute
+ */
+#include <synext.hpp>
+#include "../common.hpp"
+#include <ast/ast.hpp>
+#include <ast/expr.hpp>
+#include <ast/crate.hpp>
+#include <main_bindings.hpp>
+#include <hir/hir.hpp>  // ABI_RUST
+#include "proc_macro.hpp"
+#include <parse/lex.hpp>
+#ifdef WIN32
+#else
+# include <unistd.h>    // read/write/pipe
+# include <spawn.h>
+# include <sys/wait.h>
+#endif
+
+#define NEWNODE(_ty, ...)   ::AST::ExprNodeP(new ::AST::ExprNode##_ty(__VA_ARGS__))
+
+class Decorator_ProcMacroDerive:
+    public ExpandDecorator
+{
+public:
+    AttrStage stage() const override { return AttrStage::Post; }
+    void handle(const Span& sp, const AST::MetaItem& attr, ::AST::Crate& crate, const AST::Path& path, AST::Module& mod, AST::Item& i) const override
+    {
+        if( i.is_None() )
+            return;
+
+        if( !i.is_Function() )
+            TODO(sp, "Error for proc_macro_derive on non-Function");
+        //auto& fcn = i.as_Function();
+        auto trait_name = attr.items().at(0).name();
+        ::std::vector<::std::string>    attributes;
+        for(size_t i = 1; i < attr.items().size(); i ++)
+        {
+            if( attr.items()[i].name() == "attributes") {
+                for(const auto& si : attr.items()[i].items()) {
+                    attributes.push_back( si.name() );
+                }
+            }
+        }
+
+        // TODO: Store attributes for later use.
+        crate.m_proc_macros.push_back(::std::make_pair( FMT("derive#" << trait_name), path ));
+    }
+};
+
+STATIC_DECORATOR("proc_macro_derive", Decorator_ProcMacroDerive)
+
+
+
+void Expand_ProcMacro(::AST::Crate& crate)
+{
+    // Create the following module:
+    // ```
+    // mod `proc_macro#` {
+    //   extern crate proc_macro;
+    //   fn main() {
+    //     self::proc_macro::main(&::`proc_macro#`::MACROS);
+    //   }
+    //   static TESTS: [proc_macro::MacroDesc; _] = [
+    //     proc_macro::MacroDesc { name: "deriving_Foo", handler: ::path::to::foo }
+    //     ];
+    // }
+    // ```
+
+    // ---- main function ----
+    auto main_fn = ::AST::Function { Span(), {}, ABI_RUST, false, false, false, TypeRef(TypeRef::TagUnit(), Span()), {} };
+    {
+        auto call_node = NEWNODE(_CallPath,
+                ::AST::Path("proc_macro", { ::AST::PathNode("main") }),
+                ::make_vec1(
+                    NEWNODE(_UniOp, ::AST::ExprNode_UniOp::REF,
+                        NEWNODE(_NamedValue, ::AST::Path("", { ::AST::PathNode("proc_macro#"), ::AST::PathNode("MACROS") }))
+                        )
+                    )
+                );
+        main_fn.set_code( mv$(call_node) );
+    }
+
+
+    // ---- test list ----
+    ::std::vector< ::AST::ExprNodeP>    test_nodes;
+
+    for(const auto& desc : crate.m_proc_macros)
+    {
+        ::AST::ExprNode_StructLiteral::t_values   desc_vals;
+        // `name: "foo",`
+        desc_vals.push_back({ {}, "name", NEWNODE(_String,  desc.first) });
+        // `handler`: ::foo
+        desc_vals.push_back({ {}, "handler", NEWNODE(_NamedValue, AST::Path(desc.second)) });
+
+        test_nodes.push_back( NEWNODE(_StructLiteral,  ::AST::Path("proc_macro", { ::AST::PathNode("MacroDesc")}), nullptr, mv$(desc_vals) ) );
+    }
+    auto* tests_array = new ::AST::ExprNode_Array(mv$(test_nodes));
+
+    size_t test_count = tests_array->m_values.size();
+    auto tests_list = ::AST::Static { ::AST::Static::Class::STATIC,
+        TypeRef(TypeRef::TagSizedArray(), Span(),
+                TypeRef(Span(), ::AST::Path("proc_macro", { ::AST::PathNode("MacroDesc") })),
+                ::std::shared_ptr<::AST::ExprNode>( new ::AST::ExprNode_Integer(test_count, CORETYPE_UINT) )
+               ),
+        ::AST::Expr( mv$(tests_array) )
+        };
+
+    // ---- module ----
+    auto newmod = ::AST::Module { ::AST::Path("", { ::AST::PathNode("proc_macro#") }) };
+    // - TODO: These need to be loaded too.
+    //  > They don't actually need to exist here, just be loaded (and use absolute paths)
+    newmod.add_ext_crate(false, "proc_macro", "proc_macro", {});
+
+    newmod.add_item(false, "main", mv$(main_fn), {});
+    newmod.add_item(false, "MACROS", mv$(tests_list), {});
+
+    crate.m_root_module.add_item(false, "proc_macro#", mv$(newmod), {});
+    crate.m_lang_items["mrustc-main"] = ::AST::Path("", { AST::PathNode("proc_macro#"), AST::PathNode("main") });
+}
+
+enum class TokenClass
+{
+    Symbol = 0,
+    Ident = 1,
+    Lifetime = 2,
+    String = 3,
+    ByteString = 4, // String
+    CharLit = 5,    // v128
+    UnsignedInt = 6,
+    SignedInt = 7,
+    Float = 8,
+    Fragment = 9,
+};
+enum class FragType
+{
+    Ident = 0,
+    Tt = 1,
+
+    Path = 2,
+    Type = 3,
+
+    Expr = 4,
+    Statement = 5,
+    Block = 6,
+    Pattern = 7,
+};
+struct ProcMacroInv:
+    public TokenStream
+{
+    Span    m_parent_span;
+
+#ifdef WIN32
+    HANDLE  child_handle;
+    HANDLE  child_stdin;
+    HANDLE  child_stdout;
+#else
+    // POSIX
+    pid_t   child_pid;  // Questionably needed
+     int    child_stdin;
+     int    child_stdout;
+    // NOTE: stderr stays as our stderr
+#endif
+    bool    m_eof_hit = false;
+
+public:
+    ProcMacroInv(const Span& sp, const char* executable, const char* macro_name);
+    ProcMacroInv(const ProcMacroInv&) = delete;
+    ProcMacroInv(ProcMacroInv&&);
+    ProcMacroInv& operator=(const ProcMacroInv&) = delete;
+    ProcMacroInv& operator=(ProcMacroInv&&);
+    virtual ~ProcMacroInv();
+
+    bool check_good();
+    void send_done() {
+        send_symbol("");
+        DEBUG("Input tokens sent");
+    }
+    void send_symbol(const char* val) {
+        this->send_u8(static_cast<uint8_t>(TokenClass::Symbol));
+        this->send_bytes(val, ::std::strlen(val));
+    }
+    void send_ident(const char* val) {
+        this->send_u8(static_cast<uint8_t>(TokenClass::Ident));
+        this->send_bytes(val, ::std::strlen(val));
+    }
+    void send_lifetime(const char* val) {
+        this->send_u8(static_cast<uint8_t>(TokenClass::Lifetime));
+        this->send_bytes(val, ::std::strlen(val));
+    }
+    void send_string(const ::std::string& s) {
+        this->send_u8(static_cast<uint8_t>(TokenClass::String));
+        this->send_bytes(s.data(), s.size());
+    }
+    void send_bytestring(const ::std::string& s);
+    void send_char(uint32_t ch);
+    void send_int(eCoreType ct, int64_t v);
+    void send_float(eCoreType ct, double v);
+    //void send_fragment();
+
+    virtual Position getPosition() const override;
+    virtual Token realGetToken() override;
+    virtual Ident::Hygiene realGetHygiene() const override;
+private:
+    Token realGetToken_();
+    void send_u8(uint8_t v);
+    void send_bytes(const void* val, size_t size);
+    void send_v128u(uint64_t val);
+
+    uint8_t recv_u8();
+    ::std::string recv_bytes();
+    uint64_t recv_v128u();
+};
+
+ProcMacroInv ProcMacro_Invoke_int(const Span& sp, const ::AST::Crate& crate, const ::std::vector<::std::string>& mac_path)
+{
+    // 1. Locate macro in HIR list
+    const auto& crate_name = mac_path.front();
+    const auto& ext_crate = crate.m_extern_crates.at(crate_name);
+    // TODO: Ensure that this macro is in the listed crate.
+
+    // 2. Get executable and macro name
+    ::std::string   proc_macro_exe_name = (ext_crate.m_filename + "-plugin");
+    ::std::string   mac_name = mac_path.at(1);
+    for(size_t i = 2; i < mac_path.size(); i ++)
+    {
+        mac_name += "::";
+        mac_name += mac_path[i];
+    }
+
+    // 3. Create ProcMacroInv
+    return ProcMacroInv(sp, proc_macro_exe_name.c_str(), mac_name.c_str());
+}
+
+
+namespace {
+    struct Visitor/*:
+        public AST::NodeVisitor*/
+    {
+        ProcMacroInv&   m_pmi;
+        Visitor(ProcMacroInv& pmi):
+            m_pmi(pmi)
+        {
+        }
+
+        void visit_type(const ::TypeRef& ty)
+        {
+            // TODO: Correct handling of visit_type
+            m_pmi.send_symbol("(");
+            m_pmi.send_symbol(")");
+        }
+
+        void visit_params(const AST::GenericParams& params)
+        {
+            if( params.ty_params().size() > 0 || params.lft_params().size() > 0 )
+            {
+                bool is_first = true;
+                m_pmi.send_symbol("<");
+                // Lifetimes
+                for( const auto& p : params.lft_params() )
+                {
+                    if( !is_first )
+                        m_pmi.send_symbol(",");
+                    m_pmi.send_lifetime(p.c_str());
+                    is_first = false;
+                }
+                // Types
+                for( const auto& p : params.ty_params() )
+                {
+                    if( !is_first )
+                        m_pmi.send_symbol(",");
+                    m_pmi.send_ident(p.name().c_str());
+                    if( !p.get_default().is_wildcard() )
+                    {
+                        m_pmi.send_symbol("=");
+                        this->visit_type(p.get_default());
+                    }
+                    is_first = false;
+                }
+                m_pmi.send_symbol(">");
+            }
+        }
+        void visit_bounds(const AST::GenericParams& params)
+        {
+        }
+        void visit_nodes(const ::AST::Expr& e)
+        {
+            //e.visit_nodes(*this);
+        }
+        void visit_struct(const ::std::string& name, bool is_pub, const ::AST::Struct& str)
+        {
+            if( is_pub ) {
+                m_pmi.send_ident("pub");
+            }
+
+            m_pmi.send_ident("struct");
+            m_pmi.send_ident(name.c_str());
+            this->visit_params(str.params());
+            TU_MATCH(AST::StructData, (str.m_data), (se),
+            (Unit,
+                this->visit_bounds(str.params());
+                m_pmi.send_symbol(";");
+                ),
+            (Tuple,
+                m_pmi.send_symbol("(");
+                for( const auto& si : se.ents )
+                {
+                    if( si.m_is_public )
+                        m_pmi.send_ident("pub");
+                    this->visit_type(si.m_type);
+                    m_pmi.send_symbol(",");
+                }
+                m_pmi.send_symbol(")");
+                this->visit_bounds(str.params());
+                m_pmi.send_symbol(";");
+                ),
+            (Struct,
+                this->visit_bounds(str.params());
+                m_pmi.send_symbol("{");
+
+                for( const auto& si : se.ents )
+                {
+                    if( si.m_is_public )
+                        m_pmi.send_ident("pub");
+                    m_pmi.send_ident(si.m_name.c_str());
+                    m_pmi.send_symbol(":");
+                    this->visit_type(si.m_type);
+                    m_pmi.send_symbol(",");
+                }
+                m_pmi.send_symbol("}");
+                )
+            )
+        }
+        void visit_enum(const ::std::string& name, bool is_pub, const ::AST::Enum& str)
+        {
+        }
+        void visit_union(const ::std::string& name, bool is_pub, const ::AST::Union& str)
+        {
+        }
+    };
+}
+::std::unique_ptr<TokenStream> ProcMacro_Invoke(const Span& sp, const ::AST::Crate& crate, const ::std::vector<::std::string>& mac_path, const ::std::string& item_name, const ::AST::Struct& i)
+{
+    // 1. Create ProcMacroInv instance
+    auto pmi = ProcMacro_Invoke_int(sp, crate, mac_path);
+    if( !pmi.check_good() )
+        return ::std::unique_ptr<TokenStream>();
+    // 2. Feed item as a token stream.
+    Visitor(pmi).visit_struct(item_name, false, i);
+    pmi.send_done();
+    // 3. Return boxed invocation instance
+    return box$(pmi);
+}
+::std::unique_ptr<TokenStream> ProcMacro_Invoke(const Span& sp, const ::AST::Crate& crate, const ::std::vector<::std::string>& mac_path, const ::std::string& item_name, const ::AST::Enum& i)
+{
+    // 1. Create ProcMacroInv instance
+    auto pmi = ProcMacro_Invoke_int(sp, crate, mac_path);
+    if( !pmi.check_good() )
+        return ::std::unique_ptr<TokenStream>();
+    // 2. Feed item as a token stream.
+    Visitor(pmi).visit_enum(item_name, false, i);
+    pmi.send_done();
+    // 3. Return boxed invocation instance
+    return box$(pmi);
+}
+::std::unique_ptr<TokenStream> ProcMacro_Invoke(const Span& sp, const ::AST::Crate& crate, const ::std::vector<::std::string>& mac_path, const ::std::string& item_name, const ::AST::Union& i)
+{
+    // 1. Create ProcMacroInv instance
+    auto pmi = ProcMacro_Invoke_int(sp, crate, mac_path);
+    if( !pmi.check_good() )
+        return ::std::unique_ptr<TokenStream>();
+    // 2. Feed item as a token stream.
+    Visitor(pmi).visit_union(item_name, false, i);
+    pmi.send_done();
+    // 3. Return boxed invocation instance
+    return box$(pmi);
+}
+
+ProcMacroInv::ProcMacroInv(const Span& sp, const char* executable, const char* macro_name):
+    m_parent_span(sp)
+{
+#ifdef WIN32
+#else
+     int    stdin_pipes[2];
+    pipe(stdin_pipes);
+    this->child_stdin = stdin_pipes[1]; // Write end
+     int    stdout_pipes[2];
+    pipe(stdout_pipes);
+    this->child_stdout = stdout_pipes[0]; // Read end
+
+    posix_spawn_file_actions_t  file_actions;
+    posix_spawn_file_actions_init(&file_actions);
+    posix_spawn_file_actions_adddup2(&file_actions, stdin_pipes[0], 0);
+    posix_spawn_file_actions_adddup2(&file_actions, stdout_pipes[1], 1);
+    posix_spawn_file_actions_addclose(&file_actions, stdin_pipes[0]);
+    posix_spawn_file_actions_addclose(&file_actions, stdin_pipes[1]);
+    posix_spawn_file_actions_addclose(&file_actions, stdout_pipes[0]);
+    posix_spawn_file_actions_addclose(&file_actions, stdout_pipes[1]);
+
+    char*   argv[3] = { const_cast<char*>(executable), const_cast<char*>(macro_name), nullptr };
+    char*   envp[] = { nullptr };
+    int rv = posix_spawn(&this->child_pid, executable, &file_actions, nullptr, argv, envp);
+    if( rv != 0 )
+    {
+        BUG(sp, "Error in posix_spawn - " << rv);
+    }
+
+    posix_spawn_file_actions_destroy(&file_actions);
+    // Close the ends we don't care about.
+    close(stdin_pipes[0]);
+    close(stdout_pipes[1]);
+
+#endif
+}
+ProcMacroInv::ProcMacroInv(ProcMacroInv&& x):
+    m_parent_span(x.m_parent_span),
+#ifdef WIN32
+#else
+    child_pid(x.child_pid),
+    child_stdin(x.child_stdin),
+    child_stdout(x.child_stdout)
+#endif
+{
+#ifdef WIN32
+#else
+    x.child_pid = 0;
+#endif
+    DEBUG("");
+}
+ProcMacroInv& ProcMacroInv::operator=(ProcMacroInv&& x)
+{
+    m_parent_span = x.m_parent_span;
+#ifdef WIN32
+#else
+    child_pid = x.child_pid;
+    child_stdin = x.child_stdin;
+    child_stdout = x.child_stdout;
+
+    x.child_pid = 0;
+#endif
+    DEBUG("");
+    return *this;
+}
+ProcMacroInv::~ProcMacroInv()
+{
+#ifdef WIN32
+#else
+    if( this->child_pid != 0 )
+    {
+        DEBUG("Waiting for child " << this->child_pid << " to terminate");
+        int status;
+        waitpid(this->child_pid, &status, 0);
+        close(this->child_stdout);
+        close(this->child_stdin);
+    }
+#endif
+}
+bool ProcMacroInv::check_good()
+{
+    char    v;
+    int rv = read(this->child_stdout, &v, 1);
+    if( rv == 0 )
+    {
+        DEBUG("Unexpected EOF from child");
+        return false;
+    }
+    if( rv < 0 )
+    {
+        DEBUG("Error reading from child, rv=" << rv << " " << strerror(errno));
+        return false;
+    }
+    DEBUG("Child started, value = " << (int)v);
+    if( v != 0 )
+        return false;
+    return true;
+}
+void ProcMacroInv::send_u8(uint8_t v)
+{
+#ifdef WIN32
+#else
+    write(this->child_stdin, &v, 1);
+#endif
+}
+void ProcMacroInv::send_bytes(const void* val, size_t size)
+{
+    this->send_v128u( static_cast<uint64_t>(size) );
+#ifdef WIN32
+#else
+    write(this->child_stdin, val, size);
+#endif
+}
+void ProcMacroInv::send_v128u(uint64_t val)
+{
+    while( val >= 128 ) {
+        this->send_u8( static_cast<uint8_t>(val & 0x7F) | 0x80 );
+        val >>= 7;
+    }
+    this->send_u8( static_cast<uint8_t>(val & 0x7F) );
+}
+uint8_t ProcMacroInv::recv_u8()
+{
+    uint8_t v;
+    if( read(this->child_stdout, &v, 1) != 1 )
+        BUG(this->m_parent_span, "Unexpected EOF while reading from child process");
+    return v;
+}
+::std::string ProcMacroInv::recv_bytes()
+{
+    auto len = this->recv_v128u();
+    ASSERT_BUG(this->m_parent_span, len < SIZE_MAX, "Oversized string from child process");
+    ::std::string   val;
+    val.resize(len);
+    size_t  ofs = 0, rem = len;
+    while( rem > 0 )
+    {
+        auto n = read(this->child_stdout, &val[ofs], rem);
+        if( n == 0 ) {
+            BUG(this->m_parent_span, "Unexpected EOF while reading from child process");
+        }
+        if( n < 0 ) {
+            BUG(this->m_parent_span, "Error while reading from child process");
+        }
+        assert(static_cast<size_t>(n) <= rem);
+        ofs += n;
+        rem -= n;
+    }
+
+    return val;
+}
+uint64_t ProcMacroInv::recv_v128u()
+{
+    uint64_t    v = 0;
+    unsigned    ofs = 0;
+    for(;;)
+    {
+        auto b = recv_u8();
+        v |= static_cast<uint64_t>(b) << ofs;
+        if( (b & 0x80) == 0 )
+            break;
+        ofs += 7;
+    }
+    return v;
+}
+
+Position ProcMacroInv::getPosition() const {
+    return Position();
+}
+Token ProcMacroInv::realGetToken() {
+    auto rv = this->realGetToken_();
+    DEBUG(rv);
+    return rv;
+}
+Token ProcMacroInv::realGetToken_() {
+    if( m_eof_hit )
+        return Token(TOK_EOF);
+    uint8_t v = this->recv_u8();
+
+    switch( static_cast<TokenClass>(v) )
+    {
+    case TokenClass::Symbol: {
+        auto val = this->recv_bytes();
+        if( val == "" ) {
+            m_eof_hit = true;
+            return Token(TOK_EOF);
+        }
+        auto t = Lex_FindOperator(val);
+        ASSERT_BUG(this->m_parent_span, t != TOK_NULL, "Unknown symbol from child process - " << val);
+        return t;
+        }
+    case TokenClass::Ident: {
+        auto val = this->recv_bytes();
+        auto t = Lex_FindReservedWord(val);
+        if( t != TOK_NULL )
+            return t;
+        return Token(TOK_IDENT, mv$(val));
+        }
+    case TokenClass::Lifetime: {
+        auto val = this->recv_bytes();
+        return Token(TOK_LIFETIME, mv$(val));
+        }
+    case TokenClass::String: {
+        auto val = this->recv_bytes();
+        return Token(TOK_STRING, mv$(val));
+        }
+    case TokenClass::ByteString: {
+        auto val = this->recv_bytes();
+        return Token(TOK_BYTESTRING, mv$(val));
+        }
+    case TokenClass::CharLit: {
+        auto val = this->recv_v128u();
+        return Token(static_cast<uint64_t>(val), CORETYPE_CHAR);
+        }
+    case TokenClass::UnsignedInt: {
+        ::eCoreType ty;
+        switch(this->recv_u8())
+        {
+        case   0: ty = CORETYPE_ANY;  break;
+        case   1: ty = CORETYPE_UINT; break;
+        case   8: ty = CORETYPE_U8;  break;
+        case  16: ty = CORETYPE_U16; break;
+        case  32: ty = CORETYPE_U32; break;
+        case  64: ty = CORETYPE_U64; break;
+        case 128: ty = CORETYPE_U128; break;
+        default:    BUG(this->m_parent_span, "Invalid integer size from child process");
+        }
+        auto val = this->recv_v128u();
+        return Token(static_cast<uint64_t>(val), ty);
+        }
+    case TokenClass::SignedInt:
+    case TokenClass::Float:
+    case TokenClass::Fragment:
+        TODO(this->m_parent_span, "Handle ints/floats/fragments from child process");
+    }
+    BUG(this->m_parent_span, "Invalid token class from child process");
+
+    throw "";
+}
+Ident::Hygiene ProcMacroInv::realGetHygiene() const {
+    return Ident::Hygiene();
+}
+
diff --git a/src/expand/proc_macro.hpp b/src/expand/proc_macro.hpp
new file mode 100644
index 00000000..8c5b71c7
--- /dev/null
+++ b/src/expand/proc_macro.hpp
@@ -0,0 +1,15 @@
+/*
+ * MRustC - Rust Compiler
+ * - By John Hodge (Mutabah/thePowersGang)
+ *
+ * expand/proc_macro.hpp
+ * - Support for the `#[proc_macro_derive]` attribute
+ */
+#pragma once
+#include <parse/tokenstream.hpp>
+
+extern ::std::unique_ptr<TokenStream> ProcMacro_Invoke(const Span& sp, const ::AST::Crate& crate, const ::std::vector<::std::string>& mac_path, const ::std::string& name, const ::AST::Struct& i);
+extern ::std::unique_ptr<TokenStream> ProcMacro_Invoke(const Span& sp, const ::AST::Crate& crate, const ::std::vector<::std::string>& mac_path, const ::std::string& name, const ::AST::Enum& i);
+extern ::std::unique_ptr<TokenStream> ProcMacro_Invoke(const Span& sp, const ::AST::Crate& crate, const ::std::vector<::std::string>& mac_path, const ::std::string& name, const ::AST::Union& i);
+//extern ::std::unique_ptr<TokenStream> ProcMacro_Invoke(const Span& sp, const ::AST::Crate& crate, const ::std::vector<::std::string>& mac_path, const TokenStream& tt);
+
diff --git a/src/parse/lex.cpp b/src/parse/lex.cpp
index 5e57b418..17d7bf27 100644
--- a/src/parse/lex.cpp
+++ b/src/parse/lex.cpp
@@ -1066,3 +1066,37 @@ bool Codepoint::isxdigit() const {
     }
     return os;
 }
+
+Token Lex_FindOperator(const ::std::string& s)
+{
+    if( s == "_" )
+        return TOK_UNDERSCORE;
+    for(size_t i = 0; i < LEN(TOKENMAP); i++)
+    {
+        const auto& e = TOKENMAP[i];
+        if( s < e.chars )
+            break;
+        if( s == e.chars )
+        {
+            if( e.type < 0 )
+                break ;
+            return static_cast<eTokenType>(e.type);
+        }
+    }
+    return TOK_NULL;
+}
+Token Lex_FindReservedWord(const ::std::string& s)
+{
+    for(size_t i = 0; i < LEN(RWORDS); i++)
+    {
+        const auto& e = RWORDS[i];
+        if( s < e.chars )
+            break;
+        if( s == e.chars )
+        {
+            assert(e.type > 0);
+            return static_cast<eTokenType>(e.type);
+        }
+    }
+    return TOK_NULL;
+}
diff --git a/src/parse/lex.hpp b/src/parse/lex.hpp
index 67aa155d..55af4c56 100644
--- a/src/parse/lex.hpp
+++ b/src/parse/lex.hpp
@@ -27,6 +27,9 @@ struct Codepoint {
 extern ::std::string& operator+=(::std::string& s, const Codepoint& cp);
 extern ::std::ostream& operator<<(::std::ostream& s, const Codepoint& cp);
 
+extern Token Lex_FindOperator(const ::std::string& s);
+extern Token Lex_FindReservedWord(const ::std::string& s);
+
 typedef Codepoint   uchar;
 
 class Lexer:
diff --git a/src/parse/token.cpp b/src/parse/token.cpp
index d626659c..768a96bc 100644
--- a/src/parse/token.cpp
+++ b/src/parse/token.cpp
@@ -331,7 +331,7 @@ struct EscapedString {
     case TOK_COMMA:     return ",";
     case TOK_SEMICOLON: return ";";
     case TOK_COLON:     return ":";
-    case TOK_DOUBLE_COLON:  return ":";
+    case TOK_DOUBLE_COLON:  return "::";
     case TOK_STAR:  return "*";
     case TOK_AMP:   return "&";
     case TOK_PIPE:  return "|";