HIR Serialise - Explicitly serialise tokens (instead of using ancient code)

4 files changed, 57 insertions, 18 deletions

diff --git a/src/ast/path.hpp b/src/ast/path.hpp
index 0470084b..0cb6fcc8 100644
--- a/src/ast/path.hpp
+++ b/src/ast/path.hpp
@@ -9,7 +9,6 @@
 #include <vector>
 #include <initializer_list>
 #include <cassert>
-#include <serialise.hpp>
 #include <tagged_union.hpp>
 #include <string>
 #include "../include/span.hpp"
diff --git a/src/hir/deserialise.cpp b/src/hir/deserialise.cpp
index 4d47016f..2b9b3512 100644
--- a/src/hir/deserialise.cpp
+++ b/src/hir/deserialise.cpp
@@ -14,7 +14,7 @@
 #include "serialise_lowlevel.hpp"
 #include <typeinfo>
 
-namespace {
+//namespace {
 
     template<typename T>
     struct D
@@ -311,15 +311,29 @@ namespace {
         }
 
         ::Token deserialise_token() {
-            ::Token tok;
-            // HACK: Hand off to old serialiser code
-            auto s = m_in.read_string();
-            ::std::stringstream tmp(s);
+            auto ty = static_cast<enum eTokenType>( m_in.read_tag() );
+            auto d = deserialise_tokendata();
+            return ::Token(ty, ::std::move(d), {});
+        }
+        ::Token::Data deserialise_tokendata() {
+            auto tag = static_cast< ::Token::Data::Tag>( m_in.read_tag() );
+            switch(tag)
             {
-                Deserialiser_TextTree ser(tmp);
-                tok.deserialise( ser );
+            case ::Token::Data::TAG_None:
+                return ::Token::Data::make_None({});
+            case ::Token::Data::TAG_String:
+                return ::Token::Data::make_String( m_in.read_string() );
+            case ::Token::Data::TAG_Integer: {
+                auto dty = static_cast<eCoreType>(m_in.read_tag());
+                return ::Token::Data::make_Integer({ dty, m_in.read_u64c() });
+                }
+            case ::Token::Data::TAG_Float: {
+                auto dty = static_cast<eCoreType>(m_in.read_tag());
+                return ::Token::Data::make_Float({ dty, m_in.read_double() });
+                }
+            default:
+                throw ::std::runtime_error(FMT("Invalid Token data tag - " << tag));
             }
-            return tok;
         }
 
         ::HIR::Literal deserialise_literal();
@@ -1234,7 +1248,7 @@ namespace {
 
         return rv;
     }
-}
+//}
 
 ::HIR::CratePtr HIR_Deserialise(const ::std::string& filename, const ::std::string& loaded_name)
 {
diff --git a/src/hir/serialise.cpp b/src/hir/serialise.cpp
index b244764b..00e7aa28 100644
--- a/src/hir/serialise.cpp
+++ b/src/hir/serialise.cpp
@@ -12,7 +12,7 @@
 #include <mir/mir.hpp>
 #include "serialise_lowlevel.hpp"
 
-namespace {
+//namespace {
     class HirSerialiser
     {
         ::HIR::serialise::Writer&   m_out;
@@ -446,14 +446,31 @@ namespace {
             )
         }
         void serialise(const ::Token& tok) {
-            // HACK: Hand off to old serialiser code
-            ::std::stringstream tmp;
+            m_out.write_tag(tok.m_type);
+            serialise(tok.m_data);
+            // TODO: Position information.
+        }
+        void serialise(const ::Token::Data& td) {
+            m_out.write_tag(td.tag());
+            switch(td.tag())
             {
-                Serialiser_TextTree ser(tmp);
-                tok.serialise( ser );
+            case ::Token::Data::TAGDEAD:    throw "";
+            TU_ARM(td, None, _e) {
+                } break;
+            TU_ARM(td, String, e) {
+                m_out.write_string(e);
+                } break;
+            TU_ARM(td, Integer, e) {
+                m_out.write_tag(e.m_datatype);
+                m_out.write_u64c(e.m_intval);
+                } break;
+            TU_ARM(td, Float, e) {
+                m_out.write_tag(e.m_datatype);
+                m_out.write_double(e.m_floatval);
+                } break;
+            TU_ARM(td, Fragment, e)
+                assert(!"Serialising interpolated macro fragment");
             }
-
-            m_out.write_string(tmp.str());
         }
 
         void serialise(const ::HIR::Literal& lit)
@@ -1011,7 +1028,7 @@ namespace {
             serialise_type(at.m_default);
         }
     };
-}
+//}
 
 void HIR_Serialise(const ::std::string& filename, const ::HIR::Crate& crate)
 {
diff --git a/src/parse/token.hpp b/src/parse/token.hpp
index 71b543fc..2da64bca 100644
--- a/src/parse/token.hpp
+++ b/src/parse/token.hpp
@@ -59,6 +59,9 @@ class InterpolatedFragment;
 class Token:
     public Serialisable
 {
+    friend class HirSerialiser;
+    friend class HirDeserialiser;
+
     TAGGED_UNION(Data, None,
     (None, struct {}),
     (String, ::std::string),
@@ -77,6 +80,12 @@ class Token:
     Data    m_data;
     Position    m_pos;
 
+    Token(enum eTokenType t, Data d, Position p):
+        m_type(t),
+        m_data( ::std::move(d) ),
+        m_pos( ::std::move(p) )
+    {
+    }
 public:
     virtual ~Token();
     Token();