HIR Serialise - Replace boost with raw zlib

4 files changed, 230 insertions, 33 deletions

diff --git a/Makefile b/Makefile
index c5edb114..ce70235b 100644
--- a/Makefile
+++ b/Makefile
@@ -8,15 +8,14 @@
 # - Attempts to compile rust's libstd
 #
 # DEPENDENCIES
-# - boost_iostreams (with zlib)
-#   TODO: Replace this dependency with raw zlib.
+# - zlib
 
 EXESUF ?=
 CXX ?= g++
 V ?= @
 
 LINKFLAGS := -g
-LIBS := -lboost_iostreams
+LIBS := -lz
 CXXFLAGS := -g -Wall
 # - Only turn on -Werror when running as `tpg` (i.e. me)
 ifeq ($(shell whoami),tpg)
diff --git a/src/hir/deserialise.cpp b/src/hir/deserialise.cpp
index 5230703e..bb0449cc 100644
--- a/src/hir/deserialise.cpp
+++ b/src/hir/deserialise.cpp
@@ -872,6 +872,7 @@ namespace {
                 deserialise_mir_lvalue()
                 });
         default:
+            ::std::cerr << "Bad tag for a MIR Statement" << ::std::endl;
             throw "";
         }
     }
diff --git a/src/hir/serialise_lowlevel.cpp b/src/hir/serialise_lowlevel.cpp
index a7c1a0c7..706adaee 100644
--- a/src/hir/serialise_lowlevel.cpp
+++ b/src/hir/serialise_lowlevel.cpp
@@ -6,27 +6,157 @@
  * - HIR (De)Serialisation low-level "protocol"
  */
 #include "serialise_lowlevel.hpp"
-#include <boost/iostreams/filter/zlib.hpp>
+#include <zlib.h>
+#include <fstream>
+#include <string.h>   // memcpy
+#include <common.hpp>
 
-::HIR::serialise::Writer::Writer(const ::std::string& filename):
-    m_backing( filename )
+namespace HIR {
+namespace serialise {
+
+class WriterInner
+{
+    ::std::ofstream m_backing;
+    z_stream    m_zstream;
+    ::std::vector<unsigned char> m_buffer;
+    
+    unsigned int    m_byte_out_count = 0;
+    unsigned int    m_byte_in_count = 0;
+public:
+    WriterInner(const ::std::string& filename);
+    ~WriterInner();
+    void write(const void* buf, size_t len);
+};
+
+Writer::Writer(const ::std::string& filename):
+    m_inner( new WriterInner(filename) )
+{
+}
+Writer::~Writer()
 {
-    m_os.push( ::boost::iostreams::zlib_compressor() );
-    m_os.push( m_backing );
+    delete m_inner, m_inner = nullptr;
 }
+void Writer::write(const void* buf, size_t len)
+{
+    m_inner->write(buf, len);
+}
+
 
-void ::HIR::serialise::Writer::write(const void* buf, size_t len)
+WriterInner::WriterInner(const ::std::string& filename):
+    m_backing( filename ),
+    m_zstream(),
+    m_buffer( 16*1024 )
+    //m_buffer( 4*1024 )
 {
-    m_os.write(reinterpret_cast<const char*>(buf), len);
+    m_zstream.zalloc = Z_NULL;
+    m_zstream.zfree = Z_NULL;
+    m_zstream.opaque = Z_NULL;
+    
+    const int COMPRESSION_LEVEL = Z_BEST_COMPRESSION;
+    int ret = deflateInit(&m_zstream, COMPRESSION_LEVEL);
+    if(ret != Z_OK)
+        throw ::std::runtime_error("zlib init failure");
+
+    m_zstream.avail_out = m_buffer.size();
+    m_zstream.next_out = m_buffer.data();
+}
+WriterInner::~WriterInner()
+{
+    assert( m_zstream.avail_in == 0 );
+    
+    // Complete the compression
+    int ret;
+    do
+    {
+        ret = deflate(&m_zstream, Z_FINISH);
+        if(ret == Z_STREAM_ERROR)
+            throw ::std::runtime_error("zlib deflate stream error (cleanup)");
+        if( m_zstream.avail_out != m_buffer.size() )
+        {
+            size_t rem = m_buffer.size() - m_zstream.avail_out;
+            m_byte_out_count += rem;
+            m_backing.write( reinterpret_cast<char*>(m_buffer.data()), rem );
+            
+            m_zstream.avail_out = m_buffer.size();
+            m_zstream.next_out = m_buffer.data();
+        }
+    } while(ret == Z_OK);
+    deflateEnd(&m_zstream);
+}
+
+void WriterInner::write(const void* buf, size_t len)
+{
+    m_zstream.avail_in = len;
+    m_zstream.next_in = reinterpret_cast<unsigned char*>( const_cast<void*>(buf) );
+    
+    size_t last_avail_in = m_zstream.avail_in;
+
+    // While there's data to compress
+    while( m_zstream.avail_in > 0 )
+    {
+        assert(m_zstream.avail_out != 0);
+        
+        // Compress the data
+        int ret = deflate(&m_zstream, Z_NO_FLUSH);
+        if(ret == Z_STREAM_ERROR)
+            throw ::std::runtime_error("zlib deflate stream error");
+        
+        size_t used_this_time = last_avail_in - m_zstream.avail_in;
+        last_avail_in = m_zstream.avail_in;
+        m_byte_in_count += used_this_time;
+        
+        // If the entire input wasn't consumed, then it was likely due to a lack of output space
+        // - Flush the output buffer to the file
+        if( m_zstream.avail_in > 0 )
+        {
+            size_t bytes = m_buffer.size() - m_zstream.avail_out;
+            m_backing.write( reinterpret_cast<char*>(m_buffer.data()), bytes );
+            m_byte_out_count += bytes;
+            
+            m_zstream.avail_out = m_buffer.size();
+            m_zstream.next_out = m_buffer.data();
+        }
+    }
+
+    // Flush stream contents if the output buffer is full.
+    while( m_zstream.avail_out == 0 )
+    {
+        size_t bytes = m_buffer.size() - m_zstream.avail_out;
+        m_backing.write( reinterpret_cast<char*>(m_buffer.data()), bytes );
+        m_byte_out_count += bytes;
+        
+        m_zstream.avail_out = m_buffer.size();
+        m_zstream.next_out = m_buffer.data();
+        
+        int ret = deflate(&m_zstream, Z_NO_FLUSH);
+        if(ret == Z_STREAM_ERROR)
+            throw ::std::runtime_error("zlib deflate stream error");
+    }
 }
 
 
-::HIR::serialise::ReadBuffer::ReadBuffer(size_t cap):
+// --------------------------------------------------------------------
+class ReaderInner
+{
+    ::std::ifstream m_backing;
+    z_stream    m_zstream;
+    ::std::vector<unsigned char> m_buffer;
+
+    unsigned int    m_byte_out_count = 0;
+    unsigned int    m_byte_in_count = 0;
+public:
+    ReaderInner(const ::std::string& filename);
+    ~ReaderInner();
+    size_t read(void* buf, size_t len);
+};
+
+
+ReadBuffer::ReadBuffer(size_t cap):
     m_ofs(0)
 {
     m_backing.reserve(cap);
 }
-size_t ::HIR::serialise::ReadBuffer::read(void* dst, size_t len)
+size_t ReadBuffer::read(void* dst, size_t len)
 {
     size_t rem = m_backing.size() - m_ofs;
     if( rem >= len )
@@ -42,23 +172,26 @@ size_t ::HIR::serialise::ReadBuffer::read(void* dst, size_t len)
         return rem;
     }
 }
-void ::HIR::serialise::ReadBuffer::populate(::std::istream& is)
+void ReadBuffer::populate(ReaderInner& is)
 {
     m_backing.resize( m_backing.capacity(), 0 );
-    is.read(reinterpret_cast<char*>(m_backing.data()), m_backing.capacity());
-    m_backing.resize( is.gcount() );
+    auto len = is.read(m_backing.data(), m_backing.capacity());
+    m_backing.resize( len );
     m_ofs = 0;
 }
 
-::HIR::serialise::Reader::Reader(const ::std::string& filename):
-    m_backing( filename ),
+
+Reader::Reader(const ::std::string& filename):
+    m_inner( new ReaderInner(filename) ),
     m_buffer(1024)
 {
-    m_is.push( ::boost::iostreams::zlib_decompressor() );
-    m_is.push( m_backing );
+}
+Reader::~Reader()
+{
+    delete m_inner, m_inner = nullptr;
 }
 
-void ::HIR::serialise::Reader::read(void* buf, size_t len)
+void Reader::read(void* buf, size_t len)
 {
     auto used = m_buffer.read(buf, len);
     if( used == len ) {
@@ -69,15 +202,76 @@ void ::HIR::serialise::Reader::read(void* buf, size_t len)
     
     if( len >= m_buffer.capacity() )
     {
-        m_is.read(reinterpret_cast<char*>(buf), len);
-        if( !m_is )
-            throw "";
+        m_inner->read(buf, len);
     }
     else
     {
-        m_buffer.populate( m_is );
+        m_buffer.populate( *m_inner );
         used = m_buffer.read(buf, len);
         if( used != len )
-            throw "";
+            throw ::std::runtime_error( FMT("Reader::read - Requested " << len << " bytes from buffer, got " << used) );
     }
 }
+
+
+ReaderInner::ReaderInner(const ::std::string& filename):
+    m_backing(filename),
+    m_zstream(),
+    m_buffer(16*1024)
+{
+    m_zstream.zalloc = Z_NULL;
+    m_zstream.zfree = Z_NULL;
+    m_zstream.opaque = Z_NULL;
+    
+    int ret = inflateInit(&m_zstream);
+    if(ret != Z_OK)
+        throw ::std::runtime_error("zlib init failure");
+
+    m_zstream.avail_in = 0;
+}
+ReaderInner::~ReaderInner()
+{
+    inflateEnd(&m_zstream);
+}
+size_t ReaderInner::read(void* buf, size_t len)
+{
+    m_zstream.avail_out = len;
+    m_zstream.next_out = reinterpret_cast<unsigned char*>(buf);
+    do {
+        // Reset input buffer if empty
+        if( m_zstream.avail_in == 0 )
+        {
+            m_backing.read( reinterpret_cast<char*>(m_buffer.data()), m_buffer.size() );
+            m_zstream.avail_in = m_backing.gcount();
+            if( m_zstream.avail_in == 0 ) {
+                m_byte_out_count += len  - m_zstream.avail_out;
+                ::std::cerr << "Out of bytes, " << m_zstream.avail_out << " needed" << ::std::endl;
+                return len - m_zstream.avail_out;
+            }
+            m_zstream.next_in = const_cast<unsigned char*>(m_buffer.data());
+            
+            m_byte_in_count += m_zstream.avail_in;
+        }
+        
+        int ret = inflate(&m_zstream, Z_NO_FLUSH);
+        if(ret == Z_STREAM_ERROR)
+            throw ::std::runtime_error("zlib inflate stream error");
+        switch(ret)
+        {
+        case Z_NEED_DICT:
+            ret = Z_DATA_ERROR;
+        case Z_DATA_ERROR:
+        case Z_MEM_ERROR:
+            throw ::std::runtime_error("zlib inflate error");
+        default:
+            break;
+        }
+        
+    } while( m_zstream.avail_out > 0 );
+    m_byte_out_count += len;
+    
+    return len;
+}
+
+}   // namespace serialise
+}   // namespace HIR
diff --git a/src/hir/serialise_lowlevel.hpp b/src/hir/serialise_lowlevel.hpp
index 4571b863..b710669e 100644
--- a/src/hir/serialise_lowlevel.hpp
+++ b/src/hir/serialise_lowlevel.hpp
@@ -7,21 +7,25 @@
  */
 #pragma once
 
-#include <boost/iostreams/filtering_stream.hpp>
-#include <fstream>
 #include <vector>
+#include <string>
+#include <stddef.h>
+#include <assert.h>
 
 namespace HIR {
 namespace serialise {
 
+class WriterInner;
+class ReaderInner;
+
 class Writer
 {
-    ::std::ofstream m_backing;
-    ::boost::iostreams::filtering_ostream  m_os;
+    WriterInner*    m_inner;
 public:
     Writer(const ::std::string& path);
     Writer(const Writer&) = delete;
     Writer(Writer&&) = delete;
+    ~Writer();
     
     void write(const void* data, size_t count);
     
@@ -133,19 +137,18 @@ public:
     
     size_t capacity() const { return m_backing.capacity(); }
     size_t read(void* dst, size_t len);
-    void populate(::std::istream& is);
+    void populate(ReaderInner& is);
 };
 
 class Reader
 {
-    ::std::ifstream m_backing;
-    ::boost::iostreams::filtering_istream  m_is;
-    
+    ReaderInner*    m_inner;
     ReadBuffer  m_buffer;
 public:
     Reader(const ::std::string& path);
     Reader(const Writer&) = delete;
     Reader(Writer&&) = delete;
+    ~Reader();
 
     void read(void* dst, size_t count);