Refactor compression code: extract into compression.{hh,cc} and hide compression methods behind a consistent interface

bundle.cc

@@ -1,7 +1,6 @@
 // Copyright (c) 2012-2013 Konstantin Isakov <ikm@zbackup.org>
 // Part of ZBackup. Licensed under GNU GPLv2 or later
 
-#include <lzma.h>
 #include <stdint.h>
 
 #include "bundle.hh"
@@ -11,6 +10,7 @@
 #include "encryption.hh"
 #include "hex.hh"
 #include "message.hh"
+#include "compression.hh"
 
 namespace Bundle {
 
@@ -37,21 +37,19 @@ void Creator::write( std::string const & fileName, EncryptionKey const & key )
   header.set_version( FileFormatVersion );
   Message::serialize( header, os );
 
+  const Compression* compression = Compression::default_compression;
+  info.set_compression_method( compression->getName() );
+
   Message::serialize( info, os );
   os.writeAdler32();
 
   // Compress
 
-  uint32_t preset = 6; // TODO: make this customizable, although 6 seems to be
-                       // the best option
-	lzma_stream strm = LZMA_STREAM_INIT;
-	lzma_ret ret;
+  EnDecoder* encoder = compression->getEncoder();
 
-  ret = lzma_easy_encoder( &strm, preset, LZMA_CHECK_CRC64 );
-  CHECK( ret == LZMA_OK, "lzma_easy_encoder error: %d", (int) ret );
+  encoder->setInput( payload.data(), payload.size() );
 
-  strm.next_in = ( uint8_t const * ) payload.data();
-  strm.avail_in = payload.size();
+  // deliberately break the test: ((uint8_t*)strm.next_in)[10] = 7;
 
   for ( ; ; )
   {
@@ -60,29 +58,24 @@ void Creator::write( std::string const & fileName, EncryptionKey const & key )
       int size;
       if ( !os.Next( &data, &size ) )
       {
-        lzma_end( &strm );
+        delete encoder;
         throw exBundleWriteFailed();
       }
       if ( !size )
         continue;
-      strm.next_out = ( uint8_t * ) data;
-      strm.avail_out = size;
+      encoder->setOutput( data, size );
     }
 
     // Perform the compression
-    ret = lzma_code( &strm, LZMA_FINISH );
-
-    if ( ret == LZMA_STREAM_END )
+    if ( encoder->process(true) )
     {
-      if ( strm.avail_out )
-        os.BackUp( strm.avail_out );
+      if ( encoder->getAvailableOutput() )
+        os.BackUp( encoder->getAvailableOutput() );
       break;
     }
-
-    CHECK( ret == LZMA_OK, "lzma_code error: %d", (int) ret );
   }
 
-	lzma_end( &strm );
+	delete encoder;
 
   os.writeAdler32();
 }
@@ -109,15 +102,9 @@ Reader::Reader( string const & fileName, EncryptionKey const & key )
 
   payload.resize( payloadSize );
 
-  lzma_stream strm = LZMA_STREAM_INIT;
+  EnDecoder* decoder = Compression::findCompression( info.compression_method() )->getDecoder();
 
-  lzma_ret ret;
-
-  ret = lzma_stream_decoder( &strm, UINT64_MAX, 0 );
-  CHECK( ret == LZMA_OK,"lzma_stream_decoder error: %d", (int) ret );
-
-  strm.next_out = ( uint8_t * ) &payload[ 0 ];
-  strm.avail_out = payload.size();
+  decoder->setOutput( &payload[ 0 ], payload.size() );
 
   for ( ; ; )
   {
@@ -126,35 +113,29 @@ Reader::Reader( string const & fileName, EncryptionKey const & key )
       int size;
       if ( !is.Next( &data, &size ) )
       {
-        lzma_end( &strm );
+        delete decoder;
         throw exBundleReadFailed();
       }
       if ( !size )
         continue;
-      strm.next_in = ( uint8_t const * ) data;
-      strm.avail_in = size;
+      decoder->setInput( data, size );
     }
 
-    ret = lzma_code( &strm, LZMA_RUN );
-
-    if ( ret == LZMA_STREAM_END )
-    {
-      if ( strm.avail_in )
-        is.BackUp( strm.avail_in );
+    if ( decoder->process(false) ) {
+      if ( decoder->getAvailableInput() )
+        is.BackUp( decoder->getAvailableInput() );
       break;
     }
 
-    CHECK( ret == LZMA_OK, "lzma_code error: %d", (int) ret );
-
-    if ( !strm.avail_out && strm.avail_in )
+    if ( !decoder->getAvailableOutput() && decoder->getAvailableInput() )
     {
       // Apparently we have more data than we were expecting
-      lzma_end( &strm );
+      delete decoder;
       throw exTooMuchData();
     }
   }
 
-  lzma_end( &strm );
+  delete decoder;
 
   is.checkAdler32();
 

compression.cc

@@ -0,0 +1,109 @@
+// Copyright (c) 2012-2013 Konstantin Isakov <ikm@zbackup.org>
+// Part of ZBackup. Licensed under GNU GPLv2 or later
+
+#include <lzma.h>
+
+#include "compression.hh"
+#include "check.hh"
+
+EnDecoder::EnDecoder() { }
+EnDecoder::EnDecoder(const EnDecoder&) { }
+EnDecoder::~EnDecoder() {}
+
+Compression::~Compression() {}
+
+
+// LZMA
+
+class LZMAEnDecoder : public EnDecoder {
+protected:
+  static lzma_stream init_value;
+  lzma_stream strm;
+public:
+  LZMAEnDecoder() {
+    strm = init_value;
+  }
+
+  void setInput(const void* data, size_t size) {
+    strm.next_in  = (const uint8_t *) data;
+    strm.avail_in = size;
+  }
+
+  void setOutput(void* data, size_t size) {
+    strm.next_out  = (uint8_t *) data;
+    strm.avail_out = size;
+  }
+
+  size_t getAvailableInput() {
+    return strm.avail_in;
+  }
+
+  size_t getAvailableOutput() {
+    return strm.avail_out;
+  }
+
+  bool process(bool finish) {
+    lzma_ret ret = lzma_code( &strm, ( finish ? LZMA_FINISH : LZMA_RUN ) );
+
+    CHECK( ret == LZMA_OK || ret == LZMA_STREAM_END, "lzma_code error: %d", (int) ret );
+
+    return (ret == LZMA_STREAM_END);
+  }
+};
+lzma_stream LZMAEnDecoder::init_value = LZMA_STREAM_INIT;
+
+class LZMAEncoder : public LZMAEnDecoder {
+public:
+  LZMAEncoder() {
+    uint32_t preset = 6; // TODO: make this customizable, although 6 seems to be
+                         // the best option
+    lzma_ret ret = lzma_easy_encoder( &strm, preset, LZMA_CHECK_CRC64 );
+    CHECK( ret == LZMA_OK, "lzma_easy_encoder error: %d", (int) ret );
+  }
+};
+
+class LZMADecoder : public LZMAEnDecoder {
+public:
+  LZMADecoder() {
+    lzma_ret ret = lzma_stream_decoder( &strm, UINT64_MAX, 0 );
+    CHECK( ret == LZMA_OK,"lzma_stream_decoder error: %d", (int) ret );
+  }
+};
+
+class LZMACompression : public Compression {
+public:
+  EnDecoder* getEncoder() const {
+    return new LZMAEncoder();
+  }
+
+  EnDecoder* getDecoder() const {
+    return new LZMADecoder();
+  }
+
+  std::string getName() const { return "lzma"; }
+};
+
+
+// LZOP
+
+//TODO
+
+
+// register them
+
+static const Compression* compressions[] = {
+  new LZMACompression(),
+  NULL
+};
+
+const Compression* Compression::default_compression = compressions[0];
+
+const Compression* Compression::findCompression( const std::string& name, bool optional ) {
+  for (const Compression** c = compressions+0; *c; ++c) {
+    if ( (*c)->getName() == name ) {
+      return (*c);
+    }
+  }
+  CHECK( !optional, "Couldn't find compression method '%s'", name.c_str() );
+  return NULL;
+}

compression.hh

@@ -0,0 +1,59 @@
+// Copyright (c) 2012-2013 Konstantin Isakov <ikm@zbackup.org>
+// Part of ZBackup. Licensed under GNU GPLv2 or later
+
+#ifndef COMPRESSION_HH_INCLUDED__
+#define COMPRESSION_HH_INCLUDED__
+
+#include <string>
+
+
+// used for encoding or decoding
+class EnDecoder {
+protected:
+  EnDecoder();
+  // cannot be copied
+  EnDecoder(const EnDecoder&);
+public:
+  virtual ~EnDecoder();
+
+  // encoder can read up to size bytes from data
+  virtual void setInput (const void* data, size_t size) =0;
+  // how many bytes of the last input haven't been used, yet?
+  virtual size_t getAvailableInput() =0;
+
+  // encoder can write up to size bytes to output
+  virtual void setOutput(void* data, size_t size) =0;
+  // how many bytes of free space are remaining in the output buffer
+  virtual size_t getAvailableOutput() =0;
+
+  // process some bytes
+  // finish: will you pass more data to the encoder via setOutput?
+  // NOTE You must eventually set finish to true.
+  // returns, whether all output bytes have been written
+  virtual bool process(bool finish) =0;
+
+};
+
+// compression method
+class Compression {
+public:
+  virtual ~Compression();
+
+  // returns name of compression method
+  // This name is saved in the file header of the compressed file.
+  virtual std::string getName() const =0;
+
+  virtual EnDecoder* getEncoder() const =0;
+  virtual EnDecoder* getDecoder() const =0;
+
+  // find a compression by name
+  // If optional is false, it will either return a valid Compression object or abort
+  // the program If optional is true, it will return NULL, if it cannot find the a
+  // compression with that name.
+  // Don't delete the returned object because it is a shared instance!
+  static const Compression* findCompression( const std::string& name, bool optional = false );
+
+  static const Compression* default_compression;
+};
+
+#endif

tests/bundle/bundle.pro

@@ -24,6 +24,7 @@ SOURCES += test_bundle.cc \
     ../../bundle.cc \
     ../../message.cc \
     ../../hex.cc \
+    ../../compression.cc \
     ../../zbackup.pb.cc
 
 HEADERS += \
@@ -41,4 +42,5 @@ HEADERS += \
     ../../bundle.hh \
     ../../message.hh \
     ../../hex.hh \
+    ../../compression.hh \
     ../../zbackup.pb.h

zbackup.proto

@@ -53,6 +53,14 @@ message BundleInfo
 
   // A sequence of chunk records
   repeated ChunkRecord chunk_record = 1;
+
+  // Compression method that is used for this file
+  // If the program doesn't support that field, it will try LZMA. If it is
+  // LZMA, that will work. If it isn't, liblzma will recognize this and abort.
+  //NOTE If I was mainting this code (instead of adding this without asking for
+  //     permission or support), I would probably use a string and I would
+  //     definetly use tag 2 instead of 101.
+  optional string compression_method = 101 [default = "lzma"];
 }
 
 message FileHeader