diff --git a/bundle.cc b/bundle.cc index 1248038..b944d83 100644 --- a/bundle.cc +++ b/bundle.cc @@ -90,7 +90,8 @@ void Creator::write( std::string const & fileName, EncryptionKey const & key, } } -void Creator::write( std::string const & fileName, EncryptionKey const & key ) +void Creator::write( Config const & config, std::string const & fileName, + EncryptionKey const & key ) { EncryptedFile::OutputStream os( fileName.c_str(), key, Encryption::ZeroIv ); @@ -98,7 +99,8 @@ void Creator::write( std::string const & fileName, EncryptionKey const & key ) BundleFileHeader header; - const_sptr compression = Compression::CompressionMethod::selectedCompression; + const_sptr compression = + Compression::CompressionMethod::selectedCompression; header.set_compression_method( compression->getName() ); // The old code only support lzma, so we will bump up the version, if we're @@ -115,7 +117,8 @@ void Creator::write( std::string const & fileName, EncryptionKey const & key ) // Compress - sptr encoder = compression->createEncoder(); + sptr encoder = compression->createEncoder( + config ); encoder->setInput( payload.data(), payload.size() ); @@ -135,7 +138,7 @@ void Creator::write( std::string const & fileName, EncryptionKey const & key ) } // Perform the compression - if ( encoder->process(true) ) + if ( encoder->process( true ) ) { if ( encoder->getAvailableOutput() ) os.BackUp( encoder->getAvailableOutput() ); @@ -190,7 +193,8 @@ Reader::Reader( string const & fileName, EncryptionKey const & key, bool prohibi decoder->setInput( data, size ); } - if ( decoder->process(false) ) { + if ( decoder->process( false ) ) + { if ( decoder->getAvailableInput() ) is.BackUp( decoder->getAvailableInput() ); break; diff --git a/bundle.hh b/bundle.hh index bac8ffa..c95cc2e 100644 --- a/bundle.hh +++ b/bundle.hh @@ -17,6 +17,7 @@ #include "static_assert.hh" #include "zbackup.pb.h" #include "encrypted_file.hh" +#include "config.hh" namespace Bundle { @@ -101,7 +102,7 @@ public: /// Compresses and writes the bundle to the given file. The operation is /// time-consuming - calling this function from a worker thread could be /// warranted - void write( string const & fileName, EncryptionKey const & ); + void write( Config const &, string const & fileName, EncryptionKey const & ); void write( string const & fileName, EncryptionKey const &, Bundle::Reader & reader ); diff --git a/chunk_storage.cc b/chunk_storage.cc index bc92990..5b7a99e 100644 --- a/chunk_storage.cc +++ b/chunk_storage.cc @@ -130,7 +130,8 @@ void Writer::finishCurrentBundle() while ( runningCompressors >= maxCompressorsToRun ) runningCompressorsCondition.wait( runningCompressorsMutex ); - Compressor * compressor = new Compressor( *this, currentBundle, + Compressor * compressor = new Compressor( config, + *this, currentBundle, file->getFileName() ); currentBundle.reset(); @@ -159,10 +160,11 @@ Bundle::Id const & Writer::getCurrentBundleId() return currentBundleId; } -Writer::Compressor::Compressor( Writer & writer, +Writer::Compressor::Compressor( Config const & configIn, Writer & writer, sptr< Bundle::Creator > const & bundleCreator, string const & fileName ): - writer( writer ), bundleCreator( bundleCreator ), fileName( fileName ) + writer( writer ), bundleCreator( bundleCreator ), fileName( fileName ), + config( configIn ) { } @@ -170,7 +172,7 @@ void * Writer::Compressor::Compressor::threadFunction() throw() { try { - bundleCreator->write( fileName, writer.encryptionKey ); + bundleCreator->write( config, fileName, writer.encryptionKey ); } catch( std::exception & e ) { diff --git a/chunk_storage.hh b/chunk_storage.hh index ae26a83..742c2a6 100644 --- a/chunk_storage.hh +++ b/chunk_storage.hh @@ -68,8 +68,9 @@ private: Writer & writer; sptr< Bundle::Creator > bundleCreator; string fileName; + Config const & config; public: - Compressor( Writer &, sptr< Bundle::Creator > const &, + Compressor( Config const &, Writer &, sptr< Bundle::Creator > const &, string const & fileName ); protected: virtual void * threadFunction() throw(); diff --git a/compression.cc b/compression.cc index 151a240..750079f 100644 --- a/compression.cc +++ b/compression.cc @@ -8,11 +8,17 @@ namespace Compression { -EnDecoder::EnDecoder() { } -EnDecoder::~EnDecoder() { } +EnDecoder::EnDecoder() +{ +} -CompressionMethod::~CompressionMethod() { } +EnDecoder::~EnDecoder() +{ +} +CompressionMethod::~CompressionMethod() +{ +} // LZMA @@ -72,8 +78,17 @@ class LZMAEncoder : public LZMAEnDecoder public: LZMAEncoder() { - uint32_t preset = 6; // TODO: make this customizable, although 6 seems to be - // the best option + uint32_t preset = 6; + lzma_ret ret = lzma_easy_encoder( &strm, preset, LZMA_CHECK_CRC64 ); + CHECK( ret == LZMA_OK, "lzma_easy_encoder error: %d", (int) ret ); + } + + LZMAEncoder( Config const & config ) + { + uint32_t compressionLevel = config.GET_STORABLE( lzma, compression_level ); + uint32_t preset = ( compressionLevel > 9 ) ? + ( compressionLevel - 10 ) | LZMA_PRESET_EXTREME : + compressionLevel; lzma_ret ret = lzma_easy_encoder( &strm, preset, LZMA_CHECK_CRC64 ); CHECK( ret == LZMA_OK, "lzma_easy_encoder error: %d", (int) ret ); } @@ -92,6 +107,11 @@ public: class LZMACompression : public CompressionMethod { public: + sptr createEncoder( Config const & config ) const + { + return new LZMAEncoder( config ); + } + sptr createEncoder() const { return new LZMAEncoder(); @@ -105,7 +125,6 @@ public: std::string getName() const { return "lzma"; } }; - // LZO // liblzo implements a lot of algorithms "for unlimited backward compatibility" @@ -232,7 +251,7 @@ public: { // data has been encoded or decoded, remaining output is in accDataOut // -> copy to output - if (availOut > 0 && accDataOut.size() - posInAccDataOut > 0) + if ( availOut > 0 && accDataOut.size() - posInAccDataOut > 0 ) { size_t sz = availOut; if ( sz > accDataOut.size() - posInAccDataOut ) @@ -273,7 +292,7 @@ private: // we use our own buffer size_t bufferSize = suggestOutputSize( dataIn, availIn ); do { - accDataOut.resize(bufferSize); + accDataOut.resize( bufferSize ); size_t outputSize; //TODO doc says we mustn't modify the pointer returned by data()... @@ -312,7 +331,6 @@ protected: virtual bool doProcessNoSize( const char* dataIn, size_t availIn, char* dataOut, size_t availOut, size_t& outputSize ) =0; - bool shouldTryWith( const char* dataIn, size_t availIn, size_t availOut ) { return suggestOutputSize( dataIn, availIn ) <= availOut; @@ -388,7 +406,6 @@ protected: virtual bool doProcessNoSize( const char* dataIn, size_t availIn, char* dataOut, size_t availOut, size_t& outputSize ) =0; - bool shouldTryWith( const char*, size_t, size_t availOut ) { // If the compression doesn't use any spaces... @@ -443,7 +460,6 @@ protected: } }; - #ifdef HAVE_LIBLZO #include @@ -472,9 +488,9 @@ class LZO1X_1_Compression; class LZO1X_1_Encoder : public NoStreamAndUnknownSizeEncoder { const LZO1X_1_Compression* compression; - static size_t calcMaxCompressedSize(size_t availIn); + static size_t calcMaxCompressedSize( size_t availIn ); public: - LZO1X_1_Encoder(const LZO1X_1_Compression* compression) + LZO1X_1_Encoder( const LZO1X_1_Compression* compression ) { this->compression = compression; } @@ -499,13 +515,19 @@ class LZO1X_1_Compression : public CompressionMethod } } public: - sptr createEncoder() const + sptr< EnDecoder > createEncoder( Config const & config ) const { init(); return new LZO1X_1_Encoder(this); } - sptr createDecoder() const + sptr< EnDecoder > createEncoder() const + { + init(); + return new LZO1X_1_Encoder(this); + } + + sptr< EnDecoder > createDecoder() const { init(); return new LZO1X_1_Decoder(); @@ -513,7 +535,6 @@ public: std::string getName() const { return "lzo1x_1"; } - lzo_voidp getWorkmem( size_t size ) const { return new char[size]; @@ -531,7 +552,6 @@ public: bool LZO1X_1_Compression::initialized = false; - size_t LZO1X_1_Encoder::calcMaxCompressedSize( size_t availIn ) { // It seems that lzo1x_1_compress does NOT check whether the buffer is big enough. @@ -596,12 +616,12 @@ const_sptr< CompressionMethod > const CompressionMethod::compressions[] = { }; const_sptr< CompressionMethod > CompressionMethod::selectedCompression = - compressions[0]; + compressions[ 0 ]; const_sptr< CompressionMethod > CompressionMethod::findCompression( const std::string& name, bool optional ) { - for ( const const_sptr* c = compressions+0; *c; ++c ) + for ( const const_sptr* c = compressions + 0; *c; ++c ) { if ( (*c)->getName() == name ) { @@ -616,9 +636,15 @@ const_sptr< CompressionMethod > CompressionMethod::findCompression( } // iterator over compressions +CompressionMethod::iterator::iterator( const const_sptr< CompressionMethod > * ptr ): + ptr( ptr ) +{ +} -CompressionMethod::iterator::iterator( const const_sptr* ptr ) : ptr( ptr) { } -CompressionMethod::iterator::iterator( const iterator& it ) : ptr(it.ptr) { } +CompressionMethod::iterator::iterator( const iterator & it ): + ptr( it.ptr ) +{ +} CompressionMethod::iterator& CompressionMethod::iterator::operator =( const iterator& it ) { diff --git a/compression.hh b/compression.hh index c7c17f3..13c9d3b 100644 --- a/compression.hh +++ b/compression.hh @@ -7,6 +7,7 @@ #include "sptr.hh" #include "ex.hh" #include "nocopy.hh" +#include "config.hh" namespace Compression { @@ -48,6 +49,7 @@ public: // This name is saved in the file header of the compressed file. virtual std::string getName() const = 0; + virtual sptr< EnDecoder > createEncoder( Config const & ) const = 0; virtual sptr< EnDecoder > createEncoder() const = 0; virtual sptr< EnDecoder > createDecoder() const = 0; diff --git a/config.cc b/config.cc index ef7ecdb..984f79d 100644 --- a/config.cc +++ b/config.cc @@ -22,6 +22,9 @@ return true; \ } +// Some configurables could be just a switch +// So we introducing a macros that would indicate +// that this configurable is not a switch #define REQUIRE_VALUE \ { \ if ( !hasValue && !validate ) \ @@ -52,7 +55,9 @@ static struct Config::oChunk_max_size, Config::Storable, "Maximum chunk size used when storing chunks\n" - "Affects deduplication ratio directly" + "Affects deduplication ratio directly\n" + "Default is %s", + Utils::numberToString( defaultConfig.GET_STORABLE( chunk, max_size ) ) }, { "bundle.max_payload_size", @@ -61,13 +66,26 @@ static struct "Maximum number of bytes a bundle can hold. Only real chunk bytes are\n" "counted, not metadata. Any bundle should be able to contain at least\n" "one arbitrary single chunk, so this should not be smaller than\n" - "chunk.max_size" + "chunk.max_size\n" + "Default is %s", + Utils::numberToString( defaultConfig.GET_STORABLE( bundle, max_payload_size ) ) }, { "bundle.compression_method", Config::oBundle_compression_method, Config::Storable, - "Compression method for new bundles" + "Compression method for new bundles\n" + "Default is %s", + defaultConfig.GET_STORABLE( bundle, compression_method ) + }, + { + "lzma.compression_level", + Config::oLZMA_compression_level, + Config::Storable, + "Compression level for new LZMA-compressed files\n" + "Valid values: 0-19 (values over 9 enables extreme mode)\n" + "Default is %s", + Utils::numberToString( defaultConfig.GET_STORABLE( lzma, compression_level ) ) }, // Shortcuts for storable options @@ -75,7 +93,9 @@ static struct "compression", Config::oBundle_compression_method, Config::Storable, - "Shortcut for bundle.compression_method" + "Shortcut for bundle.compression_method\n" + "Default is %s", + defaultConfig.GET_STORABLE( bundle, compression_method ) }, // Runtime options @@ -239,6 +259,25 @@ bool Config::parseOrValidate( const char * option, const OptionType type, /* NOTREACHED */ break; + case oLZMA_compression_level: + REQUIRE_VALUE; + + if ( PARSE_OR_VALIDATE( + sscanf( optionValue, "%zu %n", &uint32Value, &n ) != 1 || + optionValue[ n ] || uint32Value > 19, + GET_STORABLE( lzma, compression_level ) > 19 ) + ) + return false; + + SKIP_ON_VALIDATION; + SET_STORABLE( lzma, compression_level, uint32Value ); + dPrintf( "storable[lzma][compression_level] = %zu\n", + GET_STORABLE( lzma, compression_level ) ); + + return true; + /* NOTREACHED */ + break; + case oBundle_compression_method: REQUIRE_VALUE; @@ -504,6 +543,7 @@ void Config::reset_storable() SET_STORABLE( chunk, max_size, GET_STORABLE( chunk, max_size ) ); SET_STORABLE( bundle, max_payload_size, GET_STORABLE( bundle, max_payload_size ) ); SET_STORABLE( bundle, compression_method, GET_STORABLE( bundle, compression_method ) ); + SET_STORABLE( lzma, compression_level, GET_STORABLE( lzma, compression_level ) ); } void Config::show() @@ -515,27 +555,3 @@ void Config::show( const ConfigInfo & config ) { printf( "%s", toString( config ).c_str() ); } - -bool Config::editInteractively( ZBackupBase * zbb ) -{ - string configData( toString( *zbb->config.storable ) ); - - if ( !zbb->spawnEditor( configData, &validateProto ) ) - return false; - - ConfigInfo newConfig; - parseProto( configData, &newConfig ); - if ( toString( *zbb->config.storable ) == toString( newConfig ) ) - { - verbosePrintf( "No changes made to config\n" ); - return false; - } - - verbosePrintf( "Updating configuration...\n" ); - zbb->config.storable->MergeFrom( newConfig ); - verbosePrintf( -"Configuration successfully updated!\n" -"Updated configuration:\n%s", toString( *zbb->config.storable ).c_str() ); - - return true; -} diff --git a/config.hh b/config.hh index 14682b0..95cd213 100644 --- a/config.hh +++ b/config.hh @@ -21,8 +21,6 @@ using std::string; using std::bitset; -class ZBackupBase; - class Config { public: @@ -55,6 +53,7 @@ public: oChunk_max_size, oBundle_max_payload_size, oBundle_compression_method, + oLZMA_compression_level, oRuntime_threads, oRuntime_cacheSize, @@ -72,10 +71,6 @@ public: static string toString( google::protobuf::Message const & ); - // Edit current configuration - // returns true if configuration is changed - static bool editInteractively( ZBackupBase * ); - // Print configuration to screen static void show( const ConfigInfo & ); void show(); @@ -96,6 +91,4 @@ private: bool want_cleanup; }; -#include "zbackup_base.hh" - #endif diff --git a/zbackup.cc b/zbackup.cc index 1afec24..c990126 100644 --- a/zbackup.cc +++ b/zbackup.cc @@ -649,7 +649,7 @@ invalid_option: { ZBackupBase zbb( ZBackupBase::deriveStorageDirFromBackupsFile( args[ fieldStorage ], true ), passwords[ 0 ], true ); - if ( Config::editInteractively( &zbb ) ) + if ( zbb.editConfigInteractively() ) zbb.saveExtendedStorageInfo(); } else diff --git a/zbackup.proto b/zbackup.proto index 713f9de..2bfc391 100644 --- a/zbackup.proto +++ b/zbackup.proto @@ -42,6 +42,12 @@ message StorageInfo optional string default_compression_method = 4 [default = "lzma", deprecated = true]; } +message LZMAConfigInfo +{ + // Compression level for new LZMA-compressed files + optional uint32 compression_level = 1 [default = 6]; +} + message ChunkConfigInfo { // Maximum chunk size used when storing chunks @@ -64,6 +70,7 @@ message ConfigInfo { required ChunkConfigInfo chunk = 1; required BundleConfigInfo bundle = 2; + required LZMAConfigInfo lzma = 3; } message ExtendedStorageInfo diff --git a/zbackup_base.cc b/zbackup_base.cc index 839ef13..c1a5614 100644 --- a/zbackup_base.cc +++ b/zbackup_base.cc @@ -116,7 +116,7 @@ ZBackupBase::ZBackupBase( string const & storageDir, string const & password, storageDir.c_str() ); } -// Update all internal variables after configuration change +// Update all internal variables according to real configuration // Dunno why someone need to store duplicate information // in deduplication utility void ZBackupBase::propagateUpdate() @@ -419,3 +419,28 @@ end: return isChanged; } + +bool ZBackupBase::editConfigInteractively() +{ + string configData( Config::toString( *config.storable ) ); + + if ( !spawnEditor( configData, &Config::validateProto ) ) + return false; + + ConfigInfo newConfig; + Config::parseProto( configData, &newConfig ); + if ( Config::toString( *config.storable ) == + Config::toString( newConfig ) ) + { + verbosePrintf( "No changes made to config\n" ); + return false; + } + + verbosePrintf( "Updating configuration...\n" ); + config.storable->MergeFrom( newConfig ); + verbosePrintf( +"Configuration successfully updated!\n" +"Updated configuration:\n%s", Config::toString( *config.storable ).c_str() ); + + return true; +} diff --git a/zbackup_base.hh b/zbackup_base.hh index 24dc047..79c645d 100644 --- a/zbackup_base.hh +++ b/zbackup_base.hh @@ -67,6 +67,10 @@ public: bool spawnEditor( std::string & data, bool( * validator ) ( string const &, string const & ) ); + // Edit current configuration + // returns true if configuration is changed + bool editConfigInteractively(); + StorageInfo storageInfo; EncryptionKey encryptionkey; ExtendedStorageInfo extendedStorageInfo;