diff --git a/backup_collector.cc b/backup_collector.cc index f1d358d..b276f2c 100644 --- a/backup_collector.cc +++ b/backup_collector.cc @@ -125,7 +125,7 @@ public: ZCollector::ZCollector( string const & storageDir, string const & password, Config & configIn ): ZBackupBase( storageDir, password, configIn ), - chunkStorageReader( storageInfo, encryptionkey, chunkIndex, getBundlesPath(), + chunkStorageReader( config, encryptionkey, chunkIndex, getBundlesPath(), config.runtime.cacheSize ) { } @@ -134,8 +134,8 @@ void ZCollector::gc() { ChunkIndex chunkReindex( encryptionkey, tmpMgr, getIndexPath(), true ); - ChunkStorage::Writer chunkStorageWriter( storageInfo, encryptionkey, tmpMgr, chunkReindex, - getBundlesPath(), getIndexPath(), config.runtime.threads ); + ChunkStorage::Writer chunkStorageWriter( config, encryptionkey, tmpMgr, + chunkReindex, getBundlesPath(), getIndexPath(), config.runtime.threads ); string fileName; string backupsPath = getBackupsPath(); diff --git a/backup_creator.cc b/backup_creator.cc index bdaace5..a4857fb 100644 --- a/backup_creator.cc +++ b/backup_creator.cc @@ -15,10 +15,10 @@ namespace { unsigned const MinChunkSize = 256; } -BackupCreator::BackupCreator( StorageInfo const & info, +BackupCreator::BackupCreator( Config const & config, ChunkIndex & chunkIndex, ChunkStorage::Writer & chunkStorageWriter ): - chunkMaxSize( info.chunk_max_size() ), + chunkMaxSize( config.GET_STORABLE( chunk, max_size ) ), chunkIndex( chunkIndex ), chunkStorageWriter( chunkStorageWriter ), ringBufferFill( 0 ), chunkToSaveFill( 0 ), diff --git a/backup_creator.hh b/backup_creator.hh index dc97a5b..44d2558 100644 --- a/backup_creator.hh +++ b/backup_creator.hh @@ -17,6 +17,7 @@ #include "rolling_hash.hh" #include "sptr.hh" #include "zbackup.pb.h" +#include "config.hh" using std::vector; using std::string; @@ -69,7 +70,7 @@ class BackupCreator: ChunkIndex::ChunkInfoInterface, NoCopy virtual ChunkId const & getChunkId(); public: - BackupCreator( StorageInfo const &, ChunkIndex &, ChunkStorage::Writer & ); + BackupCreator( Config const &, ChunkIndex &, ChunkStorage::Writer & ); /// The data is fed the following way: the user fills getInputBuffer() with /// up to getInputBufferSize() bytes, then calls handleMoreData() with the diff --git a/chunk_storage.cc b/chunk_storage.cc index d5e2652..bc92990 100644 --- a/chunk_storage.cc +++ b/chunk_storage.cc @@ -10,11 +10,11 @@ namespace ChunkStorage { -Writer::Writer( StorageInfo const & storageInfo, +Writer::Writer( Config const & configIn, EncryptionKey const & encryptionKey, TmpMgr & tmpMgr, ChunkIndex & index, string const & bundlesDir, string const & indexDir, size_t maxCompressorsToRun ): - storageInfo( storageInfo ), encryptionKey( encryptionKey ), + config( configIn ), encryptionKey( encryptionKey ), tmpMgr( tmpMgr ), index( index ), bundlesDir( bundlesDir ), indexDir( indexDir ), hasCurrentBundleId( false ), maxCompressorsToRun( maxCompressorsToRun ), runningCompressors( 0 ) @@ -34,7 +34,7 @@ bool Writer::add( ChunkId const & id, void const * data, size_t size ) { // Added to the index? Emit to the bundle then if ( getCurrentBundle().getPayloadSize() + size > - storageInfo.bundle_max_payload_size() ) + config.GET_STORABLE( bundle, max_payload_size ) ) finishCurrentBundle(); getCurrentBundle().addChunk( id.toBlob(), data, size ); @@ -192,17 +192,18 @@ void * Writer::Compressor::Compressor::threadFunction() throw() return NULL; } -Reader::Reader( StorageInfo const & storageInfo, +Reader::Reader( Config const & configIn, EncryptionKey const & encryptionKey, ChunkIndex & index, string const & bundlesDir, size_t maxCacheSizeBytes ): - storageInfo( storageInfo ), encryptionKey( encryptionKey ), + config( configIn ), encryptionKey( encryptionKey ), index( index ), bundlesDir( bundlesDir ), // We need to have at least one cached reader, otherwise we would have to // unpack a bundle each time a chunk is read, even for consecutive chunks // in the same bundle - cachedReaders( maxCacheSizeBytes < storageInfo.bundle_max_payload_size() ? - 1 : maxCacheSizeBytes / storageInfo.bundle_max_payload_size() ) + cachedReaders( + maxCacheSizeBytes < config.GET_STORABLE( bundle, max_payload_size ) ? + 1 : maxCacheSizeBytes / config.GET_STORABLE( bundle, max_payload_size ) ) { verbosePrintf( "Using up to %zu MB of RAM as cache\n", maxCacheSizeBytes / 1048576 ); diff --git a/chunk_storage.hh b/chunk_storage.hh index 6df8ae6..ae26a83 100644 --- a/chunk_storage.hh +++ b/chunk_storage.hh @@ -23,6 +23,7 @@ #include "sptr.hh" #include "tmp_mgr.hh" #include "zbackup.pb.h" +#include "config.hh" namespace ChunkStorage { @@ -40,7 +41,7 @@ public: /// All new bundles and index files are created as temp files. Call commit() /// to move them to their permanent locations. commit() is never called /// automatically! - Writer( StorageInfo const &, EncryptionKey const &, + Writer( Config const &, EncryptionKey const &, TmpMgr &, ChunkIndex & index, string const & bundlesDir, string const & indexDir, size_t maxCompressorsToRun ); @@ -90,7 +91,7 @@ private: /// Wait for all compressors to finish void waitForAllCompressorsToFinish(); - StorageInfo const & storageInfo; + Config const & config; EncryptionKey const & encryptionKey; TmpMgr & tmpMgr; ChunkIndex & index; @@ -119,7 +120,7 @@ class Reader: NoCopy public: DEF_EX_STR( exNoSuchChunk, "no such chunk found:", Ex ) - Reader( StorageInfo const &, EncryptionKey const &, ChunkIndex & index, + Reader( Config const &, EncryptionKey const &, ChunkIndex & index, string const & bundlesDir, size_t maxCacheSizeBytes ); /// Loads the given chunk from the store into the given buffer. May throw file @@ -131,7 +132,7 @@ public: Bundle::Reader & getReaderFor( Bundle::Id const & ); private: - StorageInfo const & storageInfo; + Config const & config; EncryptionKey const & encryptionKey; ChunkIndex & index; string bundlesDir; diff --git a/config.cc b/config.cc index e722c4e..054c17f 100644 --- a/config.cc +++ b/config.cc @@ -100,9 +100,17 @@ static struct } +Config::~Config() +{ + // prevent memleak + if ( default_instance ) + delete storable; +} + Config::Config() { ConfigInfo * configInfo = new ConfigInfo; + default_instance = true; storable = configInfo; dPrintf( "Config is instantiated and initialized with default values\n" ); } @@ -110,6 +118,7 @@ Config::Config() Config::Config( ConfigInfo * configInfo ) { storable = configInfo; + default_instance = false; dPrintf( "Config is instantiated and initialized with supplied ConfigInfo\n" ); } @@ -118,6 +127,7 @@ Config::Config( const Config & configIn, ConfigInfo * configInfo ) configInfo->MergeFrom( *configIn.storable ); *this = configIn; storable = configInfo; + default_instance = false; dPrintf( "Config is instantiated and initialized with supplied values\n" ); } diff --git a/config.hh b/config.hh index 3dc33a0..b7c9ffb 100644 --- a/config.hh +++ b/config.hh @@ -87,9 +87,12 @@ public: Config( const Config &, ConfigInfo * ); Config( ConfigInfo * ); Config(); + ~Config(); RuntimeConfig runtime; ConfigInfo * storable; +private: + bool default_instance; }; #include "zbackup_base.hh" diff --git a/zbackup.cc b/zbackup.cc index c6b86f9..2f93a60 100644 --- a/zbackup.cc +++ b/zbackup.cc @@ -18,7 +18,7 @@ using std::iterator; ZBackup::ZBackup( string const & storageDir, string const & password, Config & configIn ): ZBackupBase( storageDir, password, configIn ), - chunkStorageWriter( storageInfo, encryptionkey, tmpMgr, chunkIndex, + chunkStorageWriter( config, encryptionkey, tmpMgr, chunkIndex, getBundlesPath(), getIndexPath(), config.runtime.threads ) { } @@ -32,7 +32,7 @@ void ZBackup::backupFromStdin( string const & outputFileName ) throw exWontOverwrite( outputFileName ); Sha256 sha256; - BackupCreator backupCreator( storageInfo, chunkIndex, chunkStorageWriter ); + BackupCreator backupCreator( config, chunkIndex, chunkStorageWriter ); time_t startTime = time( 0 ); uint64_t totalDataSize = 0; @@ -77,7 +77,7 @@ void ZBackup::backupFromStdin( string const & outputFileName ) // Shrink the serialized data iteratively until it wouldn't shrink anymore for ( ; ; ) { - BackupCreator backupCreator( storageInfo, chunkIndex, chunkStorageWriter ); + BackupCreator backupCreator( config, chunkIndex, chunkStorageWriter ); char const * ptr = serialized.data(); size_t left = serialized.size(); while( left ) @@ -124,7 +124,7 @@ void ZBackup::backupFromStdin( string const & outputFileName ) ZRestore::ZRestore( string const & storageDir, string const & password, Config & configIn ): ZBackupBase( storageDir, password, configIn ), - chunkStorageReader( storageInfo, encryptionkey, chunkIndex, getBundlesPath(), + chunkStorageReader( config, encryptionkey, chunkIndex, getBundlesPath(), config.runtime.cacheSize ) { } diff --git a/zbackup.proto b/zbackup.proto index 62df774..713f9de 100644 --- a/zbackup.proto +++ b/zbackup.proto @@ -30,12 +30,12 @@ message EncryptionKeyInfo message StorageInfo { // Maximum chunk size used when storing chunks - required uint32 chunk_max_size = 1 [deprecated = true]; + optional uint32 chunk_max_size = 1 [deprecated = true]; // Maximum number of bytes a bundle can hold. Only real chunk bytes are // counted, not metadata. Any bundle should be able to contain at least // one arbitrary single chunk, so this should not be smaller than // chunk_max_size - required uint32 bundle_max_payload_size = 2 [deprecated = true]; + optional uint32 bundle_max_payload_size = 2 [deprecated = true]; // If present, used for encryption/decryption of all data optional EncryptionKeyInfo encryption_key = 3; // Default compression for new bundles diff --git a/zbackup_base.cc b/zbackup_base.cc index fe7c5a2..8aed669 100644 --- a/zbackup_base.cc +++ b/zbackup_base.cc @@ -154,18 +154,13 @@ void ZBackupBase::initStorage( string const & storageDir, StorageInfo storageInfo; ExtendedStorageInfo extendedStorageInfo; - // TODO: Make a proper setup of initial values - /*storageInfo.set_chunk_max_size( 65536 ); - storageInfo.set_bundle_max_payload_size( 0x200000 ); - storageInfo.set_default_compression_method( - Compression::CompressionMethod::selectedCompression->getName() );*/ - - extendedStorageInfo.mutable_config()->mutable_chunk()->set_max_size( - extendedStorageInfo.config().chunk().max_size() ); - extendedStorageInfo.mutable_config()->mutable_bundle()->set_max_payload_size( - extendedStorageInfo.config().bundle().max_payload_size() ); - extendedStorageInfo.mutable_config()->mutable_bundle()->set_compression_method( - extendedStorageInfo.config().bundle().compression_method() ); + ConfigInfo * storable = extendedStorageInfo.mutable_config(); + // TODO: Use protobuf reflection in loop + // to fill messages with default values + // without explicitly defining them + SET_STORABLE( chunk, max_size, GET_STORABLE( chunk, max_size ) ); + SET_STORABLE( bundle, max_payload_size, GET_STORABLE( bundle, max_payload_size ) ); + SET_STORABLE( bundle, compression_method, GET_STORABLE( bundle, compression_method ) ); EncryptionKey encryptionkey = EncryptionKey::noKey();