mirror of https://github.com/vitalif/zbackup
Leave deprecated StorageInfo fields alone
parent
f9fc3c0b52
commit
a5c7857408
|
@ -125,7 +125,7 @@ public:
|
||||||
ZCollector::ZCollector( string const & storageDir, string const & password,
|
ZCollector::ZCollector( string const & storageDir, string const & password,
|
||||||
Config & configIn ):
|
Config & configIn ):
|
||||||
ZBackupBase( storageDir, password, configIn ),
|
ZBackupBase( storageDir, password, configIn ),
|
||||||
chunkStorageReader( storageInfo, encryptionkey, chunkIndex, getBundlesPath(),
|
chunkStorageReader( config, encryptionkey, chunkIndex, getBundlesPath(),
|
||||||
config.runtime.cacheSize )
|
config.runtime.cacheSize )
|
||||||
{
|
{
|
||||||
}
|
}
|
||||||
|
@ -134,8 +134,8 @@ void ZCollector::gc()
|
||||||
{
|
{
|
||||||
ChunkIndex chunkReindex( encryptionkey, tmpMgr, getIndexPath(), true );
|
ChunkIndex chunkReindex( encryptionkey, tmpMgr, getIndexPath(), true );
|
||||||
|
|
||||||
ChunkStorage::Writer chunkStorageWriter( storageInfo, encryptionkey, tmpMgr, chunkReindex,
|
ChunkStorage::Writer chunkStorageWriter( config, encryptionkey, tmpMgr,
|
||||||
getBundlesPath(), getIndexPath(), config.runtime.threads );
|
chunkReindex, getBundlesPath(), getIndexPath(), config.runtime.threads );
|
||||||
|
|
||||||
string fileName;
|
string fileName;
|
||||||
string backupsPath = getBackupsPath();
|
string backupsPath = getBackupsPath();
|
||||||
|
|
|
@ -15,10 +15,10 @@ namespace {
|
||||||
unsigned const MinChunkSize = 256;
|
unsigned const MinChunkSize = 256;
|
||||||
}
|
}
|
||||||
|
|
||||||
BackupCreator::BackupCreator( StorageInfo const & info,
|
BackupCreator::BackupCreator( Config const & config,
|
||||||
ChunkIndex & chunkIndex,
|
ChunkIndex & chunkIndex,
|
||||||
ChunkStorage::Writer & chunkStorageWriter ):
|
ChunkStorage::Writer & chunkStorageWriter ):
|
||||||
chunkMaxSize( info.chunk_max_size() ),
|
chunkMaxSize( config.GET_STORABLE( chunk, max_size ) ),
|
||||||
chunkIndex( chunkIndex ), chunkStorageWriter( chunkStorageWriter ),
|
chunkIndex( chunkIndex ), chunkStorageWriter( chunkStorageWriter ),
|
||||||
ringBufferFill( 0 ),
|
ringBufferFill( 0 ),
|
||||||
chunkToSaveFill( 0 ),
|
chunkToSaveFill( 0 ),
|
||||||
|
|
|
@ -17,6 +17,7 @@
|
||||||
#include "rolling_hash.hh"
|
#include "rolling_hash.hh"
|
||||||
#include "sptr.hh"
|
#include "sptr.hh"
|
||||||
#include "zbackup.pb.h"
|
#include "zbackup.pb.h"
|
||||||
|
#include "config.hh"
|
||||||
|
|
||||||
using std::vector;
|
using std::vector;
|
||||||
using std::string;
|
using std::string;
|
||||||
|
@ -69,7 +70,7 @@ class BackupCreator: ChunkIndex::ChunkInfoInterface, NoCopy
|
||||||
virtual ChunkId const & getChunkId();
|
virtual ChunkId const & getChunkId();
|
||||||
|
|
||||||
public:
|
public:
|
||||||
BackupCreator( StorageInfo const &, ChunkIndex &, ChunkStorage::Writer & );
|
BackupCreator( Config const &, ChunkIndex &, ChunkStorage::Writer & );
|
||||||
|
|
||||||
/// The data is fed the following way: the user fills getInputBuffer() with
|
/// The data is fed the following way: the user fills getInputBuffer() with
|
||||||
/// up to getInputBufferSize() bytes, then calls handleMoreData() with the
|
/// up to getInputBufferSize() bytes, then calls handleMoreData() with the
|
||||||
|
|
|
@ -10,11 +10,11 @@
|
||||||
|
|
||||||
namespace ChunkStorage {
|
namespace ChunkStorage {
|
||||||
|
|
||||||
Writer::Writer( StorageInfo const & storageInfo,
|
Writer::Writer( Config const & configIn,
|
||||||
EncryptionKey const & encryptionKey,
|
EncryptionKey const & encryptionKey,
|
||||||
TmpMgr & tmpMgr, ChunkIndex & index, string const & bundlesDir,
|
TmpMgr & tmpMgr, ChunkIndex & index, string const & bundlesDir,
|
||||||
string const & indexDir, size_t maxCompressorsToRun ):
|
string const & indexDir, size_t maxCompressorsToRun ):
|
||||||
storageInfo( storageInfo ), encryptionKey( encryptionKey ),
|
config( configIn ), encryptionKey( encryptionKey ),
|
||||||
tmpMgr( tmpMgr ), index( index ), bundlesDir( bundlesDir ),
|
tmpMgr( tmpMgr ), index( index ), bundlesDir( bundlesDir ),
|
||||||
indexDir( indexDir ), hasCurrentBundleId( false ),
|
indexDir( indexDir ), hasCurrentBundleId( false ),
|
||||||
maxCompressorsToRun( maxCompressorsToRun ), runningCompressors( 0 )
|
maxCompressorsToRun( maxCompressorsToRun ), runningCompressors( 0 )
|
||||||
|
@ -34,7 +34,7 @@ bool Writer::add( ChunkId const & id, void const * data, size_t size )
|
||||||
{
|
{
|
||||||
// Added to the index? Emit to the bundle then
|
// Added to the index? Emit to the bundle then
|
||||||
if ( getCurrentBundle().getPayloadSize() + size >
|
if ( getCurrentBundle().getPayloadSize() + size >
|
||||||
storageInfo.bundle_max_payload_size() )
|
config.GET_STORABLE( bundle, max_payload_size ) )
|
||||||
finishCurrentBundle();
|
finishCurrentBundle();
|
||||||
|
|
||||||
getCurrentBundle().addChunk( id.toBlob(), data, size );
|
getCurrentBundle().addChunk( id.toBlob(), data, size );
|
||||||
|
@ -192,17 +192,18 @@ void * Writer::Compressor::Compressor::threadFunction() throw()
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
|
|
||||||
Reader::Reader( StorageInfo const & storageInfo,
|
Reader::Reader( Config const & configIn,
|
||||||
EncryptionKey const & encryptionKey,
|
EncryptionKey const & encryptionKey,
|
||||||
ChunkIndex & index, string const & bundlesDir,
|
ChunkIndex & index, string const & bundlesDir,
|
||||||
size_t maxCacheSizeBytes ):
|
size_t maxCacheSizeBytes ):
|
||||||
storageInfo( storageInfo ), encryptionKey( encryptionKey ),
|
config( configIn ), encryptionKey( encryptionKey ),
|
||||||
index( index ), bundlesDir( bundlesDir ),
|
index( index ), bundlesDir( bundlesDir ),
|
||||||
// We need to have at least one cached reader, otherwise we would have to
|
// We need to have at least one cached reader, otherwise we would have to
|
||||||
// unpack a bundle each time a chunk is read, even for consecutive chunks
|
// unpack a bundle each time a chunk is read, even for consecutive chunks
|
||||||
// in the same bundle
|
// in the same bundle
|
||||||
cachedReaders( maxCacheSizeBytes < storageInfo.bundle_max_payload_size() ?
|
cachedReaders(
|
||||||
1 : maxCacheSizeBytes / storageInfo.bundle_max_payload_size() )
|
maxCacheSizeBytes < config.GET_STORABLE( bundle, max_payload_size ) ?
|
||||||
|
1 : maxCacheSizeBytes / config.GET_STORABLE( bundle, max_payload_size ) )
|
||||||
{
|
{
|
||||||
verbosePrintf( "Using up to %zu MB of RAM as cache\n",
|
verbosePrintf( "Using up to %zu MB of RAM as cache\n",
|
||||||
maxCacheSizeBytes / 1048576 );
|
maxCacheSizeBytes / 1048576 );
|
||||||
|
|
|
@ -23,6 +23,7 @@
|
||||||
#include "sptr.hh"
|
#include "sptr.hh"
|
||||||
#include "tmp_mgr.hh"
|
#include "tmp_mgr.hh"
|
||||||
#include "zbackup.pb.h"
|
#include "zbackup.pb.h"
|
||||||
|
#include "config.hh"
|
||||||
|
|
||||||
namespace ChunkStorage {
|
namespace ChunkStorage {
|
||||||
|
|
||||||
|
@ -40,7 +41,7 @@ public:
|
||||||
/// All new bundles and index files are created as temp files. Call commit()
|
/// All new bundles and index files are created as temp files. Call commit()
|
||||||
/// to move them to their permanent locations. commit() is never called
|
/// to move them to their permanent locations. commit() is never called
|
||||||
/// automatically!
|
/// automatically!
|
||||||
Writer( StorageInfo const &, EncryptionKey const &,
|
Writer( Config const &, EncryptionKey const &,
|
||||||
TmpMgr &, ChunkIndex & index, string const & bundlesDir,
|
TmpMgr &, ChunkIndex & index, string const & bundlesDir,
|
||||||
string const & indexDir, size_t maxCompressorsToRun );
|
string const & indexDir, size_t maxCompressorsToRun );
|
||||||
|
|
||||||
|
@ -90,7 +91,7 @@ private:
|
||||||
/// Wait for all compressors to finish
|
/// Wait for all compressors to finish
|
||||||
void waitForAllCompressorsToFinish();
|
void waitForAllCompressorsToFinish();
|
||||||
|
|
||||||
StorageInfo const & storageInfo;
|
Config const & config;
|
||||||
EncryptionKey const & encryptionKey;
|
EncryptionKey const & encryptionKey;
|
||||||
TmpMgr & tmpMgr;
|
TmpMgr & tmpMgr;
|
||||||
ChunkIndex & index;
|
ChunkIndex & index;
|
||||||
|
@ -119,7 +120,7 @@ class Reader: NoCopy
|
||||||
public:
|
public:
|
||||||
DEF_EX_STR( exNoSuchChunk, "no such chunk found:", Ex )
|
DEF_EX_STR( exNoSuchChunk, "no such chunk found:", Ex )
|
||||||
|
|
||||||
Reader( StorageInfo const &, EncryptionKey const &, ChunkIndex & index,
|
Reader( Config const &, EncryptionKey const &, ChunkIndex & index,
|
||||||
string const & bundlesDir, size_t maxCacheSizeBytes );
|
string const & bundlesDir, size_t maxCacheSizeBytes );
|
||||||
|
|
||||||
/// Loads the given chunk from the store into the given buffer. May throw file
|
/// Loads the given chunk from the store into the given buffer. May throw file
|
||||||
|
@ -131,7 +132,7 @@ public:
|
||||||
Bundle::Reader & getReaderFor( Bundle::Id const & );
|
Bundle::Reader & getReaderFor( Bundle::Id const & );
|
||||||
|
|
||||||
private:
|
private:
|
||||||
StorageInfo const & storageInfo;
|
Config const & config;
|
||||||
EncryptionKey const & encryptionKey;
|
EncryptionKey const & encryptionKey;
|
||||||
ChunkIndex & index;
|
ChunkIndex & index;
|
||||||
string bundlesDir;
|
string bundlesDir;
|
||||||
|
|
10
config.cc
10
config.cc
|
@ -100,9 +100,17 @@ static struct
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Config::~Config()
|
||||||
|
{
|
||||||
|
// prevent memleak
|
||||||
|
if ( default_instance )
|
||||||
|
delete storable;
|
||||||
|
}
|
||||||
|
|
||||||
Config::Config()
|
Config::Config()
|
||||||
{
|
{
|
||||||
ConfigInfo * configInfo = new ConfigInfo;
|
ConfigInfo * configInfo = new ConfigInfo;
|
||||||
|
default_instance = true;
|
||||||
storable = configInfo;
|
storable = configInfo;
|
||||||
dPrintf( "Config is instantiated and initialized with default values\n" );
|
dPrintf( "Config is instantiated and initialized with default values\n" );
|
||||||
}
|
}
|
||||||
|
@ -110,6 +118,7 @@ Config::Config()
|
||||||
Config::Config( ConfigInfo * configInfo )
|
Config::Config( ConfigInfo * configInfo )
|
||||||
{
|
{
|
||||||
storable = configInfo;
|
storable = configInfo;
|
||||||
|
default_instance = false;
|
||||||
dPrintf( "Config is instantiated and initialized with supplied ConfigInfo\n" );
|
dPrintf( "Config is instantiated and initialized with supplied ConfigInfo\n" );
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -118,6 +127,7 @@ Config::Config( const Config & configIn, ConfigInfo * configInfo )
|
||||||
configInfo->MergeFrom( *configIn.storable );
|
configInfo->MergeFrom( *configIn.storable );
|
||||||
*this = configIn;
|
*this = configIn;
|
||||||
storable = configInfo;
|
storable = configInfo;
|
||||||
|
default_instance = false;
|
||||||
dPrintf( "Config is instantiated and initialized with supplied values\n" );
|
dPrintf( "Config is instantiated and initialized with supplied values\n" );
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -87,9 +87,12 @@ public:
|
||||||
Config( const Config &, ConfigInfo * );
|
Config( const Config &, ConfigInfo * );
|
||||||
Config( ConfigInfo * );
|
Config( ConfigInfo * );
|
||||||
Config();
|
Config();
|
||||||
|
~Config();
|
||||||
|
|
||||||
RuntimeConfig runtime;
|
RuntimeConfig runtime;
|
||||||
ConfigInfo * storable;
|
ConfigInfo * storable;
|
||||||
|
private:
|
||||||
|
bool default_instance;
|
||||||
};
|
};
|
||||||
|
|
||||||
#include "zbackup_base.hh"
|
#include "zbackup_base.hh"
|
||||||
|
|
|
@ -18,7 +18,7 @@ using std::iterator;
|
||||||
ZBackup::ZBackup( string const & storageDir, string const & password,
|
ZBackup::ZBackup( string const & storageDir, string const & password,
|
||||||
Config & configIn ):
|
Config & configIn ):
|
||||||
ZBackupBase( storageDir, password, configIn ),
|
ZBackupBase( storageDir, password, configIn ),
|
||||||
chunkStorageWriter( storageInfo, encryptionkey, tmpMgr, chunkIndex,
|
chunkStorageWriter( config, encryptionkey, tmpMgr, chunkIndex,
|
||||||
getBundlesPath(), getIndexPath(), config.runtime.threads )
|
getBundlesPath(), getIndexPath(), config.runtime.threads )
|
||||||
{
|
{
|
||||||
}
|
}
|
||||||
|
@ -32,7 +32,7 @@ void ZBackup::backupFromStdin( string const & outputFileName )
|
||||||
throw exWontOverwrite( outputFileName );
|
throw exWontOverwrite( outputFileName );
|
||||||
|
|
||||||
Sha256 sha256;
|
Sha256 sha256;
|
||||||
BackupCreator backupCreator( storageInfo, chunkIndex, chunkStorageWriter );
|
BackupCreator backupCreator( config, chunkIndex, chunkStorageWriter );
|
||||||
|
|
||||||
time_t startTime = time( 0 );
|
time_t startTime = time( 0 );
|
||||||
uint64_t totalDataSize = 0;
|
uint64_t totalDataSize = 0;
|
||||||
|
@ -77,7 +77,7 @@ void ZBackup::backupFromStdin( string const & outputFileName )
|
||||||
// Shrink the serialized data iteratively until it wouldn't shrink anymore
|
// Shrink the serialized data iteratively until it wouldn't shrink anymore
|
||||||
for ( ; ; )
|
for ( ; ; )
|
||||||
{
|
{
|
||||||
BackupCreator backupCreator( storageInfo, chunkIndex, chunkStorageWriter );
|
BackupCreator backupCreator( config, chunkIndex, chunkStorageWriter );
|
||||||
char const * ptr = serialized.data();
|
char const * ptr = serialized.data();
|
||||||
size_t left = serialized.size();
|
size_t left = serialized.size();
|
||||||
while( left )
|
while( left )
|
||||||
|
@ -124,7 +124,7 @@ void ZBackup::backupFromStdin( string const & outputFileName )
|
||||||
ZRestore::ZRestore( string const & storageDir, string const & password,
|
ZRestore::ZRestore( string const & storageDir, string const & password,
|
||||||
Config & configIn ):
|
Config & configIn ):
|
||||||
ZBackupBase( storageDir, password, configIn ),
|
ZBackupBase( storageDir, password, configIn ),
|
||||||
chunkStorageReader( storageInfo, encryptionkey, chunkIndex, getBundlesPath(),
|
chunkStorageReader( config, encryptionkey, chunkIndex, getBundlesPath(),
|
||||||
config.runtime.cacheSize )
|
config.runtime.cacheSize )
|
||||||
{
|
{
|
||||||
}
|
}
|
||||||
|
|
|
@ -30,12 +30,12 @@ message EncryptionKeyInfo
|
||||||
message StorageInfo
|
message StorageInfo
|
||||||
{
|
{
|
||||||
// Maximum chunk size used when storing chunks
|
// Maximum chunk size used when storing chunks
|
||||||
required uint32 chunk_max_size = 1 [deprecated = true];
|
optional uint32 chunk_max_size = 1 [deprecated = true];
|
||||||
// Maximum number of bytes a bundle can hold. Only real chunk bytes are
|
// Maximum number of bytes a bundle can hold. Only real chunk bytes are
|
||||||
// counted, not metadata. Any bundle should be able to contain at least
|
// counted, not metadata. Any bundle should be able to contain at least
|
||||||
// one arbitrary single chunk, so this should not be smaller than
|
// one arbitrary single chunk, so this should not be smaller than
|
||||||
// chunk_max_size
|
// chunk_max_size
|
||||||
required uint32 bundle_max_payload_size = 2 [deprecated = true];
|
optional uint32 bundle_max_payload_size = 2 [deprecated = true];
|
||||||
// If present, used for encryption/decryption of all data
|
// If present, used for encryption/decryption of all data
|
||||||
optional EncryptionKeyInfo encryption_key = 3;
|
optional EncryptionKeyInfo encryption_key = 3;
|
||||||
// Default compression for new bundles
|
// Default compression for new bundles
|
||||||
|
|
|
@ -154,18 +154,13 @@ void ZBackupBase::initStorage( string const & storageDir,
|
||||||
StorageInfo storageInfo;
|
StorageInfo storageInfo;
|
||||||
ExtendedStorageInfo extendedStorageInfo;
|
ExtendedStorageInfo extendedStorageInfo;
|
||||||
|
|
||||||
// TODO: Make a proper setup of initial values
|
ConfigInfo * storable = extendedStorageInfo.mutable_config();
|
||||||
/*storageInfo.set_chunk_max_size( 65536 );
|
// TODO: Use protobuf reflection in loop
|
||||||
storageInfo.set_bundle_max_payload_size( 0x200000 );
|
// to fill messages with default values
|
||||||
storageInfo.set_default_compression_method(
|
// without explicitly defining them
|
||||||
Compression::CompressionMethod::selectedCompression->getName() );*/
|
SET_STORABLE( chunk, max_size, GET_STORABLE( chunk, max_size ) );
|
||||||
|
SET_STORABLE( bundle, max_payload_size, GET_STORABLE( bundle, max_payload_size ) );
|
||||||
extendedStorageInfo.mutable_config()->mutable_chunk()->set_max_size(
|
SET_STORABLE( bundle, compression_method, GET_STORABLE( bundle, compression_method ) );
|
||||||
extendedStorageInfo.config().chunk().max_size() );
|
|
||||||
extendedStorageInfo.mutable_config()->mutable_bundle()->set_max_payload_size(
|
|
||||||
extendedStorageInfo.config().bundle().max_payload_size() );
|
|
||||||
extendedStorageInfo.mutable_config()->mutable_bundle()->set_compression_method(
|
|
||||||
extendedStorageInfo.config().bundle().compression_method() );
|
|
||||||
|
|
||||||
EncryptionKey encryptionkey = EncryptionKey::noKey();
|
EncryptionKey encryptionkey = EncryptionKey::noKey();
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue