diff --git a/chunk_index.cc b/chunk_index.cc index 747cb1f..df96fee 100644 --- a/chunk_index.cc +++ b/chunk_index.cc @@ -33,7 +33,7 @@ void ChunkIndex::loadIndex( IndexProcessor & ip ) while( lst.getNext( entry ) ) { - verbosePrintf( "Loading index file %s... ", entry.getFileName().c_str() ); + verbosePrintf( "Loading index file %s...\n", entry.getFileName().c_str() ); try { string indexFn = Dir::addPath( indexPath, entry.getFileName() ); @@ -73,7 +73,6 @@ void ChunkIndex::loadIndex( IndexProcessor & ip ) fprintf( stderr, "error: %s\n", e.what() ); continue; } - verbosePrintf( "\n" ); } verbosePrintf( "Index loaded.\n" ); @@ -106,6 +105,8 @@ ChunkIndex::ChunkIndex( EncryptionKey const & key, TmpMgr & tmpMgr, key( key ), tmpMgr( tmpMgr ), indexPath( indexPath ), storage( 65536, 1 ), lastBundleId( NULL ) { + dPrintf( "Chunk index (%s) instantiated and initialized, hasKey: %s\n", indexPath.c_str(), + key.hasKey() ? "true" : "false" ); if ( !prohibitChunkIndexLoading ) loadIndex( *this ); } diff --git a/debug.hh b/debug.hh index bd80a9f..b4e274b 100644 --- a/debug.hh +++ b/debug.hh @@ -10,7 +10,11 @@ #ifndef NDEBUG -#define dPrintf( ... ) (fprintf( stderr, __VA_ARGS__ )) +#define __FILE_BASE (strrchr(__FILE__, '/') ? strrchr(__FILE__, '/') + 1 : __FILE__) + +#define dPrintf( ... ) ({ fprintf( stderr, "[DEBUG] at %s( %s:%d ): ", __func__,\ + __FILE_BASE, __LINE__ );\ + fprintf( stderr, __VA_ARGS__ ); }) #else diff --git a/encrypted_file.cc b/encrypted_file.cc index a283fe3..f62f919 100644 --- a/encrypted_file.cc +++ b/encrypted_file.cc @@ -9,6 +9,7 @@ #include "endian.hh" #include "page_size.hh" #include "random.hh" +#include "debug.hh" namespace EncryptedFile { @@ -22,6 +23,7 @@ InputStream::InputStream( char const * fileName, EncryptionKey const & key, buffer( std::max( getPageSize(), ( unsigned ) BlockSize * 2 ) ), fill( 0 ), remainder( 0 ), backedUp( false ) { + dPrintf( "Loading %s, hasKey: %s\n", fileName, key.hasKey() ? "true" : "false" ); if ( key.hasKey() ) { memcpy( iv, iv_, sizeof( iv ) ); @@ -239,6 +241,7 @@ OutputStream::OutputStream( char const * fileName, EncryptionKey const & key, file( fileName, UnbufferedFile::WriteOnly ), filePos( 0 ), key( key ), buffer( getPageSize() ), start( buffer.data() ), avail( 0 ), backedUp( false ) { + dPrintf( "Saving %s, hasKey: %s\n", fileName, key.hasKey() ? "true" : "false" ); if ( key.hasKey() ) memcpy( iv, iv_, sizeof( iv ) ); } diff --git a/storage_info_file.cc b/storage_info_file.cc index 7ab2f22..e37f56d 100644 --- a/storage_info_file.cc +++ b/storage_info_file.cc @@ -6,6 +6,7 @@ #include "encrypted_file.hh" #include "message.hh" #include "storage_info_file.hh" +#include "debug.hh" namespace StorageInfoFile { @@ -16,6 +17,7 @@ enum void save( string const & fileName, StorageInfo const & storageInfo ) { + dPrintf( "Saving storage info...\n" ); EncryptedFile::OutputStream os( fileName.c_str(), EncryptionKey::noKey(), NULL ); FileHeader header; @@ -28,6 +30,7 @@ void save( string const & fileName, StorageInfo const & storageInfo ) void load( string const & fileName, StorageInfo & storageInfo ) { + dPrintf( "Loading storage info...\n" ); EncryptedFile::InputStream is( fileName.c_str(), EncryptionKey::noKey(), NULL ); FileHeader header; @@ -40,3 +43,47 @@ void load( string const & fileName, StorageInfo & storageInfo ) } } + +namespace ExtendedStorageInfoFile { + +enum +{ + FileFormatVersion = 1 +}; + +void save( string const & fileName, EncryptionKey const & encryptionKey, + ExtendedStorageInfo const & extendedStorageInfo ) +{ + dPrintf( "Saving extended storage info, hasKey: %s\n", + encryptionKey.hasKey() ? "true" : "false" ); + EncryptedFile::OutputStream os( fileName.c_str(), encryptionKey, + Encryption::ZeroIv ); + os.writeRandomIv(); + + FileHeader header; + header.set_version( FileFormatVersion ); + Message::serialize( header, os ); + + Message::serialize( extendedStorageInfo, os ); + os.writeAdler32(); +} + +void load( string const & fileName, EncryptionKey const & encryptionKey, + ExtendedStorageInfo & extendedStorageInfo ) +{ + dPrintf( "Loading extended storage info, hasKey: %s\n", + encryptionKey.hasKey() ? "true" : "false" ); + EncryptedFile::InputStream is( fileName.c_str(), encryptionKey, + Encryption::ZeroIv ); + is.consumeRandomIv(); + + FileHeader header; + Message::parse( header, is ); + if ( header.version() != FileFormatVersion ) + throw exUnsupportedVersion(); + + Message::parse( extendedStorageInfo, is ); + is.checkAdler32(); +} + +} diff --git a/storage_info_file.hh b/storage_info_file.hh index 53758e0..b1eefe3 100644 --- a/storage_info_file.hh +++ b/storage_info_file.hh @@ -25,4 +25,18 @@ void save( string const & fileName, StorageInfo const & ); void load( string const & fileName, StorageInfo & ); } +namespace ExtendedStorageInfoFile { + +using std::string; + +DEF_EX( Ex, "Extended storage info file exception", std::exception ) +DEF_EX( exUnsupportedVersion, "Unsupported version of the extended storage info file format", Ex ) + +/// Saves the given ExtendedStorageInfo data into the given file +void save( string const & fileName, EncryptionKey const &, ExtendedStorageInfo const & ); + +/// Loads the given ExtendedStorageInfo data from the given file +void load( string const & fileName, EncryptionKey const &, ExtendedStorageInfo & ); +} + #endif diff --git a/zbackup.cc b/zbackup.cc index d35988e..cb0958b 100644 --- a/zbackup.cc +++ b/zbackup.cc @@ -441,7 +441,8 @@ int main( int argc, char *argv[] ) ++x; if ( strcmp( argv[ x ], "lzma" ) == 0 ) { - const_sptr lzma = Compression::CompressionMethod::findCompression( "lzma" ); + const_sptr lzma = + Compression::CompressionMethod::findCompression( "lzma" ); if ( !lzma ) { fprintf( stderr, "zbackup is compiled without LZMA support, but the code " @@ -454,7 +455,8 @@ int main( int argc, char *argv[] ) else if ( strcmp( argv[ x ], "lzo" ) == 0 ) { - const_sptr lzo = Compression::CompressionMethod::findCompression( "lzo1x_1" ); + const_sptr lzo = + Compression::CompressionMethod::findCompression( "lzo1x_1" ); if ( !lzo ) { fprintf( stderr, "zbackup is compiled without LZO support, but the code " @@ -509,7 +511,8 @@ int main( int argc, char *argv[] ) " import -\n" " performs import from source to destination storage;\n" " gc - performs chunk garbage collection;\n" -" passwd - changes repository info file passphrase.\n" +" passwd - changes repository info file passphrase;\n" +" info - shows information about storage.\n" " For export/import storage path must be valid (initialized) storage.\n" "", *argv, defaultThreads, defaultCacheSizeMb ); @@ -637,18 +640,34 @@ int main( int argc, char *argv[] ) return EXIT_FAILURE; } - ZBackupBase zbb( args[ 1 ], passwords[ 0 ], true ); + ZBackupBase zbb( ZBackupBase::deriveStorageDirFromBackupsFile( args[ 1 ], true ), + passwords[ 0 ], true ); + if ( passwords[ 0 ].empty() != passwords[ 1 ].empty() ) { fprintf( stderr, -"Changing repo encryption type (non-encrypted to encrypted and vice versa) " -"is not supported yet.\n" -"Current repo type: %s\n", zbb.encryptionkey.hasKey() ? "encrypted" : "non-encrypted" ); +"Changing repo encryption type (non-encrypted to encrypted and vice versa) is possible " +"only via import/export operations.\n" +"Current repo type: %s.\n", zbb.encryptionkey.hasKey() ? "encrypted" : "non-encrypted" ); return EXIT_FAILURE; } zbb.setPassword( passwords[ 1 ] ); } else + if ( strcmp( args[ 0 ], "info" ) == 0 ) + { + // Show repo info + if ( args.size() != 2 ) + { + fprintf( stderr, "Usage: %s info \n", + *argv ); + return EXIT_FAILURE; + } + + ZBackupBase zbb( ZBackupBase::deriveStorageDirFromBackupsFile( args[ 1 ], true ), + passwords[ 0 ] ); + } + else { fprintf( stderr, "Error: unknown command line option: %s\n", args[ 0 ] ); return EXIT_FAILURE; diff --git a/zbackup.proto b/zbackup.proto index a52bea9..c3e5493 100644 --- a/zbackup.proto +++ b/zbackup.proto @@ -30,16 +30,34 @@ message EncryptionKeyInfo message StorageInfo { // Maximum chunk size used when storing chunks - required uint32 chunk_max_size = 1; + required uint32 chunk_max_size = 1 [deprecated = true]; // Maximum number of bytes a bundle can hold. Only real chunk bytes are // counted, not metadata. Any bundle should be able to contain at least // one arbitrary single chunk, so this should not be smaller than // chunk_max_size - required uint32 bundle_max_payload_size = 2; + required uint32 bundle_max_payload_size = 2 [deprecated = true]; // If present, used for encryption/decryption of all data optional EncryptionKeyInfo encryption_key = 3; // Default compression for new bundles - optional string default_compression_method = 4 [default = "lzma"]; + optional string default_compression_method = 4 [default = "lzma", deprecated = true]; +} + +message ConfigInfo +{ + required uint32 chunk_max_size = 1 [default = 65536]; + // Maximum number of bytes a bundle can hold. Only real chunk bytes are + // counted, not metadata. Any bundle should be able to contain at least + // one arbitrary single chunk, so this should not be smaller than + // chunk_max_size + required uint32 bundle_max_payload_size = 2 [default = 0x200000]; + // Default compression for new bundles + optional string default_compression_method = 3 [default = "lzma"]; +} + +message ExtendedStorageInfo +{ + // Config data storage + optional ConfigInfo config = 1; } message BundleInfo diff --git a/zbackup_base.cc b/zbackup_base.cc index b6ddfa3..20af32d 100644 --- a/zbackup_base.cc +++ b/zbackup_base.cc @@ -5,6 +5,7 @@ #include "storage_info_file.hh" #include "compression.hh" +#include "debug.hh" using std::string; @@ -27,6 +28,11 @@ string Paths::getStorageInfoPath() return string( Dir::addPath( storageDir, "info" ) ); } +string Paths::getExtendedStorageInfoPath() +{ + return string( Dir::addPath( storageDir, "info_extended" ) ); +} + string Paths::getIndexPath() { return string( Dir::addPath( storageDir, "index" ) ); @@ -41,9 +47,11 @@ ZBackupBase::ZBackupBase( string const & storageDir, string const & password ): Paths( storageDir ), storageInfo( loadStorageInfo() ), encryptionkey( password, storageInfo.has_encryption_key() ? &storageInfo.encryption_key() : 0 ), + extendedStorageInfo( loadExtendedStorageInfo( encryptionkey ) ), tmpMgr( getTmpPath() ), chunkIndex( encryptionkey, tmpMgr, getIndexPath(), false ) { + dPrintf("%s repo instantiated and initialized\n", storageDir.c_str() ); } ZBackupBase::ZBackupBase( string const & storageDir, string const & password, @@ -51,9 +59,11 @@ ZBackupBase::ZBackupBase( string const & storageDir, string const & password, Paths( storageDir ), storageInfo( loadStorageInfo() ), encryptionkey( password, storageInfo.has_encryption_key() ? &storageInfo.encryption_key() : 0 ), + extendedStorageInfo( loadExtendedStorageInfo( encryptionkey ) ), tmpMgr( getTmpPath() ), chunkIndex( encryptionkey, tmpMgr, getIndexPath(), prohibitChunkIndexLoading ) { + dPrintf("%s repo instantiated and initialized\n", storageDir.c_str() ); } StorageInfo ZBackupBase::loadStorageInfo() @@ -65,14 +75,30 @@ StorageInfo ZBackupBase::loadStorageInfo() return storageInfo; } +ExtendedStorageInfo ZBackupBase::loadExtendedStorageInfo( EncryptionKey const & encryptionkey ) +{ + ExtendedStorageInfo extendedStorageInfo; + + ExtendedStorageInfoFile::load( getExtendedStorageInfoPath(), encryptionkey, extendedStorageInfo ); + + return extendedStorageInfo; +} + void ZBackupBase::initStorage( string const & storageDir, string const & password, bool isEncrypted ) { StorageInfo storageInfo; + ExtendedStorageInfo extendedStorageInfo; + // TODO: make the following configurable storageInfo.set_chunk_max_size( 65536 ); storageInfo.set_bundle_max_payload_size( 0x200000 ); + extendedStorageInfo.mutable_config()->set_chunk_max_size( + extendedStorageInfo.config().chunk_max_size() ); + extendedStorageInfo.mutable_config()->set_bundle_max_payload_size( + extendedStorageInfo.config().bundle_max_payload_size() ); + EncryptionKey encryptionkey = EncryptionKey::noKey(); if ( isEncrypted ) @@ -98,11 +124,16 @@ void ZBackupBase::initStorage( string const & storageDir, Dir::create( paths.getIndexPath() ); string storageInfoPath( paths.getStorageInfoPath() ); + string extendedStorageInfoPath( paths.getExtendedStorageInfoPath() ); if ( File::exists( storageInfoPath ) ) throw exWontOverwrite( storageInfoPath ); + encryptionkey = EncryptionKey( password, storageInfo.has_encryption_key() ? + &storageInfo.encryption_key() : 0 ); + StorageInfoFile::save( storageInfoPath, storageInfo ); + ExtendedStorageInfoFile::save( extendedStorageInfoPath, encryptionkey, extendedStorageInfo ); } string ZBackupBase::deriveStorageDirFromBackupsFile( string const & @@ -145,3 +176,7 @@ void ZBackupBase::setPassword( string const & password ) &storageInfo.encryption_key() : 0 ); } +void ZBackupBase::saveExtendedStorageInfo() +{ + ExtendedStorageInfoFile::save( getExtendedStorageInfoPath(), encryptionkey, extendedStorageInfo ); +} diff --git a/zbackup_base.hh b/zbackup_base.hh index 0411571..008cfdb 100644 --- a/zbackup_base.hh +++ b/zbackup_base.hh @@ -21,6 +21,7 @@ struct Paths std::string getCreatePath(); std::string getBundlesPath(); std::string getStorageInfoPath(); + std::string getExtendedStorageInfoPath(); std::string getIndexPath(); std::string getBackupsPath(); }; @@ -53,15 +54,19 @@ public: void useDefaultCompressionMethod(); + void saveExtendedStorageInfo(); + void setPassword( std::string const & password ); StorageInfo storageInfo; EncryptionKey encryptionkey; + ExtendedStorageInfo extendedStorageInfo; TmpMgr tmpMgr; ChunkIndex chunkIndex; private: StorageInfo loadStorageInfo(); + ExtendedStorageInfo loadExtendedStorageInfo( EncryptionKey const & ); };