diff --git a/backup_collector.cc b/backup_collector.cc index 1b29b84..ba5f01e 100644 --- a/backup_collector.cc +++ b/backup_collector.cc @@ -16,178 +16,88 @@ using std::string; -namespace { - -class BundleCollector: public IndexProcessor +void BundleCollector::startIndex( string const & indexFn ) { -private: - Bundle::Id savedId; - int totalChunks, usedChunks, indexTotalChunks, indexUsedChunks; - int indexModifiedBundles, indexKeptBundles, indexRemovedBundles; - bool indexModified; - vector< string > filesToUnlink; + indexModified = false; + indexTotalChunks = indexUsedChunks = 0; + indexModifiedBundles = indexKeptBundles = indexRemovedBundles = 0; +} -public: - string bundlesPath; - bool verbose; - ChunkStorage::Reader *chunkStorageReader; - ChunkStorage::Writer *chunkStorageWriter; - BackupRestorer::ChunkSet usedChunkSet; - - void startIndex( string const & indexFn ) +void BundleCollector::finishIndex( string const & indexFn ) +{ + if ( indexModified ) { - indexModified = false; - indexTotalChunks = indexUsedChunks = 0; - indexModifiedBundles = indexKeptBundles = indexRemovedBundles = 0; + verbosePrintf( "Chunks: %d used / %d total, bundles: %d kept / %d modified / %d removed\n", + indexUsedChunks, indexTotalChunks, indexKeptBundles, indexModifiedBundles, indexRemovedBundles); + filesToUnlink.push_back( indexFn ); } +} - void finishIndex( string const & indexFn ) +void BundleCollector::startBundle( Bundle::Id const & bundleId ) +{ + savedId = bundleId; + totalChunks = 0; + usedChunks = 0; +} + +void BundleCollector::processChunk( ChunkId const & chunkId ) +{ + totalChunks++; + if ( usedChunkSet.find( chunkId ) != usedChunkSet.end() ) { - if ( indexModified ) - { - verbosePrintf( "Chunks: %d used / %d total, bundles: %d kept / %d modified / %d removed\n", - indexUsedChunks, indexTotalChunks, indexKeptBundles, indexModifiedBundles, indexRemovedBundles); - filesToUnlink.push_back( indexFn ); - } + usedChunks++; } +} - void startBundle( Bundle::Id const & bundleId ) +void BundleCollector::finishBundle( Bundle::Id const & bundleId, BundleInfo const & info ) +{ + string i = Bundle::generateFileName( savedId, "", false ); + indexTotalChunks += totalChunks; + indexUsedChunks += usedChunks; + if ( usedChunks == 0 ) { - savedId = bundleId; - totalChunks = 0; - usedChunks = 0; + if ( verbose ) + printf( "delete %s\n", i.c_str() ); + filesToUnlink.push_back( Dir::addPath( bundlesPath, i ) ); + indexModified = true; + indexRemovedBundles++; } - - void processChunk( ChunkId const & chunkId ) + else if ( usedChunks < totalChunks ) { - totalChunks++; - if ( usedChunkSet.find( chunkId ) != usedChunkSet.end() ) + if ( verbose ) + printf( "%s: used %d/%d\n", i.c_str(), usedChunks, totalChunks ); + filesToUnlink.push_back( Dir::addPath( bundlesPath, i ) ); + indexModified = true; + // Copy used chunks to the new index + string chunk; + size_t chunkSize; + for ( int x = info.chunk_record_size(); x--; ) { - usedChunks++; - } - } - - void finishBundle( Bundle::Id const & bundleId, BundleInfo const & info ) - { - string i = Bundle::generateFileName( savedId, "", false ); - indexTotalChunks += totalChunks; - indexUsedChunks += usedChunks; - if ( usedChunks == 0 ) - { - if ( verbose ) - printf( "delete %s\n", i.c_str() ); - filesToUnlink.push_back( Dir::addPath( bundlesPath, i ) ); - indexModified = true; - indexRemovedBundles++; - } - else if ( usedChunks < totalChunks ) - { - if ( verbose ) - printf( "%s: used %d/%d\n", i.c_str(), usedChunks, totalChunks ); - filesToUnlink.push_back( Dir::addPath( bundlesPath, i ) ); - indexModified = true; - // Copy used chunks to the new index - string chunk; - size_t chunkSize; - for ( int x = info.chunk_record_size(); x--; ) + BundleInfo_ChunkRecord const & record = info.chunk_record( x ); + ChunkId id( record.id() ); + if ( usedChunkSet.find( id ) != usedChunkSet.end() ) { - BundleInfo_ChunkRecord const & record = info.chunk_record( x ); - ChunkId id( record.id() ); - if ( usedChunkSet.find( id ) != usedChunkSet.end() ) - { - chunkStorageReader->get( id, chunk, chunkSize ); - chunkStorageWriter->add( id, chunk.data(), chunkSize ); - } + chunkStorageReader->get( id, chunk, chunkSize ); + chunkStorageWriter->add( id, chunk.data(), chunkSize ); } - indexModifiedBundles++; - } - else - { - chunkStorageWriter->addBundle( info, savedId ); - if ( verbose ) - printf( "keep %s\n", i.c_str() ); - indexKeptBundles++; } + indexModifiedBundles++; } - - void commit() + else { - for ( int i = filesToUnlink.size(); i--; ) - { - unlink( filesToUnlink[i].c_str() ); - } - filesToUnlink.clear(); - chunkStorageWriter->commit(); + chunkStorageWriter->addBundle( info, savedId ); + if ( verbose ) + printf( "keep %s\n", i.c_str() ); + indexKeptBundles++; } -}; - } -ZCollector::ZCollector( string const & storageDir, string const & password, - Config & configIn ): - ZBackupBase( storageDir, password, configIn ), - chunkStorageReader( config, encryptionkey, chunkIndex, getBundlesPath(), - config.runtime.cacheSize ) +void BundleCollector::commit() { -} - -void ZCollector::gc() -{ - ChunkIndex chunkReindex( encryptionkey, tmpMgr, getIndexPath(), true ); - - ChunkStorage::Writer chunkStorageWriter( config, encryptionkey, tmpMgr, - chunkReindex, getBundlesPath(), getIndexPath(), config.runtime.threads ); - - string fileName; - - Dir::Entry entry; - - BundleCollector collector; - collector.bundlesPath = getBundlesPath(); - collector.chunkStorageReader = &this->chunkStorageReader; - collector.chunkStorageWriter = &chunkStorageWriter; - collector.verbose = false; - - verbosePrintf( "Checking used chunks...\n" ); - - verbosePrintf( "Searching for backups...\n" ); - vector< string > backups = BackupExchanger::findOrRebuild( getBackupsPath() ); - - for ( std::vector< string >::iterator it = backups.begin(); it != backups.end(); ++it ) + for ( int i = filesToUnlink.size(); i--; ) { - string backup( Dir::addPath( getBackupsPath(), *it ) ); - - verbosePrintf( "Checking backup %s...\n", backup.c_str() ); - - BackupInfo backupInfo; - - BackupFile::load( backup, encryptionkey, backupInfo ); - - string backupData; - - BackupRestorer::restoreIterations( chunkStorageReader, backupInfo, backupData, &collector.usedChunkSet ); - - BackupRestorer::restore( chunkStorageReader, backupData, NULL, &collector.usedChunkSet ); + unlink( filesToUnlink[i].c_str() ); } - - verbosePrintf( "Checking bundles...\n" ); - - chunkIndex.loadIndex( collector ); - - collector.commit(); - - verbosePrintf( "Cleaning up...\n" ); - - string bundlesPath = getBundlesPath(); - Dir::Listing bundleLst( bundlesPath ); - while( bundleLst.getNext( entry ) ) - { - const string dirPath = Dir::addPath( bundlesPath, entry.getFileName()); - if ( entry.isDir() && Dir::isDirEmpty( dirPath ) ) - { - Dir::remove( dirPath ); - } - } - - verbosePrintf( "Garbage collection complete\n" ); + filesToUnlink.clear(); + chunkStorageWriter->commit(); } diff --git a/backup_collector.hh b/backup_collector.hh index 6ac27a2..59926db 100644 --- a/backup_collector.hh +++ b/backup_collector.hh @@ -7,15 +7,46 @@ #include "zbackup_base.hh" #include "chunk_storage.hh" -class ZCollector : public ZBackupBase +#include +#include + +#include "bundle.hh" +#include "chunk_index.hh" +#include "backup_restorer.hh" +#include "backup_file.hh" +#include "backup_exchanger.hh" + +#include "debug.hh" + +using std::string; + +class BundleCollector: public IndexProcessor { - ChunkStorage::Reader chunkStorageReader; +private: + Bundle::Id savedId; + int totalChunks, usedChunks, indexTotalChunks, indexUsedChunks; + int indexModifiedBundles, indexKeptBundles, indexRemovedBundles; + bool indexModified; + vector< string > filesToUnlink; public: - ZCollector( std::string const & storageDir, std::string const & password, - Config & configIn ); + string bundlesPath; + bool verbose; + ChunkStorage::Reader *chunkStorageReader; + ChunkStorage::Writer *chunkStorageWriter; + BackupRestorer::ChunkSet usedChunkSet; - void gc(); + void startIndex( string const & indexFn ); + + void finishIndex( string const & indexFn ); + + void startBundle( Bundle::Id const & bundleId ); + + void processChunk( ChunkId const & chunkId ); + + void finishBundle( Bundle::Id const & bundleId, BundleInfo const & info ); + + void commit(); }; #endif diff --git a/zbackup.cc b/zbackup.cc index f086485..50d1244 100644 --- a/zbackup.cc +++ b/zbackup.cc @@ -1,308 +1,10 @@ // Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS // Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE -#include "zbackup.hh" -#include "backup_creator.hh" -#include "backup_file.hh" -#include "backup_restorer.hh" +#include "zutils.hh" #include "debug.hh" -#include "sha256.hh" -#include "backup_collector.hh" -#include "config.hh" -#include "utils.hh" #include "version.hh" - -using std::vector; -using std::bitset; -using std::iterator; - -ZBackup::ZBackup( string const & storageDir, string const & password, - Config & configIn ): - ZBackupBase( storageDir, password, configIn ), - chunkStorageWriter( config, encryptionkey, tmpMgr, chunkIndex, - getBundlesPath(), getIndexPath(), config.runtime.threads ) -{ -} - -void ZBackup::backupFromStdin( string const & outputFileName ) -{ - if ( isatty( fileno( stdin ) ) ) - throw exWontReadFromTerminal(); - - if ( File::exists( outputFileName ) ) - throw exWontOverwrite( outputFileName ); - - Sha256 sha256; - BackupCreator backupCreator( config, chunkIndex, chunkStorageWriter ); - - time_t startTime = time( 0 ); - uint64_t totalDataSize = 0; - - for ( ; ; ) - { - size_t toRead = backupCreator.getInputBufferSize(); -// dPrintf( "Reading up to %u bytes from stdin\n", toRead ); - - void * inputBuffer = backupCreator.getInputBuffer(); - size_t rd = fread( inputBuffer, 1, toRead, stdin ); - - if ( !rd ) - { - if ( feof( stdin ) ) - { - dPrintf( "No more input on stdin\n" ); - break; - } - else - throw exStdinError(); - } - - sha256.add( inputBuffer, rd ); - - backupCreator.handleMoreData( rd ); - - totalDataSize += rd; - } - - // Finish up with the creator - backupCreator.finish(); - - string serialized; - backupCreator.getBackupData( serialized ); - - BackupInfo info; - - info.set_sha256( sha256.finish() ); - info.set_size( totalDataSize ); - - // Shrink the serialized data iteratively until it wouldn't shrink anymore - for ( ; ; ) - { - BackupCreator backupCreator( config, chunkIndex, chunkStorageWriter ); - char const * ptr = serialized.data(); - size_t left = serialized.size(); - while( left ) - { - size_t bufferSize = backupCreator.getInputBufferSize(); - size_t toCopy = bufferSize > left ? left : bufferSize; - - memcpy( backupCreator.getInputBuffer(), ptr, toCopy ); - backupCreator.handleMoreData( toCopy ); - ptr += toCopy; - left -= toCopy; - } - - backupCreator.finish(); - - string newGen; - backupCreator.getBackupData( newGen ); - - if ( newGen.size() < serialized.size() ) - { - serialized.swap( newGen ); - info.set_iterations( info.iterations() + 1 ); - } - else - break; - } - - dPrintf( "Iterations: %u\n", info.iterations() ); - - info.mutable_backup_data()->swap( serialized ); - - info.set_time( time( 0 ) - startTime ); - - // Commit the bundles to the disk before creating the final output file - chunkStorageWriter.commit(); - - // Now save the resulting BackupInfo - - sptr< TemporaryFile > tmpFile = tmpMgr.makeTemporaryFile(); - BackupFile::save( tmpFile->getFileName(), encryptionkey, info ); - tmpFile->moveOverTo( outputFileName ); -} - -ZRestore::ZRestore( string const & storageDir, string const & password, - Config & configIn ): - ZBackupBase( storageDir, password, configIn ), - chunkStorageReader( config, encryptionkey, chunkIndex, getBundlesPath(), - config.runtime.cacheSize ) -{ -} - -void ZRestore::restoreToStdin( string const & inputFileName ) -{ - if ( isatty( fileno( stdout ) ) ) - throw exWontWriteToTerminal(); - - BackupInfo backupInfo; - - BackupFile::load( inputFileName, encryptionkey, backupInfo ); - - string backupData; - - // Perform the iterations needed to get to the actual user backup data - BackupRestorer::restoreIterations( chunkStorageReader, backupInfo, backupData, NULL ); - - struct StdoutWriter: public DataSink - { - Sha256 sha256; - - virtual void saveData( void const * data, size_t size ) - { - sha256.add( data, size ); - if ( fwrite( data, size, 1, stdout ) != 1 ) - throw exStdoutError(); - } - } stdoutWriter; - - BackupRestorer::restore( chunkStorageReader, backupData, &stdoutWriter, NULL ); - - if ( stdoutWriter.sha256.finish() != backupInfo.sha256() ) - throw exChecksumError(); -} - -ZExchange::ZExchange( string const & srcStorageDir, string const & srcPassword, - string const & dstStorageDir, string const & dstPassword, - Config & configIn ): - srcZBackupBase( srcStorageDir, srcPassword, configIn, true ), - dstZBackupBase( dstStorageDir, dstPassword, configIn, true ), - config( configIn ) -{ -} - -void ZExchange::exchange() -{ - vector< BackupExchanger::PendingExchangeRename > pendingExchangeRenames; - - if ( config.runtime.exchange.test( BackupExchanger::bundles ) ) - { - verbosePrintf( "Searching for bundles...\n" ); - - vector< string > bundles = BackupExchanger::findOrRebuild( - srcZBackupBase.getBundlesPath(), dstZBackupBase.getBundlesPath() ); - - for ( std::vector< string >::iterator it = bundles.begin(); it != bundles.end(); ++it ) - { - verbosePrintf( "Processing bundle file %s... ", it->c_str() ); - string outputFileName ( Dir::addPath( dstZBackupBase.getBundlesPath(), *it ) ); - if ( !File::exists( outputFileName ) ) - { - sptr< Bundle::Reader > reader = new Bundle::Reader( Dir::addPath ( - srcZBackupBase.getBundlesPath(), *it ), srcZBackupBase.encryptionkey, true ); - sptr< Bundle::Creator > creator = new Bundle::Creator; - sptr< TemporaryFile > bundleTempFile = dstZBackupBase.tmpMgr.makeTemporaryFile(); - creator->write( bundleTempFile->getFileName(), dstZBackupBase.encryptionkey, *reader ); - - if ( creator.get() && reader.get() ) - { - creator.reset(); - reader.reset(); - pendingExchangeRenames.push_back( BackupExchanger::PendingExchangeRename( - bundleTempFile, outputFileName ) ); - verbosePrintf( "done.\n" ); - } - } - else - { - verbosePrintf( "file exists - skipped.\n" ); - } - } - - verbosePrintf( "Bundle exchange completed.\n" ); - } - - if ( config.runtime.exchange.test( BackupExchanger::index ) ) - { - verbosePrintf( "Searching for indicies...\n" ); - vector< string > indicies = BackupExchanger::findOrRebuild( - srcZBackupBase.getIndexPath(), dstZBackupBase.getIndexPath() ); - - for ( std::vector< string >::iterator it = indicies.begin(); it != indicies.end(); ++it ) - { - verbosePrintf( "Processing index file %s... ", it->c_str() ); - string outputFileName ( Dir::addPath( dstZBackupBase.getIndexPath(), *it ) ); - if ( !File::exists( outputFileName ) ) - { - sptr< IndexFile::Reader > reader = new IndexFile::Reader( srcZBackupBase.encryptionkey, - Dir::addPath( srcZBackupBase.getIndexPath(), *it ) ); - sptr< TemporaryFile > indexTempFile = dstZBackupBase.tmpMgr.makeTemporaryFile(); - sptr< IndexFile::Writer > writer = new IndexFile::Writer( dstZBackupBase.encryptionkey, - indexTempFile->getFileName() ); - - BundleInfo bundleInfo; - Bundle::Id bundleId; - while( reader->readNextRecord( bundleInfo, bundleId ) ) - { - writer->add( bundleInfo, bundleId ); - } - - if ( writer.get() && reader.get() ) - { - writer.reset(); - reader.reset(); - pendingExchangeRenames.push_back( BackupExchanger::PendingExchangeRename( - indexTempFile, outputFileName ) ); - verbosePrintf( "done.\n" ); - } - } - else - { - verbosePrintf( "file exists - skipped.\n" ); - } - } - - verbosePrintf( "Index exchange completed.\n" ); - } - - if ( config.runtime.exchange.test( BackupExchanger::backups ) ) - { - BackupInfo backupInfo; - - verbosePrintf( "Searching for backups...\n" ); - vector< string > backups = BackupExchanger::findOrRebuild( - srcZBackupBase.getBackupsPath(), dstZBackupBase.getBackupsPath() ); - - for ( std::vector< string >::iterator it = backups.begin(); it != backups.end(); ++it ) - { - verbosePrintf( "Processing backup file %s... ", it->c_str() ); - string outputFileName ( Dir::addPath( dstZBackupBase.getBackupsPath(), *it ) ); - if ( !File::exists( outputFileName ) ) - { - BackupFile::load( Dir::addPath( srcZBackupBase.getBackupsPath(), *it ), - srcZBackupBase.encryptionkey, backupInfo ); - sptr< TemporaryFile > tmpFile = dstZBackupBase.tmpMgr.makeTemporaryFile(); - BackupFile::save( tmpFile->getFileName(), dstZBackupBase.encryptionkey, - backupInfo ); - pendingExchangeRenames.push_back( BackupExchanger::PendingExchangeRename( - tmpFile, outputFileName ) ); - verbosePrintf( "done.\n" ); - } - else - { - verbosePrintf( "file exists - skipped.\n" ); - } - } - - verbosePrintf( "Backup exchange completed.\n" ); - } - - if ( pendingExchangeRenames.size() > 0 ) - { - verbosePrintf( "Moving files from temp directory to appropriate places... " ); - for ( size_t x = pendingExchangeRenames.size(); x--; ) - { - BackupExchanger::PendingExchangeRename & r = pendingExchangeRenames[ x ]; - r.first->moveOverTo( r.second ); - if ( r.first.get() ) - { - r.first.reset(); - } - } - pendingExchangeRenames.clear(); - verbosePrintf( "done.\n" ); - } -} +#include "utils.hh" DEF_EX( exSpecifyTwoKeys, "Specify password flag (--non-encrypted or --password-file)" " for import/export/passwd operation twice (first for source and second for destination)", std::exception ) diff --git a/zutils.cc b/zutils.cc new file mode 100644 index 0000000..5abde00 --- /dev/null +++ b/zutils.cc @@ -0,0 +1,368 @@ +// Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS +// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE + +#include "zutils.hh" +#include "backup_creator.hh" +#include "sha256.hh" +#include "backup_collector.hh" + +using std::vector; +using std::bitset; +using std::iterator; + +ZBackup::ZBackup( string const & storageDir, string const & password, + Config & configIn ): + ZBackupBase( storageDir, password, configIn ), + chunkStorageWriter( config, encryptionkey, tmpMgr, chunkIndex, + getBundlesPath(), getIndexPath(), config.runtime.threads ) +{ +} + +void ZBackup::backupFromStdin( string const & outputFileName ) +{ + if ( isatty( fileno( stdin ) ) ) + throw exWontReadFromTerminal(); + + if ( File::exists( outputFileName ) ) + throw exWontOverwrite( outputFileName ); + + Sha256 sha256; + BackupCreator backupCreator( config, chunkIndex, chunkStorageWriter ); + + time_t startTime = time( 0 ); + uint64_t totalDataSize = 0; + + for ( ; ; ) + { + size_t toRead = backupCreator.getInputBufferSize(); +// dPrintf( "Reading up to %u bytes from stdin\n", toRead ); + + void * inputBuffer = backupCreator.getInputBuffer(); + size_t rd = fread( inputBuffer, 1, toRead, stdin ); + + if ( !rd ) + { + if ( feof( stdin ) ) + { + dPrintf( "No more input on stdin\n" ); + break; + } + else + throw exStdinError(); + } + + sha256.add( inputBuffer, rd ); + + backupCreator.handleMoreData( rd ); + + totalDataSize += rd; + } + + // Finish up with the creator + backupCreator.finish(); + + string serialized; + backupCreator.getBackupData( serialized ); + + BackupInfo info; + + info.set_sha256( sha256.finish() ); + info.set_size( totalDataSize ); + + // Shrink the serialized data iteratively until it wouldn't shrink anymore + for ( ; ; ) + { + BackupCreator backupCreator( config, chunkIndex, chunkStorageWriter ); + char const * ptr = serialized.data(); + size_t left = serialized.size(); + while( left ) + { + size_t bufferSize = backupCreator.getInputBufferSize(); + size_t toCopy = bufferSize > left ? left : bufferSize; + + memcpy( backupCreator.getInputBuffer(), ptr, toCopy ); + backupCreator.handleMoreData( toCopy ); + ptr += toCopy; + left -= toCopy; + } + + backupCreator.finish(); + + string newGen; + backupCreator.getBackupData( newGen ); + + if ( newGen.size() < serialized.size() ) + { + serialized.swap( newGen ); + info.set_iterations( info.iterations() + 1 ); + } + else + break; + } + + dPrintf( "Iterations: %u\n", info.iterations() ); + + info.mutable_backup_data()->swap( serialized ); + + info.set_time( time( 0 ) - startTime ); + + // Commit the bundles to the disk before creating the final output file + chunkStorageWriter.commit(); + + // Now save the resulting BackupInfo + + sptr< TemporaryFile > tmpFile = tmpMgr.makeTemporaryFile(); + BackupFile::save( tmpFile->getFileName(), encryptionkey, info ); + tmpFile->moveOverTo( outputFileName ); +} + +ZRestore::ZRestore( string const & storageDir, string const & password, + Config & configIn ): + ZBackupBase( storageDir, password, configIn ), + chunkStorageReader( config, encryptionkey, chunkIndex, getBundlesPath(), + config.runtime.cacheSize ) +{ +} + +void ZRestore::restoreToStdin( string const & inputFileName ) +{ + if ( isatty( fileno( stdout ) ) ) + throw exWontWriteToTerminal(); + + BackupInfo backupInfo; + + BackupFile::load( inputFileName, encryptionkey, backupInfo ); + + string backupData; + + // Perform the iterations needed to get to the actual user backup data + BackupRestorer::restoreIterations( chunkStorageReader, backupInfo, backupData, NULL ); + + struct StdoutWriter: public DataSink + { + Sha256 sha256; + + virtual void saveData( void const * data, size_t size ) + { + sha256.add( data, size ); + if ( fwrite( data, size, 1, stdout ) != 1 ) + throw exStdoutError(); + } + } stdoutWriter; + + BackupRestorer::restore( chunkStorageReader, backupData, &stdoutWriter, NULL ); + + if ( stdoutWriter.sha256.finish() != backupInfo.sha256() ) + throw exChecksumError(); +} + +ZExchange::ZExchange( string const & srcStorageDir, string const & srcPassword, + string const & dstStorageDir, string const & dstPassword, + Config & configIn ): + srcZBackupBase( srcStorageDir, srcPassword, configIn, true ), + dstZBackupBase( dstStorageDir, dstPassword, configIn, true ), + config( configIn ) +{ +} + +void ZExchange::exchange() +{ + vector< BackupExchanger::PendingExchangeRename > pendingExchangeRenames; + + if ( config.runtime.exchange.test( BackupExchanger::bundles ) ) + { + verbosePrintf( "Searching for bundles...\n" ); + + vector< string > bundles = BackupExchanger::findOrRebuild( + srcZBackupBase.getBundlesPath(), dstZBackupBase.getBundlesPath() ); + + for ( std::vector< string >::iterator it = bundles.begin(); it != bundles.end(); ++it ) + { + verbosePrintf( "Processing bundle file %s... ", it->c_str() ); + string outputFileName ( Dir::addPath( dstZBackupBase.getBundlesPath(), *it ) ); + if ( !File::exists( outputFileName ) ) + { + sptr< Bundle::Reader > reader = new Bundle::Reader( Dir::addPath ( + srcZBackupBase.getBundlesPath(), *it ), srcZBackupBase.encryptionkey, true ); + sptr< Bundle::Creator > creator = new Bundle::Creator; + sptr< TemporaryFile > bundleTempFile = dstZBackupBase.tmpMgr.makeTemporaryFile(); + creator->write( bundleTempFile->getFileName(), dstZBackupBase.encryptionkey, *reader ); + + if ( creator.get() && reader.get() ) + { + creator.reset(); + reader.reset(); + pendingExchangeRenames.push_back( BackupExchanger::PendingExchangeRename( + bundleTempFile, outputFileName ) ); + verbosePrintf( "done.\n" ); + } + } + else + { + verbosePrintf( "file exists - skipped.\n" ); + } + } + + verbosePrintf( "Bundle exchange completed.\n" ); + } + + if ( config.runtime.exchange.test( BackupExchanger::index ) ) + { + verbosePrintf( "Searching for indicies...\n" ); + vector< string > indicies = BackupExchanger::findOrRebuild( + srcZBackupBase.getIndexPath(), dstZBackupBase.getIndexPath() ); + + for ( std::vector< string >::iterator it = indicies.begin(); it != indicies.end(); ++it ) + { + verbosePrintf( "Processing index file %s... ", it->c_str() ); + string outputFileName ( Dir::addPath( dstZBackupBase.getIndexPath(), *it ) ); + if ( !File::exists( outputFileName ) ) + { + sptr< IndexFile::Reader > reader = new IndexFile::Reader( srcZBackupBase.encryptionkey, + Dir::addPath( srcZBackupBase.getIndexPath(), *it ) ); + sptr< TemporaryFile > indexTempFile = dstZBackupBase.tmpMgr.makeTemporaryFile(); + sptr< IndexFile::Writer > writer = new IndexFile::Writer( dstZBackupBase.encryptionkey, + indexTempFile->getFileName() ); + + BundleInfo bundleInfo; + Bundle::Id bundleId; + while( reader->readNextRecord( bundleInfo, bundleId ) ) + { + writer->add( bundleInfo, bundleId ); + } + + if ( writer.get() && reader.get() ) + { + writer.reset(); + reader.reset(); + pendingExchangeRenames.push_back( BackupExchanger::PendingExchangeRename( + indexTempFile, outputFileName ) ); + verbosePrintf( "done.\n" ); + } + } + else + { + verbosePrintf( "file exists - skipped.\n" ); + } + } + + verbosePrintf( "Index exchange completed.\n" ); + } + + if ( config.runtime.exchange.test( BackupExchanger::backups ) ) + { + BackupInfo backupInfo; + + verbosePrintf( "Searching for backups...\n" ); + vector< string > backups = BackupExchanger::findOrRebuild( + srcZBackupBase.getBackupsPath(), dstZBackupBase.getBackupsPath() ); + + for ( std::vector< string >::iterator it = backups.begin(); it != backups.end(); ++it ) + { + verbosePrintf( "Processing backup file %s... ", it->c_str() ); + string outputFileName ( Dir::addPath( dstZBackupBase.getBackupsPath(), *it ) ); + if ( !File::exists( outputFileName ) ) + { + BackupFile::load( Dir::addPath( srcZBackupBase.getBackupsPath(), *it ), + srcZBackupBase.encryptionkey, backupInfo ); + sptr< TemporaryFile > tmpFile = dstZBackupBase.tmpMgr.makeTemporaryFile(); + BackupFile::save( tmpFile->getFileName(), dstZBackupBase.encryptionkey, + backupInfo ); + pendingExchangeRenames.push_back( BackupExchanger::PendingExchangeRename( + tmpFile, outputFileName ) ); + verbosePrintf( "done.\n" ); + } + else + { + verbosePrintf( "file exists - skipped.\n" ); + } + } + + verbosePrintf( "Backup exchange completed.\n" ); + } + + if ( pendingExchangeRenames.size() > 0 ) + { + verbosePrintf( "Moving files from temp directory to appropriate places... " ); + for ( size_t x = pendingExchangeRenames.size(); x--; ) + { + BackupExchanger::PendingExchangeRename & r = pendingExchangeRenames[ x ]; + r.first->moveOverTo( r.second ); + if ( r.first.get() ) + { + r.first.reset(); + } + } + pendingExchangeRenames.clear(); + verbosePrintf( "done.\n" ); + } +} + +ZCollector::ZCollector( string const & storageDir, string const & password, + Config & configIn ): + ZBackupBase( storageDir, password, configIn ), + chunkStorageReader( config, encryptionkey, chunkIndex, getBundlesPath(), + config.runtime.cacheSize ) +{ +} + +void ZCollector::gc() +{ + ChunkIndex chunkReindex( encryptionkey, tmpMgr, getIndexPath(), true ); + + ChunkStorage::Writer chunkStorageWriter( config, encryptionkey, tmpMgr, + chunkReindex, getBundlesPath(), getIndexPath(), config.runtime.threads ); + + string fileName; + + Dir::Entry entry; + + BundleCollector collector; + collector.bundlesPath = getBundlesPath(); + collector.chunkStorageReader = &this->chunkStorageReader; + collector.chunkStorageWriter = &chunkStorageWriter; + collector.verbose = false; + + verbosePrintf( "Checking used chunks...\n" ); + + verbosePrintf( "Searching for backups...\n" ); + vector< string > backups = BackupExchanger::findOrRebuild( getBackupsPath() ); + + for ( std::vector< string >::iterator it = backups.begin(); it != backups.end(); ++it ) + { + string backup( Dir::addPath( getBackupsPath(), *it ) ); + + verbosePrintf( "Checking backup %s...\n", backup.c_str() ); + + BackupInfo backupInfo; + + BackupFile::load( backup, encryptionkey, backupInfo ); + + string backupData; + + BackupRestorer::restoreIterations( chunkStorageReader, backupInfo, backupData, &collector.usedChunkSet ); + + BackupRestorer::restore( chunkStorageReader, backupData, NULL, &collector.usedChunkSet ); + } + + verbosePrintf( "Checking bundles...\n" ); + + chunkIndex.loadIndex( collector ); + + collector.commit(); + + verbosePrintf( "Cleaning up...\n" ); + + string bundlesPath = getBundlesPath(); + Dir::Listing bundleLst( bundlesPath ); + while( bundleLst.getNext( entry ) ) + { + const string dirPath = Dir::addPath( bundlesPath, entry.getFileName()); + if ( entry.isDir() && Dir::isDirEmpty( dirPath ) ) + { + Dir::remove( dirPath ); + } + } + + verbosePrintf( "Garbage collection complete\n" ); +} diff --git a/zbackup.hh b/zutils.hh similarity index 83% rename from zbackup.hh rename to zutils.hh index f6836ac..59d23d4 100644 --- a/zbackup.hh +++ b/zutils.hh @@ -1,16 +1,12 @@ // Copyright (c) 2012-2014 Konstantin Isakov and ZBackup contributors, see CONTRIBUTORS // Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE -#ifndef ZBACKUP_HH_INCLUDED -#define ZBACKUP_HH_INCLUDED +#ifndef ZUTILS_HH_INCLUDED +#define ZUTILS_HH_INCLUDED #include "chunk_storage.hh" #include "zbackup_base.hh" -using std::string; -using std::vector; -using std::bitset; - class ZBackup: public ZBackupBase { ChunkStorage::Writer chunkStorageWriter; @@ -35,17 +31,6 @@ public: void restoreToStdin( string const & inputFileName ); }; -class ZCollect: public ZBackupBase -{ - ChunkStorage::Reader chunkStorageReader; - -public: - ZCollect( string const & storageDir, string const & password, - Config & configIn ); - - void gc(); -}; - class ZExchange { ZBackupBase srcZBackupBase; @@ -62,4 +47,15 @@ public: void exchange(); }; +class ZCollector : public ZBackupBase +{ + ChunkStorage::Reader chunkStorageReader; + +public: + ZCollector( std::string const & storageDir, std::string const & password, + Config & configIn ); + + void gc(); +}; + #endif