vitalif
/
zbackup
mirror of https://github.com/vitalif/zbackup
1
0
Fork 0
Code Issues Releases Wiki Activity

Merge pull request #57 from Am1GO/master 

Class/headers refactoring, removed dead ZCollect
master
Am1GO 2015-02-04 14:50:39 +03:00
parent 8b11e385c6 ebe9daa056
commit a7a671a76c
5 changed files with 480 additions and 473 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

212
backup_collector.cc
View File

@ -16,178 +16,88 @@
using std::string;
namespace {
class BundleCollector: public IndexProcessor
void BundleCollector::startIndex( string const & indexFn )
{
private:
Bundle::Id savedId;
int totalChunks, usedChunks, indexTotalChunks, indexUsedChunks;
int indexModifiedBundles, indexKeptBundles, indexRemovedBundles;
bool indexModified;
vector< string > filesToUnlink;
indexModified = false;
indexTotalChunks = indexUsedChunks = 0;
indexModifiedBundles = indexKeptBundles = indexRemovedBundles = 0;
}
public:
string bundlesPath;
bool verbose;
ChunkStorage::Reader *chunkStorageReader;
ChunkStorage::Writer *chunkStorageWriter;
BackupRestorer::ChunkSet usedChunkSet;
void startIndex( string const & indexFn )
void BundleCollector::finishIndex( string const & indexFn )
{
if ( indexModified )
{
indexModified = false;
indexTotalChunks = indexUsedChunks = 0;
indexModifiedBundles = indexKeptBundles = indexRemovedBundles = 0;
verbosePrintf( "Chunks: %d used / %d total, bundles: %d kept / %d modified / %d removed\n",
indexUsedChunks, indexTotalChunks, indexKeptBundles, indexModifiedBundles, indexRemovedBundles);
filesToUnlink.push_back( indexFn );
}
}
void finishIndex( string const & indexFn )
void BundleCollector::startBundle( Bundle::Id const & bundleId )
{
savedId = bundleId;
totalChunks = 0;
usedChunks = 0;
}
void BundleCollector::processChunk( ChunkId const & chunkId )
{
totalChunks++;
if ( usedChunkSet.find( chunkId ) != usedChunkSet.end() )
{
if ( indexModified )
{
verbosePrintf( "Chunks: %d used / %d total, bundles: %d kept / %d modified / %d removed\n",
indexUsedChunks, indexTotalChunks, indexKeptBundles, indexModifiedBundles, indexRemovedBundles);
filesToUnlink.push_back( indexFn );
}
usedChunks++;
}
}
void startBundle( Bundle::Id const & bundleId )
void BundleCollector::finishBundle( Bundle::Id const & bundleId, BundleInfo const & info )
{
string i = Bundle::generateFileName( savedId, "", false );
indexTotalChunks += totalChunks;
indexUsedChunks += usedChunks;
if ( usedChunks == 0 )
{
savedId = bundleId;
totalChunks = 0;
usedChunks = 0;
if ( verbose )
printf( "delete %s\n", i.c_str() );
filesToUnlink.push_back( Dir::addPath( bundlesPath, i ) );
indexModified = true;
indexRemovedBundles++;
}
void processChunk( ChunkId const & chunkId )
else if ( usedChunks < totalChunks )
{
totalChunks++;
if ( usedChunkSet.find( chunkId ) != usedChunkSet.end() )
if ( verbose )
printf( "%s: used %d/%d\n", i.c_str(), usedChunks, totalChunks );
filesToUnlink.push_back( Dir::addPath( bundlesPath, i ) );
indexModified = true;
// Copy used chunks to the new index
string chunk;
size_t chunkSize;
for ( int x = info.chunk_record_size(); x--; )
{
usedChunks++;
}
}
void finishBundle( Bundle::Id const & bundleId, BundleInfo const & info )
{
string i = Bundle::generateFileName( savedId, "", false );
indexTotalChunks += totalChunks;
indexUsedChunks += usedChunks;
if ( usedChunks == 0 )
{
if ( verbose )
printf( "delete %s\n", i.c_str() );
filesToUnlink.push_back( Dir::addPath( bundlesPath, i ) );
indexModified = true;
indexRemovedBundles++;
}
else if ( usedChunks < totalChunks )
{
if ( verbose )
printf( "%s: used %d/%d\n", i.c_str(), usedChunks, totalChunks );
filesToUnlink.push_back( Dir::addPath( bundlesPath, i ) );
indexModified = true;
// Copy used chunks to the new index
string chunk;
size_t chunkSize;
for ( int x = info.chunk_record_size(); x--; )
BundleInfo_ChunkRecord const & record = info.chunk_record( x );
ChunkId id( record.id() );
if ( usedChunkSet.find( id ) != usedChunkSet.end() )
{
BundleInfo_ChunkRecord const & record = info.chunk_record( x );
ChunkId id( record.id() );
if ( usedChunkSet.find( id ) != usedChunkSet.end() )
{
chunkStorageReader->get( id, chunk, chunkSize );
chunkStorageWriter->add( id, chunk.data(), chunkSize );
}
chunkStorageReader->get( id, chunk, chunkSize );
chunkStorageWriter->add( id, chunk.data(), chunkSize );
}
indexModifiedBundles++;
}
else
{
chunkStorageWriter->addBundle( info, savedId );
if ( verbose )
printf( "keep %s\n", i.c_str() );
indexKeptBundles++;
}
indexModifiedBundles++;
}
void commit()
else
{
for ( int i = filesToUnlink.size(); i--; )
{
unlink( filesToUnlink[i].c_str() );
}
filesToUnlink.clear();
chunkStorageWriter->commit();
chunkStorageWriter->addBundle( info, savedId );
if ( verbose )
printf( "keep %s\n", i.c_str() );
indexKeptBundles++;
}
};
}
ZCollector::ZCollector( string const & storageDir, string const & password,
Config & configIn ):
ZBackupBase( storageDir, password, configIn ),
chunkStorageReader( config, encryptionkey, chunkIndex, getBundlesPath(),
config.runtime.cacheSize )
void BundleCollector::commit()
{
}
void ZCollector::gc()
{
ChunkIndex chunkReindex( encryptionkey, tmpMgr, getIndexPath(), true );
ChunkStorage::Writer chunkStorageWriter( config, encryptionkey, tmpMgr,
chunkReindex, getBundlesPath(), getIndexPath(), config.runtime.threads );
string fileName;
Dir::Entry entry;
BundleCollector collector;
collector.bundlesPath = getBundlesPath();
collector.chunkStorageReader = &this->chunkStorageReader;
collector.chunkStorageWriter = &chunkStorageWriter;
collector.verbose = false;
verbosePrintf( "Checking used chunks...\n" );
verbosePrintf( "Searching for backups...\n" );
vector< string > backups = BackupExchanger::findOrRebuild( getBackupsPath() );
for ( std::vector< string >::iterator it = backups.begin(); it != backups.end(); ++it )
for ( int i = filesToUnlink.size(); i--; )
{
string backup( Dir::addPath( getBackupsPath(), *it ) );
verbosePrintf( "Checking backup %s...\n", backup.c_str() );
BackupInfo backupInfo;
BackupFile::load( backup, encryptionkey, backupInfo );
string backupData;
BackupRestorer::restoreIterations( chunkStorageReader, backupInfo, backupData, &collector.usedChunkSet );
BackupRestorer::restore( chunkStorageReader, backupData, NULL, &collector.usedChunkSet );
unlink( filesToUnlink[i].c_str() );
}
verbosePrintf( "Checking bundles...\n" );
chunkIndex.loadIndex( collector );
collector.commit();
verbosePrintf( "Cleaning up...\n" );
string bundlesPath = getBundlesPath();
Dir::Listing bundleLst( bundlesPath );
while( bundleLst.getNext( entry ) )
{
const string dirPath = Dir::addPath( bundlesPath, entry.getFileName());
if ( entry.isDir() && Dir::isDirEmpty( dirPath ) )
{
Dir::remove( dirPath );
}
}
verbosePrintf( "Garbage collection complete\n" );
filesToUnlink.clear();
chunkStorageWriter->commit();
}

41
backup_collector.hh
View File

@ -7,15 +7,46 @@
#include "zbackup_base.hh"
#include "chunk_storage.hh"
class ZCollector : public ZBackupBase
#include <string>
#include <vector>
#include "bundle.hh"
#include "chunk_index.hh"
#include "backup_restorer.hh"
#include "backup_file.hh"
#include "backup_exchanger.hh"
#include "debug.hh"
using std::string;
class BundleCollector: public IndexProcessor
{
ChunkStorage::Reader chunkStorageReader;
private:
Bundle::Id savedId;
int totalChunks, usedChunks, indexTotalChunks, indexUsedChunks;
int indexModifiedBundles, indexKeptBundles, indexRemovedBundles;
bool indexModified;
vector< string > filesToUnlink;
public:
ZCollector( std::string const & storageDir, std::string const & password,
Config & configIn );
string bundlesPath;
bool verbose;
ChunkStorage::Reader *chunkStorageReader;
ChunkStorage::Writer *chunkStorageWriter;
BackupRestorer::ChunkSet usedChunkSet;
void gc();
void startIndex( string const & indexFn );
void finishIndex( string const & indexFn );
void startBundle( Bundle::Id const & bundleId );
void processChunk( ChunkId const & chunkId );
void finishBundle( Bundle::Id const & bundleId, BundleInfo const & info );
void commit();
};
#endif

302
zbackup.cc
View File

@ -1,308 +1,10 @@
// Copyright (c) 2012-2014 Konstantin Isakov <ikm@zbackup.org> and ZBackup contributors, see CONTRIBUTORS
// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE
#include "zbackup.hh"
#include "backup_creator.hh"
#include "backup_file.hh"
#include "backup_restorer.hh"
#include "zutils.hh"
#include "debug.hh"
#include "sha256.hh"
#include "backup_collector.hh"
#include "config.hh"
#include "utils.hh"
#include "version.hh"
using std::vector;
using std::bitset;
using std::iterator;
ZBackup::ZBackup( string const & storageDir, string const & password,
Config & configIn ):
ZBackupBase( storageDir, password, configIn ),
chunkStorageWriter( config, encryptionkey, tmpMgr, chunkIndex,
getBundlesPath(), getIndexPath(), config.runtime.threads )
{
}
void ZBackup::backupFromStdin( string const & outputFileName )
{
if ( isatty( fileno( stdin ) ) )
throw exWontReadFromTerminal();
if ( File::exists( outputFileName ) )
throw exWontOverwrite( outputFileName );
Sha256 sha256;
BackupCreator backupCreator( config, chunkIndex, chunkStorageWriter );
time_t startTime = time( 0 );
uint64_t totalDataSize = 0;
for ( ; ; )
{
size_t toRead = backupCreator.getInputBufferSize();
// dPrintf( "Reading up to %u bytes from stdin\n", toRead );
void * inputBuffer = backupCreator.getInputBuffer();
size_t rd = fread( inputBuffer, 1, toRead, stdin );
if ( !rd )
{
if ( feof( stdin ) )
{
dPrintf( "No more input on stdin\n" );
break;
}
else
throw exStdinError();
}
sha256.add( inputBuffer, rd );
backupCreator.handleMoreData( rd );
totalDataSize += rd;
}
// Finish up with the creator
backupCreator.finish();
string serialized;
backupCreator.getBackupData( serialized );
BackupInfo info;
info.set_sha256( sha256.finish() );
info.set_size( totalDataSize );
// Shrink the serialized data iteratively until it wouldn't shrink anymore
for ( ; ; )
{
BackupCreator backupCreator( config, chunkIndex, chunkStorageWriter );
char const * ptr = serialized.data();
size_t left = serialized.size();
while( left )
{
size_t bufferSize = backupCreator.getInputBufferSize();
size_t toCopy = bufferSize > left ? left : bufferSize;
memcpy( backupCreator.getInputBuffer(), ptr, toCopy );
backupCreator.handleMoreData( toCopy );
ptr += toCopy;
left -= toCopy;
}
backupCreator.finish();
string newGen;
backupCreator.getBackupData( newGen );
if ( newGen.size() < serialized.size() )
{
serialized.swap( newGen );
info.set_iterations( info.iterations() + 1 );
}
else
break;
}
dPrintf( "Iterations: %u\n", info.iterations() );
info.mutable_backup_data()->swap( serialized );
info.set_time( time( 0 ) - startTime );
// Commit the bundles to the disk before creating the final output file
chunkStorageWriter.commit();
// Now save the resulting BackupInfo
sptr< TemporaryFile > tmpFile = tmpMgr.makeTemporaryFile();
BackupFile::save( tmpFile->getFileName(), encryptionkey, info );
tmpFile->moveOverTo( outputFileName );
}
ZRestore::ZRestore( string const & storageDir, string const & password,
Config & configIn ):
ZBackupBase( storageDir, password, configIn ),
chunkStorageReader( config, encryptionkey, chunkIndex, getBundlesPath(),
config.runtime.cacheSize )
{
}
void ZRestore::restoreToStdin( string const & inputFileName )
{
if ( isatty( fileno( stdout ) ) )
throw exWontWriteToTerminal();
BackupInfo backupInfo;
BackupFile::load( inputFileName, encryptionkey, backupInfo );
string backupData;
// Perform the iterations needed to get to the actual user backup data
BackupRestorer::restoreIterations( chunkStorageReader, backupInfo, backupData, NULL );
struct StdoutWriter: public DataSink
{
Sha256 sha256;
virtual void saveData( void const * data, size_t size )
{
sha256.add( data, size );
if ( fwrite( data, size, 1, stdout ) != 1 )
throw exStdoutError();
}
} stdoutWriter;
BackupRestorer::restore( chunkStorageReader, backupData, &stdoutWriter, NULL );
if ( stdoutWriter.sha256.finish() != backupInfo.sha256() )
throw exChecksumError();
}
ZExchange::ZExchange( string const & srcStorageDir, string const & srcPassword,
string const & dstStorageDir, string const & dstPassword,
Config & configIn ):
srcZBackupBase( srcStorageDir, srcPassword, configIn, true ),
dstZBackupBase( dstStorageDir, dstPassword, configIn, true ),
config( configIn )
{
}
void ZExchange::exchange()
{
vector< BackupExchanger::PendingExchangeRename > pendingExchangeRenames;
if ( config.runtime.exchange.test( BackupExchanger::bundles ) )
{
verbosePrintf( "Searching for bundles...\n" );
vector< string > bundles = BackupExchanger::findOrRebuild(
srcZBackupBase.getBundlesPath(), dstZBackupBase.getBundlesPath() );
for ( std::vector< string >::iterator it = bundles.begin(); it != bundles.end(); ++it )
{
verbosePrintf( "Processing bundle file %s... ", it->c_str() );
string outputFileName ( Dir::addPath( dstZBackupBase.getBundlesPath(), *it ) );
if ( !File::exists( outputFileName ) )
{
sptr< Bundle::Reader > reader = new Bundle::Reader( Dir::addPath (
srcZBackupBase.getBundlesPath(), *it ), srcZBackupBase.encryptionkey, true );
sptr< Bundle::Creator > creator = new Bundle::Creator;
sptr< TemporaryFile > bundleTempFile = dstZBackupBase.tmpMgr.makeTemporaryFile();
creator->write( bundleTempFile->getFileName(), dstZBackupBase.encryptionkey, *reader );
if ( creator.get() && reader.get() )
{
creator.reset();
reader.reset();
pendingExchangeRenames.push_back( BackupExchanger::PendingExchangeRename(
bundleTempFile, outputFileName ) );
verbosePrintf( "done.\n" );
}
}
else
{
verbosePrintf( "file exists - skipped.\n" );
}
}
verbosePrintf( "Bundle exchange completed.\n" );
}
if ( config.runtime.exchange.test( BackupExchanger::index ) )
{
verbosePrintf( "Searching for indicies...\n" );
vector< string > indicies = BackupExchanger::findOrRebuild(
srcZBackupBase.getIndexPath(), dstZBackupBase.getIndexPath() );
for ( std::vector< string >::iterator it = indicies.begin(); it != indicies.end(); ++it )
{
verbosePrintf( "Processing index file %s... ", it->c_str() );
string outputFileName ( Dir::addPath( dstZBackupBase.getIndexPath(), *it ) );
if ( !File::exists( outputFileName ) )
{
sptr< IndexFile::Reader > reader = new IndexFile::Reader( srcZBackupBase.encryptionkey,
Dir::addPath( srcZBackupBase.getIndexPath(), *it ) );
sptr< TemporaryFile > indexTempFile = dstZBackupBase.tmpMgr.makeTemporaryFile();
sptr< IndexFile::Writer > writer = new IndexFile::Writer( dstZBackupBase.encryptionkey,
indexTempFile->getFileName() );
BundleInfo bundleInfo;
Bundle::Id bundleId;
while( reader->readNextRecord( bundleInfo, bundleId ) )
{
writer->add( bundleInfo, bundleId );
}
if ( writer.get() && reader.get() )
{
writer.reset();
reader.reset();
pendingExchangeRenames.push_back( BackupExchanger::PendingExchangeRename(
indexTempFile, outputFileName ) );
verbosePrintf( "done.\n" );
}
}
else
{
verbosePrintf( "file exists - skipped.\n" );
}
}
verbosePrintf( "Index exchange completed.\n" );
}
if ( config.runtime.exchange.test( BackupExchanger::backups ) )
{
BackupInfo backupInfo;
verbosePrintf( "Searching for backups...\n" );
vector< string > backups = BackupExchanger::findOrRebuild(
srcZBackupBase.getBackupsPath(), dstZBackupBase.getBackupsPath() );
for ( std::vector< string >::iterator it = backups.begin(); it != backups.end(); ++it )
{
verbosePrintf( "Processing backup file %s... ", it->c_str() );
string outputFileName ( Dir::addPath( dstZBackupBase.getBackupsPath(), *it ) );
if ( !File::exists( outputFileName ) )
{
BackupFile::load( Dir::addPath( srcZBackupBase.getBackupsPath(), *it ),
srcZBackupBase.encryptionkey, backupInfo );
sptr< TemporaryFile > tmpFile = dstZBackupBase.tmpMgr.makeTemporaryFile();
BackupFile::save( tmpFile->getFileName(), dstZBackupBase.encryptionkey,
backupInfo );
pendingExchangeRenames.push_back( BackupExchanger::PendingExchangeRename(
tmpFile, outputFileName ) );
verbosePrintf( "done.\n" );
}
else
{
verbosePrintf( "file exists - skipped.\n" );
}
}
verbosePrintf( "Backup exchange completed.\n" );
}
if ( pendingExchangeRenames.size() > 0 )
{
verbosePrintf( "Moving files from temp directory to appropriate places... " );
for ( size_t x = pendingExchangeRenames.size(); x--; )
{
BackupExchanger::PendingExchangeRename & r = pendingExchangeRenames[ x ];
r.first->moveOverTo( r.second );
if ( r.first.get() )
{
r.first.reset();
}
}
pendingExchangeRenames.clear();
verbosePrintf( "done.\n" );
}
}
#include "utils.hh"
DEF_EX( exSpecifyTwoKeys, "Specify password flag (--non-encrypted or --password-file)"
" for import/export/passwd operation twice (first for source and second for destination)", std::exception )

368
zutils.cc Normal file
View File

@ -0,0 +1,368 @@
// Copyright (c) 2012-2014 Konstantin Isakov <ikm@zbackup.org> and ZBackup contributors, see CONTRIBUTORS
// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE
#include "zutils.hh"
#include "backup_creator.hh"
#include "sha256.hh"
#include "backup_collector.hh"
using std::vector;
using std::bitset;
using std::iterator;
ZBackup::ZBackup( string const & storageDir, string const & password,
Config & configIn ):
ZBackupBase( storageDir, password, configIn ),
chunkStorageWriter( config, encryptionkey, tmpMgr, chunkIndex,
getBundlesPath(), getIndexPath(), config.runtime.threads )
{
}
void ZBackup::backupFromStdin( string const & outputFileName )
{
if ( isatty( fileno( stdin ) ) )
throw exWontReadFromTerminal();
if ( File::exists( outputFileName ) )
throw exWontOverwrite( outputFileName );
Sha256 sha256;
BackupCreator backupCreator( config, chunkIndex, chunkStorageWriter );
time_t startTime = time( 0 );
uint64_t totalDataSize = 0;
for ( ; ; )
{
size_t toRead = backupCreator.getInputBufferSize();
// dPrintf( "Reading up to %u bytes from stdin\n", toRead );
void * inputBuffer = backupCreator.getInputBuffer();
size_t rd = fread( inputBuffer, 1, toRead, stdin );
if ( !rd )
{
if ( feof( stdin ) )
{
dPrintf( "No more input on stdin\n" );
break;
}
else
throw exStdinError();
}
sha256.add( inputBuffer, rd );
backupCreator.handleMoreData( rd );
totalDataSize += rd;
}
// Finish up with the creator
backupCreator.finish();
string serialized;
backupCreator.getBackupData( serialized );
BackupInfo info;
info.set_sha256( sha256.finish() );
info.set_size( totalDataSize );
// Shrink the serialized data iteratively until it wouldn't shrink anymore
for ( ; ; )
{
BackupCreator backupCreator( config, chunkIndex, chunkStorageWriter );
char const * ptr = serialized.data();
size_t left = serialized.size();
while( left )
{
size_t bufferSize = backupCreator.getInputBufferSize();
size_t toCopy = bufferSize > left ? left : bufferSize;
memcpy( backupCreator.getInputBuffer(), ptr, toCopy );
backupCreator.handleMoreData( toCopy );
ptr += toCopy;
left -= toCopy;
}
backupCreator.finish();
string newGen;
backupCreator.getBackupData( newGen );
if ( newGen.size() < serialized.size() )
{
serialized.swap( newGen );
info.set_iterations( info.iterations() + 1 );
}
else
break;
}
dPrintf( "Iterations: %u\n", info.iterations() );
info.mutable_backup_data()->swap( serialized );
info.set_time( time( 0 ) - startTime );
// Commit the bundles to the disk before creating the final output file
chunkStorageWriter.commit();
// Now save the resulting BackupInfo
sptr< TemporaryFile > tmpFile = tmpMgr.makeTemporaryFile();
BackupFile::save( tmpFile->getFileName(), encryptionkey, info );
tmpFile->moveOverTo( outputFileName );
}
ZRestore::ZRestore( string const & storageDir, string const & password,
Config & configIn ):
ZBackupBase( storageDir, password, configIn ),
chunkStorageReader( config, encryptionkey, chunkIndex, getBundlesPath(),
config.runtime.cacheSize )
{
}
void ZRestore::restoreToStdin( string const & inputFileName )
{
if ( isatty( fileno( stdout ) ) )
throw exWontWriteToTerminal();
BackupInfo backupInfo;
BackupFile::load( inputFileName, encryptionkey, backupInfo );
string backupData;
// Perform the iterations needed to get to the actual user backup data
BackupRestorer::restoreIterations( chunkStorageReader, backupInfo, backupData, NULL );
struct StdoutWriter: public DataSink
{
Sha256 sha256;
virtual void saveData( void const * data, size_t size )
{
sha256.add( data, size );
if ( fwrite( data, size, 1, stdout ) != 1 )
throw exStdoutError();
}
} stdoutWriter;
BackupRestorer::restore( chunkStorageReader, backupData, &stdoutWriter, NULL );
if ( stdoutWriter.sha256.finish() != backupInfo.sha256() )
throw exChecksumError();
}
ZExchange::ZExchange( string const & srcStorageDir, string const & srcPassword,
string const & dstStorageDir, string const & dstPassword,
Config & configIn ):
srcZBackupBase( srcStorageDir, srcPassword, configIn, true ),
dstZBackupBase( dstStorageDir, dstPassword, configIn, true ),
config( configIn )
{
}
void ZExchange::exchange()
{
vector< BackupExchanger::PendingExchangeRename > pendingExchangeRenames;
if ( config.runtime.exchange.test( BackupExchanger::bundles ) )
{
verbosePrintf( "Searching for bundles...\n" );
vector< string > bundles = BackupExchanger::findOrRebuild(
srcZBackupBase.getBundlesPath(), dstZBackupBase.getBundlesPath() );
for ( std::vector< string >::iterator it = bundles.begin(); it != bundles.end(); ++it )
{
verbosePrintf( "Processing bundle file %s... ", it->c_str() );
string outputFileName ( Dir::addPath( dstZBackupBase.getBundlesPath(), *it ) );
if ( !File::exists( outputFileName ) )
{
sptr< Bundle::Reader > reader = new Bundle::Reader( Dir::addPath (
srcZBackupBase.getBundlesPath(), *it ), srcZBackupBase.encryptionkey, true );
sptr< Bundle::Creator > creator = new Bundle::Creator;
sptr< TemporaryFile > bundleTempFile = dstZBackupBase.tmpMgr.makeTemporaryFile();
creator->write( bundleTempFile->getFileName(), dstZBackupBase.encryptionkey, *reader );
if ( creator.get() && reader.get() )
{
creator.reset();
reader.reset();
pendingExchangeRenames.push_back( BackupExchanger::PendingExchangeRename(
bundleTempFile, outputFileName ) );
verbosePrintf( "done.\n" );
}
}
else
{
verbosePrintf( "file exists - skipped.\n" );
}
}
verbosePrintf( "Bundle exchange completed.\n" );
}
if ( config.runtime.exchange.test( BackupExchanger::index ) )
{
verbosePrintf( "Searching for indicies...\n" );
vector< string > indicies = BackupExchanger::findOrRebuild(
srcZBackupBase.getIndexPath(), dstZBackupBase.getIndexPath() );
for ( std::vector< string >::iterator it = indicies.begin(); it != indicies.end(); ++it )
{
verbosePrintf( "Processing index file %s... ", it->c_str() );
string outputFileName ( Dir::addPath( dstZBackupBase.getIndexPath(), *it ) );
if ( !File::exists( outputFileName ) )
{
sptr< IndexFile::Reader > reader = new IndexFile::Reader( srcZBackupBase.encryptionkey,
Dir::addPath( srcZBackupBase.getIndexPath(), *it ) );
sptr< TemporaryFile > indexTempFile = dstZBackupBase.tmpMgr.makeTemporaryFile();
sptr< IndexFile::Writer > writer = new IndexFile::Writer( dstZBackupBase.encryptionkey,
indexTempFile->getFileName() );
BundleInfo bundleInfo;
Bundle::Id bundleId;
while( reader->readNextRecord( bundleInfo, bundleId ) )
{
writer->add( bundleInfo, bundleId );
}
if ( writer.get() && reader.get() )
{
writer.reset();
reader.reset();
pendingExchangeRenames.push_back( BackupExchanger::PendingExchangeRename(
indexTempFile, outputFileName ) );
verbosePrintf( "done.\n" );
}
}
else
{
verbosePrintf( "file exists - skipped.\n" );
}
}
verbosePrintf( "Index exchange completed.\n" );
}
if ( config.runtime.exchange.test( BackupExchanger::backups ) )
{
BackupInfo backupInfo;
verbosePrintf( "Searching for backups...\n" );
vector< string > backups = BackupExchanger::findOrRebuild(
srcZBackupBase.getBackupsPath(), dstZBackupBase.getBackupsPath() );
for ( std::vector< string >::iterator it = backups.begin(); it != backups.end(); ++it )
{
verbosePrintf( "Processing backup file %s... ", it->c_str() );
string outputFileName ( Dir::addPath( dstZBackupBase.getBackupsPath(), *it ) );
if ( !File::exists( outputFileName ) )
{
BackupFile::load( Dir::addPath( srcZBackupBase.getBackupsPath(), *it ),
srcZBackupBase.encryptionkey, backupInfo );
sptr< TemporaryFile > tmpFile = dstZBackupBase.tmpMgr.makeTemporaryFile();
BackupFile::save( tmpFile->getFileName(), dstZBackupBase.encryptionkey,
backupInfo );
pendingExchangeRenames.push_back( BackupExchanger::PendingExchangeRename(
tmpFile, outputFileName ) );
verbosePrintf( "done.\n" );
}
else
{
verbosePrintf( "file exists - skipped.\n" );
}
}
verbosePrintf( "Backup exchange completed.\n" );
}
if ( pendingExchangeRenames.size() > 0 )
{
verbosePrintf( "Moving files from temp directory to appropriate places... " );
for ( size_t x = pendingExchangeRenames.size(); x--; )
{
BackupExchanger::PendingExchangeRename & r = pendingExchangeRenames[ x ];
r.first->moveOverTo( r.second );
if ( r.first.get() )
{
r.first.reset();
}
}
pendingExchangeRenames.clear();
verbosePrintf( "done.\n" );
}
}
ZCollector::ZCollector( string const & storageDir, string const & password,
Config & configIn ):
ZBackupBase( storageDir, password, configIn ),
chunkStorageReader( config, encryptionkey, chunkIndex, getBundlesPath(),
config.runtime.cacheSize )
{
}
void ZCollector::gc()
{
ChunkIndex chunkReindex( encryptionkey, tmpMgr, getIndexPath(), true );
ChunkStorage::Writer chunkStorageWriter( config, encryptionkey, tmpMgr,
chunkReindex, getBundlesPath(), getIndexPath(), config.runtime.threads );
string fileName;
Dir::Entry entry;
BundleCollector collector;
collector.bundlesPath = getBundlesPath();
collector.chunkStorageReader = &this->chunkStorageReader;
collector.chunkStorageWriter = &chunkStorageWriter;
collector.verbose = false;
verbosePrintf( "Checking used chunks...\n" );
verbosePrintf( "Searching for backups...\n" );
vector< string > backups = BackupExchanger::findOrRebuild( getBackupsPath() );
for ( std::vector< string >::iterator it = backups.begin(); it != backups.end(); ++it )
{
string backup( Dir::addPath( getBackupsPath(), *it ) );
verbosePrintf( "Checking backup %s...\n", backup.c_str() );
BackupInfo backupInfo;
BackupFile::load( backup, encryptionkey, backupInfo );
string backupData;
BackupRestorer::restoreIterations( chunkStorageReader, backupInfo, backupData, &collector.usedChunkSet );
BackupRestorer::restore( chunkStorageReader, backupData, NULL, &collector.usedChunkSet );
}
verbosePrintf( "Checking bundles...\n" );
chunkIndex.loadIndex( collector );
collector.commit();
verbosePrintf( "Cleaning up...\n" );
string bundlesPath = getBundlesPath();
Dir::Listing bundleLst( bundlesPath );
while( bundleLst.getNext( entry ) )
{
const string dirPath = Dir::addPath( bundlesPath, entry.getFileName());
if ( entry.isDir() && Dir::isDirEmpty( dirPath ) )
{
Dir::remove( dirPath );
}
}
verbosePrintf( "Garbage collection complete\n" );
}

30
zbackup.hh → zutils.hh
View File

@ -1,16 +1,12 @@
// Copyright (c) 2012-2014 Konstantin Isakov <ikm@zbackup.org> and ZBackup contributors, see CONTRIBUTORS
// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE
#ifndef ZBACKUP_HH_INCLUDED
#define ZBACKUP_HH_INCLUDED
#ifndef ZUTILS_HH_INCLUDED
#define ZUTILS_HH_INCLUDED
#include "chunk_storage.hh"
#include "zbackup_base.hh"
using std::string;
using std::vector;
using std::bitset;
class ZBackup: public ZBackupBase
{
ChunkStorage::Writer chunkStorageWriter;
@ -35,17 +31,6 @@ public:
void restoreToStdin( string const & inputFileName );
};
class ZCollect: public ZBackupBase
{
ChunkStorage::Reader chunkStorageReader;
public:
ZCollect( string const & storageDir, string const & password,
Config & configIn );
void gc();
};
class ZExchange
{
ZBackupBase srcZBackupBase;
@ -62,4 +47,15 @@ public:
void exchange();
};
class ZCollector : public ZBackupBase
{
ChunkStorage::Reader chunkStorageReader;
public:
ZCollector( std::string const & storageDir, std::string const & password,
Config & configIn );
void gc();
};
#endif