vitalif
/
zbackup
mirror of https://github.com/vitalif/zbackup
1
0
Fork 0
Code Issues Releases Wiki Activity

Initial bundle support for backup exchange 

Adler32 calculation for destination is in development
master
Am1GO 2014-10-01 13:43:46 +04:00
parent 83bdb77c4d
commit 0a61cc8a17
5 changed files with 116 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
bundle.cc
View File

@ -7,7 +7,6 @@
#include "bundle.hh"
#include "check.hh"
#include "dir.hh"
#include "encrypted_file.hh"
#include "encryption.hh"
#include "hex.hh"
#include "message.hh"
@ -27,6 +26,26 @@ void Creator::addChunk( string const & id, void const * data, size_t size )
payload.append( ( char const * ) data, size );
}
void Creator::write( std::string const & fileName, EncryptionKey const & key,
Reader & reader )
{
EncryptedFile::OutputStream os( fileName.c_str(), key, Encryption::ZeroIv );
os.writeRandomIv();
FileHeader header;
header.set_version( FileFormatVersion );
Message::serialize( header, os );
Message::serialize( reader.getBundleInfo(), os );
os.writeAdler32();
const void * buf;
int size;
while ( reader.is.Next( &buf, &size ) )
os.write( buf, ( int ) size );
}
void Creator::write( std::string const & fileName, EncryptionKey const & key )
{
EncryptedFile::OutputStream os( fileName.c_str(), key, Encryption::ZeroIv );
@ -87,10 +106,9 @@ void Creator::write( std::string const & fileName, EncryptionKey const & key )
os.writeAdler32();
}
Reader::Reader( string const & fileName, EncryptionKey const & key )
Reader::Reader( string const & fileName, EncryptionKey const & key, bool prohibitProcessing ):
is( fileName.c_str(), key, Encryption::ZeroIv )
{
EncryptedFile::InputStream is( fileName.c_str(), key, Encryption::ZeroIv );
is.consumeRandomIv();
FileHeader header;
@ -99,7 +117,6 @@ Reader::Reader( string const & fileName, EncryptionKey const & key )
if ( header.version() != FileFormatVersion )
throw exUnsupportedVersion();
BundleInfo info;
Message::parse( info, is );
is.checkAdler32();
@ -109,6 +126,9 @@ Reader::Reader( string const & fileName, EncryptionKey const & key )
payload.resize( payloadSize );
if ( prohibitProcessing )
return;
lzma_stream strm = LZMA_STREAM_INIT;
lzma_ret ret;

59
bundle.hh
View File

@ -16,6 +16,7 @@
#include "nocopy.hh"
#include "static_assert.hh"
#include "zbackup.pb.h"
#include "encrypted_file.hh"
namespace Bundle {
@ -44,6 +45,38 @@ struct Id
STATIC_ASSERT( sizeof( Id ) == IdSize );
/// Reads the bundle and allows accessing chunks
class Reader: NoCopy
{
BundleInfo info;
/// Unpacked payload
string payload;
/// Maps chunk id blob to its contents and size
typedef map< string, pair< char const *, size_t > > Chunks;
Chunks chunks;
public:
DEF_EX( Ex, "Bundle reader exception", std::exception )
DEF_EX( exBundleReadFailed, "Bundle read failed", Ex )
DEF_EX( exUnsupportedVersion, "Unsupported version of the index file format", Ex )
DEF_EX( exTooMuchData, "More data than expected in a bundle", Ex )
DEF_EX( exDuplicateChunks, "Chunks with the same id found in a bundle", Ex )
Reader( string const & fileName, EncryptionKey const & key,
bool prohibitProcessing = false );
/// Reads the chunk into chunkData and returns true, or returns false if there
/// was no such chunk in the bundle. chunkData may be enlarged but won't
/// be shrunk. The size of the actual chunk would be stored in chunkDataSize
bool get( string const & chunkId, string & chunkData, size_t & chunkDataSize );
BundleInfo getBundleInfo()
{ return info; }
string getPayload()
{ return payload; }
EncryptedFile::InputStream is;
};
/// Creates a bundle by adding chunks to it until it's full, then compressing
/// it and writing out to disk
class Creator
@ -66,36 +99,14 @@ public:
/// time-consuming - calling this function from a worker thread could be
/// warranted
void write( string const & fileName, EncryptionKey const & );
void write( string const & fileName, EncryptionKey const &,
Bundle::Reader & reader );
/// Returns the current BundleInfo record - this is used for index files
BundleInfo const & getCurrentBundleInfo() const
{ return info; }
};
/// Reads the bundle and allows accessing chunks
class Reader: NoCopy
{
/// Unpacked payload
string payload;
/// Maps chunk id blob to its contents and size
typedef map< string, pair< char const *, size_t > > Chunks;
Chunks chunks;
public:
DEF_EX( Ex, "Bundle reader exception", std::exception )
DEF_EX( exBundleReadFailed, "Bundle read failed", Ex )
DEF_EX( exUnsupportedVersion, "Unsupported version of the index file format", Ex )
DEF_EX( exTooMuchData, "More data than expected in a bundle", Ex )
DEF_EX( exDuplicateChunks, "Chunks with the same id found in a bundle", Ex )
Reader( string const & fileName, EncryptionKey const & );
/// Reads the chunk into chunkData and returns true, or returns false if there
/// was no such chunk in the bundle. chunkData may be enlarged but won't
/// be shrunk. The size of the actual chunk would be stored in chunkDataSize
bool get( string const & chunkId, string & chunkData, size_t & chunkDataSize );
};
/// Generates a full file name for a bundle with the given id. If createDirs
/// is true, any intermediate directories will be created if they don't exist
/// already

2
encrypted_file.cc
View File

@ -9,7 +9,6 @@
#include "endian.hh"
#include "page_size.hh"
#include "random.hh"
#include "debug.hh"
namespace EncryptedFile {
@ -23,7 +22,6 @@ InputStream::InputStream( char const * fileName, EncryptionKey const & key,
buffer( std::max( getPageSize(), ( unsigned ) BlockSize * 2 ) ),
fill( 0 ), remainder( 0 ), backedUp( false )
{
dPrintf( "encrypted fileName: %s\n", fileName );
if ( key.hasKey() )
{
memcpy( iv, iv_, sizeof( iv ) );

79
zbackup.cc
View File

@ -25,6 +25,7 @@
#include "storage_info_file.hh"
#include "zbackup.hh"
#include "index_file.hh"
#include "bundle.hh"
using std::vector;
using std::bitset;
@ -322,27 +323,32 @@ ZExchange::ZExchange( string const & srcStorageDir, string const & srcPassword,
void ZExchange::exchange( string const & srcPath, string const & dstPath,
bitset< BackupExchanger::Flags > const & exchange )
{
if ( exchange.test( BackupExchanger::backups ) )
if ( exchange.test( BackupExchanger::bundles ) )
{
BackupInfo backupInfo;
verbosePrintf( "Searching for bundles...\n" );
verbosePrintf( "Searching for backups...\n" );
vector< string > backups = BackupExchanger::recreateDirectories(
srcZBackupBase.getBackupsPath(), dstZBackupBase.getBackupsPath() );
vector< string > bundles = BackupExchanger::recreateDirectories(
srcZBackupBase.getBundlesPath(), dstZBackupBase.getBundlesPath() );
for ( std::vector< string >::iterator it = backups.begin(); it != backups.end(); ++it )
for ( std::vector< string >::iterator it = bundles.begin(); it != bundles.end(); ++it )
{
verbosePrintf( "Processing backup file %s... ", it->c_str() );
string outputFileName ( Dir::addPath( dstZBackupBase.getBackupsPath(), *it ) );
verbosePrintf( "Processing bundle file %s... ", it->c_str() );
string outputFileName ( Dir::addPath( dstZBackupBase.getBundlesPath(), *it ) );
if ( !File::exists( outputFileName ) )
{
BackupFile::load( Dir::addPath( srcZBackupBase.getBackupsPath(), *it ),
srcZBackupBase.encryptionkey, backupInfo );
sptr< TemporaryFile > tmpFile = dstZBackupBase.tmpMgr.makeTemporaryFile();
BackupFile::save( tmpFile->getFileName(), dstZBackupBase.encryptionkey,
backupInfo );
tmpFile->moveOverTo( outputFileName );
verbosePrintf( "done.\n" );
sptr< Bundle::Reader > reader = new Bundle::Reader( Dir::addPath (
srcZBackupBase.getBundlesPath(), *it ), srcZBackupBase.encryptionkey, true );
sptr< Bundle::Creator > creator = new Bundle::Creator;
sptr< TemporaryFile > bundleTempFile = dstZBackupBase.tmpMgr.makeTemporaryFile();
creator->write( bundleTempFile->getFileName(), dstZBackupBase.encryptionkey, *reader );
if ( creator.get() )
{
creator.reset();
bundleTempFile->moveOverTo( outputFileName );
bundleTempFile.reset();
verbosePrintf( "done.\n" );
}
}
else
{
@ -350,24 +356,16 @@ void ZExchange::exchange( string const & srcPath, string const & dstPath,
}
}
verbosePrintf( "Backup exchange completed.\n" );
}
if ( exchange.test( BackupExchanger::bundles ) )
{
verbosePrintf( "Searching for bundles...\n" );
verbosePrintf( "NOT IMPLEMENTED!\n" );
verbosePrintf( "Bundle exchange completed.\n" );
}
if ( exchange.test( BackupExchanger::index ) )
{
verbosePrintf( "Searching for indicies...\n" );
vector< string > backups = BackupExchanger::recreateDirectories(
vector< string > indicies = BackupExchanger::recreateDirectories(
srcZBackupBase.getIndexPath(), dstZBackupBase.getIndexPath() );
for ( std::vector< string >::iterator it = backups.begin(); it != backups.end(); ++it )
for ( std::vector< string >::iterator it = indicies.begin(); it != indicies.end(); ++it )
{
verbosePrintf( "Processing index file %s... ", it->c_str() );
string outputFileName ( Dir::addPath( dstZBackupBase.getIndexPath(), *it ) );
@ -403,6 +401,37 @@ void ZExchange::exchange( string const & srcPath, string const & dstPath,
verbosePrintf( "Index exchange completed.\n" );
}
if ( exchange.test( BackupExchanger::backups ) )
{
BackupInfo backupInfo;
verbosePrintf( "Searching for backups...\n" );
vector< string > backups = BackupExchanger::recreateDirectories(
srcZBackupBase.getBackupsPath(), dstZBackupBase.getBackupsPath() );
for ( std::vector< string >::iterator it = backups.begin(); it != backups.end(); ++it )
{
verbosePrintf( "Processing backup file %s... ", it->c_str() );
string outputFileName ( Dir::addPath( dstZBackupBase.getBackupsPath(), *it ) );
if ( !File::exists( outputFileName ) )
{
BackupFile::load( Dir::addPath( srcZBackupBase.getBackupsPath(), *it ),
srcZBackupBase.encryptionkey, backupInfo );
sptr< TemporaryFile > tmpFile = dstZBackupBase.tmpMgr.makeTemporaryFile();
BackupFile::save( tmpFile->getFileName(), dstZBackupBase.encryptionkey,
backupInfo );
tmpFile->moveOverTo( outputFileName );
verbosePrintf( "done.\n" );
}
else
{
verbosePrintf( "file exists - skipped.\n" );
}
}
verbosePrintf( "Backup exchange completed.\n" );
}
}
DEF_EX( exExchangeWithLessThanTwoKeys, "Specify password flag (--non-encrypted or --password-file)"

3
zbackup.hh
View File

@ -109,7 +109,8 @@ public:
bool prohibitChunkIndexLoading );
/// Exchanges the data between storages
void exchange( string const & srcFileName, string const & dstFileName, bitset< BackupExchanger::Flags > const & exchange );
void exchange( string const & srcFileName, string const & dstFileName,
bitset< BackupExchanger::Flags > const & exchange );
};
#endif