2014-12-11 10:50:15 +03:00
|
|
|
// Copyright (c) 2012-2014 Konstantin Isakov <ikm@zbackup.org> and ZBackup contributors, see CONTRIBUTORS
|
2014-10-10 13:50:55 +04:00
|
|
|
// Part of ZBackup. Licensed under GNU GPLv2 or later + OpenSSL, see LICENSE
|
2013-07-18 21:33:25 +04:00
|
|
|
|
|
|
|
#include <stdint.h>
|
|
|
|
|
|
|
|
#include "bundle.hh"
|
|
|
|
#include "check.hh"
|
|
|
|
#include "dir.hh"
|
|
|
|
#include "encryption.hh"
|
|
|
|
#include "hex.hh"
|
|
|
|
#include "message.hh"
|
2014-10-01 23:01:59 +04:00
|
|
|
#include "adler32.hh"
|
2013-10-03 05:51:24 +04:00
|
|
|
#include "compression.hh"
|
2013-07-18 21:33:25 +04:00
|
|
|
|
|
|
|
namespace Bundle {
|
|
|
|
|
|
|
|
enum
|
|
|
|
{
|
2013-10-09 01:29:05 +04:00
|
|
|
FileFormatVersion = 1,
|
|
|
|
|
|
|
|
// This means, we don't use LZMA in this file.
|
|
|
|
FileFormatVersionNotLZMA,
|
|
|
|
|
|
|
|
// <- add more versions here
|
|
|
|
|
|
|
|
// This is the first version, we do not support.
|
|
|
|
FileFormatVersionFirstUnsupported
|
2013-07-18 21:33:25 +04:00
|
|
|
};
|
|
|
|
|
|
|
|
void Creator::addChunk( string const & id, void const * data, size_t size )
|
|
|
|
{
|
|
|
|
BundleInfo_ChunkRecord * record = info.add_chunk_record();
|
|
|
|
record->set_id( id );
|
|
|
|
record->set_size( size );
|
|
|
|
payload.append( ( char const * ) data, size );
|
|
|
|
}
|
|
|
|
|
2014-10-01 13:43:46 +04:00
|
|
|
void Creator::write( std::string const & fileName, EncryptionKey const & key,
|
|
|
|
Reader & reader )
|
|
|
|
{
|
|
|
|
EncryptedFile::OutputStream os( fileName.c_str(), key, Encryption::ZeroIv );
|
|
|
|
|
|
|
|
os.writeRandomIv();
|
|
|
|
|
2014-12-09 17:42:41 +03:00
|
|
|
Message::serialize( reader.getBundleHeader(), os );
|
2014-10-01 13:43:46 +04:00
|
|
|
|
|
|
|
Message::serialize( reader.getBundleInfo(), os );
|
|
|
|
os.writeAdler32();
|
|
|
|
|
2014-10-01 23:01:59 +04:00
|
|
|
void * bufPrev = NULL;
|
|
|
|
const void * bufCurr = NULL;
|
|
|
|
int sizePrev = 0, sizeCurr = 0;
|
|
|
|
bool readPrev = false, readCurr = false;
|
|
|
|
|
|
|
|
for ( ; ; )
|
|
|
|
{
|
2015-07-31 10:52:11 +03:00
|
|
|
bool readCurr = reader.is->Next( &bufCurr, &sizeCurr );
|
2014-10-01 23:01:59 +04:00
|
|
|
|
|
|
|
if ( readCurr )
|
|
|
|
{
|
|
|
|
if ( readPrev )
|
|
|
|
{
|
|
|
|
os.write( bufPrev, sizePrev );
|
|
|
|
|
|
|
|
readPrev = readCurr;
|
|
|
|
free( bufPrev );
|
|
|
|
bufPrev = malloc( sizeCurr );
|
|
|
|
memcpy( bufPrev, bufCurr, sizeCurr );
|
|
|
|
sizePrev = sizeCurr;
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
readPrev = readCurr;
|
|
|
|
bufPrev = malloc( sizeCurr );
|
|
|
|
memcpy( bufPrev, bufCurr, sizeCurr );
|
|
|
|
sizePrev = sizeCurr;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
if ( readPrev )
|
|
|
|
{
|
|
|
|
sizePrev -= sizeof( Adler32::Value );
|
|
|
|
os.write( bufPrev, sizePrev );
|
|
|
|
os.writeAdler32();
|
|
|
|
free ( bufPrev );
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2015-07-31 10:52:11 +03:00
|
|
|
|
|
|
|
if ( reader.is.get() )
|
|
|
|
reader.is.reset();
|
2014-10-01 13:43:46 +04:00
|
|
|
}
|
|
|
|
|
2015-01-23 14:32:40 +03:00
|
|
|
void Creator::write( Config const & config, std::string const & fileName,
|
|
|
|
EncryptionKey const & key )
|
2013-07-18 21:33:25 +04:00
|
|
|
{
|
|
|
|
EncryptedFile::OutputStream os( fileName.c_str(), key, Encryption::ZeroIv );
|
|
|
|
|
|
|
|
os.writeRandomIv();
|
|
|
|
|
2013-10-08 03:43:33 +04:00
|
|
|
BundleFileHeader header;
|
2013-10-09 01:29:05 +04:00
|
|
|
|
2015-01-23 14:32:40 +03:00
|
|
|
const_sptr<Compression::CompressionMethod> compression =
|
|
|
|
Compression::CompressionMethod::selectedCompression;
|
2013-10-08 03:43:33 +04:00
|
|
|
header.set_compression_method( compression->getName() );
|
2013-10-09 01:29:05 +04:00
|
|
|
|
|
|
|
// The old code only support lzma, so we will bump up the version, if we're
|
|
|
|
// using lzma. This will make it fail cleanly.
|
|
|
|
if ( compression->getName() == "lzma" )
|
|
|
|
header.set_version( FileFormatVersion );
|
|
|
|
else
|
|
|
|
header.set_version( FileFormatVersionNotLZMA );
|
|
|
|
|
2013-07-18 21:33:25 +04:00
|
|
|
Message::serialize( header, os );
|
|
|
|
|
|
|
|
Message::serialize( info, os );
|
|
|
|
os.writeAdler32();
|
|
|
|
|
|
|
|
// Compress
|
|
|
|
|
2015-01-23 14:32:40 +03:00
|
|
|
sptr<Compression::EnDecoder> encoder = compression->createEncoder(
|
|
|
|
config );
|
2013-07-18 21:33:25 +04:00
|
|
|
|
2013-10-03 05:51:24 +04:00
|
|
|
encoder->setInput( payload.data(), payload.size() );
|
2013-07-18 21:33:25 +04:00
|
|
|
|
|
|
|
for ( ; ; )
|
|
|
|
{
|
|
|
|
{
|
|
|
|
void * data;
|
|
|
|
int size;
|
|
|
|
if ( !os.Next( &data, &size ) )
|
|
|
|
{
|
2014-12-16 15:25:02 +03:00
|
|
|
encoder.reset();
|
2013-07-18 21:33:25 +04:00
|
|
|
throw exBundleWriteFailed();
|
|
|
|
}
|
|
|
|
if ( !size )
|
|
|
|
continue;
|
2013-10-03 05:51:24 +04:00
|
|
|
encoder->setOutput( data, size );
|
2013-07-18 21:33:25 +04:00
|
|
|
}
|
|
|
|
|
|
|
|
// Perform the compression
|
2015-01-23 14:32:40 +03:00
|
|
|
if ( encoder->process( true ) )
|
2013-07-18 21:33:25 +04:00
|
|
|
{
|
2013-10-03 05:51:24 +04:00
|
|
|
if ( encoder->getAvailableOutput() )
|
|
|
|
os.BackUp( encoder->getAvailableOutput() );
|
2013-07-18 21:33:25 +04:00
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2014-12-16 15:25:02 +03:00
|
|
|
encoder.reset();
|
|
|
|
|
2013-07-18 21:33:25 +04:00
|
|
|
os.writeAdler32();
|
|
|
|
}
|
|
|
|
|
2015-07-31 10:52:11 +03:00
|
|
|
Reader::Reader( string const & fileName, EncryptionKey const & key, bool keepStream )
|
2013-07-18 21:33:25 +04:00
|
|
|
{
|
2015-07-31 10:52:11 +03:00
|
|
|
is = new EncryptedFile::InputStream( fileName.c_str(), key, Encryption::ZeroIv );
|
|
|
|
is->consumeRandomIv();
|
2013-07-18 21:33:25 +04:00
|
|
|
|
2015-07-31 10:52:11 +03:00
|
|
|
Message::parse( header, *is );
|
2013-07-18 21:33:25 +04:00
|
|
|
|
2013-10-09 01:29:05 +04:00
|
|
|
if ( header.version() >= FileFormatVersionFirstUnsupported )
|
2013-07-18 21:33:25 +04:00
|
|
|
throw exUnsupportedVersion();
|
|
|
|
|
2015-07-31 10:52:11 +03:00
|
|
|
Message::parse( info, *is );
|
|
|
|
is->checkAdler32();
|
2013-07-18 21:33:25 +04:00
|
|
|
|
|
|
|
size_t payloadSize = 0;
|
|
|
|
for ( int x = info.chunk_record_size(); x--; )
|
|
|
|
payloadSize += info.chunk_record( x ).size();
|
|
|
|
|
|
|
|
payload.resize( payloadSize );
|
|
|
|
|
2015-07-31 10:52:11 +03:00
|
|
|
if ( keepStream )
|
2014-10-01 13:43:46 +04:00
|
|
|
return;
|
|
|
|
|
2014-12-09 15:21:51 +03:00
|
|
|
sptr<Compression::EnDecoder> decoder = Compression::CompressionMethod::findCompression(
|
|
|
|
header.compression_method() )->createDecoder();
|
2013-07-18 21:33:25 +04:00
|
|
|
|
2013-10-03 05:51:24 +04:00
|
|
|
decoder->setOutput( &payload[ 0 ], payload.size() );
|
2013-07-18 21:33:25 +04:00
|
|
|
|
|
|
|
for ( ; ; )
|
|
|
|
{
|
|
|
|
{
|
|
|
|
void const * data;
|
|
|
|
int size;
|
2015-07-31 10:52:11 +03:00
|
|
|
if ( !is->Next( &data, &size ) )
|
2013-07-18 21:33:25 +04:00
|
|
|
{
|
2014-12-16 15:25:02 +03:00
|
|
|
decoder.reset();
|
2013-07-18 21:33:25 +04:00
|
|
|
throw exBundleReadFailed();
|
|
|
|
}
|
|
|
|
if ( !size )
|
|
|
|
continue;
|
2013-10-03 05:51:24 +04:00
|
|
|
decoder->setInput( data, size );
|
2013-07-18 21:33:25 +04:00
|
|
|
}
|
|
|
|
|
2015-01-23 14:32:40 +03:00
|
|
|
if ( decoder->process( false ) )
|
|
|
|
{
|
2013-10-03 05:51:24 +04:00
|
|
|
if ( decoder->getAvailableInput() )
|
2015-07-31 10:52:11 +03:00
|
|
|
is->BackUp( decoder->getAvailableInput() );
|
2013-07-18 21:33:25 +04:00
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
2013-10-03 05:51:24 +04:00
|
|
|
if ( !decoder->getAvailableOutput() && decoder->getAvailableInput() )
|
2013-07-18 21:33:25 +04:00
|
|
|
{
|
|
|
|
// Apparently we have more data than we were expecting
|
2014-12-16 15:25:02 +03:00
|
|
|
decoder.reset();
|
2013-07-18 21:33:25 +04:00
|
|
|
throw exTooMuchData();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2014-12-16 15:25:02 +03:00
|
|
|
decoder.reset();
|
|
|
|
|
2015-07-31 10:52:11 +03:00
|
|
|
is->checkAdler32();
|
|
|
|
if ( is.get() )
|
|
|
|
is.reset();
|
2013-07-18 21:33:25 +04:00
|
|
|
|
|
|
|
// Populate the map
|
|
|
|
char const * next = payload.data();
|
|
|
|
for ( int x = 0, count = info.chunk_record_size(); x < count; ++x )
|
|
|
|
{
|
|
|
|
BundleInfo_ChunkRecord const & record = info.chunk_record( x );
|
|
|
|
pair< Chunks::iterator, bool > res =
|
|
|
|
chunks.insert( Chunks::value_type( record.id(),
|
|
|
|
Chunks::mapped_type( next,
|
|
|
|
record.size() ) ) );
|
|
|
|
if ( !res.second )
|
|
|
|
throw exDuplicateChunks(); // Duplicate key encountered
|
|
|
|
next += record.size();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
bool Reader::get( string const & chunkId, string & chunkData,
|
|
|
|
size_t & chunkDataSize )
|
|
|
|
{
|
|
|
|
Chunks::iterator i = chunks.find( chunkId );
|
|
|
|
if ( i != chunks.end() )
|
|
|
|
{
|
|
|
|
size_t sz = i->second.second;
|
|
|
|
if ( chunkData.size() < sz )
|
|
|
|
chunkData.resize( sz );
|
|
|
|
memcpy( &chunkData[ 0 ], i->second.first, sz );
|
|
|
|
|
|
|
|
chunkDataSize = sz;
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
else
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
string generateFileName( Id const & id, string const & bundlesDir,
|
|
|
|
bool createDirs )
|
|
|
|
{
|
|
|
|
string hex( toHex( ( unsigned char * ) &id, sizeof( id ) ) );
|
|
|
|
|
|
|
|
// TODO: make this scheme more flexible and allow it to scale, or at least
|
|
|
|
// be configurable
|
|
|
|
string level1( Dir::addPath( bundlesDir, hex.substr( 0, 2 ) ) );
|
|
|
|
|
|
|
|
if ( createDirs && !Dir::exists( level1 ) )
|
|
|
|
Dir::create( level1 );
|
|
|
|
|
|
|
|
return string( Dir::addPath( level1, hex ) );
|
|
|
|
}
|
|
|
|
}
|