mirror of https://github.com/vitalif/zbackup
Index pseudo-GC implementation
parent
3ff24945a2
commit
a064d9a1d1
|
@ -7,7 +7,7 @@ using std::string;
|
||||||
|
|
||||||
void BundleCollector::startIndex( string const & indexFn )
|
void BundleCollector::startIndex( string const & indexFn )
|
||||||
{
|
{
|
||||||
indexModified = false;
|
indexModified = indexNecessary = false;
|
||||||
indexTotalChunks = indexUsedChunks = 0;
|
indexTotalChunks = indexUsedChunks = 0;
|
||||||
indexModifiedBundles = indexKeptBundles = indexRemovedBundles = 0;
|
indexModifiedBundles = indexKeptBundles = indexRemovedBundles = 0;
|
||||||
}
|
}
|
||||||
|
@ -25,6 +25,8 @@ void BundleCollector::finishIndex( string const & indexFn )
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
chunkStorageWriter->reset();
|
chunkStorageWriter->reset();
|
||||||
|
if ( indexGC && !indexNecessary )
|
||||||
|
filesToUnlink.push_back( indexFn );
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -37,10 +39,18 @@ void BundleCollector::startBundle( Bundle::Id const & bundleId )
|
||||||
|
|
||||||
void BundleCollector::processChunk( ChunkId const & chunkId )
|
void BundleCollector::processChunk( ChunkId const & chunkId )
|
||||||
{
|
{
|
||||||
|
if ( indexGC )
|
||||||
|
{
|
||||||
|
if ( overallChunkSet.find ( chunkId ) == overallChunkSet.end() )
|
||||||
|
overallChunkSet.insert( chunkId );
|
||||||
|
else
|
||||||
|
return;
|
||||||
|
}
|
||||||
totalChunks++;
|
totalChunks++;
|
||||||
if ( usedChunkSet.find( chunkId ) != usedChunkSet.end() )
|
if ( usedChunkSet.find( chunkId ) != usedChunkSet.end() )
|
||||||
{
|
{
|
||||||
usedChunks++;
|
usedChunks++;
|
||||||
|
indexNecessary = true;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -49,16 +59,16 @@ void BundleCollector::finishBundle( Bundle::Id const & bundleId, BundleInfo cons
|
||||||
string i = Bundle::generateFileName( savedId, "", false );
|
string i = Bundle::generateFileName( savedId, "", false );
|
||||||
indexTotalChunks += totalChunks;
|
indexTotalChunks += totalChunks;
|
||||||
indexUsedChunks += usedChunks;
|
indexUsedChunks += usedChunks;
|
||||||
if ( usedChunks == 0 )
|
if ( 0 == usedChunks && 0 != totalChunks )
|
||||||
{
|
{
|
||||||
verbosePrintf( "Deleting %s bundle\n", i.c_str() );
|
dPrintf( "Deleting %s bundle\n", i.c_str() );
|
||||||
filesToUnlink.push_back( Dir::addPath( bundlesPath, i ) );
|
filesToUnlink.push_back( Dir::addPath( bundlesPath, i ) );
|
||||||
indexModified = true;
|
indexModified = true;
|
||||||
indexRemovedBundles++;
|
indexRemovedBundles++;
|
||||||
}
|
}
|
||||||
else if ( usedChunks < totalChunks )
|
else if ( usedChunks < totalChunks )
|
||||||
{
|
{
|
||||||
verbosePrintf( "%s: used %d/%d chunks\n", i.c_str(), usedChunks, totalChunks );
|
dPrintf( "%s: used %d/%d chunks\n", i.c_str(), usedChunks, totalChunks );
|
||||||
filesToUnlink.push_back( Dir::addPath( bundlesPath, i ) );
|
filesToUnlink.push_back( Dir::addPath( bundlesPath, i ) );
|
||||||
indexModified = true;
|
indexModified = true;
|
||||||
// Copy used chunks to the new index
|
// Copy used chunks to the new index
|
||||||
|
@ -79,7 +89,7 @@ void BundleCollector::finishBundle( Bundle::Id const & bundleId, BundleInfo cons
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
chunkStorageWriter->addBundle( info, savedId );
|
chunkStorageWriter->addBundle( info, savedId );
|
||||||
verbosePrintf( "Keeping %s bundle\n", i.c_str() );
|
dPrintf( "Keeping %s bundle\n", i.c_str() );
|
||||||
indexKeptBundles++;
|
indexKeptBundles++;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -88,6 +98,7 @@ void BundleCollector::commit()
|
||||||
{
|
{
|
||||||
for ( int i = filesToUnlink.size(); i--; )
|
for ( int i = filesToUnlink.size(); i--; )
|
||||||
{
|
{
|
||||||
|
dPrintf( "Unlinking %s\n", filesToUnlink[i].c_str() );
|
||||||
unlink( filesToUnlink[i].c_str() );
|
unlink( filesToUnlink[i].c_str() );
|
||||||
}
|
}
|
||||||
filesToUnlink.clear();
|
filesToUnlink.clear();
|
||||||
|
|
|
@ -20,14 +20,16 @@ private:
|
||||||
Bundle::Id savedId;
|
Bundle::Id savedId;
|
||||||
int totalChunks, usedChunks, indexTotalChunks, indexUsedChunks;
|
int totalChunks, usedChunks, indexTotalChunks, indexUsedChunks;
|
||||||
int indexModifiedBundles, indexKeptBundles, indexRemovedBundles;
|
int indexModifiedBundles, indexKeptBundles, indexRemovedBundles;
|
||||||
bool indexModified;
|
bool indexModified, indexNecessary;
|
||||||
vector< string > filesToUnlink;
|
vector< string > filesToUnlink;
|
||||||
|
BackupRestorer::ChunkSet overallChunkSet;
|
||||||
|
|
||||||
public:
|
public:
|
||||||
string bundlesPath;
|
string bundlesPath;
|
||||||
ChunkStorage::Reader *chunkStorageReader;
|
ChunkStorage::Reader *chunkStorageReader;
|
||||||
ChunkStorage::Writer *chunkStorageWriter;
|
ChunkStorage::Writer *chunkStorageWriter;
|
||||||
BackupRestorer::ChunkSet usedChunkSet;
|
BackupRestorer::ChunkSet usedChunkSet;
|
||||||
|
bool indexGC;
|
||||||
|
|
||||||
void startIndex( string const & indexFn );
|
void startIndex( string const & indexFn );
|
||||||
|
|
||||||
|
|
|
@ -87,7 +87,7 @@ public:
|
||||||
DEF_EX( Ex, "Chunk index exception", std::exception )
|
DEF_EX( Ex, "Chunk index exception", std::exception )
|
||||||
DEF_EX( exIncorrectChunkIdSize, "Incorrect chunk id size encountered", Ex )
|
DEF_EX( exIncorrectChunkIdSize, "Incorrect chunk id size encountered", Ex )
|
||||||
|
|
||||||
ChunkIndex( EncryptionKey const &, TmpMgr &, string const & indexPath, bool prohibitChunkIndexLoading );
|
ChunkIndex( EncryptionKey const &, TmpMgr &, string const & indexPath, bool );
|
||||||
|
|
||||||
struct ChunkInfoInterface
|
struct ChunkInfoInterface
|
||||||
{
|
{
|
||||||
|
|
19
config.cc
19
config.cc
|
@ -123,6 +123,16 @@ void Config::prefillKeywords()
|
||||||
"No default value, you should specify it explicitly"
|
"No default value, you should specify it explicitly"
|
||||||
},
|
},
|
||||||
|
|
||||||
|
{
|
||||||
|
"gc-indexes",
|
||||||
|
Config::oRuntime_gcIndexes,
|
||||||
|
Config::Runtime,
|
||||||
|
"Purge duplicated indexes from repo during\n"
|
||||||
|
"garbage collection\n"
|
||||||
|
"Normally you would not need this\n"
|
||||||
|
"No value, specify to enable"
|
||||||
|
},
|
||||||
|
|
||||||
{ "", Config::oBadOption, Config::None }
|
{ "", Config::oBadOption, Config::None }
|
||||||
};
|
};
|
||||||
|
|
||||||
|
@ -456,6 +466,15 @@ bool Config::parseOrValidate( const string & option, const OptionType type,
|
||||||
/* NOTREACHED */
|
/* NOTREACHED */
|
||||||
break;
|
break;
|
||||||
|
|
||||||
|
case oRuntime_gcIndexes:
|
||||||
|
runtime.gcIndexes = true;
|
||||||
|
|
||||||
|
dPrintf( "runtime[gcIndexes] = true\n" );
|
||||||
|
|
||||||
|
return true;
|
||||||
|
/* NOTREACHED */
|
||||||
|
break;
|
||||||
|
|
||||||
case oBadOption:
|
case oBadOption:
|
||||||
default:
|
default:
|
||||||
return false;
|
return false;
|
||||||
|
|
|
@ -29,11 +29,13 @@ public:
|
||||||
size_t threads;
|
size_t threads;
|
||||||
size_t cacheSize;
|
size_t cacheSize;
|
||||||
bitset< BackupExchanger::Flags > exchange;
|
bitset< BackupExchanger::Flags > exchange;
|
||||||
|
bool gcIndexes;
|
||||||
|
|
||||||
// Default runtime config
|
// Default runtime config
|
||||||
RuntimeConfig():
|
RuntimeConfig():
|
||||||
threads( getNumberOfCpus() ),
|
threads( getNumberOfCpus() ),
|
||||||
cacheSize( 40 * 1024 * 1024 ) // 40 MB
|
cacheSize( 40 * 1024 * 1024 ), // 40 MB
|
||||||
|
gcIndexes ( false )
|
||||||
{
|
{
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
@ -58,6 +60,7 @@ public:
|
||||||
oRuntime_threads,
|
oRuntime_threads,
|
||||||
oRuntime_cacheSize,
|
oRuntime_cacheSize,
|
||||||
oRuntime_exchange,
|
oRuntime_exchange,
|
||||||
|
oRuntime_gcIndexes,
|
||||||
|
|
||||||
oDeprecated, oUnsupported
|
oDeprecated, oUnsupported
|
||||||
} OpCodes;
|
} OpCodes;
|
||||||
|
|
36
zbackup.cc
36
zbackup.cc
|
@ -280,42 +280,16 @@ invalid_option:
|
||||||
if ( strcmp( args[ 0 ], "gc" ) == 0 )
|
if ( strcmp( args[ 0 ], "gc" ) == 0 )
|
||||||
{
|
{
|
||||||
// Perform the garbage collection
|
// Perform the garbage collection
|
||||||
if ( args.size() < 2 || args.size() > 3 )
|
if ( args.size() != 2 )
|
||||||
{
|
{
|
||||||
fprintf( stderr, "Usage: %s %s [chunks|indexes] <storage path>\n",
|
fprintf( stderr, "Usage: %s %s <storage path>\n",
|
||||||
*argv, args[ 0 ] );
|
*argv, args[ 0 ] );
|
||||||
return EXIT_FAILURE;
|
return EXIT_FAILURE;
|
||||||
}
|
}
|
||||||
|
|
||||||
int fieldStorage = 1;
|
ZCollector zc( ZBackupBase::deriveStorageDirFromBackupsFile( args[ 1 ], true ),
|
||||||
int fieldAction = 2;
|
passwords[ 0 ], config );
|
||||||
|
zc.gc();
|
||||||
if ( args.size() == 3 )
|
|
||||||
{
|
|
||||||
fieldStorage = 2;
|
|
||||||
fieldAction = 1;
|
|
||||||
}
|
|
||||||
|
|
||||||
if ( args.size() > 2 && strcmp( args[ fieldAction ], "chunks" ) == 0 )
|
|
||||||
{
|
|
||||||
ZCollector zc( ZBackupBase::deriveStorageDirFromBackupsFile( args[ fieldStorage ], true ),
|
|
||||||
passwords[ 0 ], config );
|
|
||||||
zc.gcChunks();
|
|
||||||
}
|
|
||||||
else
|
|
||||||
if ( args.size() > 2 && strcmp( args[ fieldAction ], "indexes" ) == 0 )
|
|
||||||
{
|
|
||||||
ZCollector zc( ZBackupBase::deriveStorageDirFromBackupsFile( args[ fieldStorage ], true ),
|
|
||||||
passwords[ 0 ], config );
|
|
||||||
fprintf( stderr, "NOT IMPLEMENTED YET!\n" );
|
|
||||||
zc.gcIndexes();
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
ZCollector zc( ZBackupBase::deriveStorageDirFromBackupsFile( args[ fieldStorage ], true ),
|
|
||||||
passwords[ 0 ], config );
|
|
||||||
zc.gcChunks();
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
if ( strcmp( args[ 0 ], "passwd" ) == 0 )
|
if ( strcmp( args[ 0 ], "passwd" ) == 0 )
|
||||||
|
|
|
@ -307,7 +307,7 @@ ZCollector::ZCollector( string const & storageDir, string const & password,
|
||||||
{
|
{
|
||||||
}
|
}
|
||||||
|
|
||||||
void ZCollector::gcChunks()
|
void ZCollector::gc()
|
||||||
{
|
{
|
||||||
ChunkIndex chunkReindex( encryptionkey, tmpMgr, getIndexPath(), true );
|
ChunkIndex chunkReindex( encryptionkey, tmpMgr, getIndexPath(), true );
|
||||||
|
|
||||||
|
@ -322,6 +322,7 @@ void ZCollector::gcChunks()
|
||||||
collector.bundlesPath = getBundlesPath();
|
collector.bundlesPath = getBundlesPath();
|
||||||
collector.chunkStorageReader = &this->chunkStorageReader;
|
collector.chunkStorageReader = &this->chunkStorageReader;
|
||||||
collector.chunkStorageWriter = &chunkStorageWriter;
|
collector.chunkStorageWriter = &chunkStorageWriter;
|
||||||
|
collector.indexGC = config.runtime.gcIndexes;
|
||||||
|
|
||||||
verbosePrintf( "Checking used chunks...\n" );
|
verbosePrintf( "Checking used chunks...\n" );
|
||||||
|
|
||||||
|
@ -366,7 +367,3 @@ void ZCollector::gcChunks()
|
||||||
|
|
||||||
verbosePrintf( "Garbage collection complete\n" );
|
verbosePrintf( "Garbage collection complete\n" );
|
||||||
}
|
}
|
||||||
|
|
||||||
void ZCollector::gcIndexes()
|
|
||||||
{
|
|
||||||
}
|
|
||||||
|
|
|
@ -55,8 +55,7 @@ public:
|
||||||
ZCollector( std::string const & storageDir, std::string const & password,
|
ZCollector( std::string const & storageDir, std::string const & password,
|
||||||
Config & configIn );
|
Config & configIn );
|
||||||
|
|
||||||
void gcChunks();
|
void gc();
|
||||||
void gcIndexes();
|
|
||||||
};
|
};
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
|
|
Loading…
Reference in New Issue