diff --git a/backup_collector.cc b/backup_collector.cc index 5c99c4d..dcb9fb9 100644 --- a/backup_collector.cc +++ b/backup_collector.cc @@ -7,7 +7,7 @@ using std::string; void BundleCollector::startIndex( string const & indexFn ) { - indexModified = false; + indexModified = indexNecessary = false; indexTotalChunks = indexUsedChunks = 0; indexModifiedBundles = indexKeptBundles = indexRemovedBundles = 0; } @@ -25,6 +25,8 @@ void BundleCollector::finishIndex( string const & indexFn ) else { chunkStorageWriter->reset(); + if ( indexGC && !indexNecessary ) + filesToUnlink.push_back( indexFn ); } } @@ -37,10 +39,18 @@ void BundleCollector::startBundle( Bundle::Id const & bundleId ) void BundleCollector::processChunk( ChunkId const & chunkId ) { + if ( indexGC ) + { + if ( overallChunkSet.find ( chunkId ) == overallChunkSet.end() ) + overallChunkSet.insert( chunkId ); + else + return; + } totalChunks++; if ( usedChunkSet.find( chunkId ) != usedChunkSet.end() ) { usedChunks++; + indexNecessary = true; } } @@ -49,16 +59,16 @@ void BundleCollector::finishBundle( Bundle::Id const & bundleId, BundleInfo cons string i = Bundle::generateFileName( savedId, "", false ); indexTotalChunks += totalChunks; indexUsedChunks += usedChunks; - if ( usedChunks == 0 ) + if ( 0 == usedChunks && 0 != totalChunks ) { - verbosePrintf( "Deleting %s bundle\n", i.c_str() ); + dPrintf( "Deleting %s bundle\n", i.c_str() ); filesToUnlink.push_back( Dir::addPath( bundlesPath, i ) ); indexModified = true; indexRemovedBundles++; } else if ( usedChunks < totalChunks ) { - verbosePrintf( "%s: used %d/%d chunks\n", i.c_str(), usedChunks, totalChunks ); + dPrintf( "%s: used %d/%d chunks\n", i.c_str(), usedChunks, totalChunks ); filesToUnlink.push_back( Dir::addPath( bundlesPath, i ) ); indexModified = true; // Copy used chunks to the new index @@ -79,7 +89,7 @@ void BundleCollector::finishBundle( Bundle::Id const & bundleId, BundleInfo cons else { chunkStorageWriter->addBundle( info, savedId ); - verbosePrintf( "Keeping %s bundle\n", i.c_str() ); + dPrintf( "Keeping %s bundle\n", i.c_str() ); indexKeptBundles++; } } @@ -88,6 +98,7 @@ void BundleCollector::commit() { for ( int i = filesToUnlink.size(); i--; ) { + dPrintf( "Unlinking %s\n", filesToUnlink[i].c_str() ); unlink( filesToUnlink[i].c_str() ); } filesToUnlink.clear(); diff --git a/backup_collector.hh b/backup_collector.hh index 92b1d6c..3d6a50e 100644 --- a/backup_collector.hh +++ b/backup_collector.hh @@ -20,14 +20,16 @@ private: Bundle::Id savedId; int totalChunks, usedChunks, indexTotalChunks, indexUsedChunks; int indexModifiedBundles, indexKeptBundles, indexRemovedBundles; - bool indexModified; + bool indexModified, indexNecessary; vector< string > filesToUnlink; + BackupRestorer::ChunkSet overallChunkSet; public: string bundlesPath; ChunkStorage::Reader *chunkStorageReader; ChunkStorage::Writer *chunkStorageWriter; BackupRestorer::ChunkSet usedChunkSet; + bool indexGC; void startIndex( string const & indexFn ); diff --git a/chunk_index.hh b/chunk_index.hh index bd34a18..09108a4 100644 --- a/chunk_index.hh +++ b/chunk_index.hh @@ -87,7 +87,7 @@ public: DEF_EX( Ex, "Chunk index exception", std::exception ) DEF_EX( exIncorrectChunkIdSize, "Incorrect chunk id size encountered", Ex ) - ChunkIndex( EncryptionKey const &, TmpMgr &, string const & indexPath, bool prohibitChunkIndexLoading ); + ChunkIndex( EncryptionKey const &, TmpMgr &, string const & indexPath, bool ); struct ChunkInfoInterface { diff --git a/config.cc b/config.cc index cd37f00..4454a87 100644 --- a/config.cc +++ b/config.cc @@ -123,6 +123,16 @@ void Config::prefillKeywords() "No default value, you should specify it explicitly" }, + { + "gc-indexes", + Config::oRuntime_gcIndexes, + Config::Runtime, + "Purge duplicated indexes from repo during\n" + "garbage collection\n" + "Normally you would not need this\n" + "No value, specify to enable" + }, + { "", Config::oBadOption, Config::None } }; @@ -456,6 +466,15 @@ bool Config::parseOrValidate( const string & option, const OptionType type, /* NOTREACHED */ break; + case oRuntime_gcIndexes: + runtime.gcIndexes = true; + + dPrintf( "runtime[gcIndexes] = true\n" ); + + return true; + /* NOTREACHED */ + break; + case oBadOption: default: return false; diff --git a/config.hh b/config.hh index 871501d..0003008 100644 --- a/config.hh +++ b/config.hh @@ -29,11 +29,13 @@ public: size_t threads; size_t cacheSize; bitset< BackupExchanger::Flags > exchange; + bool gcIndexes; // Default runtime config RuntimeConfig(): threads( getNumberOfCpus() ), - cacheSize( 40 * 1024 * 1024 ) // 40 MB + cacheSize( 40 * 1024 * 1024 ), // 40 MB + gcIndexes ( false ) { } }; @@ -58,6 +60,7 @@ public: oRuntime_threads, oRuntime_cacheSize, oRuntime_exchange, + oRuntime_gcIndexes, oDeprecated, oUnsupported } OpCodes; diff --git a/zbackup.cc b/zbackup.cc index 1180977..59a7eb2 100644 --- a/zbackup.cc +++ b/zbackup.cc @@ -280,42 +280,16 @@ invalid_option: if ( strcmp( args[ 0 ], "gc" ) == 0 ) { // Perform the garbage collection - if ( args.size() < 2 || args.size() > 3 ) + if ( args.size() != 2 ) { - fprintf( stderr, "Usage: %s %s [chunks|indexes] \n", + fprintf( stderr, "Usage: %s %s \n", *argv, args[ 0 ] ); return EXIT_FAILURE; } - int fieldStorage = 1; - int fieldAction = 2; - - if ( args.size() == 3 ) - { - fieldStorage = 2; - fieldAction = 1; - } - - if ( args.size() > 2 && strcmp( args[ fieldAction ], "chunks" ) == 0 ) - { - ZCollector zc( ZBackupBase::deriveStorageDirFromBackupsFile( args[ fieldStorage ], true ), - passwords[ 0 ], config ); - zc.gcChunks(); - } - else - if ( args.size() > 2 && strcmp( args[ fieldAction ], "indexes" ) == 0 ) - { - ZCollector zc( ZBackupBase::deriveStorageDirFromBackupsFile( args[ fieldStorage ], true ), - passwords[ 0 ], config ); - fprintf( stderr, "NOT IMPLEMENTED YET!\n" ); - zc.gcIndexes(); - } - else - { - ZCollector zc( ZBackupBase::deriveStorageDirFromBackupsFile( args[ fieldStorage ], true ), - passwords[ 0 ], config ); - zc.gcChunks(); - } + ZCollector zc( ZBackupBase::deriveStorageDirFromBackupsFile( args[ 1 ], true ), + passwords[ 0 ], config ); + zc.gc(); } else if ( strcmp( args[ 0 ], "passwd" ) == 0 ) diff --git a/zutils.cc b/zutils.cc index 9818e86..5ed6529 100644 --- a/zutils.cc +++ b/zutils.cc @@ -307,7 +307,7 @@ ZCollector::ZCollector( string const & storageDir, string const & password, { } -void ZCollector::gcChunks() +void ZCollector::gc() { ChunkIndex chunkReindex( encryptionkey, tmpMgr, getIndexPath(), true ); @@ -322,6 +322,7 @@ void ZCollector::gcChunks() collector.bundlesPath = getBundlesPath(); collector.chunkStorageReader = &this->chunkStorageReader; collector.chunkStorageWriter = &chunkStorageWriter; + collector.indexGC = config.runtime.gcIndexes; verbosePrintf( "Checking used chunks...\n" ); @@ -366,7 +367,3 @@ void ZCollector::gcChunks() verbosePrintf( "Garbage collection complete\n" ); } - -void ZCollector::gcIndexes() -{ -} diff --git a/zutils.hh b/zutils.hh index 7201ff2..59d23d4 100644 --- a/zutils.hh +++ b/zutils.hh @@ -55,8 +55,7 @@ public: ZCollector( std::string const & storageDir, std::string const & password, Config & configIn ); - void gcChunks(); - void gcIndexes(); + void gc(); }; #endif