Index pseudo-GC implementation

backup_collector.cc

@@ -7,7 +7,7 @@ using std::string;
 
 void BundleCollector::startIndex( string const & indexFn )
 {
-  indexModified = false;
+  indexModified = indexNecessary = false;
   indexTotalChunks = indexUsedChunks = 0;
   indexModifiedBundles = indexKeptBundles = indexRemovedBundles = 0;
 }
@@ -25,6 +25,8 @@ void BundleCollector::finishIndex( string const & indexFn )
   else
   {
     chunkStorageWriter->reset();
+    if ( indexGC && !indexNecessary )
+      filesToUnlink.push_back( indexFn );
   }
 }
 
@@ -37,10 +39,18 @@ void BundleCollector::startBundle( Bundle::Id const & bundleId )
 
 void BundleCollector::processChunk( ChunkId const & chunkId )
 {
+  if ( indexGC )
+  {
+    if ( overallChunkSet.find ( chunkId ) == overallChunkSet.end() )
+      overallChunkSet.insert( chunkId );
+    else
+      return;
+  }
   totalChunks++;
   if ( usedChunkSet.find( chunkId ) != usedChunkSet.end() )
   {
     usedChunks++;
+    indexNecessary = true;
   }
 }
 
@@ -49,16 +59,16 @@ void BundleCollector::finishBundle( Bundle::Id const & bundleId, BundleInfo cons
   string i = Bundle::generateFileName( savedId, "", false );
   indexTotalChunks += totalChunks;
   indexUsedChunks += usedChunks;
-  if ( usedChunks == 0 )
+  if ( 0 == usedChunks && 0 != totalChunks )
   {
-    verbosePrintf( "Deleting %s bundle\n", i.c_str() );
+    dPrintf( "Deleting %s bundle\n", i.c_str() );
     filesToUnlink.push_back( Dir::addPath( bundlesPath, i ) );
     indexModified = true;
     indexRemovedBundles++;
   }
   else if ( usedChunks < totalChunks )
   {
-    verbosePrintf( "%s: used %d/%d chunks\n", i.c_str(), usedChunks, totalChunks );
+    dPrintf( "%s: used %d/%d chunks\n", i.c_str(), usedChunks, totalChunks );
     filesToUnlink.push_back( Dir::addPath( bundlesPath, i ) );
     indexModified = true;
     // Copy used chunks to the new index
@@ -79,7 +89,7 @@ void BundleCollector::finishBundle( Bundle::Id const & bundleId, BundleInfo cons
   else
   {
     chunkStorageWriter->addBundle( info, savedId );
-    verbosePrintf( "Keeping %s bundle\n", i.c_str() );
+    dPrintf( "Keeping %s bundle\n", i.c_str() );
     indexKeptBundles++;
   }
 }
@@ -88,6 +98,7 @@ void BundleCollector::commit()
 {
   for ( int i = filesToUnlink.size(); i--; )
   {
+    dPrintf( "Unlinking %s\n", filesToUnlink[i].c_str() );
     unlink( filesToUnlink[i].c_str() );
   }
   filesToUnlink.clear();

backup_collector.hh

@@ -20,14 +20,16 @@ private:
   Bundle::Id savedId;
   int totalChunks, usedChunks, indexTotalChunks, indexUsedChunks;
   int indexModifiedBundles, indexKeptBundles, indexRemovedBundles;
-  bool indexModified;
+  bool indexModified, indexNecessary;
   vector< string > filesToUnlink;
+  BackupRestorer::ChunkSet overallChunkSet;
 
 public:
   string bundlesPath;
   ChunkStorage::Reader *chunkStorageReader;
   ChunkStorage::Writer *chunkStorageWriter;
   BackupRestorer::ChunkSet usedChunkSet;
+  bool indexGC;
 
   void startIndex( string const & indexFn );
 

chunk_index.hh

@@ -87,7 +87,7 @@ public:
   DEF_EX( Ex, "Chunk index exception", std::exception )
   DEF_EX( exIncorrectChunkIdSize, "Incorrect chunk id size encountered", Ex )
 
-  ChunkIndex( EncryptionKey const &, TmpMgr &, string const & indexPath, bool prohibitChunkIndexLoading );
+  ChunkIndex( EncryptionKey const &, TmpMgr &, string const & indexPath, bool );
 
   struct ChunkInfoInterface
   {

config.cc

@@ -123,6 +123,16 @@ void Config::prefillKeywords()
       "No default value, you should specify it explicitly"
     },
 
+    {
+      "gc-indexes",
+      Config::oRuntime_gcIndexes,
+      Config::Runtime,
+      "Purge duplicated indexes from repo during\n"
+      "garbage collection\n"
+      "Normally you would not need this\n"
+      "No value, specify to enable"
+    },
+
     { "", Config::oBadOption, Config::None }
   };
 
@@ -456,6 +466,15 @@ bool Config::parseOrValidate( const string & option, const OptionType type,
       /* NOTREACHED */
       break;
 
+    case oRuntime_gcIndexes:
+      runtime.gcIndexes = true;
+
+      dPrintf( "runtime[gcIndexes] = true\n" );
+
+      return true;
+      /* NOTREACHED */
+      break;
+
     case oBadOption:
     default:
       return false;

config.hh

@@ -29,11 +29,13 @@ public:
     size_t threads;
     size_t cacheSize;
     bitset< BackupExchanger::Flags > exchange;
+    bool gcIndexes;
 
     // Default runtime config
     RuntimeConfig():
       threads( getNumberOfCpus() ),
-      cacheSize( 40 * 1024 * 1024 ) // 40 MB
+      cacheSize( 40 * 1024 * 1024 ), // 40 MB
+      gcIndexes ( false )
     {
     }
   };
@@ -58,6 +60,7 @@ public:
     oRuntime_threads,
     oRuntime_cacheSize,
     oRuntime_exchange,
+    oRuntime_gcIndexes,
 
     oDeprecated, oUnsupported
   } OpCodes;

zbackup.cc

@@ -280,42 +280,16 @@ invalid_option:
     if ( strcmp( args[ 0 ], "gc" ) == 0 )
     {
       // Perform the garbage collection
-      if ( args.size() < 2 || args.size() > 3 )
+      if ( args.size() != 2 )
       {
-        fprintf( stderr, "Usage: %s %s [chunks|indexes] <storage path>\n",
+        fprintf( stderr, "Usage: %s %s <storage path>\n",
                  *argv, args[ 0 ] );
         return EXIT_FAILURE;
       }
 
-      int fieldStorage = 1;
-      int fieldAction = 2;
-
-      if ( args.size() == 3 )
-      {
-        fieldStorage = 2;
-        fieldAction = 1;
-      }
-
-      if ( args.size() > 2 && strcmp( args[ fieldAction ], "chunks" ) == 0 )
-      {
-        ZCollector zc( ZBackupBase::deriveStorageDirFromBackupsFile( args[ fieldStorage ], true ),
-            passwords[ 0 ], config );
-        zc.gcChunks();
-      }
-      else
-      if ( args.size() > 2 && strcmp( args[ fieldAction ], "indexes" ) == 0 )
-      {
-        ZCollector zc( ZBackupBase::deriveStorageDirFromBackupsFile( args[ fieldStorage ], true ),
-            passwords[ 0 ], config );
-        fprintf( stderr, "NOT IMPLEMENTED YET!\n" );
-        zc.gcIndexes();
-      }
-      else
-      {
-        ZCollector zc( ZBackupBase::deriveStorageDirFromBackupsFile( args[ fieldStorage ], true ),
-            passwords[ 0 ], config );
-        zc.gcChunks();
-      }
+      ZCollector zc( ZBackupBase::deriveStorageDirFromBackupsFile( args[ 1 ], true ),
+          passwords[ 0 ], config );
+      zc.gc();
     }
     else
     if ( strcmp( args[ 0 ], "passwd" ) == 0 )

zutils.cc

@@ -307,7 +307,7 @@ ZCollector::ZCollector( string const & storageDir, string const & password,
 {
 }
 
-void ZCollector::gcChunks()
+void ZCollector::gc()
 {
   ChunkIndex chunkReindex( encryptionkey, tmpMgr, getIndexPath(), true );
 
@@ -322,6 +322,7 @@ void ZCollector::gcChunks()
   collector.bundlesPath = getBundlesPath();
   collector.chunkStorageReader = &this->chunkStorageReader;
   collector.chunkStorageWriter = &chunkStorageWriter;
+  collector.indexGC = config.runtime.gcIndexes;
 
   verbosePrintf( "Checking used chunks...\n" );
 
@@ -366,7 +367,3 @@ void ZCollector::gcChunks()
 
   verbosePrintf( "Garbage collection complete\n" );
 }
-
-void ZCollector::gcIndexes()
-{
-}

zutils.hh

@@ -55,8 +55,7 @@ public:
   ZCollector( std::string const & storageDir, std::string const & password,
               Config & configIn );
 
-  void gcChunks();
-  void gcIndexes();
+  void gc();
 };
 
 #endif