For Issue #271, implement 'purge' commands for both cvsdbadmin and

svndbadmin.  Teach the 'rebuild' commands to first purge existing data
and then crawl the repository.  Also, drop support for the 'rev'
parameter to 'svndbadmin rebuild', adding instead a '--force' option
to 'svndbadmin update'.

Suggested, and SQL commands offered, by Mark <mark@mitsein.net>.

* lib/cvsdb.py
  (CheckinDatabase.sql_delete, CheckinDatabase.PurgeRepository): New.

* bin/svndbadmin
  (handle_revision): Add 'force' parameter, used to force update of
    commits already recorded in the database.
  (main): Add 'force' parameter, passed on to handle_revision().
    Handle the new 'purge' command, and teach 'rebuild' to also purge.
  (usage): Update usage info.
  (__main__): Add support for 'update --force' and 'purge', and drop
    support for 'rebuild rev'.  Add a KeyboardInterrupt handler.

* bin/cvsdbadmin
  (usage): Add 'purge' usage info.
  (__main__): Rework command-line parameter handling.  Add support for
    'purge' command, and make 'rebuild' first do a purge.

* CHANGES
  Note this change.

git-svn-id: http://viewvc.tigris.org/svn/viewvc/trunk@1493 8cb11bc2-c004-0410-86c3-e597b4017df7

CHANGES

@@ -6,8 +6,9 @@ Version 1.1.0 (released ??-???-????)
   * add daemon mode to standalone.py
   * rework and update helper application configuration options
   * teach standalone.py to recognize Subversion repositories via -r option
-  * open relative paths in "viewvc.conf" using the location of that file,
+  * now interpret relative paths in "viewvc.conf" as relative to that file
-    instead of the location of the ViewVC "lib" directory, as a base
+  * add 'purge' subcommand to cvsdbadmin and svndbadmin (issue #271)
+  * fix orphaned data bug in cvsdbadmin/svndbadmin rebuild (issue #271)
 
 Version 1.0.3 (released 13-Oct-2006)
 

bin/cvsdbadmin

@@ -123,15 +123,18 @@ def usage():
     sys.stderr.write("""
 Usage: 1. %s [[-q] -q] rebuild REPOSITORY
        2. %s [[-q] -q] update REPOSITORY
+       3. %s [[-q] -q] purge REPOSITORY
 
 1.  Rebuild the commit database information for REPOSITORY.
 
 2.  Update the commit database information for all unrecorded commits
     in REPOSITORY.
 
+3.  Purge information specific to REPOSITORY from the database.
+
 Use the -q flag to cause this script to be less verbose; use it twice to
 invoke a peaceful state of noiselessness.
-""" % (cmd, cmd))
+""" % (cmd, cmd, cmd))
     sys.exit(1)
 
 
@@ -139,7 +142,7 @@ invoke a peaceful state of noiselessness.
 if __name__ == '__main__':
     args = sys.argv
 
-    ## check the quietness level (0 = verbose, 1 = new commits, 2 = silent)
+    # check the quietness level (0 = verbose, 1 = new commits, 2 = silent)
     quiet_level = 0
     while 1:
         try:
@@ -149,30 +152,30 @@ if __name__ == '__main__':
         except ValueError:
             break
 
-    ## check that a command was given
+    # validate the command
     if len(args) <= 2:
         usage()
-
+    command = args[1].lower()
-    ## set the handler function for the command
+    if command not in ('rebuild', 'update', 'purge'):
-    command = args[1]
+        sys.stderr.write('ERROR: unknown command %s\n' % command)
-    if string.lower(command) == 'rebuild':
-        update = 0
-    elif string.lower(command) == 'update':
-        update = 1
-    else:
-        print 'ERROR: unknown command %s' % (command)
         usage()
 
-    # get repository path
+    # get repository and path, and do the work
     root, path_parts = RootPath(args[2], quiet_level)
-
-    ## run command
     try:
-        ## connect to the database we are updating
         cfg = viewvc.load_config(CONF_PATHNAME)
         db = cvsdb.ConnectDatabase(cfg)
-        repository = vclib.bincvs.BinCVSRepository(None, root, cfg.utilities)
+
-        RecurseUpdate(db, repository, path_parts, update, quiet_level)
+        if command in ('rebuild', 'purge'):
+            if quiet_level < 2:
+                print "Purging existing data for repository root `%s'" % root
+            db.PurgeRepository(root)
+
+        if command in ('rebuild', 'update'):
+            repository = vclib.bincvs.BinCVSRepository(None, root,
+                                                       cfg.utilities)
+            RecurseUpdate(db, repository, path_parts,
+                          command == 'update', quiet_level)
     except KeyboardInterrupt:
         print
         print '** break **'

bin/svndbadmin

@@ -197,7 +197,7 @@ class SvnRev:
         return self.rev_roots[rev]
 
 
-def handle_revision(db, command, repo, rev, verbose):
+def handle_revision(db, command, repo, rev, verbose, force=0):
     """Adds a particular revision of the repository to the checkin database."""
     revision = repo[rev]
     committed = 0
@@ -231,7 +231,7 @@ def handle_revision(db, command, repo, rev, verbose):
 
         if command == 'update':
             result = db.CheckCommit(commit)
-            if result:
+            if result and not force:
                 continue # already recorded
 
         # commit to database
@@ -244,38 +244,46 @@ def handle_revision(db, command, repo, rev, verbose):
         else:
             print "skipped (already recorded)."
 
-def main(pool, command, repository, rev=None, verbose=0):
+def main(pool, command, repository, rev=None, verbose=0, force=0):
     cfg = viewvc.load_config(CONF_PATHNAME)
     db = cvsdb.ConnectDatabase(cfg)
 
+    if command in ('rebuild', 'purge'):
+        if verbose:
+            print "Purging commit info for repository root `%s'" % repository
+        db.PurgeRepository(repository)
+
     repo = SvnRepo(repository, pool)
-    if rev:
+    if command == 'rebuild' or (command == 'update' and not rev):
-        handle_revision(db, command, repo, rev, verbose)
-    else:
         for rev in range(repo.rev_max+1):
             handle_revision(db, command, repo, rev, verbose)
+    elif command == 'update':
+        handle_revision(db, command, repo, rev, verbose, force)
 
 def usage():
     cmd = os.path.basename(sys.argv[0])
     sys.stderr.write("""
-Usage: 1. %s [-v] rebuild REPOSITORY [REVISION]
+Usage: 1. %s [-v] rebuild REPOSITORY
-       2. %s [-v] update REPOSITORY [REVISION]
+       2. %s [-v] update REPOSITORY [REVISION] [--force]
+       3. %s [-v] purge REPOSITORY
 
-1.  Rebuild the commit database information for REPOSITORY across all revisions
+1.  Rebuild the commit database information for REPOSITORY across all
-    or, optionally, only for the specified REVISION.
+    revisions.
 
 2.  Update the commit database information for REPOSITORY across all revisions
     or, optionally, only for the specified REVISION.  This is just like
-    rebuilding, except that no commit information will be stored for
+    rebuilding, except that, unless --force is specified, no commit information
-    commits already present in the database.
+    will be stored for commits already present in the database.
+
+3.  Purge information specific to REPOSITORY from the database.
 
 Use the -v flag to cause this script to give progress information as it works.
-""" % (cmd, cmd))
+""" % (cmd, cmd, cmd))
     sys.exit(1)
 
 if __name__ == '__main__':
     verbose = 0
-    
+    force = 0
     args = sys.argv
     try:
         index = args.index('-v')
@@ -283,12 +291,18 @@ if __name__ == '__main__':
         del args[index]
     except ValueError:
         pass
+    try:
+        index = args.index('--force')
+        force = 1
+        del args[index]
+    except ValueError:
+        pass
         
     if len(args) < 3:
         usage()
 
     command = string.lower(args[1])
-    if command not in ('rebuild', 'update'):
+    if command not in ('rebuild', 'update', 'purge'):
         sys.stderr.write('ERROR: unknown command %s\n' % command)
         usage()
 
@@ -298,6 +312,12 @@ if __name__ == '__main__':
         usage()
 
     if len(sys.argv) > 3:
+        if command == 'rebuild':
+            sys.stderr.write('ERROR: rebuild no longer accepts a revision '
+                             'number argument.  Usage update --force.')
+            usage()
+        elif command != 'update':
+            usage()
         rev = sys.argv[3]
         try:
             rev = int(rev)
@@ -307,5 +327,10 @@ if __name__ == '__main__':
     else:
         rev = None
 
+    try:
         repository = cvsdb.CleanRepository(os.path.abspath(repository))
-    svn.core.run_app(main, command, repository, rev, verbose)
+        svn.core.run_app(main, command, repository, rev, verbose, force)
+    except KeyboardInterrupt:
+        print
+        print '** break **'
+    sys.exit(0)

lib/cvsdb.py

@@ -427,6 +427,40 @@ class CheckinDatabase:
 
         return commit
 
+    def sql_delete(self, table, key, value):
+        sql = "DELETE FROM %s WHERE %s=%%s" % (table, key)
+        sql_args = (value, )
+        cursor = self.db.cursor()
+        cursor.execute(sql, sql_args)
+        
+    def PurgeRepository(self, repository):
+        rep_id = self.GetRepositoryID(repository)
+        if not rep_id:
+            raise Exception, "Unknown repository '%s'" % (repository)
+        
+        sql = "SELECT * FROM checkins WHERE repositoryid=%s"
+        sql_args = (rep_id, )
+        cursor = self.db.cursor()
+        cursor.execute(sql, sql_args)
+        checkins = []
+        while 1:
+            try:
+                (ci_type, ci_when, who_id, repository_id,
+                 dir_id, file_id, revision, sticky_tag, branch_id,
+                 plus_count, minus_count, description_id) = cursor.fetchone()
+            except TypeError:
+                break
+            checkins.append([file_id, dir_id, branch_id, description_id])
+
+        #self.sql_delete('repositories', 'id', rep_id)
+        self.sql_delete('checkins', 'repositoryid', rep_id)
+        for checkin in checkins:
+            self.sql_delete('files', 'id', checkin[0])
+            self.sql_delete('dirs', 'id', checkin[1])
+            self.sql_delete('branches', 'id', checkin[2])
+            self.sql_delete('descs', 'id', checkin[3])
+
+
 ## the Commit class holds data on one commit, the representation is as
 ## close as possible to how it should be committed and retrieved to the
 ## database engine