etcdserver/*, wal/*: Remove orphaned .snap.db files during Release

etcdserver/api/snap/snapshotter.go

@@ -22,6 +22,7 @@ import (
 	"os"
 	"path/filepath"
 	"sort"
+	"strconv"
 	"strings"
 	"time"
 
@@ -265,3 +266,31 @@ func (s *Snapshotter) cleanupSnapdir(filenames []string) error {
 	}
 	return nil
 }
+
+func (s *Snapshotter) ReleaseSnapDBs(snap raftpb.Snapshot) error {
+	dir, err := os.Open(s.dir)
+	if err != nil {
+		return err
+	}
+	defer dir.Close()
+	filenames, err := dir.Readdirnames(-1)
+	if err != nil {
+		return err
+	}
+	for _, filename := range filenames {
+		if strings.HasSuffix(filename, ".snap.db") {
+			hexIndex := strings.TrimSuffix(filepath.Base(filename), ".snap.db")
+			index, err := strconv.ParseUint(hexIndex, 16, 64)
+			if err != nil {
+				return fmt.Errorf("failed to parse index from .snap.db filename '%s': %w", filename, err)
+			}
+			if index < snap.Metadata.Index {
+				s.lg.Info("found orphaned .snap.db file; deleting", zap.String("path", filename))
+				if rmErr := os.Remove(filepath.Join(s.dir, filename)); rmErr != nil && !os.IsNotExist(rmErr) {
+					return fmt.Errorf("failed to remove orphaned defragmentation file %s: %v", filename, rmErr)
+				}
+			}
+		}
+	}
+	return nil
+}

etcdserver/api/snap/snapshotter_test.go

@@ -16,6 +16,7 @@ package snap
 
 import (
 	"fmt"
+	"go.etcd.io/etcd/v3/pkg/fileutil"
 	"hash/crc32"
 	"io/ioutil"
 	"os"
@@ -266,3 +267,42 @@ func TestAllSnapshotBroken(t *testing.T) {
 		t.Errorf("err = %v, want %v", err, ErrNoSnapshot)
 	}
 }
+
+func TestReleaseSnapDBs(t *testing.T) {
+	dir := filepath.Join(os.TempDir(), "snapshot")
+	err := os.Mkdir(dir, 0700)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(dir)
+
+	snapIndices := []uint64{100, 200, 300, 400}
+	for _, index := range snapIndices {
+		filename := filepath.Join(dir, fmt.Sprintf("%016x.snap.db", index))
+		if err := ioutil.WriteFile(filename, []byte("snap file\n"), 0644); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	ss := New(zap.NewExample(), dir)
+
+	if err := ss.ReleaseSnapDBs(raftpb.Snapshot{Metadata: raftpb.SnapshotMetadata{Index: 300}}); err != nil {
+		t.Fatal(err)
+	}
+
+	deleted := []uint64{100, 200}
+	for _, index := range deleted {
+		filename := filepath.Join(dir, fmt.Sprintf("%016x.snap.db", index))
+		if fileutil.Exist(filename) {
+			t.Errorf("expected %s (index: %d)  to be deleted, but it still exists", filename, index)
+		}
+	}
+
+	retained := []uint64{300, 400}
+	for _, index := range retained {
+		filename := filepath.Join(dir, fmt.Sprintf("%016x.snap.db", index))
+		if !fileutil.Exist(filename) {
+			t.Errorf("expected %s (index: %d) to be retained, but it no longer exists", filename, index)
+		}
+	}
+}

etcdserver/storage.go

@@ -69,9 +69,14 @@ func (st *storage) SaveSnap(snap raftpb.Snapshot) error {
 	return st.WAL.SaveSnapshot(walsnap)
 }
 
-// Release release the locks to the wal files that are older than the provided wal for the given snap.
+// Release releases resources older than the given snap and are no longer needed:
+// - releases the locks to the wal files that are older than the provided wal for the given snap.
+// - deletes any .snap.db files that are older than the given snap.
 func (st *storage) Release(snap raftpb.Snapshot) error {
-	return st.WAL.ReleaseLockTo(snap.Metadata.Index)
+	if err := st.WAL.ReleaseLockTo(snap.Metadata.Index); err != nil {
+		return err
+	}
+	return st.Snapshotter.ReleaseSnapDBs(snap)
 }
 
 // readWAL reads the WAL at the given snap and returns the wal, its latest HardState and cluster ID, and all entries that appear