diff --git a/etcdserver/raft.go b/etcdserver/raft.go index 6c411f327..1361f59c4 100644 --- a/etcdserver/raft.go +++ b/etcdserver/raft.go @@ -245,6 +245,13 @@ func (r *raftNode) start(rh *raftReadyHandler) { // gofail: var raftAfterSave struct{} if !raft.IsEmptySnap(rd.Snapshot) { + // Force WAL to fsync its hard state before Release() releases + // old data from the WAL. Otherwise could get an error like: + // panic: tocommit(107) is out of range [lastIndex(84)]. Was the raft log corrupted, truncated, or lost? + if err := r.storage.Sync(); err != nil { + log.Fatal(err) + } + // etcdserver now claim the snapshot has been persisted onto the disk notifyc <- struct{}{} diff --git a/etcdserver/server_test.go b/etcdserver/server_test.go index 21116c659..1f3bf349b 100644 --- a/etcdserver/server_test.go +++ b/etcdserver/server_test.go @@ -995,7 +995,7 @@ func TestSnapshot(t *testing.T) { ch := make(chan struct{}, 2) go func() { - gaction, _ := p.Wait(1) + gaction, _ := p.Wait(2) defer func() { ch <- struct{}{} }() if len(gaction) != 2 { @@ -1111,6 +1111,10 @@ func TestSnapshotOrdering(t *testing.T) { } // unblock SaveSnapshot, etcdserver now permitted to move snapshot file + if ac := <-p.Chan(); ac.Name != "Sync" { + t.Fatalf("expected Sync, got %+v", ac) + } + if ac := <-p.Chan(); ac.Name != "Release" { t.Fatalf("expected Release, got %+v", ac) } diff --git a/etcdserver/storage.go b/etcdserver/storage.go index 0e8390b14..3376a825a 100644 --- a/etcdserver/storage.go +++ b/etcdserver/storage.go @@ -38,6 +38,8 @@ type Storage interface { Close() error // Release release release the locked wal files since they will not be used. Release(snap raftpb.Snapshot) error + // Sync WAL + Sync() error } type storage struct { diff --git a/pkg/mock/mockstorage/storage_recorder.go b/pkg/mock/mockstorage/storage_recorder.go index d1e59a5b8..d30486687 100644 --- a/pkg/mock/mockstorage/storage_recorder.go +++ b/pkg/mock/mockstorage/storage_recorder.go @@ -52,4 +52,9 @@ func (p *storageRecorder) Release(st raftpb.Snapshot) error { return nil } +func (p *storageRecorder) Sync() error { + p.Record(testutil.Action{Name: "Sync"}) + return nil +} + func (p *storageRecorder) Close() error { return nil } diff --git a/wal/wal.go b/wal/wal.go index d05215291..17df41e26 100644 --- a/wal/wal.go +++ b/wal/wal.go @@ -723,6 +723,10 @@ func (w *WAL) sync() error { return err } +func (w *WAL) Sync() error { + return w.sync() +} + // ReleaseLockTo releases the locks, which has smaller index than the given index // except the largest one among them. // For example, if WAL is holding lock 1,2,3,4,5,6, ReleaseLockTo(4) will release