Skip to content

Commit 6c6354d

Browse files
authored
Treat WAL TruncateBefore below first index as a no-op (#4355)
TestConcurrentTruncateBeforeWithAsyncWrites fails with "out of range truncate error". The test reads `FirstOffset` and then calls `TruncateBefore(first+1)`, but background pruning runs every millisecond and can advance the first index past that value in between. tidwall/wal's `TruncateFront` rejects an index below the current first index with `ErrOutOfRange`, so a request that is already satisfied comes back as an error. In production only the pebbledb MVCC store runs a background pruner (`KeepRecent` and `PruneInterval`) alongside its own `PruneWALBeforeVersion`, which computes its truncation offset from an earlier `FirstOffset` read. Losing that race logs a spurious "failed to prune state store changelog WAL" error from the snapshot manager and does nothing worse. memiavl opens its WAL without a pruner, and `tryTruncateWAL` already skips indexes at or below the first index. `handleTruncate` now treats a `TruncateBefore` whose index is below the current first index as a successful no-op, since every entry before that index is already gone. Both `TruncateBefore` and `TruncateAll` go through this check, and so does `prune`, all on the main loop, so the check and the truncation cannot race. The `TruncateBefore` godoc on `WAL` and `GenericWAL` now states the no-op contract. `TestTruncateBefore` and `TestTruncateAll` cover it, for both a non-empty log and an emptied `AllowEmpty` log. The concurrent test also waits for pruning to reach its fixed point before it reads the range, so it no longer reads the first entry while the pruner may still move it. Flaked in: https://github.com/sei-protocol/sei-chain/actions/runs/35518364976/job/106098042389
1 parent 1c2b3a2 commit 6c6354d

3 files changed

Lines changed: 31 additions & 8 deletions

File tree

‎sei-db/wal/types.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,8 @@ type GenericWAL[T any] interface {
1111
// Write will append a new entry to the end of the log.
1212
Write(entry T) error
1313

14-
// TruncateBefore will remove all entries that are before the provided `offset`
14+
// TruncateBefore removes every entry below `offset`. An `offset` at or below
15+
// the current first entry is a no-op.
1516
TruncateBefore(offset uint64) error
1617

1718
// TruncateAfter will remove all entries that are after the provided `offset`

‎sei-db/wal/wal.go‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -339,8 +339,9 @@ func (walLog *WAL[T]) TruncateAfter(index uint64) error {
339339
return walLog.sendTruncate(false, index)
340340
}
341341

342-
// TruncateBefore will remove all entries that are before the provided `index`.
343-
// In other words the entry at `index` becomes the first entry in the log.
342+
// TruncateBefore removes every entry below `index`, so the entry at `index`
343+
// becomes the first entry in the log. An `index` at or below the current first
344+
// entry is a no-op.
344345
func (walLog *WAL[T]) TruncateBefore(index uint64) error {
345346
backgroundErr := walLog.asyncError.Load()
346347
if backgroundErr != nil {
@@ -399,6 +400,11 @@ func (walLog *WAL[T]) sendTruncate(before bool, index uint64) error {
399400
func (walLog *WAL[T]) handleTruncate(req *truncateRequest) {
400401
var err error
401402
if req.before {
403+
// Entries below the first index are already gone, e.g. removed by pruning.
404+
if first, ferr := walLog.log.FirstIndex(); ferr == nil && req.index < first {
405+
req.errChan <- nil
406+
return
407+
}
402408
err = walLog.log.TruncateFront(req.index)
403409
} else {
404410
err = walLog.log.TruncateBack(req.index)

‎sei-db/wal/wal_test.go‎

Lines changed: 21 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -249,6 +249,12 @@ func TestTruncateBefore(t *testing.T) {
249249
entry, err := changelog.ReadAt(2)
250250
require.NoError(t, err)
251251
require.Equal(t, []byte("hello1"), entry.Changesets[0].Changeset.Pairs[0].Key)
252+
253+
// Truncating below the current first index is a no-op.
254+
require.NoError(t, changelog.TruncateBefore(1))
255+
firstIndex, err = changelog.FirstOffset()
256+
require.NoError(t, err)
257+
require.Equal(t, uint64(2), firstIndex)
252258
}
253259

254260
func TestCloseSyncMode(t *testing.T) {
@@ -588,18 +594,19 @@ func TestConcurrentCloseWithInFlightAsyncWrites(t *testing.T) {
588594
}
589595

590596
func TestConcurrentTruncateBeforeWithAsyncWrites(t *testing.T) {
597+
const (
598+
totalWrites = 50
599+
keepRecent = 10
600+
)
601+
591602
dir := t.TempDir()
592603
changelog, err := NewWAL(t.Context(), marshalEntry, unmarshalEntry, dir, Config{
593604
WriteBufferSize: 10,
594-
KeepRecent: 10,
605+
KeepRecent: keepRecent,
595606
PruneInterval: 1 * time.Millisecond,
596607
})
597608
require.NoError(t, err)
598609

599-
const (
600-
totalWrites = 50
601-
)
602-
603610
// Write a bunch of entries (async writes). We'll wait until they're all persisted.
604611
for i := 1; i <= totalWrites; i++ {
605612
entry := proto.ChangelogEntry{
@@ -636,6 +643,12 @@ func TestConcurrentTruncateBeforeWithAsyncWrites(t *testing.T) {
636643
return err == nil && first >= firstBefore+1
637644
}, 3*time.Second, 10*time.Millisecond, "manual truncation did not take effect")
638645

646+
// Wait for pruning to reach its fixed point so the range read below is stable.
647+
require.Eventually(t, func() bool {
648+
first, err := changelog.FirstOffset()
649+
return err == nil && first >= totalWrites-keepRecent
650+
}, 3*time.Second, 10*time.Millisecond, "background pruning did not settle")
651+
639652
// Read first + last entries to ensure no corruption (decode succeeds; expected structure).
640653
first, err := changelog.FirstOffset()
641654
require.NoError(t, err)
@@ -679,6 +692,9 @@ func TestTruncateAll(t *testing.T) {
679692
// With AllowEmpty, FirstIndex returns LastIndex+1 when empty.
680693
require.True(t, first > last, "expected empty WAL (first > last)")
681694

695+
// Truncating below the first index of an emptied log is a no-op.
696+
require.NoError(t, changelog.TruncateBefore(1))
697+
682698
// Can write new entries after truncating all.
683699
entry := proto.ChangelogEntry{
684700
Changesets: []*proto.NamedChangeSet{{Name: "after", Changeset: proto.ChangeSet{Pairs: MockKVPairs("k", "v")}}},

0 commit comments

Comments
 (0)