From c6308d5645b283e7f4faa31c3345b9f4b4a404ab Mon Sep 17 00:00:00 2001 From: blindchaser Date: Mon, 3 Aug 2026 00:51:05 -0400 Subject: [PATCH 1/3] fix(seidb): stop evm-logical-digest replay mode from mutating a live memiavl --memiavl-open-mode=replay relied on memiavl's ReadOnly option, which only skips the LOCK file. The changelog is still opened read-write, and sei-db's wal.open() answers wal.ErrCorrupt with os.Truncate on the node's segment. On a live node a torn tail is the writer mid-append, not corruption. Truncating it drops committed versions, and since the writer's fd keeps its old offset, its next append leaves a zero-filled hole that binary.Uvarint reads as valid empty records rather than an error. The node keeps producing blocks and finds out at its next restart. Clone instead, as openFlatKVReadOnly already does for the other backend: hardlink the snapshot, byte-copy the changelog, verify it still covers snapshotVersion+1, retry if the writer prunes mid-clone, then open the clone normally. Every source access is now a read. Also reject a clone that falls short of --height, since a tail repair costs the trailing version and a silently early digest looks like real divergence; and export memiavl's SeekSnapshotDir rather than reimplementing its snapshot layout in the tool. Replay mode now needs a writable dbDir and room for the changelog copy, as the FlatKV path already did. memiavl.OpenDB(ReadOnly) is still reachable from wasm snapshot export and storev2/rootmulti proof queries; fixing it at the WAL layer is tracked separately. Co-authored-by: Cursor --- sei-db/state_db/sc/memiavl/db.go | 22 +++ .../seidb/operations/evm_logical_digest.go | 44 ++--- .../tools/cmd/seidb/operations/flatkv_open.go | 20 ++- .../cmd/seidb/operations/flatkv_open_test.go | 4 +- .../cmd/seidb/operations/memiavl_open.go | 140 +++++++++++++++ .../cmd/seidb/operations/memiavl_open_test.go | 159 ++++++++++++++++++ 6 files changed, 351 insertions(+), 38 deletions(-) create mode 100644 sei-db/tools/cmd/seidb/operations/memiavl_open.go create mode 100644 sei-db/tools/cmd/seidb/operations/memiavl_open_test.go diff --git a/sei-db/state_db/sc/memiavl/db.go b/sei-db/state_db/sc/memiavl/db.go index ddd18f675c..2b352c3e9f 100644 --- a/sei-db/state_db/sc/memiavl/db.go +++ b/sei-db/state_db/sc/memiavl/db.go @@ -1205,6 +1205,28 @@ func seekSnapshot(root string, targetVersion int64) (int64, error) { return snapshotVersion, nil } +// SeekSnapshotDir returns the directory name and version of the snapshot that +// OpenDB would start from for targetVersion: the "current" link when +// targetVersion is 0, otherwise the newest snapshot at or below it. +// +// Exported for readers that need to resolve a snapshot without opening the DB, +// so they inherit this package's layout rules instead of restating them. +func SeekSnapshotDir(root string, targetVersion int64) (string, int64, error) { + if targetVersion == 0 { + version, err := currentVersion(root) + if err != nil { + return "", 0, fmt.Errorf("read current snapshot: %w", err) + } + return snapshotName(version), version, nil + } + + version, err := seekSnapshot(root, targetVersion) + if err != nil { + return "", 0, err + } + return snapshotName(version), version, nil +} + // GetEarliestVersion returns the earliest snapshot name in the db func GetEarliestVersion(root string) (int64, error) { var found int64 diff --git a/sei-db/tools/cmd/seidb/operations/evm_logical_digest.go b/sei-db/tools/cmd/seidb/operations/evm_logical_digest.go index 79ecd140c6..aceed4cc98 100644 --- a/sei-db/tools/cmd/seidb/operations/evm_logical_digest.go +++ b/sei-db/tools/cmd/seidb/operations/evm_logical_digest.go @@ -81,12 +81,14 @@ const ( // that exact height (or --height 0 for the current symlink). This is the // preferred mode whenever the target height lines up with an existing // snapshot boundary. -// - replay (SLOW): opens a read-only DB, replays the changelog up to -// --height, then walks the in-memory/mmap tree. Roughly an order of +// - replay (SLOW): clones the newest snapshot at or below --height plus the +// changelog into a temp directory, replays the changelog up to --height in +// that clone, then walks the in-memory/mmap tree. Roughly an order of // magnitude slower than snapshot (changelog replay + per-leaf tree walk -// instead of a sequential file read). Use it only when no snapshot exists -// at the target height — e.g. nodes whose snapshot rewrite lags the tip, so -// an arbitrary comparison height has no snapshot- on disk. +// instead of a sequential file read) and it byte-copies the changelog, so +// it needs free space alongside the source. Use it only when no snapshot +// exists at the target height — e.g. nodes whose snapshot rewrite lags the +// tip, so an arbitrary comparison height has no snapshot- on disk. // // The flatkv side is always a pebble WAL-replay-to-height and is fast // regardless. So when comparing across nodes, pick a height that is an existing @@ -165,7 +167,7 @@ func EvmLogicalDigestCmd() *cobra.Command { cmd.Flags().String("flatkv-dir", "", "Composite mode: flatkv data dir") cmd.Flags().String("memiavl-dir", "", "Composite mode: memiavl root dir (contains current/ and snapshot-* )") cmd.Flags().Int64("height", 0, "Target version. flatkv WAL-replays to it; memiavl resolves snapshot-/evm (0 = current symlink)") - cmd.Flags().String("memiavl-open-mode", memiavlOpenModeSnapshot, "memiavl read mode: snapshot (FAST: sequential scan of the completed snapshot kvs file; requires an on-disk snapshot at --height, or --height 0 for current) | replay (SLOW, ~10x: replays changelog to --height then walks the mmap tree; use only when no snapshot exists at the target height). Prefer snapshot when --height matches an existing snapshot boundary") + cmd.Flags().String("memiavl-open-mode", memiavlOpenModeSnapshot, "memiavl read mode: snapshot (FAST: sequential scan of the completed snapshot kvs file; requires an on-disk snapshot at --height, or --height 0 for current) | replay (SLOW, ~10x: clones the snapshot + changelog to a temp dir, replays to --height there, then walks the mmap tree; use only when no snapshot exists at the target height). Prefer snapshot when --height matches an existing snapshot boundary") cmd.Flags().String("memiavl-normalization", memiavlNormSemantic, "memiavl digest/inspect normalization: semantic/independent (raw EVM key/value decoder) | translator (current migration mapping)") cmd.Flags().String("inspect-bucket", "", "Inspect one normalized bucket (account|code|storage|misc) instead of printing the global digest") cmd.Flags().Int("key-offset", 0, "Inspect mode: byte offset into physical key before applying --key-prefix / sharding") @@ -465,7 +467,7 @@ func digestCompositeMigrateEVM(flatKVDir, memIAVLDir string, height int64, findT requestedHeight: height, version: opened.Version(), } - var memReplayDB *memiavl.DB + var memReplayDB *openedMemIAVL var memEvmSnapshotDir string var memVersion int64 switch memiavlOpenMode { @@ -481,13 +483,13 @@ func digestCompositeMigrateEVM(flatKVDir, memIAVLDir string, height int64, findT ctx.source = fmt.Sprintf("flatkv clone version=%d + memiavl snapshot=%s", opened.Version(), memEvmSnapshotDir) ctx.normalization = fmt.Sprintf("flatkv rows plus memiavl rows not migrated by boundary=%s version_known=%t migration_version=%d memiavl_version=%d", boundary.String(), versionKnown, migrationVersion, memVersion) case memiavlOpenModeReplay: - memReplayDB, err = openMemiAVLReplayReadOnly(memIAVLDir, height) + memReplayDB, err = openMemiAVLReplay(memIAVLDir, height) if err != nil { return err } defer func() { _ = memReplayDB.Close() }() memVersion = memReplayDB.Version() - ctx.source = fmt.Sprintf("flatkv clone version=%d + memiavl read-only replay dir=%s", opened.Version(), memIAVLDir) + ctx.source = fmt.Sprintf("flatkv clone version=%d + memiavl clone replay dir=%s", opened.Version(), memIAVLDir) ctx.normalization = fmt.Sprintf("flatkv rows plus replayed memiavl rows not migrated by boundary=%s version_known=%t migration_version=%d memiavl_version=%d", boundary.String(), versionKnown, migrationVersion, memVersion) default: return fmt.Errorf("unknown --memiavl-open-mode %q (want snapshot|replay)", memiavlOpenMode) @@ -504,7 +506,7 @@ func digestCompositeMigrateEVM(flatKVDir, memIAVLDir string, height int64, findT if boundary.Status() != migration.MigrationComplete { if memReplayDB != nil { if err := consumeCompositeMemiavl(func(fn func(rawKey, rawVal []byte) error) error { - return scanMemiavlReplayEVMLeaves(memReplayDB, fn) + return scanMemiavlReplayEVMLeaves(memReplayDB.DB, fn) }, "memiavl-replay", boundary, &d, accounts); err != nil { return err } @@ -1104,20 +1106,8 @@ func digestMemIAVL(dbDir string, height int64, findTarget []byte, normalization } } -func openMemiAVLReplayReadOnly(dbDir string, height int64) (*memiavl.DB, error) { - db, err := memiavl.OpenDB(height, memiavl.Options{ - Dir: dbDir, - ReadOnly: true, - ZeroCopy: true, - }) - if err != nil { - return nil, fmt.Errorf("open memiavl read-only replay: %w", err) - } - return db, nil -} - func digestMemIAVLReplay(dbDir string, height int64, findTarget []byte, normalization string) error { - db, err := openMemiAVLReplayReadOnly(dbDir, height) + db, err := openMemiAVLReplay(dbDir, height) if err != nil { return err } @@ -1125,9 +1115,9 @@ func digestMemIAVLReplay(dbDir string, height int64, findTarget []byte, normaliz switch normalization { case "", memiavlNormSemantic, memiavlNormIndependent: - return digestMemIAVLReplaySemantic(dbDir, height, db, findTarget) + return digestMemIAVLReplaySemantic(dbDir, height, db.DB, findTarget) case memiavlNormTranslator: - return digestMemIAVLReplayTranslator(dbDir, height, db, findTarget) + return digestMemIAVLReplayTranslator(dbDir, height, db.DB, findTarget) default: return fmt.Errorf("unknown --memiavl-normalization %q (want semantic|independent|translator)", normalization) } @@ -1291,7 +1281,7 @@ func digestMemIAVLReplaySemantic(dbDir string, height int64, db *memiavl.DB, fin backend: "memiavl", mode: "semantic-replay", dbDir: dbDir, - source: "read-only memiavl DB opened from snapshot + changelog replay", + source: "isolated memiavl clone opened from snapshot + changelog replay", normalization: "independent semantic decoder for replayed memiavl EVM keys; does not call flatkv.ImportTranslator", requestedHeight: height, version: db.Version(), @@ -1308,7 +1298,7 @@ func digestMemIAVLReplayTranslator(dbDir string, height int64, db *memiavl.DB, f backend: "memiavl", mode: "translator-replay", dbDir: dbDir, - source: "read-only memiavl DB opened from snapshot + changelog replay", + source: "isolated memiavl clone opened from snapshot + changelog replay", normalization: "replayed memiavl leaves translated with flatkv.ImportTranslator, then reduced to logical payload", requestedHeight: height, version: db.Version(), diff --git a/sei-db/tools/cmd/seidb/operations/flatkv_open.go b/sei-db/tools/cmd/seidb/operations/flatkv_open.go index 04c9ebe138..e722de0086 100644 --- a/sei-db/tools/cmd/seidb/operations/flatkv_open.go +++ b/sei-db/tools/cmd/seidb/operations/flatkv_open.go @@ -29,11 +29,10 @@ const ( maxCloneRetries = 3 ) -// errSourceChurning marks transient races where the source FlatKV directory -// mutates (snapshot pruned, WAL truncated) between our reads. It is the -// sentinel that prepareFlatKVToolingCloneWith uses to decide whether to -// retry instead of bailing out. -var errSourceChurning = errors.New("flatkv source kept churning during clone") +// errSourceChurning marks transient races where the source directory mutates +// (snapshot pruned, WAL truncated) between our reads. It is the sentinel that +// retryToolingClone uses to decide whether to retry instead of bailing out. +var errSourceChurning = errors.New("source kept churning during clone") // openedFlatKV wraps a temp-cloned FlatKV store used by tooling. // @@ -108,10 +107,13 @@ func openFlatKVReadOnly(dbDir string, height int64) (*openedFlatKV, error) { } func prepareFlatKVToolingClone(dbDir string, height int64) (string, error) { - return prepareFlatKVToolingCloneWith(dbDir, height, tryPrepareFlatKVToolingClone) + return retryToolingClone(dbDir, height, tryPrepareFlatKVToolingClone) } -func prepareFlatKVToolingCloneWith(dbDir string, height int64, tryClone func(string, int64) (string, error)) (string, error) { +// retryToolingClone runs tryClone, retrying while the live writer keeps +// mutating the source out from under us. Shared by the FlatKV and memiavl +// tooling clones, which race the same writer in the same ways. +func retryToolingClone(dbDir string, height int64, tryClone func(string, int64) (string, error)) (string, error) { var lastErr error for attempt := 0; attempt < maxCloneRetries; attempt++ { tempDir, err := tryClone(dbDir, height) @@ -302,7 +304,7 @@ func isFlatKVSnapshotName(name string) bool { // error: snapshots can be many GB, and the previous behavior of falling back // to a byte-copy on tmpfs (the historical $TMPDIR default) routinely OOM'd // nodes and exhausted /tmp. Callers must ensure the tool clone dir lives on -// the same filesystem as the source FlatKV directory. +// the same filesystem as the source directory. // // Hardlinking is safe because: // - snapshot-N files are immutable after Pebble Checkpoint + Rename. @@ -368,7 +370,7 @@ func linkOnly(src, dst string) error { if err := os.Link(src, dst); err != nil { if isCrossDeviceLinkError(err) { return fmt.Errorf("hardlink %s -> %s failed across filesystems; "+ - "FlatKV tooling requires the temp clone to share a filesystem with the source: %w", + "seidb tooling requires the temp clone to share a filesystem with the source: %w", src, dst, err) } return err diff --git a/sei-db/tools/cmd/seidb/operations/flatkv_open_test.go b/sei-db/tools/cmd/seidb/operations/flatkv_open_test.go index 469a085e1b..fcc7dd5f5d 100644 --- a/sei-db/tools/cmd/seidb/operations/flatkv_open_test.go +++ b/sei-db/tools/cmd/seidb/operations/flatkv_open_test.go @@ -123,7 +123,7 @@ func TestPrepareFlatKVToolingCloneMissingCurrentAndSnapshot(t *testing.T) { func TestPrepareFlatKVToolingCloneRetriesENOENT(t *testing.T) { var attempts int - cloneDir, err := prepareFlatKVToolingCloneWith(t.TempDir(), 0, func(string, int64) (string, error) { + cloneDir, err := retryToolingClone(t.TempDir(), 0, func(string, int64) (string, error) { attempts++ if attempts < maxCloneRetries { return "", fmt.Errorf("source vanished: %w", os.ErrNotExist) @@ -135,7 +135,7 @@ func TestPrepareFlatKVToolingCloneRetriesENOENT(t *testing.T) { require.Equal(t, maxCloneRetries, attempts) attempts = 0 - _, err = prepareFlatKVToolingCloneWith(t.TempDir(), 0, func(string, int64) (string, error) { + _, err = retryToolingClone(t.TempDir(), 0, func(string, int64) (string, error) { attempts++ return "", errors.New("permission denied") }) diff --git a/sei-db/tools/cmd/seidb/operations/memiavl_open.go b/sei-db/tools/cmd/seidb/operations/memiavl_open.go new file mode 100644 index 0000000000..ee68720e8f --- /dev/null +++ b/sei-db/tools/cmd/seidb/operations/memiavl_open.go @@ -0,0 +1,140 @@ +package operations + +import ( + "errors" + "fmt" + "os" + "path/filepath" + + "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/memiavl" +) + +// openedMemIAVL is a memiavl DB opened against a temp clone of the source. +type openedMemIAVL struct { + *memiavl.DB + tempDir string +} + +func (o *openedMemIAVL) Close() error { + var err error + if o.DB != nil { + err = o.DB.Close() + } + if o.tempDir != "" { + if rmErr := os.RemoveAll(o.tempDir); rmErr != nil { + if err != nil { + return fmt.Errorf("%w; cleanup temp dir: %w", err, rmErr) + } + return fmt.Errorf("cleanup temp dir: %w", rmErr) + } + } + return err +} + +// openMemiAVLReplay opens memiavl at the given height (0 means latest) by +// replaying the changelog on top of the newest snapshot at or below it. +// +// It clones the source rather than using memiavl's ReadOnly option, which is +// weaker than it reads: ReadOnly skips the LOCK file but the changelog is still +// opened read-write, and that open "repairs" a torn tail record by truncating +// the segment. On a live node a torn tail is the writer mid-append, not +// corruption, so a read-only replay could truncate committed versions out from +// under a running seid. Repairing a torn tail in a private copy is harmless. +// This mirrors openFlatKVReadOnly. +func openMemiAVLReplay(dbDir string, height int64) (*openedMemIAVL, error) { + tempDir, err := prepareMemIAVLToolingClone(dbDir, height) + if err != nil { + return nil, err + } + + db, err := memiavl.OpenDB(height, memiavl.Options{ + Dir: tempDir, + ZeroCopy: true, + }) + if err != nil { + _ = os.RemoveAll(tempDir) + return nil, fmt.Errorf("open memiavl clone at version %d: %w", height, err) + } + opened := &openedMemIAVL{DB: db, tempDir: tempDir} + + // memiavl replays whatever the changelog holds and reports success even + // when that falls short of the requested height. Every tail repair inside + // the clone costs the trailing version, and a digest computed one version + // early is indistinguishable from real divergence when comparing nodes. + if reached := db.Version(); height > 0 && reached != height { + err := fmt.Errorf("memiavl clone version mismatch: requested %d, reached %d "+ + "(changelog does not cover the target height)", height, reached) + if closeErr := opened.Close(); closeErr != nil { + return nil, errors.Join(err, fmt.Errorf("close clone: %w", closeErr)) + } + return nil, err + } + + return opened, nil +} + +func prepareMemIAVLToolingClone(dbDir string, height int64) (string, error) { + return retryToolingClone(dbDir, height, tryPrepareMemIAVLToolingClone) +} + +// tryPrepareMemIAVLToolingClone mirrors tryPrepareFlatKVToolingClone: the two +// layouts are the same shape (current -> snapshot-N/, changelog/, LOCK) and both +// publish snapshots by rename and drop them wholesale, so the same +// hardlink-the-snapshot / byte-copy-the-changelog split applies. +func tryPrepareMemIAVLToolingClone(dbDir string, height int64) (string, error) { + snapshotName, snapshotVersion, err := memiavl.SeekSnapshotDir(dbDir, height) + if err != nil { + return "", err + } + + // The clone must sit inside dbDir to share a filesystem with the source + // snapshot: dbDir is often its own mount point, so a sibling directory is + // not enough and hardlinks would fail across the boundary. + if err := os.MkdirAll(dbDir, 0o750); err != nil { + return "", fmt.Errorf("ensure clone root %s: %w", dbDir, err) + } + // Not a "-tmp" suffix: memiavl's removeTmpDirs deletes every "*-tmp" + // directory under its root when a node opens the DB read-write. + tempDir, err := os.MkdirTemp(dbDir, ".seidb-memiavl-tool-*") + if err != nil { + return "", fmt.Errorf("create temp dir under %s: %w", dbDir, err) + } + cleanup := func(err error) (string, error) { + _ = os.RemoveAll(tempDir) + return "", err + } + + srcSnapshotDir := filepath.Join(dbDir, snapshotName) + dstSnapshotDir := filepath.Join(tempDir, snapshotName) + if err := cloneDirRecursive(srcSnapshotDir, dstSnapshotDir); err != nil { + return cleanup(fmt.Errorf("clone snapshot %s: %w", snapshotName, err)) + } + + if err := os.Symlink(snapshotName, filepath.Join(tempDir, "current")); err != nil { + return cleanup(fmt.Errorf("create current symlink: %w", err)) + } + + srcChangelogDir := filepath.Join(dbDir, "changelog") + info, err := os.Stat(srcChangelogDir) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return cleanup(fmt.Errorf("stat changelog: %w", err)) + } + if err == nil && !info.IsDir() { + return cleanup(fmt.Errorf("changelog path is not a directory: %s", srcChangelogDir)) + } + if err == nil { + dstChangelogDir := filepath.Join(tempDir, "changelog") + if err := copyDirRecursive(srcChangelogDir, dstChangelogDir); err != nil { + return cleanup(fmt.Errorf("clone changelog: %w", err)) + } + // A live writer can roll a new snapshot between our snapshot clone and + // our changelog copy, then prune the changelog up to that newer + // version — leaving a copy that no longer covers snapshotVersion+1 and + // a catchup that would silently skip versions. Retryable. + if err := verifyClonedWALCovers(dstChangelogDir, snapshotVersion); err != nil { + return cleanup(err) + } + } + + return tempDir, nil +} diff --git a/sei-db/tools/cmd/seidb/operations/memiavl_open_test.go b/sei-db/tools/cmd/seidb/operations/memiavl_open_test.go new file mode 100644 index 0000000000..6110b8e4f5 --- /dev/null +++ b/sei-db/tools/cmd/seidb/operations/memiavl_open_test.go @@ -0,0 +1,159 @@ +package operations + +import ( + "os" + "path/filepath" + "sort" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/sei-protocol/sei-chain/sei-db/common/keys" + "github.com/sei-protocol/sei-chain/sei-db/common/utils" + "github.com/sei-protocol/sei-chain/sei-db/proto" +) + +// newMemiavlSourceDir builds a memiavl directory with `versions` committed +// blocks and returns the directory a tool would be pointed at. +func newMemiavlSourceDir(t *testing.T, versions int) string { + t.Helper() + homeDir := t.TempDir() + store := newTestMemiavlStore(t, homeDir) + for i := 1; i <= versions; i++ { + require.NoError(t, store.ApplyChangeSets([]*proto.NamedChangeSet{{ + Name: keys.EVMStoreKey, + Changeset: proto.ChangeSet{Pairs: []*proto.KVPair{noncePair(addrN(byte(i)), uint64(i))}}, + }})) + v, err := store.Commit() + require.NoError(t, err) + require.Equal(t, int64(i), v) + } + require.NoError(t, store.Close()) + return utils.GetCosmosSCStorePath(homeDir) +} + +// snapshotDirState records every regular file under root by relative path and +// content so a later comparison catches truncation, appends, and deletions. +func snapshotDirState(t *testing.T, root string) map[string][]byte { + t.Helper() + state := make(map[string][]byte) + require.NoError(t, filepath.Walk(root, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if !info.Mode().IsRegular() { + return nil + } + rel, err := filepath.Rel(root, path) + if err != nil { + return err + } + bz, err := os.ReadFile(filepath.Clean(path)) + if err != nil { + return err + } + state[rel] = bz + return nil + })) + return state +} + +// lastChangelogSegment returns the newest tidwall segment file in the memiavl +// changelog. Segment names are fixed-width, so lexical order is index order. +func lastChangelogSegment(t *testing.T, dbDir string) string { + t.Helper() + changelogDir := filepath.Join(dbDir, "changelog") + entries, err := os.ReadDir(changelogDir) + require.NoError(t, err) + var names []string + for _, e := range entries { + if !e.IsDir() && len(e.Name()) >= 20 { + names = append(names, e.Name()) + } + } + require.NotEmpty(t, names, "memiavl changelog should have at least one segment") + sort.Strings(names) + return filepath.Join(changelogDir, names[len(names)-1]) +} + +// TestOpenMemiAVLReplayLeavesSourceUntouched is the regression test for the +// audited hazard: a replay that advertises itself as read-only used to hand the +// live changelog straight to memiavl, whose WAL open truncates a torn tail. A +// torn tail on a running node is just the writer mid-append, so the "repair" +// destroyed committed versions. The tool must now repair only its own copy. +func TestOpenMemiAVLReplayLeavesSourceUntouched(t *testing.T) { + dbDir := newMemiavlSourceDir(t, 3) + + // A single length-prefix byte declaring a 16-byte record that never + // arrived is exactly what a reader observes while the writer is + // partway through appending a block. + segment := lastChangelogSegment(t, dbDir) + intact, err := os.ReadFile(filepath.Clean(segment)) + require.NoError(t, err) + torn := append(append([]byte{}, intact...), 0x10) + require.NoError(t, os.WriteFile(segment, torn, 0o600)) + + before := snapshotDirState(t, dbDir) + + db, err := openMemiAVLReplay(dbDir, 0) + require.NoError(t, err, "replay must tolerate a torn tail by repairing its own clone") + require.Equal(t, int64(3), db.Version()) + require.NoError(t, db.Close()) + + require.Equal(t, before, snapshotDirState(t, dbDir), + "replay must not add, remove, truncate, or rewrite any file in the source directory") + + after, err := os.ReadFile(filepath.Clean(segment)) + require.NoError(t, err) + require.Equal(t, torn, after, "the torn tail must still be there for the live writer to finish") +} + +// TestOpenMemiAVLReplayWorksWhileWriterHoldsLock pins the other half of the +// contract: avoiding the mutation must not cost us the ability to read a live +// node. The clone is independent, so the source LOCK is irrelevant to us. +func TestOpenMemiAVLReplayWorksWhileWriterHoldsLock(t *testing.T) { + homeDir := t.TempDir() + writer := newTestMemiavlStore(t, homeDir) + require.NoError(t, writer.ApplyChangeSets([]*proto.NamedChangeSet{{ + Name: keys.EVMStoreKey, + Changeset: proto.ChangeSet{Pairs: []*proto.KVPair{noncePair(addrN(0xA1), 1)}}, + }})) + _, err := writer.Commit() + require.NoError(t, err) + defer func() { require.NoError(t, writer.Close()) }() + + dbDir := utils.GetCosmosSCStorePath(homeDir) + require.FileExists(t, filepath.Join(dbDir, "LOCK")) + + db, err := openMemiAVLReplay(dbDir, 0) + require.NoError(t, err, "tooling clone must not contend for the live writer's lock") + require.Equal(t, int64(1), db.Version()) + require.NoError(t, db.Close()) +} + +// TestOpenMemiAVLReplayRejectsShortChangelog guards the failure mode this +// design makes routine: repairing a torn tail inside the clone silently costs +// the trailing version, and memiavl reports success anyway. For a tool whose +// whole job is comparing digests across nodes, quietly digesting one version +// early is indistinguishable from a real state divergence. +func TestOpenMemiAVLReplayRejectsShortChangelog(t *testing.T) { + dbDir := newMemiavlSourceDir(t, 3) + + segment := lastChangelogSegment(t, dbDir) + intact, err := os.ReadFile(filepath.Clean(segment)) + require.NoError(t, err) + // Lop off the tail so the final committed record is torn and the + // repaired clone can only reach version 2. + require.NoError(t, os.WriteFile(segment, intact[:len(intact)-8], 0o600)) + + _, err = openMemiAVLReplay(dbDir, 3) + require.Error(t, err) + require.Contains(t, err.Error(), "requested 3, reached 2") + + // The rejected clone must not be left behind in the node's data dir. + entries, err := os.ReadDir(dbDir) + require.NoError(t, err) + for _, e := range entries { + require.NotContains(t, e.Name(), ".seidb-memiavl-tool-") + } +} From 87b886c1983fcfb918ff7ec61fb75fc5bb13a667 Mon Sep 17 00:00:00 2001 From: blindchaser Date: Mon, 3 Aug 2026 11:36:02 -0400 Subject: [PATCH 2/3] fix(seidb): address review findings on the tooling clone path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The clone-based replay open regressed nodes bootstrapped with a non-default initial height: memiavl keeps its bootstrap snapshot named snapshot-0 even after SetInitialVersion(N), while the first changelog entry is version N, so verifyClonedWALCovers treated the valid successor as a gap and replay mode failed deterministically with "source kept churning" until the first snapshot rewrite. A freshly recovered chain (genesis initial_height > 1) is exactly that shape. Derive the snapshot's successor from the cloned snapshot's metadata via utils.NextVersion instead of assuming snapshotVersion+1; FlatKV keeps the +1 rule because SetInitialVersion(N) seeds snapshot- there. Also from review: - Reap abandoned clones. A SIGKILLed tool leaves its clone inside the data dir, where hardlinks pin snapshot inodes so pruning frees no disk and the deliberate non-"-tmp" name escapes memiavl's cleanup. Every clone now carries a flock-held owner marker (OFD/flock locks die with the process); clone creation sweeps siblings whose lock is acquirable, and marker-less directories once they are older than the mkdir-to-lock window can explain. - Document that --height 0 against a live node is best-effort: the clone is a consistent committed prefix as of the copy instant, and a detected torn-tail repair now prints a warning telling the operator to trust the report's version line. Cross-node comparisons should use an explicit common height. - Drop the MkdirAll that could never do useful work (snapshot selection already reads dbDir and fails first when it is missing). - Rename SeekSnapshotDir to SeekSnapshotName — it returns a name relative to root, not a path — and pin the exported contract with a unit test, along with ReadMetadata which the successor derivation needs. Co-authored-by: Cursor --- sei-db/state_db/sc/memiavl/db.go | 11 +- sei-db/state_db/sc/memiavl/db_test.go | 35 +++++ sei-db/state_db/sc/memiavl/multitree.go | 10 +- .../seidb/operations/evm_logical_digest.go | 2 +- .../tools/cmd/seidb/operations/flatkv_open.go | 121 +++++++++------ .../cmd/seidb/operations/flatkv_open_test.go | 34 ++--- .../cmd/seidb/operations/memiavl_open.go | 86 +++++++---- .../cmd/seidb/operations/memiavl_open_test.go | 36 +++++ .../tools/cmd/seidb/operations/tool_clone.go | 139 ++++++++++++++++++ .../cmd/seidb/operations/tool_clone_test.go | 75 ++++++++++ 10 files changed, 444 insertions(+), 105 deletions(-) create mode 100644 sei-db/tools/cmd/seidb/operations/tool_clone.go create mode 100644 sei-db/tools/cmd/seidb/operations/tool_clone_test.go diff --git a/sei-db/state_db/sc/memiavl/db.go b/sei-db/state_db/sc/memiavl/db.go index 2b352c3e9f..a924036130 100644 --- a/sei-db/state_db/sc/memiavl/db.go +++ b/sei-db/state_db/sc/memiavl/db.go @@ -1205,13 +1205,14 @@ func seekSnapshot(root string, targetVersion int64) (int64, error) { return snapshotVersion, nil } -// SeekSnapshotDir returns the directory name and version of the snapshot that -// OpenDB would start from for targetVersion: the "current" link when -// targetVersion is 0, otherwise the newest snapshot at or below it. +// SeekSnapshotName returns the directory name (relative to root — callers +// join it themselves) and version of the snapshot that OpenDB would start +// from for targetVersion: the "current" link when targetVersion is 0, +// otherwise the newest snapshot at or below it. // // Exported for readers that need to resolve a snapshot without opening the DB, // so they inherit this package's layout rules instead of restating them. -func SeekSnapshotDir(root string, targetVersion int64) (string, int64, error) { +func SeekSnapshotName(root string, targetVersion int64) (string, int64, error) { if targetVersion == 0 { version, err := currentVersion(root) if err != nil { @@ -1349,7 +1350,7 @@ func isSnapshotName(name string) bool { // it's needed for upgrade module to check store upgrades, // it returns 0 if db doesn't exist or is empty. func GetLatestVersion(dir string) (int64, error) { - metadata, err := readMetadata(currentPath(dir)) + metadata, err := ReadMetadata(currentPath(dir)) if err != nil { if os.IsNotExist(err) { return 0, nil diff --git a/sei-db/state_db/sc/memiavl/db_test.go b/sei-db/state_db/sc/memiavl/db_test.go index 3cddd00f77..d047b29b11 100644 --- a/sei-db/state_db/sc/memiavl/db_test.go +++ b/sei-db/state_db/sc/memiavl/db_test.go @@ -1116,3 +1116,38 @@ func TestUpdateCurrentSymlinkClearsStaleTmp(t *testing.T) { require.NoError(t, err) require.Equal(t, "snapshot-1", target) } + +// TestSeekSnapshotName pins the exported snapshot-resolution contract that +// external readers (seidb tooling) rely on: the returned name is relative to +// root, targetVersion 0 resolves through the current link, a positive target +// selects the newest snapshot at or below it, and a target older than the +// earliest snapshot reports pruning instead of guessing. +func TestSeekSnapshotName(t *testing.T) { + root := t.TempDir() + for _, v := range []int64{5, 10} { + require.NoError(t, os.Mkdir(filepath.Join(root, snapshotName(v)), 0o750)) + } + require.NoError(t, os.Symlink(snapshotName(10), currentPath(root))) + + name, version, err := SeekSnapshotName(root, 0) + require.NoError(t, err) + require.Equal(t, snapshotName(10), name) + require.Equal(t, int64(10), version) + + name, version, err = SeekSnapshotName(root, 7) + require.NoError(t, err) + require.Equal(t, snapshotName(5), name) + require.Equal(t, int64(5), version) + + name, version, err = SeekSnapshotName(root, 10) + require.NoError(t, err) + require.Equal(t, snapshotName(10), name) + require.Equal(t, int64(10), version) + + _, _, err = SeekSnapshotName(root, 3) + require.Error(t, err) + require.Contains(t, err.Error(), "target version is pruned") + + _, _, err = SeekSnapshotName(filepath.Join(root, "does-not-exist"), 0) + require.Error(t, err) +} diff --git a/sei-db/state_db/sc/memiavl/multitree.go b/sei-db/state_db/sc/memiavl/multitree.go index 6e9101e339..f0f85e417a 100644 --- a/sei-db/state_db/sc/memiavl/multitree.go +++ b/sei-db/state_db/sc/memiavl/multitree.go @@ -76,7 +76,7 @@ func NewEmptyMultiTree(initialVersion uint32) *MultiTree { func LoadMultiTree(ctx context.Context, dir string, opts Options) (*MultiTree, error) { startTime := time.Now() - metadata, err := readMetadata(dir) + metadata, err := ReadMetadata(dir) if err != nil { return nil, err } @@ -600,7 +600,13 @@ func (t *MultiTree) ReplaceWith(other *MultiTree) error { return errors.Join(errs...) } -func readMetadata(dir string) (*proto.MultiTreeMetadata, error) { +// ReadMetadata loads a snapshot directory's MultiTreeMetadata (commit info + +// initial version). Exported for readers that need snapshot metadata without +// opening the DB — e.g. seidb tooling, whose changelog-coverage check must +// know the initial version because a DB bootstrapped with initial version N +// keeps its first snapshot named snapshot-0 while the first changelog entry +// is version N, not 1. +func ReadMetadata(dir string) (*proto.MultiTreeMetadata, error) { // load commit info bz, err := os.ReadFile(filepath.Join(filepath.Clean(dir), MetadataFileName)) if err != nil { diff --git a/sei-db/tools/cmd/seidb/operations/evm_logical_digest.go b/sei-db/tools/cmd/seidb/operations/evm_logical_digest.go index aceed4cc98..143acc1e50 100644 --- a/sei-db/tools/cmd/seidb/operations/evm_logical_digest.go +++ b/sei-db/tools/cmd/seidb/operations/evm_logical_digest.go @@ -166,7 +166,7 @@ func EvmLogicalDigestCmd() *cobra.Command { cmd.Flags().StringP("db-dir", "d", "", "For flatkv: the flatkv data dir. For memiavl: the memiavl root dir (contains current/ and snapshot-* )") cmd.Flags().String("flatkv-dir", "", "Composite mode: flatkv data dir") cmd.Flags().String("memiavl-dir", "", "Composite mode: memiavl root dir (contains current/ and snapshot-* )") - cmd.Flags().Int64("height", 0, "Target version. flatkv WAL-replays to it; memiavl resolves snapshot-/evm (0 = current symlink)") + cmd.Flags().Int64("height", 0, "Target version. flatkv WAL-replays to it; memiavl resolves snapshot-/evm (0 = current symlink). On a live node 0 = latest is best-effort — the printed version line records what was actually digested; always pass an explicit common height when comparing nodes") cmd.Flags().String("memiavl-open-mode", memiavlOpenModeSnapshot, "memiavl read mode: snapshot (FAST: sequential scan of the completed snapshot kvs file; requires an on-disk snapshot at --height, or --height 0 for current) | replay (SLOW, ~10x: clones the snapshot + changelog to a temp dir, replays to --height there, then walks the mmap tree; use only when no snapshot exists at the target height). Prefer snapshot when --height matches an existing snapshot boundary") cmd.Flags().String("memiavl-normalization", memiavlNormSemantic, "memiavl digest/inspect normalization: semantic/independent (raw EVM key/value decoder) | translator (current migration mapping)") cmd.Flags().String("inspect-bucket", "", "Inspect one normalized bucket (account|code|storage|misc) instead of printing the global digest") diff --git a/sei-db/tools/cmd/seidb/operations/flatkv_open.go b/sei-db/tools/cmd/seidb/operations/flatkv_open.go index e722de0086..8f0d3ec086 100644 --- a/sei-db/tools/cmd/seidb/operations/flatkv_open.go +++ b/sei-db/tools/cmd/seidb/operations/flatkv_open.go @@ -40,7 +40,7 @@ var errSourceChurning = errors.New("source kept churning during clone") // WAL so they do not compete with a live node for the FlatKV writer lock. type openedFlatKV struct { *flatkv.CommitStore - tempDir string + clone *toolClone } func (o *openedFlatKV) Close() error { @@ -48,13 +48,11 @@ func (o *openedFlatKV) Close() error { if o.CommitStore != nil { err = o.CommitStore.Close() } - if o.tempDir != "" { - if rmErr := os.RemoveAll(o.tempDir); rmErr != nil { - if err != nil { - return fmt.Errorf("%w; cleanup temp dir: %w", err, rmErr) - } - return fmt.Errorf("cleanup temp dir: %w", rmErr) + if rmErr := o.clone.Remove(); rmErr != nil { + if err != nil { + return fmt.Errorf("%w; %w", err, rmErr) } + return rmErr } return err } @@ -78,54 +76,73 @@ func (o *openedFlatKV) Close() error { // os.ReadDir and os.Link calls, we surface ENOENT, re-select the // snapshot, and retry up to maxCloneRetries times. // -// height=0 means latest version. +// height=0 means the latest version, best-effort on a live node: the clone is +// a consistent committed prefix as of the copy instant, and a torn-tail +// repair inside the clone (surfaced as a warning) can land it one version +// behind the source tip. Tools print the version actually opened; treat that +// line as authoritative when comparing across nodes. func openFlatKVReadOnly(dbDir string, height int64) (*openedFlatKV, error) { - tempDir, err := prepareFlatKVToolingClone(dbDir, height) + clone, err := prepareFlatKVToolingClone(dbDir, height) if err != nil { return nil, err } + warnIfCloneRepaired(clone, "flatkv", height) cfg := config.DefaultConfig() - cfg.DataDir = tempDir + cfg.DataDir = clone.dir store, err := flatkv.NewCommitStore(context.Background(), cfg) if err != nil { - _ = os.RemoveAll(tempDir) + _ = clone.Remove() return nil, fmt.Errorf("failed to create FlatKV store: %w", err) } if _, err := store.LoadVersion(height, false); err != nil { _ = store.Close() - _ = os.RemoveAll(tempDir) + _ = clone.Remove() return nil, fmt.Errorf("failed to open FlatKV at version %d: %w", height, err) } return &openedFlatKV{ CommitStore: store, - tempDir: tempDir, + clone: clone, }, nil } -func prepareFlatKVToolingClone(dbDir string, height int64) (string, error) { +// warnIfCloneRepaired tells the operator that the cloned changelog had a torn +// tail (the byte-copy raced the live writer mid-append) and was repaired +// inside the clone. For an explicit --height the reached-version checks catch +// any resulting shortfall; for height 0 ("latest") there is no target to +// check against, so the printed version line is the only record of what was +// actually digested. +func warnIfCloneRepaired(clone *toolClone, backend string, height int64) { + if !clone.walRepaired || height != 0 { + return + } + fmt.Fprintf(os.Stderr, "warning: cloned %s changelog had a torn tail (live writer mid-append) and was repaired in the clone; "+ + "the opened version may trail the source tip by one — trust the printed version line\n", backend) +} + +func prepareFlatKVToolingClone(dbDir string, height int64) (*toolClone, error) { return retryToolingClone(dbDir, height, tryPrepareFlatKVToolingClone) } // retryToolingClone runs tryClone, retrying while the live writer keeps // mutating the source out from under us. Shared by the FlatKV and memiavl // tooling clones, which race the same writer in the same ways. -func retryToolingClone(dbDir string, height int64, tryClone func(string, int64) (string, error)) (string, error) { +func retryToolingClone(dbDir string, height int64, tryClone func(string, int64) (*toolClone, error)) (*toolClone, error) { var lastErr error for attempt := 0; attempt < maxCloneRetries; attempt++ { - tempDir, err := tryClone(dbDir, height) + clone, err := tryClone(dbDir, height) if err == nil { - return tempDir, nil + return clone, nil } if !isCloneRetryableError(err) { - return "", err + return nil, err } lastErr = err } - return "", fmt.Errorf("clone aborted after %d retries, source kept churning: %w", maxCloneRetries, lastErr) + return nil, fmt.Errorf("clone aborted after %d retries, source kept churning: %w", maxCloneRetries, lastErr) } // isCloneRetryableError reports whether err indicates a transient race with @@ -136,39 +153,37 @@ func isCloneRetryableError(err error) bool { return errors.Is(err, os.ErrNotExist) || errors.Is(err, errSourceChurning) } -func tryPrepareFlatKVToolingClone(dbDir string, height int64) (string, error) { +func tryPrepareFlatKVToolingClone(dbDir string, height int64) (*toolClone, error) { snapshotName, err := selectFlatKVSnapshot(dbDir, height) if err != nil { - return "", err + return nil, err } snapshotVersion, err := strconv.ParseInt(snapshotName[len(flatkvSnapshotPrefix):], 10, 64) if err != nil { - return "", fmt.Errorf("parse snapshot version from %q: %w", snapshotName, err) + return nil, fmt.Errorf("parse snapshot version from %q: %w", snapshotName, err) } - // Place the temp clone inside dbDir so it is on the exact same mounted - // filesystem as the source snapshots. A sibling directory is not enough: - // dbDir itself is often a mount point on dedicated data volumes. - cloneRoot := dbDir - if err := os.MkdirAll(cloneRoot, 0o750); err != nil { - return "", fmt.Errorf("ensure clone root %s: %w", cloneRoot, err) - } - tempDir, err := os.MkdirTemp(cloneRoot, ".seidb-flatkv-tool-*") + // The clone must sit inside dbDir so it is on the exact same mounted + // filesystem as the source snapshots (dbDir is often its own mount + // point, so a sibling directory is not enough and hardlinks would fail + // across the boundary). selectFlatKVSnapshot already read dbDir, so it + // is known to exist. + clone, err := newToolClone(dbDir, ".seidb-flatkv-tool-") if err != nil { - return "", fmt.Errorf("create temp dir under %s: %w", cloneRoot, err) + return nil, err } - cleanup := func(err error) (string, error) { - _ = os.RemoveAll(tempDir) - return "", err + cleanup := func(err error) (*toolClone, error) { + _ = clone.Remove() + return nil, err } srcSnapshotDir := filepath.Join(dbDir, snapshotName) - dstSnapshotDir := filepath.Join(tempDir, snapshotName) + dstSnapshotDir := filepath.Join(clone.dir, snapshotName) if err := cloneDirRecursive(srcSnapshotDir, dstSnapshotDir); err != nil { return cleanup(fmt.Errorf("clone snapshot %s: %w", snapshotName, err)) } - if err := os.Symlink(snapshotName, filepath.Join(tempDir, "current")); err != nil { + if err := os.Symlink(snapshotName, filepath.Join(clone.dir, "current")); err != nil { return cleanup(fmt.Errorf("create current symlink: %w", err)) } @@ -181,29 +196,39 @@ func tryPrepareFlatKVToolingClone(dbDir string, height int64) (string, error) { return cleanup(fmt.Errorf("changelog path is not a directory: %s", srcChangelogDir)) } if err == nil { - dstChangelogDir := filepath.Join(tempDir, "changelog") + dstChangelogDir := filepath.Join(clone.dir, "changelog") if err := copyDirRecursive(srcChangelogDir, dstChangelogDir); err != nil { return cleanup(fmt.Errorf("clone changelog: %w", err)) } // Detect the snapshot/WAL race: a live writer can roll a new // snapshot between our snapshot clone and our changelog copy and // then truncateWAL up to that newer snapshot's version. If that - // happened, the cloned WAL no longer covers snapshotVersion+1, - // and a downstream catchup would silently jump over missing - // versions. Surface it as a retryable error so the outer loop - // re-selects the snapshot and tries again. - if err := verifyClonedWALCovers(dstChangelogDir, snapshotVersion); err != nil { + // happened, the cloned WAL no longer covers the snapshot's + // successor version, and a downstream catchup would silently jump + // over missing versions. Surface it as a retryable error so the + // outer loop re-selects the snapshot and tries again. + // + // FlatKV snapshots are always named with a real committed version — + // SetInitialVersion(N) seeds committedVersion N-1 and writes + // snapshot- — so the successor is unconditionally + // snapshotVersion+1 here (unlike memiavl, whose bootstrap + // snapshot-0 hides a configurable initial version). + sizeBefore := changelogByteSize(dstChangelogDir) + if err := verifyClonedWALCovers(dstChangelogDir, snapshotVersion, snapshotVersion+1); err != nil { return cleanup(err) } + clone.walRepaired = changelogByteSize(dstChangelogDir) < sizeBefore } - return tempDir, nil + return clone, nil } // verifyClonedWALCovers opens the cloned WAL just long enough to ensure it // either is empty, ends at or before snapshotVersion (no replay needed), or -// starts at or before snapshotVersion+1 (catchup can resume cleanly). -func verifyClonedWALCovers(dstChangelogDir string, snapshotVersion int64) error { +// starts at or before firstNeeded — the first version catchup must replay on +// top of the snapshot (snapshotVersion+1, except for a memiavl bootstrap +// snapshot-0 whose successor is the configured initial version). +func verifyClonedWALCovers(dstChangelogDir string, snapshotVersion, firstNeeded int64) error { walLog, err := wal.NewChangelogWAL(dstChangelogDir, wal.Config{}) if err != nil { return fmt.Errorf("open cloned changelog for validation: %w", err) @@ -234,11 +259,11 @@ func verifyClonedWALCovers(dstChangelogDir string, snapshotVersion int64) error if lastVer <= snapshotVersion { return nil } - if firstVer <= snapshotVersion+1 { + if firstVer <= firstNeeded { return nil } - return fmt.Errorf("%w: cloned WAL starts at version %d but snapshot is %d (truncated past snapshot mid-clone)", - errSourceChurning, firstVer, snapshotVersion) + return fmt.Errorf("%w: cloned WAL starts at version %d but catchup needs %d over snapshot %d (truncated past snapshot mid-clone)", + errSourceChurning, firstVer, firstNeeded, snapshotVersion) } func readWALEntryVersion(walLog wal.ChangelogWAL, off uint64) (int64, error) { diff --git a/sei-db/tools/cmd/seidb/operations/flatkv_open_test.go b/sei-db/tools/cmd/seidb/operations/flatkv_open_test.go index fcc7dd5f5d..839007c7ed 100644 --- a/sei-db/tools/cmd/seidb/operations/flatkv_open_test.go +++ b/sei-db/tools/cmd/seidb/operations/flatkv_open_test.go @@ -55,17 +55,17 @@ func TestPrepareFlatKVToolingCloneHardlinksSnapshotAndCopiesChangelog(t *testing srcChangelogFile := filepath.Join(dbDir, "changelog", "000001.log") require.NoError(t, os.WriteFile(srcChangelogFile, []byte("wal-data"), 0o600)) - cloneDir, err := prepareFlatKVToolingClone(dbDir, 0) + clone, err := prepareFlatKVToolingClone(dbDir, 0) require.NoError(t, err) - defer os.RemoveAll(cloneDir) //nolint:errcheck // test cleanup + defer clone.Remove() //nolint:errcheck // test cleanup - target, err := os.Readlink(filepath.Join(cloneDir, "current")) + target, err := os.Readlink(filepath.Join(clone.dir, "current")) require.NoError(t, err) require.Equal(t, snapshot, target) - require.FileExists(t, filepath.Join(cloneDir, snapshot, "account", "000001.sst")) - require.NoFileExists(t, filepath.Join(cloneDir, snapshot, "LOCK")) - dstSnapshotFile := filepath.Join(cloneDir, snapshot, "account", "000001.sst") - dstChangelogFile := filepath.Join(cloneDir, "changelog", "000001.log") + require.FileExists(t, filepath.Join(clone.dir, snapshot, "account", "000001.sst")) + require.NoFileExists(t, filepath.Join(clone.dir, snapshot, "LOCK")) + dstSnapshotFile := filepath.Join(clone.dir, snapshot, "account", "000001.sst") + dstChangelogFile := filepath.Join(clone.dir, "changelog", "000001.log") require.FileExists(t, dstChangelogFile) srcSnapshotInfo, err := os.Stat(srcSnapshotFile) @@ -123,21 +123,21 @@ func TestPrepareFlatKVToolingCloneMissingCurrentAndSnapshot(t *testing.T) { func TestPrepareFlatKVToolingCloneRetriesENOENT(t *testing.T) { var attempts int - cloneDir, err := retryToolingClone(t.TempDir(), 0, func(string, int64) (string, error) { + clone, err := retryToolingClone(t.TempDir(), 0, func(string, int64) (*toolClone, error) { attempts++ if attempts < maxCloneRetries { - return "", fmt.Errorf("source vanished: %w", os.ErrNotExist) + return nil, fmt.Errorf("source vanished: %w", os.ErrNotExist) } - return t.TempDir(), nil + return &toolClone{dir: t.TempDir()}, nil }) require.NoError(t, err) - require.NotEmpty(t, cloneDir) + require.NotEmpty(t, clone.dir) require.Equal(t, maxCloneRetries, attempts) attempts = 0 - _, err = retryToolingClone(t.TempDir(), 0, func(string, int64) (string, error) { + _, err = retryToolingClone(t.TempDir(), 0, func(string, int64) (*toolClone, error) { attempts++ - return "", errors.New("permission denied") + return nil, errors.New("permission denied") }) require.Error(t, err) require.Equal(t, 1, attempts) @@ -160,15 +160,15 @@ func TestPrepareFlatKVToolingClonePlacesTempDirInsideDBDir(t *testing.T) { require.NoError(t, store.WriteSnapshot("")) require.NoError(t, store.Close()) - cloneDir, err := prepareFlatKVToolingClone(dbDir, 0) + clone, err := prepareFlatKVToolingClone(dbDir, 0) require.NoError(t, err) - defer os.RemoveAll(cloneDir) //nolint:errcheck // test cleanup + defer clone.Remove() //nolint:errcheck // test cleanup - rel, err := filepath.Rel(dbDir, cloneDir) + rel, err := filepath.Rel(dbDir, clone.dir) require.NoError(t, err) require.NotEqual(t, ".", rel) require.False(t, strings.HasPrefix(rel, ".."), "tooling clone must be created inside dbDir to stay on dbDir's mounted filesystem") - require.Contains(t, filepath.Base(cloneDir), ".seidb-flatkv-tool-") + require.Contains(t, filepath.Base(clone.dir), ".seidb-flatkv-tool-") } // TestPrepareFlatKVToolingCloneDetectsWALTruncationRace simulates the audited diff --git a/sei-db/tools/cmd/seidb/operations/memiavl_open.go b/sei-db/tools/cmd/seidb/operations/memiavl_open.go index ee68720e8f..c7b8b96987 100644 --- a/sei-db/tools/cmd/seidb/operations/memiavl_open.go +++ b/sei-db/tools/cmd/seidb/operations/memiavl_open.go @@ -3,16 +3,18 @@ package operations import ( "errors" "fmt" + "math" "os" "path/filepath" + "github.com/sei-protocol/sei-chain/sei-db/common/utils" "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/memiavl" ) // openedMemIAVL is a memiavl DB opened against a temp clone of the source. type openedMemIAVL struct { *memiavl.DB - tempDir string + clone *toolClone } func (o *openedMemIAVL) Close() error { @@ -20,13 +22,11 @@ func (o *openedMemIAVL) Close() error { if o.DB != nil { err = o.DB.Close() } - if o.tempDir != "" { - if rmErr := os.RemoveAll(o.tempDir); rmErr != nil { - if err != nil { - return fmt.Errorf("%w; cleanup temp dir: %w", err, rmErr) - } - return fmt.Errorf("cleanup temp dir: %w", rmErr) + if rmErr := o.clone.Remove(); rmErr != nil { + if err != nil { + return fmt.Errorf("%w; %w", err, rmErr) } + return rmErr } return err } @@ -41,21 +41,29 @@ func (o *openedMemIAVL) Close() error { // corruption, so a read-only replay could truncate committed versions out from // under a running seid. Repairing a torn tail in a private copy is harmless. // This mirrors openFlatKVReadOnly. +// +// height=0 ("latest") is best-effort on a live node: the clone is a +// consistent committed prefix as of the copy instant, and a torn-tail repair +// inside the clone (surfaced as a warning) can land it one version behind the +// source tip. There is no target version to check against, so the report's +// printed version line is the authoritative record of what was digested; +// cross-node comparisons should always use an explicit common --height. func openMemiAVLReplay(dbDir string, height int64) (*openedMemIAVL, error) { - tempDir, err := prepareMemIAVLToolingClone(dbDir, height) + clone, err := prepareMemIAVLToolingClone(dbDir, height) if err != nil { return nil, err } + warnIfCloneRepaired(clone, "memiavl", height) db, err := memiavl.OpenDB(height, memiavl.Options{ - Dir: tempDir, + Dir: clone.dir, ZeroCopy: true, }) if err != nil { - _ = os.RemoveAll(tempDir) + _ = clone.Remove() return nil, fmt.Errorf("open memiavl clone at version %d: %w", height, err) } - opened := &openedMemIAVL{DB: db, tempDir: tempDir} + opened := &openedMemIAVL{DB: db, clone: clone} // memiavl replays whatever the changelog holds and reports success even // when that falls short of the requested height. Every tail repair inside @@ -73,7 +81,7 @@ func openMemiAVLReplay(dbDir string, height int64) (*openedMemIAVL, error) { return opened, nil } -func prepareMemIAVLToolingClone(dbDir string, height int64) (string, error) { +func prepareMemIAVLToolingClone(dbDir string, height int64) (*toolClone, error) { return retryToolingClone(dbDir, height, tryPrepareMemIAVLToolingClone) } @@ -81,39 +89,51 @@ func prepareMemIAVLToolingClone(dbDir string, height int64) (string, error) { // layouts are the same shape (current -> snapshot-N/, changelog/, LOCK) and both // publish snapshots by rename and drop them wholesale, so the same // hardlink-the-snapshot / byte-copy-the-changelog split applies. -func tryPrepareMemIAVLToolingClone(dbDir string, height int64) (string, error) { - snapshotName, snapshotVersion, err := memiavl.SeekSnapshotDir(dbDir, height) +func tryPrepareMemIAVLToolingClone(dbDir string, height int64) (*toolClone, error) { + snapshotName, snapshotVersion, err := memiavl.SeekSnapshotName(dbDir, height) if err != nil { - return "", err + return nil, err } // The clone must sit inside dbDir to share a filesystem with the source // snapshot: dbDir is often its own mount point, so a sibling directory is // not enough and hardlinks would fail across the boundary. - if err := os.MkdirAll(dbDir, 0o750); err != nil { - return "", fmt.Errorf("ensure clone root %s: %w", dbDir, err) - } - // Not a "-tmp" suffix: memiavl's removeTmpDirs deletes every "*-tmp" - // directory under its root when a node opens the DB read-write. - tempDir, err := os.MkdirTemp(dbDir, ".seidb-memiavl-tool-*") + // SeekSnapshotName already read dbDir, so it is known to exist. + clone, err := newToolClone(dbDir, ".seidb-memiavl-tool-") if err != nil { - return "", fmt.Errorf("create temp dir under %s: %w", dbDir, err) + return nil, err } - cleanup := func(err error) (string, error) { - _ = os.RemoveAll(tempDir) - return "", err + cleanup := func(err error) (*toolClone, error) { + _ = clone.Remove() + return nil, err } srcSnapshotDir := filepath.Join(dbDir, snapshotName) - dstSnapshotDir := filepath.Join(tempDir, snapshotName) + dstSnapshotDir := filepath.Join(clone.dir, snapshotName) if err := cloneDirRecursive(srcSnapshotDir, dstSnapshotDir); err != nil { return cleanup(fmt.Errorf("clone snapshot %s: %w", snapshotName, err)) } - if err := os.Symlink(snapshotName, filepath.Join(tempDir, "current")); err != nil { + if err := os.Symlink(snapshotName, filepath.Join(clone.dir, "current")); err != nil { return cleanup(fmt.Errorf("create current symlink: %w", err)) } + // The version parsed from the snapshot directory name is not enough to + // know which changelog version catchup resumes from: memiavl bootstraps + // every DB as snapshot-0 even when it was initialized with + // SetInitialVersion(N), in which case the first changelog entry is + // version N, not 1. Read the initial version from the cloned snapshot's + // metadata (immune to source pruning — the files are hard links) and + // derive the successor the same way memiavl itself does. + metadata, err := memiavl.ReadMetadata(dstSnapshotDir) + if err != nil { + return cleanup(fmt.Errorf("read cloned snapshot metadata: %w", err)) + } + if metadata.InitialVersion < 0 || metadata.InitialVersion > math.MaxUint32 { + return cleanup(fmt.Errorf("cloned snapshot has invalid initial version: %d", metadata.InitialVersion)) + } + firstNeeded := utils.NextVersion(snapshotVersion, uint32(metadata.InitialVersion)) + srcChangelogDir := filepath.Join(dbDir, "changelog") info, err := os.Stat(srcChangelogDir) if err != nil && !errors.Is(err, os.ErrNotExist) { @@ -123,18 +143,20 @@ func tryPrepareMemIAVLToolingClone(dbDir string, height int64) (string, error) { return cleanup(fmt.Errorf("changelog path is not a directory: %s", srcChangelogDir)) } if err == nil { - dstChangelogDir := filepath.Join(tempDir, "changelog") + dstChangelogDir := filepath.Join(clone.dir, "changelog") if err := copyDirRecursive(srcChangelogDir, dstChangelogDir); err != nil { return cleanup(fmt.Errorf("clone changelog: %w", err)) } // A live writer can roll a new snapshot between our snapshot clone and // our changelog copy, then prune the changelog up to that newer - // version — leaving a copy that no longer covers snapshotVersion+1 and - // a catchup that would silently skip versions. Retryable. - if err := verifyClonedWALCovers(dstChangelogDir, snapshotVersion); err != nil { + // version — leaving a copy that no longer covers the snapshot's + // successor and a catchup that would silently skip versions. Retryable. + sizeBefore := changelogByteSize(dstChangelogDir) + if err := verifyClonedWALCovers(dstChangelogDir, snapshotVersion, firstNeeded); err != nil { return cleanup(err) } + clone.walRepaired = changelogByteSize(dstChangelogDir) < sizeBefore } - return tempDir, nil + return clone, nil } diff --git a/sei-db/tools/cmd/seidb/operations/memiavl_open_test.go b/sei-db/tools/cmd/seidb/operations/memiavl_open_test.go index 6110b8e4f5..b2bae9ba90 100644 --- a/sei-db/tools/cmd/seidb/operations/memiavl_open_test.go +++ b/sei-db/tools/cmd/seidb/operations/memiavl_open_test.go @@ -131,6 +131,42 @@ func TestOpenMemiAVLReplayWorksWhileWriterHoldsLock(t *testing.T) { require.NoError(t, db.Close()) } +// TestOpenMemiAVLReplayAfterSetInitialVersion pins the non-default +// initial-height layout: memiavl bootstraps every DB as snapshot-0 even when +// SetInitialVersion(100) makes the first changelog entry version 100, so the +// clone's WAL-coverage check must derive the snapshot's successor from the +// snapshot metadata instead of assuming version 1 follows snapshot-0. This is +// exactly the shape of a freshly recovered chain whose genesis initial_height +// is greater than 1 and whose first snapshot rewrite has not happened yet; +// before the fix, replay mode failed deterministically on such nodes with +// "source kept churning". +func TestOpenMemiAVLReplayAfterSetInitialVersion(t *testing.T) { + homeDir := t.TempDir() + store := newTestMemiavlStore(t, homeDir) + require.NoError(t, store.SetInitialVersion(100)) + for i := 0; i < 3; i++ { + require.NoError(t, store.ApplyChangeSets([]*proto.NamedChangeSet{{ + Name: keys.EVMStoreKey, + Changeset: proto.ChangeSet{Pairs: []*proto.KVPair{noncePair(addrN(byte(i+1)), uint64(i+1))}}, + }})) + v, err := store.Commit() + require.NoError(t, err) + require.Equal(t, int64(100+i), v) + } + require.NoError(t, store.Close()) + dbDir := utils.GetCosmosSCStorePath(homeDir) + + historical, err := openMemiAVLReplay(dbDir, 101) + require.NoError(t, err, "coverage check must accept a snapshot-0 whose successor is the initial version") + require.Equal(t, int64(101), historical.Version()) + require.NoError(t, historical.Close()) + + latest, err := openMemiAVLReplay(dbDir, 0) + require.NoError(t, err) + require.Equal(t, int64(102), latest.Version()) + require.NoError(t, latest.Close()) +} + // TestOpenMemiAVLReplayRejectsShortChangelog guards the failure mode this // design makes routine: repairing a torn tail inside the clone silently costs // the trailing version, and memiavl reports success anyway. For a tool whose diff --git a/sei-db/tools/cmd/seidb/operations/tool_clone.go b/sei-db/tools/cmd/seidb/operations/tool_clone.go new file mode 100644 index 0000000000..82010d948f --- /dev/null +++ b/sei-db/tools/cmd/seidb/operations/tool_clone.go @@ -0,0 +1,139 @@ +package operations + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "time" + + "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/memiavl" +) + +const ( + // toolCloneOwnerLockName is the flock-held marker file every tooling + // clone carries. While the creating process lives, the lock is held and + // the clone is off-limits; once the process dies (including SIGKILL, + // where no deferred cleanup runs), the kernel releases the flock and the + // next tool invocation can reap the directory. + toolCloneOwnerLockName = ".seidb-tool-owner.lock" + + // staleUnmarkedCloneAge guards the marker-less case: a crash in the + // window between MkdirTemp and the owner-lock creation, where liveness + // cannot be probed. Age is a poor liveness signal for marked clones + // (a mainnet-scale digest can legitimately run for hours), so it is + // used only when no marker exists at all. + staleUnmarkedCloneAge = 24 * time.Hour +) + +// toolClone is a private, disposable copy of a live store's snapshot + +// changelog that seidb tooling operates on instead of the live directory. +// It lives inside the source dbDir (to share its filesystem for hardlinks), +// which is exactly why leaking one is costly: its hardlinks pin snapshot +// inodes, so the live node's snapshot pruning frees no disk space until the +// clone is removed. +type toolClone struct { + dir string + ownerLock memiavl.FileLock + + // walRepaired records that validating the cloned changelog shrank it: + // the byte-copy caught the live writer mid-append and the WAL open + // repaired the torn tail inside the clone, costing the trailing record. + walRepaired bool +} + +// newToolClone reaps abandoned sibling clones, creates a fresh clone +// directory under dbDir with the given prefix, and marks it owned via flock +// before any expensive cloning starts. +// +// The prefix deliberately has no "-tmp" suffix: memiavl's removeTmpDirs +// deletes every "*-tmp" directory under its root when a node opens the DB +// read-write, and these clones must never be reaped by a process that cannot +// see whether the owning tool is still alive. +func newToolClone(dbDir, prefix string) (*toolClone, error) { + sweepStaleToolClones(dbDir, prefix) + + dir, err := os.MkdirTemp(dbDir, prefix+"*") + if err != nil { + return nil, fmt.Errorf("create temp dir under %s: %w", dbDir, err) + } + ownerLock, err := memiavl.LockFile(filepath.Join(dir, toolCloneOwnerLockName)) + if err != nil { + _ = os.RemoveAll(dir) + return nil, fmt.Errorf("acquire clone owner lock in %s: %w", dir, err) + } + return &toolClone{dir: dir, ownerLock: ownerLock}, nil +} + +// Remove releases the ownership lock and deletes the clone directory. +func (c *toolClone) Remove() error { + if c == nil { + return nil + } + if c.ownerLock != nil { + _ = c.ownerLock.Unlock() + _ = c.ownerLock.Destroy() + c.ownerLock = nil + } + if c.dir == "" { + return nil + } + if err := os.RemoveAll(c.dir); err != nil { + return fmt.Errorf("cleanup temp dir: %w", err) + } + c.dir = "" + return nil +} + +// sweepStaleToolClones removes abandoned clone directories under dbDir whose +// owner is provably gone: either the owner flock is acquirable (the creating +// process died), or no marker exists and the directory is old enough that the +// mkdir-to-lock window cannot explain it. Clones whose lock is still held — +// a concurrently running tool — are left alone. Best-effort by design: a +// failed sweep must never block the read the tool was invoked for. +func sweepStaleToolClones(dbDir, prefix string) { + entries, err := os.ReadDir(dbDir) + if err != nil { + return + } + for _, entry := range entries { + if !entry.IsDir() || !strings.HasPrefix(entry.Name(), prefix) { + continue + } + dir := filepath.Join(dbDir, entry.Name()) + lockPath := filepath.Join(dir, toolCloneOwnerLockName) + if _, err := os.Stat(lockPath); errors.Is(err, os.ErrNotExist) { + if info, err := entry.Info(); err == nil && time.Since(info.ModTime()) > staleUnmarkedCloneAge { + _ = os.RemoveAll(dir) + } + continue + } + lock, err := memiavl.LockFile(lockPath) + if err != nil { + // Lock held (owner alive) or unreadable — leave the clone alone. + continue + } + _ = lock.Unlock() + _ = lock.Destroy() + _ = os.RemoveAll(dir) + } +} + +// changelogByteSize sums the sizes of the regular files in a cloned changelog +// directory. Comparing it before and after the WAL-coverage validation open +// detects a torn-tail repair inside the clone (the only mutation that open +// can perform), which callers surface as a warning for latest-height reads. +func changelogByteSize(dir string) int64 { + var total int64 + _ = filepath.Walk(dir, func(_ string, info os.FileInfo, err error) error { + if err != nil { + return nil //nolint:nilerr // best-effort size probe + } + if info.Mode().IsRegular() { + total += info.Size() + } + return nil + }) + return total +} diff --git a/sei-db/tools/cmd/seidb/operations/tool_clone_test.go b/sei-db/tools/cmd/seidb/operations/tool_clone_test.go new file mode 100644 index 0000000000..5ddf6941fb --- /dev/null +++ b/sei-db/tools/cmd/seidb/operations/tool_clone_test.go @@ -0,0 +1,75 @@ +package operations + +import ( + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/memiavl" +) + +// TestSweepStaleToolClones pins the reaping policy for abandoned tooling +// clones: a clone whose owner lock is acquirable was orphaned by a dead +// process (SIGKILL runs no deferred cleanup) and must be removed, a clone +// whose lock is held belongs to a live tool and must survive, and a +// marker-less clone is removed only once it is old enough that the +// mkdir-to-lock window cannot explain it. +func TestSweepStaleToolClones(t *testing.T) { + dbDir := t.TempDir() + prefix := ".seidb-flatkv-tool-" + + staleUnmarked := filepath.Join(dbDir, prefix+"stale-unmarked") + require.NoError(t, os.Mkdir(staleUnmarked, 0o750)) + old := time.Now().Add(-2 * staleUnmarkedCloneAge) + require.NoError(t, os.Chtimes(staleUnmarked, old, old)) + + freshUnmarked := filepath.Join(dbDir, prefix+"fresh-unmarked") + require.NoError(t, os.Mkdir(freshUnmarked, 0o750)) + + held := filepath.Join(dbDir, prefix+"held") + require.NoError(t, os.Mkdir(held, 0o750)) + heldLock, err := memiavl.LockFile(filepath.Join(held, toolCloneOwnerLockName)) + require.NoError(t, err) + + orphaned := filepath.Join(dbDir, prefix+"orphaned") + require.NoError(t, os.Mkdir(orphaned, 0o750)) + require.NoError(t, os.WriteFile(filepath.Join(orphaned, toolCloneOwnerLockName), nil, 0o600)) + + otherPrefix := filepath.Join(dbDir, ".seidb-memiavl-tool-orphaned") + require.NoError(t, os.Mkdir(otherPrefix, 0o750)) + require.NoError(t, os.WriteFile(filepath.Join(otherPrefix, toolCloneOwnerLockName), nil, 0o600)) + + sweepStaleToolClones(dbDir, prefix) + + require.NoDirExists(t, staleUnmarked, "old marker-less clone must be reaped") + require.DirExists(t, freshUnmarked, "fresh marker-less clone may still be mid-creation") + require.DirExists(t, held, "a held owner lock proves the owning tool is alive") + require.NoDirExists(t, orphaned, "an acquirable owner lock proves the owner died") + require.DirExists(t, otherPrefix, "the sweep must not touch other clone families") + + require.NoError(t, heldLock.Unlock()) + sweepStaleToolClones(dbDir, prefix) + require.NoDirExists(t, held, "once the owner releases the lock the clone is reapable") +} + +// TestNewToolCloneOwnershipLifecycle checks that a live clone defends itself +// against a concurrent sweep and that Remove releases everything. +func TestNewToolCloneOwnershipLifecycle(t *testing.T) { + dbDir := t.TempDir() + prefix := ".seidb-memiavl-tool-" + + clone, err := newToolClone(dbDir, prefix) + require.NoError(t, err) + require.DirExists(t, clone.dir) + require.FileExists(t, filepath.Join(clone.dir, toolCloneOwnerLockName)) + + sweepStaleToolClones(dbDir, prefix) + require.DirExists(t, clone.dir, "an owned clone must survive a concurrent sweep") + + require.NoError(t, clone.Remove()) + require.NoDirExists(t, clone.dir) + require.NoError(t, clone.Remove(), "Remove must be idempotent") +} From 69281d76c61d3046f7ced6301767db13e0be0961 Mon Sep 17 00:00:00 2001 From: blindchaser Date: Fri, 7 Aug 2026 10:46:32 -0400 Subject: [PATCH 3/3] fix(seidb): keep main's WAL-coverage check untouched in the merge The merge resolution had folded main's verifyClonedWALCovers and the branch's memiavl equivalent into a shared helper. Main is the shared truth for the FlatKV path, so restore its function verbatim and let the memiavl verifier stand on its own: the two backends read different WAL formats and cannot share an implementation anyway. Co-authored-by: Cursor --- .../tools/cmd/seidb/operations/flatkv_open.go | 40 ++++++------------- .../cmd/seidb/operations/memiavl_open.go | 23 ++++++++--- 2 files changed, 31 insertions(+), 32 deletions(-) diff --git a/sei-db/tools/cmd/seidb/operations/flatkv_open.go b/sei-db/tools/cmd/seidb/operations/flatkv_open.go index 7bc0eb8795..7b30691e51 100644 --- a/sei-db/tools/cmd/seidb/operations/flatkv_open.go +++ b/sei-db/tools/cmd/seidb/operations/flatkv_open.go @@ -232,7 +232,7 @@ func tryPrepareFlatKVToolingClone(dbDir string, height int64) (*toolClone, error // snapshotVersion+1 here (unlike memiavl, whose bootstrap // snapshot-0 hides a configurable initial version). sizeBefore := changelogByteSize(dstChangelogDir) - if err := verifyClonedFlatKVWALCovers(dstChangelogDir, snapshotVersion); err != nil { + if err := verifyClonedWALCovers(dstChangelogDir, snapshotVersion); err != nil { return cleanup(err) } clone.walRepaired = changelogByteSize(dstChangelogDir) < sizeBefore @@ -241,19 +241,12 @@ func tryPrepareFlatKVToolingClone(dbDir string, height int64) (*toolClone, error return clone, nil } -// verifyClonedFlatKVWALCovers inspects the cloned WAL just long enough to -// ensure it either is empty, ends at or before snapshotVersion (no replay -// needed), or starts at or before snapshotVersion+1 (catchup can resume -// cleanly). The state WAL is keyed by block number, so its stored range is -// directly the version range; GetRange reads it offline without a live WAL -// instance. -// -// FlatKV snapshots are always named with a real committed version — -// SetInitialVersion(N) seeds committedVersion N-1 and writes snapshot- — -// so the first version catchup needs is unconditionally snapshotVersion+1, -// unlike memiavl, whose bootstrap snapshot-0 hides a configurable initial -// version. -func verifyClonedFlatKVWALCovers(dstChangelogDir string, snapshotVersion int64) error { +// verifyClonedWALCovers inspects the cloned WAL just long enough to ensure it +// either is empty, ends at or before snapshotVersion (no replay needed), or +// starts at or before snapshotVersion+1 (catchup can resume cleanly). The state +// WAL is keyed by block number, so its stored range is directly the version +// range; GetRange reads it offline without a live WAL instance. +func verifyClonedWALCovers(dstChangelogDir string, snapshotVersion int64) error { ok, firstVer, lastVer, err := statewal.GetRange(statewal.DefaultConfig(dstChangelogDir, flatkvStateWALName)) if err != nil { return fmt.Errorf("read cloned changelog range: %w", err) @@ -261,21 +254,14 @@ func verifyClonedFlatKVWALCovers(dstChangelogDir string, snapshotVersion int64) if !ok { return nil } - //nolint:gosec // version fits int64 - return checkClonedWALCoverage(int64(firstVer), int64(lastVer), snapshotVersion, snapshotVersion+1) -} - -// checkClonedWALCoverage is the shared verdict for both backends: an empty or -// fully-superseded WAL needs no replay, and otherwise the WAL must begin at or -// before firstNeeded — the first version catchup replays on top of the -// snapshot. A later start means the source truncated past our snapshot -// mid-clone, so the caller should retry with a freshly selected snapshot. -func checkClonedWALCoverage(firstVer, lastVer, snapshotVersion, firstNeeded int64) error { - if lastVer <= snapshotVersion || firstVer <= firstNeeded { + if int64(lastVer) <= snapshotVersion { //nolint:gosec // version fits int64 + return nil + } + if int64(firstVer) <= snapshotVersion+1 { //nolint:gosec // version fits int64 return nil } - return fmt.Errorf("%w: cloned WAL starts at version %d but catchup needs %d over snapshot %d (truncated past snapshot mid-clone)", - errSourceChurning, firstVer, firstNeeded, snapshotVersion) + return fmt.Errorf("%w: cloned WAL starts at version %d but snapshot is %d (truncated past snapshot mid-clone)", + errSourceChurning, firstVer, snapshotVersion) } func selectFlatKVSnapshot(dbDir string, height int64) (string, error) { diff --git a/sei-db/tools/cmd/seidb/operations/memiavl_open.go b/sei-db/tools/cmd/seidb/operations/memiavl_open.go index 0700036ba8..b63279cbcb 100644 --- a/sei-db/tools/cmd/seidb/operations/memiavl_open.go +++ b/sei-db/tools/cmd/seidb/operations/memiavl_open.go @@ -164,10 +164,15 @@ func tryPrepareMemIAVLToolingClone(dbDir string, height int64) (*toolClone, erro } // verifyClonedMemIAVLWALCovers is the memiavl counterpart to -// verifyClonedFlatKVWALCovers. memiavl still stores its changelog in the -// offset-indexed changelog WAL, where the offset says nothing about the -// version, so the range has to be read by replaying the first and last entries -// rather than from sealed file names as the block-keyed state WAL allows. +// verifyClonedWALCovers: it ensures the cloned changelog either is empty, ends +// at or before snapshotVersion (no replay needed), or starts at or before +// firstNeeded (catchup can resume cleanly). +// +// It cannot share the FlatKV implementation. FlatKV moved to the block-keyed +// state WAL, whose stored range comes straight from sealed file names, while +// memiavl still keeps its changelog in the offset-indexed changelog WAL, where +// the offset says nothing about the version. The range therefore has to be read +// by replaying the first and last entries. func verifyClonedMemIAVLWALCovers(dstChangelogDir string, snapshotVersion, firstNeeded int64) error { walLog, err := wal.NewChangelogWAL(dstChangelogDir, wal.Config{}) if err != nil { @@ -195,7 +200,15 @@ func verifyClonedMemIAVLWALCovers(dstChangelogDir string, snapshotVersion, first if err != nil { return fmt.Errorf("read last cloned changelog entry: %w", err) } - return checkClonedWALCoverage(firstVer, lastVer, snapshotVersion, firstNeeded) + + if lastVer <= snapshotVersion { + return nil + } + if firstVer <= firstNeeded { + return nil + } + return fmt.Errorf("%w: cloned WAL starts at version %d but catchup needs %d over snapshot %d (truncated past snapshot mid-clone)", + errSourceChurning, firstVer, firstNeeded, snapshotVersion) } func readWALEntryVersion(walLog wal.ChangelogWAL, off uint64) (int64, error) {