diff --git a/sei-db/state_db/sc/memiavl/db.go b/sei-db/state_db/sc/memiavl/db.go index ddd18f675c..a924036130 100644 --- a/sei-db/state_db/sc/memiavl/db.go +++ b/sei-db/state_db/sc/memiavl/db.go @@ -1205,6 +1205,29 @@ func seekSnapshot(root string, targetVersion int64) (int64, error) { return snapshotVersion, nil } +// SeekSnapshotName returns the directory name (relative to root — callers +// join it themselves) and version of the snapshot that OpenDB would start +// from for targetVersion: the "current" link when targetVersion is 0, +// otherwise the newest snapshot at or below it. +// +// Exported for readers that need to resolve a snapshot without opening the DB, +// so they inherit this package's layout rules instead of restating them. +func SeekSnapshotName(root string, targetVersion int64) (string, int64, error) { + if targetVersion == 0 { + version, err := currentVersion(root) + if err != nil { + return "", 0, fmt.Errorf("read current snapshot: %w", err) + } + return snapshotName(version), version, nil + } + + version, err := seekSnapshot(root, targetVersion) + if err != nil { + return "", 0, err + } + return snapshotName(version), version, nil +} + // GetEarliestVersion returns the earliest snapshot name in the db func GetEarliestVersion(root string) (int64, error) { var found int64 @@ -1327,7 +1350,7 @@ func isSnapshotName(name string) bool { // it's needed for upgrade module to check store upgrades, // it returns 0 if db doesn't exist or is empty. func GetLatestVersion(dir string) (int64, error) { - metadata, err := readMetadata(currentPath(dir)) + metadata, err := ReadMetadata(currentPath(dir)) if err != nil { if os.IsNotExist(err) { return 0, nil diff --git a/sei-db/state_db/sc/memiavl/db_test.go b/sei-db/state_db/sc/memiavl/db_test.go index 3cddd00f77..d047b29b11 100644 --- a/sei-db/state_db/sc/memiavl/db_test.go +++ b/sei-db/state_db/sc/memiavl/db_test.go @@ -1116,3 +1116,38 @@ func TestUpdateCurrentSymlinkClearsStaleTmp(t *testing.T) { require.NoError(t, err) require.Equal(t, "snapshot-1", target) } + +// TestSeekSnapshotName pins the exported snapshot-resolution contract that +// external readers (seidb tooling) rely on: the returned name is relative to +// root, targetVersion 0 resolves through the current link, a positive target +// selects the newest snapshot at or below it, and a target older than the +// earliest snapshot reports pruning instead of guessing. +func TestSeekSnapshotName(t *testing.T) { + root := t.TempDir() + for _, v := range []int64{5, 10} { + require.NoError(t, os.Mkdir(filepath.Join(root, snapshotName(v)), 0o750)) + } + require.NoError(t, os.Symlink(snapshotName(10), currentPath(root))) + + name, version, err := SeekSnapshotName(root, 0) + require.NoError(t, err) + require.Equal(t, snapshotName(10), name) + require.Equal(t, int64(10), version) + + name, version, err = SeekSnapshotName(root, 7) + require.NoError(t, err) + require.Equal(t, snapshotName(5), name) + require.Equal(t, int64(5), version) + + name, version, err = SeekSnapshotName(root, 10) + require.NoError(t, err) + require.Equal(t, snapshotName(10), name) + require.Equal(t, int64(10), version) + + _, _, err = SeekSnapshotName(root, 3) + require.Error(t, err) + require.Contains(t, err.Error(), "target version is pruned") + + _, _, err = SeekSnapshotName(filepath.Join(root, "does-not-exist"), 0) + require.Error(t, err) +} diff --git a/sei-db/state_db/sc/memiavl/multitree.go b/sei-db/state_db/sc/memiavl/multitree.go index 6e9101e339..f0f85e417a 100644 --- a/sei-db/state_db/sc/memiavl/multitree.go +++ b/sei-db/state_db/sc/memiavl/multitree.go @@ -76,7 +76,7 @@ func NewEmptyMultiTree(initialVersion uint32) *MultiTree { func LoadMultiTree(ctx context.Context, dir string, opts Options) (*MultiTree, error) { startTime := time.Now() - metadata, err := readMetadata(dir) + metadata, err := ReadMetadata(dir) if err != nil { return nil, err } @@ -600,7 +600,13 @@ func (t *MultiTree) ReplaceWith(other *MultiTree) error { return errors.Join(errs...) } -func readMetadata(dir string) (*proto.MultiTreeMetadata, error) { +// ReadMetadata loads a snapshot directory's MultiTreeMetadata (commit info + +// initial version). Exported for readers that need snapshot metadata without +// opening the DB — e.g. seidb tooling, whose changelog-coverage check must +// know the initial version because a DB bootstrapped with initial version N +// keeps its first snapshot named snapshot-0 while the first changelog entry +// is version N, not 1. +func ReadMetadata(dir string) (*proto.MultiTreeMetadata, error) { // load commit info bz, err := os.ReadFile(filepath.Join(filepath.Clean(dir), MetadataFileName)) if err != nil { diff --git a/sei-db/tools/cmd/seidb/operations/evm_logical_digest.go b/sei-db/tools/cmd/seidb/operations/evm_logical_digest.go index 79ecd140c6..143acc1e50 100644 --- a/sei-db/tools/cmd/seidb/operations/evm_logical_digest.go +++ b/sei-db/tools/cmd/seidb/operations/evm_logical_digest.go @@ -81,12 +81,14 @@ const ( // that exact height (or --height 0 for the current symlink). This is the // preferred mode whenever the target height lines up with an existing // snapshot boundary. -// - replay (SLOW): opens a read-only DB, replays the changelog up to -// --height, then walks the in-memory/mmap tree. Roughly an order of +// - replay (SLOW): clones the newest snapshot at or below --height plus the +// changelog into a temp directory, replays the changelog up to --height in +// that clone, then walks the in-memory/mmap tree. Roughly an order of // magnitude slower than snapshot (changelog replay + per-leaf tree walk -// instead of a sequential file read). Use it only when no snapshot exists -// at the target height — e.g. nodes whose snapshot rewrite lags the tip, so -// an arbitrary comparison height has no snapshot- on disk. +// instead of a sequential file read) and it byte-copies the changelog, so +// it needs free space alongside the source. Use it only when no snapshot +// exists at the target height — e.g. nodes whose snapshot rewrite lags the +// tip, so an arbitrary comparison height has no snapshot- on disk. // // The flatkv side is always a pebble WAL-replay-to-height and is fast // regardless. So when comparing across nodes, pick a height that is an existing @@ -164,8 +166,8 @@ func EvmLogicalDigestCmd() *cobra.Command { cmd.Flags().StringP("db-dir", "d", "", "For flatkv: the flatkv data dir. For memiavl: the memiavl root dir (contains current/ and snapshot-* )") cmd.Flags().String("flatkv-dir", "", "Composite mode: flatkv data dir") cmd.Flags().String("memiavl-dir", "", "Composite mode: memiavl root dir (contains current/ and snapshot-* )") - cmd.Flags().Int64("height", 0, "Target version. flatkv WAL-replays to it; memiavl resolves snapshot-/evm (0 = current symlink)") - cmd.Flags().String("memiavl-open-mode", memiavlOpenModeSnapshot, "memiavl read mode: snapshot (FAST: sequential scan of the completed snapshot kvs file; requires an on-disk snapshot at --height, or --height 0 for current) | replay (SLOW, ~10x: replays changelog to --height then walks the mmap tree; use only when no snapshot exists at the target height). Prefer snapshot when --height matches an existing snapshot boundary") + cmd.Flags().Int64("height", 0, "Target version. flatkv WAL-replays to it; memiavl resolves snapshot-/evm (0 = current symlink). On a live node 0 = latest is best-effort — the printed version line records what was actually digested; always pass an explicit common height when comparing nodes") + cmd.Flags().String("memiavl-open-mode", memiavlOpenModeSnapshot, "memiavl read mode: snapshot (FAST: sequential scan of the completed snapshot kvs file; requires an on-disk snapshot at --height, or --height 0 for current) | replay (SLOW, ~10x: clones the snapshot + changelog to a temp dir, replays to --height there, then walks the mmap tree; use only when no snapshot exists at the target height). Prefer snapshot when --height matches an existing snapshot boundary") cmd.Flags().String("memiavl-normalization", memiavlNormSemantic, "memiavl digest/inspect normalization: semantic/independent (raw EVM key/value decoder) | translator (current migration mapping)") cmd.Flags().String("inspect-bucket", "", "Inspect one normalized bucket (account|code|storage|misc) instead of printing the global digest") cmd.Flags().Int("key-offset", 0, "Inspect mode: byte offset into physical key before applying --key-prefix / sharding") @@ -465,7 +467,7 @@ func digestCompositeMigrateEVM(flatKVDir, memIAVLDir string, height int64, findT requestedHeight: height, version: opened.Version(), } - var memReplayDB *memiavl.DB + var memReplayDB *openedMemIAVL var memEvmSnapshotDir string var memVersion int64 switch memiavlOpenMode { @@ -481,13 +483,13 @@ func digestCompositeMigrateEVM(flatKVDir, memIAVLDir string, height int64, findT ctx.source = fmt.Sprintf("flatkv clone version=%d + memiavl snapshot=%s", opened.Version(), memEvmSnapshotDir) ctx.normalization = fmt.Sprintf("flatkv rows plus memiavl rows not migrated by boundary=%s version_known=%t migration_version=%d memiavl_version=%d", boundary.String(), versionKnown, migrationVersion, memVersion) case memiavlOpenModeReplay: - memReplayDB, err = openMemiAVLReplayReadOnly(memIAVLDir, height) + memReplayDB, err = openMemiAVLReplay(memIAVLDir, height) if err != nil { return err } defer func() { _ = memReplayDB.Close() }() memVersion = memReplayDB.Version() - ctx.source = fmt.Sprintf("flatkv clone version=%d + memiavl read-only replay dir=%s", opened.Version(), memIAVLDir) + ctx.source = fmt.Sprintf("flatkv clone version=%d + memiavl clone replay dir=%s", opened.Version(), memIAVLDir) ctx.normalization = fmt.Sprintf("flatkv rows plus replayed memiavl rows not migrated by boundary=%s version_known=%t migration_version=%d memiavl_version=%d", boundary.String(), versionKnown, migrationVersion, memVersion) default: return fmt.Errorf("unknown --memiavl-open-mode %q (want snapshot|replay)", memiavlOpenMode) @@ -504,7 +506,7 @@ func digestCompositeMigrateEVM(flatKVDir, memIAVLDir string, height int64, findT if boundary.Status() != migration.MigrationComplete { if memReplayDB != nil { if err := consumeCompositeMemiavl(func(fn func(rawKey, rawVal []byte) error) error { - return scanMemiavlReplayEVMLeaves(memReplayDB, fn) + return scanMemiavlReplayEVMLeaves(memReplayDB.DB, fn) }, "memiavl-replay", boundary, &d, accounts); err != nil { return err } @@ -1104,20 +1106,8 @@ func digestMemIAVL(dbDir string, height int64, findTarget []byte, normalization } } -func openMemiAVLReplayReadOnly(dbDir string, height int64) (*memiavl.DB, error) { - db, err := memiavl.OpenDB(height, memiavl.Options{ - Dir: dbDir, - ReadOnly: true, - ZeroCopy: true, - }) - if err != nil { - return nil, fmt.Errorf("open memiavl read-only replay: %w", err) - } - return db, nil -} - func digestMemIAVLReplay(dbDir string, height int64, findTarget []byte, normalization string) error { - db, err := openMemiAVLReplayReadOnly(dbDir, height) + db, err := openMemiAVLReplay(dbDir, height) if err != nil { return err } @@ -1125,9 +1115,9 @@ func digestMemIAVLReplay(dbDir string, height int64, findTarget []byte, normaliz switch normalization { case "", memiavlNormSemantic, memiavlNormIndependent: - return digestMemIAVLReplaySemantic(dbDir, height, db, findTarget) + return digestMemIAVLReplaySemantic(dbDir, height, db.DB, findTarget) case memiavlNormTranslator: - return digestMemIAVLReplayTranslator(dbDir, height, db, findTarget) + return digestMemIAVLReplayTranslator(dbDir, height, db.DB, findTarget) default: return fmt.Errorf("unknown --memiavl-normalization %q (want semantic|independent|translator)", normalization) } @@ -1291,7 +1281,7 @@ func digestMemIAVLReplaySemantic(dbDir string, height int64, db *memiavl.DB, fin backend: "memiavl", mode: "semantic-replay", dbDir: dbDir, - source: "read-only memiavl DB opened from snapshot + changelog replay", + source: "isolated memiavl clone opened from snapshot + changelog replay", normalization: "independent semantic decoder for replayed memiavl EVM keys; does not call flatkv.ImportTranslator", requestedHeight: height, version: db.Version(), @@ -1308,7 +1298,7 @@ func digestMemIAVLReplayTranslator(dbDir string, height int64, db *memiavl.DB, f backend: "memiavl", mode: "translator-replay", dbDir: dbDir, - source: "read-only memiavl DB opened from snapshot + changelog replay", + source: "isolated memiavl clone opened from snapshot + changelog replay", normalization: "replayed memiavl leaves translated with flatkv.ImportTranslator, then reduced to logical payload", requestedHeight: height, version: db.Version(), diff --git a/sei-db/tools/cmd/seidb/operations/flatkv_open.go b/sei-db/tools/cmd/seidb/operations/flatkv_open.go index 60c53c8f18..7b30691e51 100644 --- a/sei-db/tools/cmd/seidb/operations/flatkv_open.go +++ b/sei-db/tools/cmd/seidb/operations/flatkv_open.go @@ -32,11 +32,10 @@ const ( maxCloneRetries = 3 ) -// errSourceChurning marks transient races where the source FlatKV directory -// mutates (snapshot pruned, WAL truncated) between our reads. It is the -// sentinel that prepareFlatKVToolingCloneWith uses to decide whether to -// retry instead of bailing out. -var errSourceChurning = errors.New("flatkv source kept churning during clone") +// errSourceChurning marks transient races where the source directory mutates +// (snapshot pruned, WAL truncated) between our reads. It is the sentinel that +// retryToolingClone uses to decide whether to retry instead of bailing out. +var errSourceChurning = errors.New("source kept churning during clone") // openedFlatKV wraps a temp-cloned FlatKV store used by tooling. // @@ -44,7 +43,7 @@ var errSourceChurning = errors.New("flatkv source kept churning during clone") // WAL so they do not compete with a live node for the FlatKV writer lock. type openedFlatKV struct { flatkv.Store - tempDir string + clone *toolClone } func (o *openedFlatKV) Close() error { @@ -52,13 +51,11 @@ func (o *openedFlatKV) Close() error { if o.Store != nil { err = o.Store.Close() } - if o.tempDir != "" { - if rmErr := os.RemoveAll(o.tempDir); rmErr != nil { - if err != nil { - return fmt.Errorf("%w; cleanup temp dir: %w", err, rmErr) - } - return fmt.Errorf("cleanup temp dir: %w", rmErr) + if rmErr := o.clone.Remove(); rmErr != nil { + if err != nil { + return fmt.Errorf("%w; %w", err, rmErr) } + return rmErr } return err } @@ -82,25 +79,30 @@ func (o *openedFlatKV) Close() error { // os.ReadDir and os.Link calls, we surface ENOENT, re-select the // snapshot, and retry up to maxCloneRetries times. // -// height=0 means latest version. +// height=0 means the latest version, best-effort on a live node: the clone is +// a consistent committed prefix as of the copy instant, and a torn-tail +// repair inside the clone (surfaced as a warning) can land it one version +// behind the source tip. Tools print the version actually opened; treat that +// line as authoritative when comparing across nodes. func openFlatKVReadOnly(dbDir string, height int64) (*openedFlatKV, error) { - tempDir, err := prepareFlatKVToolingClone(dbDir, height) + clone, err := prepareFlatKVToolingClone(dbDir, height) if err != nil { return nil, err } + warnIfCloneRepaired(clone, "flatkv", height) cfg := config.DefaultConfig() - cfg.DataDir = tempDir + cfg.DataDir = clone.dir stateWAL, err := flatkv.OpenStateWAL(cfg) if err != nil { - _ = os.RemoveAll(tempDir) + _ = clone.Remove() return nil, fmt.Errorf("failed to open FlatKV state WAL: %w", err) } primary, err := flatkv.NewCommitStore(context.Background(), cfg, stateWAL) if err != nil { _ = stateWAL.Close() - _ = os.RemoveAll(tempDir) + _ = clone.Remove() return nil, fmt.Errorf("failed to create FlatKV store: %w", err) } @@ -110,38 +112,55 @@ func openFlatKVReadOnly(dbDir string, height int64) (*openedFlatKV, error) { view, err := primary.LoadVersionReadOnly(height) if err != nil { _ = primary.Close() - _ = os.RemoveAll(tempDir) + _ = clone.Remove() return nil, fmt.Errorf("failed to open FlatKV at version %d: %w", height, err) } if err := primary.Close(); err != nil { _ = view.Close() - _ = os.RemoveAll(tempDir) + _ = clone.Remove() return nil, fmt.Errorf("failed to close FlatKV clone writer: %w", err) } return &openedFlatKV{ - Store: view, - tempDir: tempDir, + Store: view, + clone: clone, }, nil } -func prepareFlatKVToolingClone(dbDir string, height int64) (string, error) { - return prepareFlatKVToolingCloneWith(dbDir, height, tryPrepareFlatKVToolingClone) +// warnIfCloneRepaired tells the operator that the cloned changelog had a torn +// tail (the byte-copy raced the live writer mid-append) and was repaired +// inside the clone. For an explicit --height the reached-version checks catch +// any resulting shortfall; for height 0 ("latest") there is no target to +// check against, so the printed version line is the only record of what was +// actually digested. +func warnIfCloneRepaired(clone *toolClone, backend string, height int64) { + if !clone.walRepaired || height != 0 { + return + } + fmt.Fprintf(os.Stderr, "warning: cloned %s changelog had a torn tail (live writer mid-append) and was repaired in the clone; "+ + "the opened version may trail the source tip by one — trust the printed version line\n", backend) +} + +func prepareFlatKVToolingClone(dbDir string, height int64) (*toolClone, error) { + return retryToolingClone(dbDir, height, tryPrepareFlatKVToolingClone) } -func prepareFlatKVToolingCloneWith(dbDir string, height int64, tryClone func(string, int64) (string, error)) (string, error) { +// retryToolingClone runs tryClone, retrying while the live writer keeps +// mutating the source out from under us. Shared by the FlatKV and memiavl +// tooling clones, which race the same writer in the same ways. +func retryToolingClone(dbDir string, height int64, tryClone func(string, int64) (*toolClone, error)) (*toolClone, error) { var lastErr error for attempt := 0; attempt < maxCloneRetries; attempt++ { - tempDir, err := tryClone(dbDir, height) + clone, err := tryClone(dbDir, height) if err == nil { - return tempDir, nil + return clone, nil } if !isCloneRetryableError(err) { - return "", err + return nil, err } lastErr = err } - return "", fmt.Errorf("clone aborted after %d retries, source kept churning: %w", maxCloneRetries, lastErr) + return nil, fmt.Errorf("clone aborted after %d retries, source kept churning: %w", maxCloneRetries, lastErr) } // isCloneRetryableError reports whether err indicates a transient race with @@ -152,39 +171,37 @@ func isCloneRetryableError(err error) bool { return errors.Is(err, os.ErrNotExist) || errors.Is(err, errSourceChurning) } -func tryPrepareFlatKVToolingClone(dbDir string, height int64) (string, error) { +func tryPrepareFlatKVToolingClone(dbDir string, height int64) (*toolClone, error) { snapshotName, err := selectFlatKVSnapshot(dbDir, height) if err != nil { - return "", err + return nil, err } snapshotVersion, err := strconv.ParseInt(snapshotName[len(flatkvSnapshotPrefix):], 10, 64) if err != nil { - return "", fmt.Errorf("parse snapshot version from %q: %w", snapshotName, err) + return nil, fmt.Errorf("parse snapshot version from %q: %w", snapshotName, err) } - // Place the temp clone inside dbDir so it is on the exact same mounted - // filesystem as the source snapshots. A sibling directory is not enough: - // dbDir itself is often a mount point on dedicated data volumes. - cloneRoot := dbDir - if err := os.MkdirAll(cloneRoot, 0o750); err != nil { - return "", fmt.Errorf("ensure clone root %s: %w", cloneRoot, err) - } - tempDir, err := os.MkdirTemp(cloneRoot, ".seidb-flatkv-tool-*") + // The clone must sit inside dbDir so it is on the exact same mounted + // filesystem as the source snapshots (dbDir is often its own mount + // point, so a sibling directory is not enough and hardlinks would fail + // across the boundary). selectFlatKVSnapshot already read dbDir, so it + // is known to exist. + clone, err := newToolClone(dbDir, ".seidb-flatkv-tool-") if err != nil { - return "", fmt.Errorf("create temp dir under %s: %w", cloneRoot, err) + return nil, err } - cleanup := func(err error) (string, error) { - _ = os.RemoveAll(tempDir) - return "", err + cleanup := func(err error) (*toolClone, error) { + _ = clone.Remove() + return nil, err } srcSnapshotDir := filepath.Join(dbDir, snapshotName) - dstSnapshotDir := filepath.Join(tempDir, snapshotName) + dstSnapshotDir := filepath.Join(clone.dir, snapshotName) if err := cloneDirRecursive(srcSnapshotDir, dstSnapshotDir); err != nil { return cleanup(fmt.Errorf("clone snapshot %s: %w", snapshotName, err)) } - if err := os.Symlink(snapshotName, filepath.Join(tempDir, "current")); err != nil { + if err := os.Symlink(snapshotName, filepath.Join(clone.dir, "current")); err != nil { return cleanup(fmt.Errorf("create current symlink: %w", err)) } @@ -197,23 +214,31 @@ func tryPrepareFlatKVToolingClone(dbDir string, height int64) (string, error) { return cleanup(fmt.Errorf("changelog path is not a directory: %s", srcChangelogDir)) } if err == nil { - dstChangelogDir := filepath.Join(tempDir, "changelog") + dstChangelogDir := filepath.Join(clone.dir, "changelog") if err := copyDirRecursive(srcChangelogDir, dstChangelogDir); err != nil { return cleanup(fmt.Errorf("clone changelog: %w", err)) } // Detect the snapshot/WAL race: a live writer can roll a new // snapshot between our snapshot clone and our changelog copy and // then truncateWAL up to that newer snapshot's version. If that - // happened, the cloned WAL no longer covers snapshotVersion+1, - // and a downstream catchup would silently jump over missing - // versions. Surface it as a retryable error so the outer loop - // re-selects the snapshot and tries again. + // happened, the cloned WAL no longer covers the snapshot's + // successor version, and a downstream catchup would silently jump + // over missing versions. Surface it as a retryable error so the + // outer loop re-selects the snapshot and tries again. + // + // FlatKV snapshots are always named with a real committed version — + // SetInitialVersion(N) seeds committedVersion N-1 and writes + // snapshot- — so the successor is unconditionally + // snapshotVersion+1 here (unlike memiavl, whose bootstrap + // snapshot-0 hides a configurable initial version). + sizeBefore := changelogByteSize(dstChangelogDir) if err := verifyClonedWALCovers(dstChangelogDir, snapshotVersion); err != nil { return cleanup(err) } + clone.walRepaired = changelogByteSize(dstChangelogDir) < sizeBefore } - return tempDir, nil + return clone, nil } // verifyClonedWALCovers inspects the cloned WAL just long enough to ensure it @@ -293,7 +318,7 @@ func isFlatKVSnapshotName(name string) bool { // error: snapshots can be many GB, and the previous behavior of falling back // to a byte-copy on tmpfs (the historical $TMPDIR default) routinely OOM'd // nodes and exhausted /tmp. Callers must ensure the tool clone dir lives on -// the same filesystem as the source FlatKV directory. +// the same filesystem as the source directory. // // Hardlinking is safe because: // - snapshot-N files are immutable after Pebble Checkpoint + Rename. @@ -359,7 +384,7 @@ func linkOnly(src, dst string) error { if err := os.Link(src, dst); err != nil { if isCrossDeviceLinkError(err) { return fmt.Errorf("hardlink %s -> %s failed across filesystems; "+ - "FlatKV tooling requires the temp clone to share a filesystem with the source: %w", + "seidb tooling requires the temp clone to share a filesystem with the source: %w", src, dst, err) } return err diff --git a/sei-db/tools/cmd/seidb/operations/flatkv_open_test.go b/sei-db/tools/cmd/seidb/operations/flatkv_open_test.go index ecd04238ee..59b0274951 100644 --- a/sei-db/tools/cmd/seidb/operations/flatkv_open_test.go +++ b/sei-db/tools/cmd/seidb/operations/flatkv_open_test.go @@ -55,17 +55,17 @@ func TestPrepareFlatKVToolingCloneHardlinksSnapshotAndCopiesChangelog(t *testing srcChangelogFile := filepath.Join(dbDir, "changelog", "000001.log") require.NoError(t, os.WriteFile(srcChangelogFile, []byte("wal-data"), 0o600)) - cloneDir, err := prepareFlatKVToolingClone(dbDir, 0) + clone, err := prepareFlatKVToolingClone(dbDir, 0) require.NoError(t, err) - defer os.RemoveAll(cloneDir) //nolint:errcheck // test cleanup + defer clone.Remove() //nolint:errcheck // test cleanup - target, err := os.Readlink(filepath.Join(cloneDir, "current")) + target, err := os.Readlink(filepath.Join(clone.dir, "current")) require.NoError(t, err) require.Equal(t, snapshot, target) - require.FileExists(t, filepath.Join(cloneDir, snapshot, "account", "000001.sst")) - require.NoFileExists(t, filepath.Join(cloneDir, snapshot, "LOCK")) - dstSnapshotFile := filepath.Join(cloneDir, snapshot, "account", "000001.sst") - dstChangelogFile := filepath.Join(cloneDir, "changelog", "000001.log") + require.FileExists(t, filepath.Join(clone.dir, snapshot, "account", "000001.sst")) + require.NoFileExists(t, filepath.Join(clone.dir, snapshot, "LOCK")) + dstSnapshotFile := filepath.Join(clone.dir, snapshot, "account", "000001.sst") + dstChangelogFile := filepath.Join(clone.dir, "changelog", "000001.log") require.FileExists(t, dstChangelogFile) srcSnapshotInfo, err := os.Stat(srcSnapshotFile) @@ -123,21 +123,21 @@ func TestPrepareFlatKVToolingCloneMissingCurrentAndSnapshot(t *testing.T) { func TestPrepareFlatKVToolingCloneRetriesENOENT(t *testing.T) { var attempts int - cloneDir, err := prepareFlatKVToolingCloneWith(t.TempDir(), 0, func(string, int64) (string, error) { + clone, err := retryToolingClone(t.TempDir(), 0, func(string, int64) (*toolClone, error) { attempts++ if attempts < maxCloneRetries { - return "", fmt.Errorf("source vanished: %w", os.ErrNotExist) + return nil, fmt.Errorf("source vanished: %w", os.ErrNotExist) } - return t.TempDir(), nil + return &toolClone{dir: t.TempDir()}, nil }) require.NoError(t, err) - require.NotEmpty(t, cloneDir) + require.NotEmpty(t, clone.dir) require.Equal(t, maxCloneRetries, attempts) attempts = 0 - _, err = prepareFlatKVToolingCloneWith(t.TempDir(), 0, func(string, int64) (string, error) { + _, err = retryToolingClone(t.TempDir(), 0, func(string, int64) (*toolClone, error) { attempts++ - return "", errors.New("permission denied") + return nil, errors.New("permission denied") }) require.Error(t, err) require.Equal(t, 1, attempts) @@ -160,15 +160,15 @@ func TestPrepareFlatKVToolingClonePlacesTempDirInsideDBDir(t *testing.T) { require.NoError(t, store.WriteSnapshot("")) require.NoError(t, store.Close()) - cloneDir, err := prepareFlatKVToolingClone(dbDir, 0) + clone, err := prepareFlatKVToolingClone(dbDir, 0) require.NoError(t, err) - defer os.RemoveAll(cloneDir) //nolint:errcheck // test cleanup + defer clone.Remove() //nolint:errcheck // test cleanup - rel, err := filepath.Rel(dbDir, cloneDir) + rel, err := filepath.Rel(dbDir, clone.dir) require.NoError(t, err) require.NotEqual(t, ".", rel) require.False(t, strings.HasPrefix(rel, ".."), "tooling clone must be created inside dbDir to stay on dbDir's mounted filesystem") - require.Contains(t, filepath.Base(cloneDir), ".seidb-flatkv-tool-") + require.Contains(t, filepath.Base(clone.dir), ".seidb-flatkv-tool-") } // TestPrepareFlatKVToolingCloneDetectsWALTruncationRace simulates the audited diff --git a/sei-db/tools/cmd/seidb/operations/memiavl_open.go b/sei-db/tools/cmd/seidb/operations/memiavl_open.go new file mode 100644 index 0000000000..b63279cbcb --- /dev/null +++ b/sei-db/tools/cmd/seidb/operations/memiavl_open.go @@ -0,0 +1,221 @@ +package operations + +import ( + "errors" + "fmt" + "math" + "os" + "path/filepath" + + "github.com/sei-protocol/sei-chain/sei-db/common/utils" + "github.com/sei-protocol/sei-chain/sei-db/proto" + "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/memiavl" + "github.com/sei-protocol/sei-chain/sei-db/wal" +) + +// openedMemIAVL is a memiavl DB opened against a temp clone of the source. +type openedMemIAVL struct { + *memiavl.DB + clone *toolClone +} + +func (o *openedMemIAVL) Close() error { + var err error + if o.DB != nil { + err = o.DB.Close() + } + if rmErr := o.clone.Remove(); rmErr != nil { + if err != nil { + return fmt.Errorf("%w; %w", err, rmErr) + } + return rmErr + } + return err +} + +// openMemiAVLReplay opens memiavl at the given height (0 means latest) by +// replaying the changelog on top of the newest snapshot at or below it. +// +// It clones the source rather than using memiavl's ReadOnly option, which is +// weaker than it reads: ReadOnly skips the LOCK file but the changelog is still +// opened read-write, and that open "repairs" a torn tail record by truncating +// the segment. On a live node a torn tail is the writer mid-append, not +// corruption, so a read-only replay could truncate committed versions out from +// under a running seid. Repairing a torn tail in a private copy is harmless. +// This mirrors openFlatKVReadOnly. +// +// height=0 ("latest") is best-effort on a live node: the clone is a +// consistent committed prefix as of the copy instant, and a torn-tail repair +// inside the clone (surfaced as a warning) can land it one version behind the +// source tip. There is no target version to check against, so the report's +// printed version line is the authoritative record of what was digested; +// cross-node comparisons should always use an explicit common --height. +func openMemiAVLReplay(dbDir string, height int64) (*openedMemIAVL, error) { + clone, err := prepareMemIAVLToolingClone(dbDir, height) + if err != nil { + return nil, err + } + warnIfCloneRepaired(clone, "memiavl", height) + + db, err := memiavl.OpenDB(height, memiavl.Options{ + Dir: clone.dir, + ZeroCopy: true, + }) + if err != nil { + _ = clone.Remove() + return nil, fmt.Errorf("open memiavl clone at version %d: %w", height, err) + } + opened := &openedMemIAVL{DB: db, clone: clone} + + // memiavl replays whatever the changelog holds and reports success even + // when that falls short of the requested height. Every tail repair inside + // the clone costs the trailing version, and a digest computed one version + // early is indistinguishable from real divergence when comparing nodes. + if reached := db.Version(); height > 0 && reached != height { + err := fmt.Errorf("memiavl clone version mismatch: requested %d, reached %d "+ + "(changelog does not cover the target height)", height, reached) + if closeErr := opened.Close(); closeErr != nil { + return nil, errors.Join(err, fmt.Errorf("close clone: %w", closeErr)) + } + return nil, err + } + + return opened, nil +} + +func prepareMemIAVLToolingClone(dbDir string, height int64) (*toolClone, error) { + return retryToolingClone(dbDir, height, tryPrepareMemIAVLToolingClone) +} + +// tryPrepareMemIAVLToolingClone mirrors tryPrepareFlatKVToolingClone: the two +// layouts are the same shape (current -> snapshot-N/, changelog/, LOCK) and both +// publish snapshots by rename and drop them wholesale, so the same +// hardlink-the-snapshot / byte-copy-the-changelog split applies. +func tryPrepareMemIAVLToolingClone(dbDir string, height int64) (*toolClone, error) { + snapshotName, snapshotVersion, err := memiavl.SeekSnapshotName(dbDir, height) + if err != nil { + return nil, err + } + + // The clone must sit inside dbDir to share a filesystem with the source + // snapshot: dbDir is often its own mount point, so a sibling directory is + // not enough and hardlinks would fail across the boundary. + // SeekSnapshotName already read dbDir, so it is known to exist. + clone, err := newToolClone(dbDir, ".seidb-memiavl-tool-") + if err != nil { + return nil, err + } + cleanup := func(err error) (*toolClone, error) { + _ = clone.Remove() + return nil, err + } + + srcSnapshotDir := filepath.Join(dbDir, snapshotName) + dstSnapshotDir := filepath.Join(clone.dir, snapshotName) + if err := cloneDirRecursive(srcSnapshotDir, dstSnapshotDir); err != nil { + return cleanup(fmt.Errorf("clone snapshot %s: %w", snapshotName, err)) + } + + if err := os.Symlink(snapshotName, filepath.Join(clone.dir, "current")); err != nil { + return cleanup(fmt.Errorf("create current symlink: %w", err)) + } + + // The version parsed from the snapshot directory name is not enough to + // know which changelog version catchup resumes from: memiavl bootstraps + // every DB as snapshot-0 even when it was initialized with + // SetInitialVersion(N), in which case the first changelog entry is + // version N, not 1. Read the initial version from the cloned snapshot's + // metadata (immune to source pruning — the files are hard links) and + // derive the successor the same way memiavl itself does. + metadata, err := memiavl.ReadMetadata(dstSnapshotDir) + if err != nil { + return cleanup(fmt.Errorf("read cloned snapshot metadata: %w", err)) + } + if metadata.InitialVersion < 0 || metadata.InitialVersion > math.MaxUint32 { + return cleanup(fmt.Errorf("cloned snapshot has invalid initial version: %d", metadata.InitialVersion)) + } + firstNeeded := utils.NextVersion(snapshotVersion, uint32(metadata.InitialVersion)) + + srcChangelogDir := filepath.Join(dbDir, "changelog") + info, err := os.Stat(srcChangelogDir) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return cleanup(fmt.Errorf("stat changelog: %w", err)) + } + if err == nil && !info.IsDir() { + return cleanup(fmt.Errorf("changelog path is not a directory: %s", srcChangelogDir)) + } + if err == nil { + dstChangelogDir := filepath.Join(clone.dir, "changelog") + if err := copyDirRecursive(srcChangelogDir, dstChangelogDir); err != nil { + return cleanup(fmt.Errorf("clone changelog: %w", err)) + } + // A live writer can roll a new snapshot between our snapshot clone and + // our changelog copy, then prune the changelog up to that newer + // version — leaving a copy that no longer covers the snapshot's + // successor and a catchup that would silently skip versions. Retryable. + sizeBefore := changelogByteSize(dstChangelogDir) + if err := verifyClonedMemIAVLWALCovers(dstChangelogDir, snapshotVersion, firstNeeded); err != nil { + return cleanup(err) + } + clone.walRepaired = changelogByteSize(dstChangelogDir) < sizeBefore + } + + return clone, nil +} + +// verifyClonedMemIAVLWALCovers is the memiavl counterpart to +// verifyClonedWALCovers: it ensures the cloned changelog either is empty, ends +// at or before snapshotVersion (no replay needed), or starts at or before +// firstNeeded (catchup can resume cleanly). +// +// It cannot share the FlatKV implementation. FlatKV moved to the block-keyed +// state WAL, whose stored range comes straight from sealed file names, while +// memiavl still keeps its changelog in the offset-indexed changelog WAL, where +// the offset says nothing about the version. The range therefore has to be read +// by replaying the first and last entries. +func verifyClonedMemIAVLWALCovers(dstChangelogDir string, snapshotVersion, firstNeeded int64) error { + walLog, err := wal.NewChangelogWAL(dstChangelogDir, wal.Config{}) + if err != nil { + return fmt.Errorf("open cloned changelog for validation: %w", err) + } + defer func() { _ = walLog.Close() }() + + firstOff, err := walLog.FirstOffset() + if err != nil { + return fmt.Errorf("cloned changelog first offset: %w", err) + } + lastOff, err := walLog.LastOffset() + if err != nil { + return fmt.Errorf("cloned changelog last offset: %w", err) + } + if firstOff == 0 || lastOff == 0 || firstOff > lastOff { + return nil + } + + firstVer, err := readWALEntryVersion(walLog, firstOff) + if err != nil { + return fmt.Errorf("read first cloned changelog entry: %w", err) + } + lastVer, err := readWALEntryVersion(walLog, lastOff) + if err != nil { + return fmt.Errorf("read last cloned changelog entry: %w", err) + } + + if lastVer <= snapshotVersion { + return nil + } + if firstVer <= firstNeeded { + return nil + } + return fmt.Errorf("%w: cloned WAL starts at version %d but catchup needs %d over snapshot %d (truncated past snapshot mid-clone)", + errSourceChurning, firstVer, firstNeeded, snapshotVersion) +} + +func readWALEntryVersion(walLog wal.ChangelogWAL, off uint64) (int64, error) { + var ver int64 + err := walLog.Replay(off, off, func(_ uint64, entry proto.ChangelogEntry) error { + ver = entry.Version + return nil + }) + return ver, err +} diff --git a/sei-db/tools/cmd/seidb/operations/memiavl_open_test.go b/sei-db/tools/cmd/seidb/operations/memiavl_open_test.go new file mode 100644 index 0000000000..b2bae9ba90 --- /dev/null +++ b/sei-db/tools/cmd/seidb/operations/memiavl_open_test.go @@ -0,0 +1,195 @@ +package operations + +import ( + "os" + "path/filepath" + "sort" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/sei-protocol/sei-chain/sei-db/common/keys" + "github.com/sei-protocol/sei-chain/sei-db/common/utils" + "github.com/sei-protocol/sei-chain/sei-db/proto" +) + +// newMemiavlSourceDir builds a memiavl directory with `versions` committed +// blocks and returns the directory a tool would be pointed at. +func newMemiavlSourceDir(t *testing.T, versions int) string { + t.Helper() + homeDir := t.TempDir() + store := newTestMemiavlStore(t, homeDir) + for i := 1; i <= versions; i++ { + require.NoError(t, store.ApplyChangeSets([]*proto.NamedChangeSet{{ + Name: keys.EVMStoreKey, + Changeset: proto.ChangeSet{Pairs: []*proto.KVPair{noncePair(addrN(byte(i)), uint64(i))}}, + }})) + v, err := store.Commit() + require.NoError(t, err) + require.Equal(t, int64(i), v) + } + require.NoError(t, store.Close()) + return utils.GetCosmosSCStorePath(homeDir) +} + +// snapshotDirState records every regular file under root by relative path and +// content so a later comparison catches truncation, appends, and deletions. +func snapshotDirState(t *testing.T, root string) map[string][]byte { + t.Helper() + state := make(map[string][]byte) + require.NoError(t, filepath.Walk(root, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if !info.Mode().IsRegular() { + return nil + } + rel, err := filepath.Rel(root, path) + if err != nil { + return err + } + bz, err := os.ReadFile(filepath.Clean(path)) + if err != nil { + return err + } + state[rel] = bz + return nil + })) + return state +} + +// lastChangelogSegment returns the newest tidwall segment file in the memiavl +// changelog. Segment names are fixed-width, so lexical order is index order. +func lastChangelogSegment(t *testing.T, dbDir string) string { + t.Helper() + changelogDir := filepath.Join(dbDir, "changelog") + entries, err := os.ReadDir(changelogDir) + require.NoError(t, err) + var names []string + for _, e := range entries { + if !e.IsDir() && len(e.Name()) >= 20 { + names = append(names, e.Name()) + } + } + require.NotEmpty(t, names, "memiavl changelog should have at least one segment") + sort.Strings(names) + return filepath.Join(changelogDir, names[len(names)-1]) +} + +// TestOpenMemiAVLReplayLeavesSourceUntouched is the regression test for the +// audited hazard: a replay that advertises itself as read-only used to hand the +// live changelog straight to memiavl, whose WAL open truncates a torn tail. A +// torn tail on a running node is just the writer mid-append, so the "repair" +// destroyed committed versions. The tool must now repair only its own copy. +func TestOpenMemiAVLReplayLeavesSourceUntouched(t *testing.T) { + dbDir := newMemiavlSourceDir(t, 3) + + // A single length-prefix byte declaring a 16-byte record that never + // arrived is exactly what a reader observes while the writer is + // partway through appending a block. + segment := lastChangelogSegment(t, dbDir) + intact, err := os.ReadFile(filepath.Clean(segment)) + require.NoError(t, err) + torn := append(append([]byte{}, intact...), 0x10) + require.NoError(t, os.WriteFile(segment, torn, 0o600)) + + before := snapshotDirState(t, dbDir) + + db, err := openMemiAVLReplay(dbDir, 0) + require.NoError(t, err, "replay must tolerate a torn tail by repairing its own clone") + require.Equal(t, int64(3), db.Version()) + require.NoError(t, db.Close()) + + require.Equal(t, before, snapshotDirState(t, dbDir), + "replay must not add, remove, truncate, or rewrite any file in the source directory") + + after, err := os.ReadFile(filepath.Clean(segment)) + require.NoError(t, err) + require.Equal(t, torn, after, "the torn tail must still be there for the live writer to finish") +} + +// TestOpenMemiAVLReplayWorksWhileWriterHoldsLock pins the other half of the +// contract: avoiding the mutation must not cost us the ability to read a live +// node. The clone is independent, so the source LOCK is irrelevant to us. +func TestOpenMemiAVLReplayWorksWhileWriterHoldsLock(t *testing.T) { + homeDir := t.TempDir() + writer := newTestMemiavlStore(t, homeDir) + require.NoError(t, writer.ApplyChangeSets([]*proto.NamedChangeSet{{ + Name: keys.EVMStoreKey, + Changeset: proto.ChangeSet{Pairs: []*proto.KVPair{noncePair(addrN(0xA1), 1)}}, + }})) + _, err := writer.Commit() + require.NoError(t, err) + defer func() { require.NoError(t, writer.Close()) }() + + dbDir := utils.GetCosmosSCStorePath(homeDir) + require.FileExists(t, filepath.Join(dbDir, "LOCK")) + + db, err := openMemiAVLReplay(dbDir, 0) + require.NoError(t, err, "tooling clone must not contend for the live writer's lock") + require.Equal(t, int64(1), db.Version()) + require.NoError(t, db.Close()) +} + +// TestOpenMemiAVLReplayAfterSetInitialVersion pins the non-default +// initial-height layout: memiavl bootstraps every DB as snapshot-0 even when +// SetInitialVersion(100) makes the first changelog entry version 100, so the +// clone's WAL-coverage check must derive the snapshot's successor from the +// snapshot metadata instead of assuming version 1 follows snapshot-0. This is +// exactly the shape of a freshly recovered chain whose genesis initial_height +// is greater than 1 and whose first snapshot rewrite has not happened yet; +// before the fix, replay mode failed deterministically on such nodes with +// "source kept churning". +func TestOpenMemiAVLReplayAfterSetInitialVersion(t *testing.T) { + homeDir := t.TempDir() + store := newTestMemiavlStore(t, homeDir) + require.NoError(t, store.SetInitialVersion(100)) + for i := 0; i < 3; i++ { + require.NoError(t, store.ApplyChangeSets([]*proto.NamedChangeSet{{ + Name: keys.EVMStoreKey, + Changeset: proto.ChangeSet{Pairs: []*proto.KVPair{noncePair(addrN(byte(i+1)), uint64(i+1))}}, + }})) + v, err := store.Commit() + require.NoError(t, err) + require.Equal(t, int64(100+i), v) + } + require.NoError(t, store.Close()) + dbDir := utils.GetCosmosSCStorePath(homeDir) + + historical, err := openMemiAVLReplay(dbDir, 101) + require.NoError(t, err, "coverage check must accept a snapshot-0 whose successor is the initial version") + require.Equal(t, int64(101), historical.Version()) + require.NoError(t, historical.Close()) + + latest, err := openMemiAVLReplay(dbDir, 0) + require.NoError(t, err) + require.Equal(t, int64(102), latest.Version()) + require.NoError(t, latest.Close()) +} + +// TestOpenMemiAVLReplayRejectsShortChangelog guards the failure mode this +// design makes routine: repairing a torn tail inside the clone silently costs +// the trailing version, and memiavl reports success anyway. For a tool whose +// whole job is comparing digests across nodes, quietly digesting one version +// early is indistinguishable from a real state divergence. +func TestOpenMemiAVLReplayRejectsShortChangelog(t *testing.T) { + dbDir := newMemiavlSourceDir(t, 3) + + segment := lastChangelogSegment(t, dbDir) + intact, err := os.ReadFile(filepath.Clean(segment)) + require.NoError(t, err) + // Lop off the tail so the final committed record is torn and the + // repaired clone can only reach version 2. + require.NoError(t, os.WriteFile(segment, intact[:len(intact)-8], 0o600)) + + _, err = openMemiAVLReplay(dbDir, 3) + require.Error(t, err) + require.Contains(t, err.Error(), "requested 3, reached 2") + + // The rejected clone must not be left behind in the node's data dir. + entries, err := os.ReadDir(dbDir) + require.NoError(t, err) + for _, e := range entries { + require.NotContains(t, e.Name(), ".seidb-memiavl-tool-") + } +} diff --git a/sei-db/tools/cmd/seidb/operations/tool_clone.go b/sei-db/tools/cmd/seidb/operations/tool_clone.go new file mode 100644 index 0000000000..82010d948f --- /dev/null +++ b/sei-db/tools/cmd/seidb/operations/tool_clone.go @@ -0,0 +1,139 @@ +package operations + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "time" + + "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/memiavl" +) + +const ( + // toolCloneOwnerLockName is the flock-held marker file every tooling + // clone carries. While the creating process lives, the lock is held and + // the clone is off-limits; once the process dies (including SIGKILL, + // where no deferred cleanup runs), the kernel releases the flock and the + // next tool invocation can reap the directory. + toolCloneOwnerLockName = ".seidb-tool-owner.lock" + + // staleUnmarkedCloneAge guards the marker-less case: a crash in the + // window between MkdirTemp and the owner-lock creation, where liveness + // cannot be probed. Age is a poor liveness signal for marked clones + // (a mainnet-scale digest can legitimately run for hours), so it is + // used only when no marker exists at all. + staleUnmarkedCloneAge = 24 * time.Hour +) + +// toolClone is a private, disposable copy of a live store's snapshot + +// changelog that seidb tooling operates on instead of the live directory. +// It lives inside the source dbDir (to share its filesystem for hardlinks), +// which is exactly why leaking one is costly: its hardlinks pin snapshot +// inodes, so the live node's snapshot pruning frees no disk space until the +// clone is removed. +type toolClone struct { + dir string + ownerLock memiavl.FileLock + + // walRepaired records that validating the cloned changelog shrank it: + // the byte-copy caught the live writer mid-append and the WAL open + // repaired the torn tail inside the clone, costing the trailing record. + walRepaired bool +} + +// newToolClone reaps abandoned sibling clones, creates a fresh clone +// directory under dbDir with the given prefix, and marks it owned via flock +// before any expensive cloning starts. +// +// The prefix deliberately has no "-tmp" suffix: memiavl's removeTmpDirs +// deletes every "*-tmp" directory under its root when a node opens the DB +// read-write, and these clones must never be reaped by a process that cannot +// see whether the owning tool is still alive. +func newToolClone(dbDir, prefix string) (*toolClone, error) { + sweepStaleToolClones(dbDir, prefix) + + dir, err := os.MkdirTemp(dbDir, prefix+"*") + if err != nil { + return nil, fmt.Errorf("create temp dir under %s: %w", dbDir, err) + } + ownerLock, err := memiavl.LockFile(filepath.Join(dir, toolCloneOwnerLockName)) + if err != nil { + _ = os.RemoveAll(dir) + return nil, fmt.Errorf("acquire clone owner lock in %s: %w", dir, err) + } + return &toolClone{dir: dir, ownerLock: ownerLock}, nil +} + +// Remove releases the ownership lock and deletes the clone directory. +func (c *toolClone) Remove() error { + if c == nil { + return nil + } + if c.ownerLock != nil { + _ = c.ownerLock.Unlock() + _ = c.ownerLock.Destroy() + c.ownerLock = nil + } + if c.dir == "" { + return nil + } + if err := os.RemoveAll(c.dir); err != nil { + return fmt.Errorf("cleanup temp dir: %w", err) + } + c.dir = "" + return nil +} + +// sweepStaleToolClones removes abandoned clone directories under dbDir whose +// owner is provably gone: either the owner flock is acquirable (the creating +// process died), or no marker exists and the directory is old enough that the +// mkdir-to-lock window cannot explain it. Clones whose lock is still held — +// a concurrently running tool — are left alone. Best-effort by design: a +// failed sweep must never block the read the tool was invoked for. +func sweepStaleToolClones(dbDir, prefix string) { + entries, err := os.ReadDir(dbDir) + if err != nil { + return + } + for _, entry := range entries { + if !entry.IsDir() || !strings.HasPrefix(entry.Name(), prefix) { + continue + } + dir := filepath.Join(dbDir, entry.Name()) + lockPath := filepath.Join(dir, toolCloneOwnerLockName) + if _, err := os.Stat(lockPath); errors.Is(err, os.ErrNotExist) { + if info, err := entry.Info(); err == nil && time.Since(info.ModTime()) > staleUnmarkedCloneAge { + _ = os.RemoveAll(dir) + } + continue + } + lock, err := memiavl.LockFile(lockPath) + if err != nil { + // Lock held (owner alive) or unreadable — leave the clone alone. + continue + } + _ = lock.Unlock() + _ = lock.Destroy() + _ = os.RemoveAll(dir) + } +} + +// changelogByteSize sums the sizes of the regular files in a cloned changelog +// directory. Comparing it before and after the WAL-coverage validation open +// detects a torn-tail repair inside the clone (the only mutation that open +// can perform), which callers surface as a warning for latest-height reads. +func changelogByteSize(dir string) int64 { + var total int64 + _ = filepath.Walk(dir, func(_ string, info os.FileInfo, err error) error { + if err != nil { + return nil //nolint:nilerr // best-effort size probe + } + if info.Mode().IsRegular() { + total += info.Size() + } + return nil + }) + return total +} diff --git a/sei-db/tools/cmd/seidb/operations/tool_clone_test.go b/sei-db/tools/cmd/seidb/operations/tool_clone_test.go new file mode 100644 index 0000000000..5ddf6941fb --- /dev/null +++ b/sei-db/tools/cmd/seidb/operations/tool_clone_test.go @@ -0,0 +1,75 @@ +package operations + +import ( + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/memiavl" +) + +// TestSweepStaleToolClones pins the reaping policy for abandoned tooling +// clones: a clone whose owner lock is acquirable was orphaned by a dead +// process (SIGKILL runs no deferred cleanup) and must be removed, a clone +// whose lock is held belongs to a live tool and must survive, and a +// marker-less clone is removed only once it is old enough that the +// mkdir-to-lock window cannot explain it. +func TestSweepStaleToolClones(t *testing.T) { + dbDir := t.TempDir() + prefix := ".seidb-flatkv-tool-" + + staleUnmarked := filepath.Join(dbDir, prefix+"stale-unmarked") + require.NoError(t, os.Mkdir(staleUnmarked, 0o750)) + old := time.Now().Add(-2 * staleUnmarkedCloneAge) + require.NoError(t, os.Chtimes(staleUnmarked, old, old)) + + freshUnmarked := filepath.Join(dbDir, prefix+"fresh-unmarked") + require.NoError(t, os.Mkdir(freshUnmarked, 0o750)) + + held := filepath.Join(dbDir, prefix+"held") + require.NoError(t, os.Mkdir(held, 0o750)) + heldLock, err := memiavl.LockFile(filepath.Join(held, toolCloneOwnerLockName)) + require.NoError(t, err) + + orphaned := filepath.Join(dbDir, prefix+"orphaned") + require.NoError(t, os.Mkdir(orphaned, 0o750)) + require.NoError(t, os.WriteFile(filepath.Join(orphaned, toolCloneOwnerLockName), nil, 0o600)) + + otherPrefix := filepath.Join(dbDir, ".seidb-memiavl-tool-orphaned") + require.NoError(t, os.Mkdir(otherPrefix, 0o750)) + require.NoError(t, os.WriteFile(filepath.Join(otherPrefix, toolCloneOwnerLockName), nil, 0o600)) + + sweepStaleToolClones(dbDir, prefix) + + require.NoDirExists(t, staleUnmarked, "old marker-less clone must be reaped") + require.DirExists(t, freshUnmarked, "fresh marker-less clone may still be mid-creation") + require.DirExists(t, held, "a held owner lock proves the owning tool is alive") + require.NoDirExists(t, orphaned, "an acquirable owner lock proves the owner died") + require.DirExists(t, otherPrefix, "the sweep must not touch other clone families") + + require.NoError(t, heldLock.Unlock()) + sweepStaleToolClones(dbDir, prefix) + require.NoDirExists(t, held, "once the owner releases the lock the clone is reapable") +} + +// TestNewToolCloneOwnershipLifecycle checks that a live clone defends itself +// against a concurrent sweep and that Remove releases everything. +func TestNewToolCloneOwnershipLifecycle(t *testing.T) { + dbDir := t.TempDir() + prefix := ".seidb-memiavl-tool-" + + clone, err := newToolClone(dbDir, prefix) + require.NoError(t, err) + require.DirExists(t, clone.dir) + require.FileExists(t, filepath.Join(clone.dir, toolCloneOwnerLockName)) + + sweepStaleToolClones(dbDir, prefix) + require.DirExists(t, clone.dir, "an owned clone must survive a concurrent sweep") + + require.NoError(t, clone.Remove()) + require.NoDirExists(t, clone.dir) + require.NoError(t, clone.Remove(), "Remove must be idempotent") +}