Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 24 additions & 1 deletion sei-db/state_db/sc/memiavl/db.go
Original file line number Diff line number Diff line change
Expand Up @@ -1205,6 +1205,29 @@ func seekSnapshot(root string, targetVersion int64) (int64, error) {
return snapshotVersion, nil
}

// SeekSnapshotName returns the directory name (relative to root — callers
// join it themselves) and version of the snapshot that OpenDB would start
// from for targetVersion: the "current" link when targetVersion is 0,
// otherwise the newest snapshot at or below it.
//
// Exported for readers that need to resolve a snapshot without opening the DB,
// so they inherit this package's layout rules instead of restating them.
func SeekSnapshotName(root string, targetVersion int64) (string, int64, error) {
if targetVersion == 0 {
version, err := currentVersion(root)
if err != nil {
return "", 0, fmt.Errorf("read current snapshot: %w", err)
}
return snapshotName(version), version, nil
}

version, err := seekSnapshot(root, targetVersion)
if err != nil {
return "", 0, err
}
return snapshotName(version), version, nil
}

// GetEarliestVersion returns the earliest snapshot name in the db
func GetEarliestVersion(root string) (int64, error) {
var found int64
Expand Down Expand Up @@ -1327,7 +1350,7 @@ func isSnapshotName(name string) bool {
// it's needed for upgrade module to check store upgrades,
// it returns 0 if db doesn't exist or is empty.
func GetLatestVersion(dir string) (int64, error) {
metadata, err := readMetadata(currentPath(dir))
metadata, err := ReadMetadata(currentPath(dir))
if err != nil {
if os.IsNotExist(err) {
return 0, nil
Expand Down
35 changes: 35 additions & 0 deletions sei-db/state_db/sc/memiavl/db_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -1116,3 +1116,38 @@ func TestUpdateCurrentSymlinkClearsStaleTmp(t *testing.T) {
require.NoError(t, err)
require.Equal(t, "snapshot-1", target)
}

// TestSeekSnapshotName pins the exported snapshot-resolution contract that
// external readers (seidb tooling) rely on: the returned name is relative to
// root, targetVersion 0 resolves through the current link, a positive target
// selects the newest snapshot at or below it, and a target older than the
// earliest snapshot reports pruning instead of guessing.
func TestSeekSnapshotName(t *testing.T) {
root := t.TempDir()
for _, v := range []int64{5, 10} {
require.NoError(t, os.Mkdir(filepath.Join(root, snapshotName(v)), 0o750))
}
require.NoError(t, os.Symlink(snapshotName(10), currentPath(root)))

name, version, err := SeekSnapshotName(root, 0)
require.NoError(t, err)
require.Equal(t, snapshotName(10), name)
require.Equal(t, int64(10), version)

name, version, err = SeekSnapshotName(root, 7)
require.NoError(t, err)
require.Equal(t, snapshotName(5), name)
require.Equal(t, int64(5), version)

name, version, err = SeekSnapshotName(root, 10)
require.NoError(t, err)
require.Equal(t, snapshotName(10), name)
require.Equal(t, int64(10), version)

_, _, err = SeekSnapshotName(root, 3)
require.Error(t, err)
require.Contains(t, err.Error(), "target version is pruned")

_, _, err = SeekSnapshotName(filepath.Join(root, "does-not-exist"), 0)
require.Error(t, err)
}
10 changes: 8 additions & 2 deletions sei-db/state_db/sc/memiavl/multitree.go
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ func NewEmptyMultiTree(initialVersion uint32) *MultiTree {

func LoadMultiTree(ctx context.Context, dir string, opts Options) (*MultiTree, error) {
startTime := time.Now()
metadata, err := readMetadata(dir)
metadata, err := ReadMetadata(dir)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -600,7 +600,13 @@ func (t *MultiTree) ReplaceWith(other *MultiTree) error {
return errors.Join(errs...)
}

func readMetadata(dir string) (*proto.MultiTreeMetadata, error) {
// ReadMetadata loads a snapshot directory's MultiTreeMetadata (commit info +
// initial version). Exported for readers that need snapshot metadata without
// opening the DB — e.g. seidb tooling, whose changelog-coverage check must
// know the initial version because a DB bootstrapped with initial version N
// keeps its first snapshot named snapshot-0 while the first changelog entry
// is version N, not 1.
func ReadMetadata(dir string) (*proto.MultiTreeMetadata, error) {
// load commit info
bz, err := os.ReadFile(filepath.Join(filepath.Clean(dir), MetadataFileName))
if err != nil {
Expand Down
46 changes: 18 additions & 28 deletions sei-db/tools/cmd/seidb/operations/evm_logical_digest.go
Original file line number Diff line number Diff line change
Expand Up @@ -81,12 +81,14 @@ const (
// that exact height (or --height 0 for the current symlink). This is the
// preferred mode whenever the target height lines up with an existing
// snapshot boundary.
// - replay (SLOW): opens a read-only DB, replays the changelog up to
// --height, then walks the in-memory/mmap tree. Roughly an order of
// - replay (SLOW): clones the newest snapshot at or below --height plus the
// changelog into a temp directory, replays the changelog up to --height in
// that clone, then walks the in-memory/mmap tree. Roughly an order of
// magnitude slower than snapshot (changelog replay + per-leaf tree walk
// instead of a sequential file read). Use it only when no snapshot exists
// at the target height — e.g. nodes whose snapshot rewrite lags the tip, so
// an arbitrary comparison height has no snapshot-<height> on disk.
// instead of a sequential file read) and it byte-copies the changelog, so
// it needs free space alongside the source. Use it only when no snapshot
// exists at the target height — e.g. nodes whose snapshot rewrite lags the
// tip, so an arbitrary comparison height has no snapshot-<height> on disk.
//
// The flatkv side is always a pebble WAL-replay-to-height and is fast
// regardless. So when comparing across nodes, pick a height that is an existing
Expand Down Expand Up @@ -164,8 +166,8 @@ func EvmLogicalDigestCmd() *cobra.Command {
cmd.Flags().StringP("db-dir", "d", "", "For flatkv: the flatkv data dir. For memiavl: the memiavl root dir (contains current/ and snapshot-* )")
cmd.Flags().String("flatkv-dir", "", "Composite mode: flatkv data dir")
cmd.Flags().String("memiavl-dir", "", "Composite mode: memiavl root dir (contains current/ and snapshot-* )")
cmd.Flags().Int64("height", 0, "Target version. flatkv WAL-replays to it; memiavl resolves snapshot-<height>/evm (0 = current symlink)")
cmd.Flags().String("memiavl-open-mode", memiavlOpenModeSnapshot, "memiavl read mode: snapshot (FAST: sequential scan of the completed snapshot kvs file; requires an on-disk snapshot at --height, or --height 0 for current) | replay (SLOW, ~10x: replays changelog to --height then walks the mmap tree; use only when no snapshot exists at the target height). Prefer snapshot when --height matches an existing snapshot boundary")
cmd.Flags().Int64("height", 0, "Target version. flatkv WAL-replays to it; memiavl resolves snapshot-<height>/evm (0 = current symlink). On a live node 0 = latest is best-effort — the printed version line records what was actually digested; always pass an explicit common height when comparing nodes")
cmd.Flags().String("memiavl-open-mode", memiavlOpenModeSnapshot, "memiavl read mode: snapshot (FAST: sequential scan of the completed snapshot kvs file; requires an on-disk snapshot at --height, or --height 0 for current) | replay (SLOW, ~10x: clones the snapshot + changelog to a temp dir, replays to --height there, then walks the mmap tree; use only when no snapshot exists at the target height). Prefer snapshot when --height matches an existing snapshot boundary")
cmd.Flags().String("memiavl-normalization", memiavlNormSemantic, "memiavl digest/inspect normalization: semantic/independent (raw EVM key/value decoder) | translator (current migration mapping)")
cmd.Flags().String("inspect-bucket", "", "Inspect one normalized bucket (account|code|storage|misc) instead of printing the global digest")
cmd.Flags().Int("key-offset", 0, "Inspect mode: byte offset into physical key before applying --key-prefix / sharding")
Expand Down Expand Up @@ -465,7 +467,7 @@ func digestCompositeMigrateEVM(flatKVDir, memIAVLDir string, height int64, findT
requestedHeight: height,
version: opened.Version(),
}
var memReplayDB *memiavl.DB
var memReplayDB *openedMemIAVL
var memEvmSnapshotDir string
var memVersion int64
switch memiavlOpenMode {
Expand All @@ -481,13 +483,13 @@ func digestCompositeMigrateEVM(flatKVDir, memIAVLDir string, height int64, findT
ctx.source = fmt.Sprintf("flatkv clone version=%d + memiavl snapshot=%s", opened.Version(), memEvmSnapshotDir)
ctx.normalization = fmt.Sprintf("flatkv rows plus memiavl rows not migrated by boundary=%s version_known=%t migration_version=%d memiavl_version=%d", boundary.String(), versionKnown, migrationVersion, memVersion)
case memiavlOpenModeReplay:
memReplayDB, err = openMemiAVLReplayReadOnly(memIAVLDir, height)
memReplayDB, err = openMemiAVLReplay(memIAVLDir, height)
if err != nil {
return err
}
defer func() { _ = memReplayDB.Close() }()
memVersion = memReplayDB.Version()
ctx.source = fmt.Sprintf("flatkv clone version=%d + memiavl read-only replay dir=%s", opened.Version(), memIAVLDir)
ctx.source = fmt.Sprintf("flatkv clone version=%d + memiavl clone replay dir=%s", opened.Version(), memIAVLDir)
ctx.normalization = fmt.Sprintf("flatkv rows plus replayed memiavl rows not migrated by boundary=%s version_known=%t migration_version=%d memiavl_version=%d", boundary.String(), versionKnown, migrationVersion, memVersion)
default:
return fmt.Errorf("unknown --memiavl-open-mode %q (want snapshot|replay)", memiavlOpenMode)
Expand All @@ -504,7 +506,7 @@ func digestCompositeMigrateEVM(flatKVDir, memIAVLDir string, height int64, findT
if boundary.Status() != migration.MigrationComplete {
if memReplayDB != nil {
if err := consumeCompositeMemiavl(func(fn func(rawKey, rawVal []byte) error) error {
return scanMemiavlReplayEVMLeaves(memReplayDB, fn)
return scanMemiavlReplayEVMLeaves(memReplayDB.DB, fn)
}, "memiavl-replay", boundary, &d, accounts); err != nil {
return err
}
Expand Down Expand Up @@ -1104,30 +1106,18 @@ func digestMemIAVL(dbDir string, height int64, findTarget []byte, normalization
}
}

func openMemiAVLReplayReadOnly(dbDir string, height int64) (*memiavl.DB, error) {
db, err := memiavl.OpenDB(height, memiavl.Options{
Dir: dbDir,
ReadOnly: true,
ZeroCopy: true,
})
if err != nil {
return nil, fmt.Errorf("open memiavl read-only replay: %w", err)
}
return db, nil
}

func digestMemIAVLReplay(dbDir string, height int64, findTarget []byte, normalization string) error {
db, err := openMemiAVLReplayReadOnly(dbDir, height)
db, err := openMemiAVLReplay(dbDir, height)
if err != nil {
return err
}
defer func() { _ = db.Close() }()

switch normalization {
case "", memiavlNormSemantic, memiavlNormIndependent:
return digestMemIAVLReplaySemantic(dbDir, height, db, findTarget)
return digestMemIAVLReplaySemantic(dbDir, height, db.DB, findTarget)
case memiavlNormTranslator:
return digestMemIAVLReplayTranslator(dbDir, height, db, findTarget)
return digestMemIAVLReplayTranslator(dbDir, height, db.DB, findTarget)
default:
return fmt.Errorf("unknown --memiavl-normalization %q (want semantic|independent|translator)", normalization)
}
Expand Down Expand Up @@ -1291,7 +1281,7 @@ func digestMemIAVLReplaySemantic(dbDir string, height int64, db *memiavl.DB, fin
backend: "memiavl",
mode: "semantic-replay",
dbDir: dbDir,
source: "read-only memiavl DB opened from snapshot + changelog replay",
source: "isolated memiavl clone opened from snapshot + changelog replay",
normalization: "independent semantic decoder for replayed memiavl EVM keys; does not call flatkv.ImportTranslator",
requestedHeight: height,
version: db.Version(),
Expand All @@ -1308,7 +1298,7 @@ func digestMemIAVLReplayTranslator(dbDir string, height int64, db *memiavl.DB, f
backend: "memiavl",
mode: "translator-replay",
dbDir: dbDir,
source: "read-only memiavl DB opened from snapshot + changelog replay",
source: "isolated memiavl clone opened from snapshot + changelog replay",
normalization: "replayed memiavl leaves translated with flatkv.ImportTranslator, then reduced to logical payload",
requestedHeight: height,
version: db.Version(),
Expand Down
Loading
Loading