diff --git a/giga/deps/tasks/metrics.go b/giga/deps/tasks/metrics.go new file mode 100644 index 0000000000..0070ae03b0 --- /dev/null +++ b/giga/deps/tasks/metrics.go @@ -0,0 +1,37 @@ +package tasks + +import ( + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/metric" +) + +// taskMetrics mirrors sei-cosmos/tasks/metrics.go's instruments of the same +// name/scope so the sei-cosmos and giga fork schedulers merge into single +// scheduler_retries/scheduler_incarnations series. Keep description/unit +// byte-identical across both declarations or the OTel SDK stops deduping. +var ( + meter = otel.Meter("seicosmos_tasks") + + taskMetrics = struct { + retries metric.Int64Counter + incarnations metric.Int64Counter + }{ + retries: must(meter.Int64Counter( + "scheduler_retries", + metric.WithDescription("Number of OCC scheduler transaction retries"), + metric.WithUnit("{count}"), + )), + incarnations: must(meter.Int64Counter( + "scheduler_incarnations", + metric.WithDescription("Sum of per-round maximum incarnations in the OCC scheduler"), + metric.WithUnit("{count}"), + )), + } +) + +func must[V any](v V, err error) V { + if err != nil { + panic(err) + } + return v +} diff --git a/giga/deps/tasks/scheduler.go b/giga/deps/tasks/scheduler.go index 21c3f39452..f797314fa3 100644 --- a/giga/deps/tasks/scheduler.go +++ b/giga/deps/tasks/scheduler.go @@ -5,6 +5,8 @@ import ( "crypto/sha256" "encoding/hex" "fmt" + "os" + "runtime/debug" "sort" "strings" "sync" @@ -279,9 +281,18 @@ type schedulerMetrics struct { retries int } -func (s *scheduler) emitMetrics() { +func (s *scheduler) emitMetrics(ctx context.Context) { + defer func() { + if e := recover(); e != nil { + fmt.Fprintf(os.Stderr, "telemetry panic: %v\n%s", e, debug.Stack()) + } + }() + // TODO(PLT-353): remove once scheduler_retries verified telemetry.IncrCounter(float32(s.metrics.retries), "scheduler", "retries") + taskMetrics.retries.Add(ctx, int64(s.metrics.retries)) + // TODO(PLT-353): remove once scheduler_incarnations verified telemetry.IncrCounter(float32(s.metrics.maxIncarnation), "scheduler", "incarnations") + taskMetrics.incarnations.Add(ctx, int64(s.metrics.maxIncarnation)) } func (s *scheduler) ProcessAll(ctx sdk.Context, reqs []*sdk.DeliverTxEntry) ([]types.ResponseDeliverTx, error) { @@ -295,7 +306,7 @@ func (s *scheduler) ProcessAll(ctx sdk.Context, reqs []*sdk.DeliverTxEntry) ([]t s.conflictKeyCounts = make(map[string]int) s.executeCh = make(chan func(), len(tasks)) s.validateCh = make(chan func(), len(tasks)) - defer s.emitMetrics() + defer s.emitMetrics(ctx.Context()) // default to number of tasks if workers is negative or 0 by this point workers := s.workers diff --git a/giga/deps/xbank/keeper/metrics.go b/giga/deps/xbank/keeper/metrics.go new file mode 100644 index 0000000000..fa66d97f27 --- /dev/null +++ b/giga/deps/xbank/keeper/metrics.go @@ -0,0 +1,63 @@ +package keeper + +import ( + "context" + "fmt" + "os" + "runtime/debug" + + "github.com/sei-protocol/sei-chain/sei-cosmos/telemetry" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/metric" +) + +// BankNewAccount* must stay byte-identical to the same consts in +// sei-cosmos/x/bank/keeper and utils/metrics so the three Int64Counter +// declarations merge into one series. +const ( + BankNewAccountMeter = "seicosmos_x_bank_keeper" + BankNewAccountName = "bank_new_account" + BankNewAccountDescription = "Number of new accounts created during bank transfers" + BankNewAccountUnit = "{count}" +) + +// bankMetrics.newAccount mirrors sei-cosmos/x/bank/keeper/metrics.go's +// instrument of the same name/scope so the two dual-emit paths merge into a +// single bank_new_account series. +var ( + meter = otel.Meter(BankNewAccountMeter) + + bankMetrics = struct { + newAccount metric.Int64Counter + }{ + newAccount: must(meter.Int64Counter( + BankNewAccountName, + metric.WithDescription(BankNewAccountDescription), + metric.WithUnit(BankNewAccountUnit), + )), + } +) + +func must[V any](v V, err error) V { + if err != nil { + panic(err) + } + return v +} + +// recordNewAccounts dual-emits the legacy new-account counter and its OTel +// counterpart (bank_new_account). Runs from consensus-critical send paths, so +// a telemetry fault here must not panic into the caller. +func recordNewAccounts(ctx context.Context, count int64) { + if count <= 0 { + return + } + defer func() { + if e := recover(); e != nil { + fmt.Fprintf(os.Stderr, "telemetry panic: %v\n%s", e, debug.Stack()) + } + }() + // TODO(PLT-353): remove once bank_new_account verified + telemetry.IncrCounter(float32(count), "new", "account") + bankMetrics.newAccount.Add(ctx, count) +} diff --git a/giga/deps/xbank/keeper/send.go b/giga/deps/xbank/keeper/send.go index a6879e9d45..f4c1b8152a 100644 --- a/giga/deps/xbank/keeper/send.go +++ b/giga/deps/xbank/keeper/send.go @@ -7,7 +7,6 @@ import ( "github.com/sei-protocol/sei-chain/giga/deps/xbank/types" "github.com/sei-protocol/sei-chain/sei-cosmos/codec" "github.com/sei-protocol/sei-chain/sei-cosmos/store/prefix" - "github.com/sei-protocol/sei-chain/sei-cosmos/telemetry" sdk "github.com/sei-protocol/sei-chain/sei-cosmos/types" sdkerrors "github.com/sei-protocol/sei-chain/sei-cosmos/types/errors" cosmosbanktypes "github.com/sei-protocol/sei-chain/sei-cosmos/x/bank/types" @@ -145,7 +144,9 @@ func (k BaseSendKeeper) InputOutputCoins(ctx sdk.Context, inputs []types.Input, // such as delegated fee messages. accExists := k.ak.HasAccount(ctx, outAddress) if !accExists { - defer telemetry.IncrCounter(1, "new", "account") + defer func() { + recordNewAccounts(ctx.Context(), 1) + }() k.ak.SetAccount(ctx, k.ak.NewAccountWithAddress(ctx, outAddress)) } } @@ -166,7 +167,9 @@ func (k BaseSendKeeper) SendCoins(ctx sdk.Context, fromAddr sdk.AccAddress, toAd // such as delegated fee messages. accExists := k.ak.HasAccount(ctx, toAddr) if !accExists { - defer telemetry.IncrCounter(1, "new", "account") + defer func() { + recordNewAccounts(ctx.Context(), 1) + }() k.ak.SetAccount(ctx, k.ak.NewAccountWithAddress(ctx, toAddr)) } diff --git a/precompiles/bank/bank.go b/precompiles/bank/bank.go index 7478d3284e..12da37456d 100644 --- a/precompiles/bank/bank.go +++ b/precompiles/bank/bank.go @@ -287,7 +287,7 @@ func (p PrecompileExecutor) sendNative(ctx sdk.Context, method *abi.Method, args } accExists := p.accountKeeper.HasAccount(ctx, receiverSeiAddr) if !accExists { - defer metrics.SafeTelemetryIncrCounter(1, "new", "account") + defer metrics.RecordBankNewAccount(ctx.Context()) p.accountKeeper.SetAccount(ctx, p.accountKeeper.NewAccountWithAddress(ctx, receiverSeiAddr)) } diff --git a/sei-cosmos/tasks/metrics.go b/sei-cosmos/tasks/metrics.go index 79c0b2f697..2f86f25f04 100644 --- a/sei-cosmos/tasks/metrics.go +++ b/sei-cosmos/tasks/metrics.go @@ -5,6 +5,10 @@ import ( "go.opentelemetry.io/otel/metric" ) +// taskMetrics mirrors giga/deps/tasks/metrics.go's instruments of the same +// name/scope so the sei-cosmos and giga fork schedulers merge into single +// scheduler_retries/scheduler_incarnations series. Keep description/unit +// byte-identical across both declarations or the OTel SDK stops deduping. var ( meter = otel.Meter("seicosmos_tasks") @@ -19,7 +23,7 @@ var ( )), incarnations: must(meter.Int64Counter( "scheduler_incarnations", - metric.WithDescription("Maximum incarnation seen in OCC scheduler round"), + metric.WithDescription("Sum of per-round maximum incarnations in the OCC scheduler"), metric.WithUnit("{count}"), )), } diff --git a/sei-cosmos/tasks/scheduler.go b/sei-cosmos/tasks/scheduler.go index edcfa901f8..5e490269f2 100644 --- a/sei-cosmos/tasks/scheduler.go +++ b/sei-cosmos/tasks/scheduler.go @@ -5,6 +5,8 @@ import ( "crypto/sha256" "encoding/hex" "fmt" + "os" + "runtime/debug" "sort" "strings" "sync" @@ -265,13 +267,18 @@ type schedulerMetrics struct { retries int } -func (s *scheduler) emitMetrics() { - taskMetrics.retries.Add(context.Background(), int64(s.metrics.retries)) +func (s *scheduler) emitMetrics(ctx context.Context) { + defer func() { + if e := recover(); e != nil { + fmt.Fprintf(os.Stderr, "telemetry panic: %v\n%s", e, debug.Stack()) + } + }() // TODO(PLT-353): remove once scheduler_retries verified telemetry.IncrCounter(float32(s.metrics.retries), "scheduler", "retries") - taskMetrics.incarnations.Add(context.Background(), int64(s.metrics.maxIncarnation)) + taskMetrics.retries.Add(ctx, int64(s.metrics.retries)) // TODO(PLT-353): remove once scheduler_incarnations verified telemetry.IncrCounter(float32(s.metrics.maxIncarnation), "scheduler", "incarnations") + taskMetrics.incarnations.Add(ctx, int64(s.metrics.maxIncarnation)) } func (s *scheduler) ProcessAll(ctx sdk.Context, reqs []*sdk.DeliverTxEntry) ([]types.ResponseDeliverTx, error) { @@ -285,7 +292,7 @@ func (s *scheduler) ProcessAll(ctx sdk.Context, reqs []*sdk.DeliverTxEntry) ([]t s.conflictKeyCounts = make(map[string]int) s.executeCh = make(chan func(), len(tasks)) s.validateCh = make(chan func(), len(tasks)) - defer s.emitMetrics() + defer s.emitMetrics(ctx.Context()) // default to number of tasks if workers is negative or 0 by this point workers := s.workers diff --git a/sei-cosmos/x/bank/keeper/metrics.go b/sei-cosmos/x/bank/keeper/metrics.go index 8672fae087..f013ac16e0 100644 --- a/sei-cosmos/x/bank/keeper/metrics.go +++ b/sei-cosmos/x/bank/keeper/metrics.go @@ -5,8 +5,18 @@ import ( "go.opentelemetry.io/otel/metric" ) +// BankNewAccount* must stay byte-identical to the same consts in +// utils/metrics and giga/deps/xbank/keeper so the three Int64Counter +// declarations merge into one series. +const ( + BankNewAccountMeter = "seicosmos_x_bank_keeper" + BankNewAccountName = "bank_new_account" + BankNewAccountDescription = "Number of new accounts created during bank transfers" + BankNewAccountUnit = "{count}" +) + var ( - meter = otel.Meter("seicosmos_x_bank_keeper") + meter = otel.Meter(BankNewAccountMeter) bankMetrics = struct { sendAmount metric.Int64Gauge @@ -17,10 +27,14 @@ var ( metric.WithDescription("Amount sent in the last MsgSend transaction by denomination"), metric.WithUnit("{utoken}"), )), + // newAccount is mirrored by utils/metrics.bankNewAccountCounter (bank + // precompiles) and giga/deps/xbank/keeper.bankMetrics.newAccount (the + // Giga fork) on the same name/scope, so all three merge into one + // bank_new_account series. newAccount: must(meter.Int64Counter( - "bank_new_account", - metric.WithDescription("Number of new accounts created during bank transfers"), - metric.WithUnit("{count}"), + BankNewAccountName, + metric.WithDescription(BankNewAccountDescription), + metric.WithUnit(BankNewAccountUnit), )), } ) diff --git a/sei-cosmos/x/bank/keeper/send.go b/sei-cosmos/x/bank/keeper/send.go index e5ae9b5ebb..06facbb02c 100644 --- a/sei-cosmos/x/bank/keeper/send.go +++ b/sei-cosmos/x/bank/keeper/send.go @@ -2,6 +2,10 @@ package keeper import ( "bytes" + "context" + "fmt" + "os" + "runtime/debug" "strings" "github.com/sei-protocol/sei-chain/sei-cosmos/codec" @@ -146,9 +150,7 @@ func (k BaseSendKeeper) InputOutputCoins(ctx sdk.Context, inputs []types.Input, accExists := k.ak.HasAccount(ctx, outAddress) if !accExists { defer func() { - bankMetrics.newAccount.Add(ctx.Context(), 1) - // TODO(PLT-353): remove once bank_new_account verified - telemetry.IncrCounter(1, "new", "account") + recordNewAccounts(ctx.Context(), 1) }() k.ak.SetAccount(ctx, k.ak.NewAccountWithAddress(ctx, outAddress)) } @@ -171,9 +173,7 @@ func (k BaseSendKeeper) SendCoins(ctx sdk.Context, fromAddr sdk.AccAddress, toAd accExists := k.ak.HasAccount(ctx, toAddr) if !accExists { defer func() { - bankMetrics.newAccount.Add(ctx.Context(), 1) - // TODO(PLT-353): remove once bank_new_account verified - telemetry.IncrCounter(1, "new", "account") + recordNewAccounts(ctx.Context(), 1) }() k.ak.SetAccount(ctx, k.ak.NewAccountWithAddress(ctx, toAddr)) } @@ -181,6 +181,23 @@ func (k BaseSendKeeper) SendCoins(ctx sdk.Context, fromAddr sdk.AccAddress, toAd return nil } +// recordNewAccounts dual-emits the legacy new-account counter and its OTel +// counterpart (bank_new_account). Runs from consensus-critical send paths, so +// a telemetry fault here must not panic into the caller. +func recordNewAccounts(ctx context.Context, count int64) { + if count <= 0 { + return + } + defer func() { + if e := recover(); e != nil { + fmt.Fprintf(os.Stderr, "telemetry panic: %v\n%s", e, debug.Stack()) + } + }() + // TODO(PLT-353): remove once bank_new_account verified + telemetry.IncrCounter(float32(count), "new", "account") + bankMetrics.newAccount.Add(ctx, count) +} + func (k BaseSendKeeper) SendCoinsWithoutAccCreation(ctx sdk.Context, fromAddr sdk.AccAddress, toAddr sdk.AccAddress, amt sdk.Coins) error { return k.sendCoinsWithoutAccCreation(ctx, fromAddr, toAddr, amt, true) } diff --git a/utils/metrics/bank_new_account_mirror_test.go b/utils/metrics/bank_new_account_mirror_test.go new file mode 100644 index 0000000000..ecd0b029b7 --- /dev/null +++ b/utils/metrics/bank_new_account_mirror_test.go @@ -0,0 +1,25 @@ +package metrics_test + +import ( + "testing" + + "github.com/stretchr/testify/require" + + gigabank "github.com/sei-protocol/sei-chain/giga/deps/xbank/keeper" + cosmosbank "github.com/sei-protocol/sei-chain/sei-cosmos/x/bank/keeper" + util "github.com/sei-protocol/sei-chain/utils/metrics" +) + +// TestBankNewAccountInstrumentMirror pins the three mirrored bank_new_account +// instrument declarations to the same meter/name/description/unit. +func TestBankNewAccountInstrumentMirror(t *testing.T) { + require.Equal(t, util.BankNewAccountMeter, cosmosbank.BankNewAccountMeter) + require.Equal(t, util.BankNewAccountName, cosmosbank.BankNewAccountName) + require.Equal(t, util.BankNewAccountDescription, cosmosbank.BankNewAccountDescription) + require.Equal(t, util.BankNewAccountUnit, cosmosbank.BankNewAccountUnit) + + require.Equal(t, util.BankNewAccountMeter, gigabank.BankNewAccountMeter) + require.Equal(t, util.BankNewAccountName, gigabank.BankNewAccountName) + require.Equal(t, util.BankNewAccountDescription, gigabank.BankNewAccountDescription) + require.Equal(t, util.BankNewAccountUnit, gigabank.BankNewAccountUnit) +} diff --git a/utils/metrics/metrics_util.go b/utils/metrics/metrics_util.go index e66f68255f..578df5ea39 100644 --- a/utils/metrics/metrics_util.go +++ b/utils/metrics/metrics_util.go @@ -1,9 +1,11 @@ package metrics import ( + "context" "errors" "fmt" "math/big" + "os" "runtime/debug" "strconv" "time" @@ -15,10 +17,38 @@ import ( "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/exporters/prometheus" + "go.opentelemetry.io/otel/metric" sdk "go.opentelemetry.io/otel/sdk/metric" "go.opentelemetry.io/otel/sdk/resource" ) +// BankNewAccount* must stay byte-identical to the same consts in +// sei-cosmos/x/bank/keeper and giga/deps/xbank/keeper so the three +// Int64Counter declarations merge into one series. +const ( + BankNewAccountMeter = "seicosmos_x_bank_keeper" + BankNewAccountName = "bank_new_account" + BankNewAccountDescription = "Number of new accounts created during bank transfers" + BankNewAccountUnit = "{count}" +) + +// bankNewAccountCounter mirrors sei-cosmos/x/bank/keeper/metrics.go's +// instrument of the same name/scope (and giga/deps/xbank/keeper/metrics.go's) +// so precompile-originated and keeper-originated new-account events merge +// into a single bank_new_account series. +var bankNewAccountCounter = mustCounter(otel.Meter(BankNewAccountMeter).Int64Counter( + BankNewAccountName, + metric.WithDescription(BankNewAccountDescription), + metric.WithUnit(BankNewAccountUnit), +)) + +func mustCounter(c metric.Int64Counter, err error) metric.Int64Counter { + if err != nil { + panic(err) + } + return c +} + func SetupOtelMetricsProvider(chainID string) error { if chainID == "" { return fmt.Errorf("chainID must not be empty") @@ -52,6 +82,21 @@ func SetupOtelMetricsProvider(chainID string) error { return nil } +// RecordBankNewAccount dual-emits the legacy new-account counter and its OTel +// counterpart (bank_new_account). Call from defer when creating an account. +// Runs during precompile execution, so a telemetry fault here must not panic +// into a consensus-critical path. +func RecordBankNewAccount(ctx context.Context) { + defer func() { + if e := recover(); e != nil { + fmt.Fprintf(os.Stderr, "telemetry panic: %v\n%s", e, debug.Stack()) + } + }() + // TODO(PLT-353): remove once bank_new_account verified + telemetry.IncrCounter(1, "new", "account") + bankNewAccountCounter.Add(ctx, 1) +} + func SafeTelemetryIncrCounter(val float32, keys ...string) { defer func() { if e := recover(); e != nil {