Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
123 commits
Select commit Hold shift + click to select a range
999953c
chore: subscribe sequentially and remove possible LRU cache race cond…
thlorenz Oct 30, 2025
a0774c2
fix: cleanup subscriptions and resubscribe on upstream connection close
thlorenz Oct 30, 2025
fb6e73e
feat: sub metrics via LRU cache
thlorenz Oct 31, 2025
dae6cc4
feat: clients also return subscription count
thlorenz Oct 31, 2025
7ecfc4a
chore: warn if LRU cache count is not matching pubsub count
thlorenz Oct 31, 2025
533ccde
chore: fix unsub on already evicted + metric counts
thlorenz Nov 2, 2025
638622d
chore: log discrepant account
thlorenz Nov 2, 2025
f6a0270
chore: don't remove pubkey from LRU cache if unsub fails
thlorenz Nov 2, 2025
402e0b7
chore: only removing sub when unsubscribe completed
thlorenz Nov 2, 2025
8d31a09
chore: improve subs logging
thlorenz Nov 2, 2025
754663f
chore: simplify unsub and remove invalid resub
thlorenz Nov 2, 2025
7943a7d
chore: eliminate sub/unsub race condition
thlorenz Nov 2, 2025
361b0da
chore: prevent overwriting existing sub
thlorenz Nov 2, 2025
33430a3
chore: tracing fetch + subs for cleaner debug logs
thlorenz Nov 2, 2025
98351e7
chore: minor comments
thlorenz Nov 3, 2025
56e05f5
chore: update correct metric + log on info for now
thlorenz Nov 3, 2025
399c37d
chore: add metrics query to test context
thlorenz Nov 3, 2025
d12b74f
chore: add manual ix test to diagnose subscriptions
thlorenz Nov 3, 2025
0d638f4
chore: merge bmuddha/fix/ws-reconnects, adjusting the changes
thlorenz Nov 3, 2025
5ec5287
chore: adding warn logs when recverr occurs
thlorenz Nov 4, 2025
e64166c
chore: fix max log level override
thlorenz Nov 4, 2025
07b170e
chore: more robust handling of fetch failure
thlorenz Nov 4, 2025
0c7a7cd
chore: fix recycle connections deadlock
thlorenz Nov 5, 2025
21f91d1
chore: clippy
thlorenz Nov 5, 2025
cf69692
fix: the extra task was overkill and not awaited
thlorenz Nov 5, 2025
c193be6
chore: log delegation issues on debug
thlorenz Nov 5, 2025
9dce1a9
fix: rely on cancellation tokens, remove join_set to fix endles recyc…
thlorenz Nov 5, 2025
9f0a972
chore: recycle with backoff
thlorenz Nov 5, 2025
7d635c4
chore: try to resub before recycle
thlorenz Nov 5, 2025
bf366ae
chore: pubsub client includes client id in all logs
thlorenz Nov 5, 2025
a4984d4
feat: better orchestrated reconnection logic
thlorenz Nov 6, 2025
fece0cd
chore: test reconnection logic
thlorenz Nov 6, 2025
cf9082a
Merge branch 'master' into thlorenz/subscription-metrics
thlorenz Nov 6, 2025
6283d23
chore: fmt ix tests
thlorenz Nov 6, 2025
889c46e
chore: clarifying comment
thlorenz Nov 6, 2025
22c1817
chore: fix reconnect ix test
thlorenz Nov 6, 2025
1058443
chore: logging precise info about which kind of accounts were in bank…
thlorenz Nov 7, 2025
31b49e5
chore: defensive refetch of accounts we should have been watching
thlorenz Nov 7, 2025
a81e9bb
chore: ensure we don't count removed accounts twice
thlorenz Nov 7, 2025
7fd5ec6
chore: more logs
thlorenz Nov 7, 2025
2bdd9ff
chore: debug fetched accounts
thlorenz Nov 7, 2025
908733f
chore: more info when removing account from bank
thlorenz Nov 7, 2025
37b9d32
chore: triaging those empty accounts
thlorenz Nov 7, 2025
b284277
chore: minor cleanup
thlorenz Nov 7, 2025
c9c7f8d
fix: rollback LRU entry when eviction unsubscribe fails
thlorenz Nov 7, 2025
fdbaed2
fix: error handling in subscription limits test to propagate task pan…
thlorenz Nov 7, 2025
405d4ff
fix: incorrect non-empty account count in chainlink logging
thlorenz Nov 7, 2025
3e7b400
chore: lint + fmt
thlorenz Nov 7, 2025
3bc011d
Merge branch 'master' into thlorenz/subscription-metrics
thlorenz Nov 7, 2025
61d475a
chore: less fetch account chatter on debug
thlorenz Nov 7, 2025
14ecd09
fix: don't persist empty accounts after txn execution
bmuddha Nov 7, 2025
8714d6a
chore: fix stale comments
thlorenz Nov 7, 2025
ab75627
fix: preserve concurrent fetch waiters in remote account provider
thlorenz Nov 7, 2025
4ef27a0
fix: log level issue
thlorenz Nov 7, 2025
7dd4665
fix: account overwrite issue
thlorenz Nov 7, 2025
ef698ee
chore: remove redundant fetch_sub call in magicblock-chainlink/src/ch…
thlorenz Nov 7, 2025
ca4f9c7
chore: stop spamming debug with not found accounts
thlorenz Nov 7, 2025
7b5a481
chore: less frequent sub metric update but with more info
thlorenz Nov 8, 2025
765c31f
chore: adding metrics for account fetches
thlorenz Nov 10, 2025
7490687
chore: log more info for mincontext slot fetch failure
thlorenz Nov 10, 2025
0279b24
chore: fix metrics suffixes
thlorenz Nov 10, 2025
5ebf091
fix(metrics): drain the connection of request body
bmuddha Nov 10, 2025
d9d5ee5
fix: found vs not-found metric counting
thlorenz Nov 10, 2025
77c33e1
fix: use more realistic slot for program deploy
thlorenz Nov 10, 2025
2cea109
chore: blacklist native token program
thlorenz Nov 10, 2025
c0e98fb
Merge branch 'master' into thlorenz/subscription-metrics
thlorenz Nov 13, 2025
9e9e14a
chore: enable ledger size metric
thlorenz Nov 13, 2025
917f177
chore: fix clippy
thlorenz Nov 13, 2025
1f2d684
chore: more logs around undelegate requests
thlorenz Nov 13, 2025
0b1b245
chore: verifying completed undelegation in intent tests
thlorenz Nov 13, 2025
7880a7d
feat: add undelegation requested/completed metrics
thlorenz Nov 13, 2025
9f8a0b2
fix: move delete onto separate thread
taco-paco Nov 13, 2025
021ef0d
chore: fix conditition to inc delegation complete
thlorenz Nov 13, 2025
a8900e1
fix: use the latest SVM with gasless feepayer check
bmuddha Nov 14, 2025
0a042a3
ci: trigger synchronize
lucacillario Nov 14, 2025
8f2e12b
fix: check for privileged mode when filtering empty accounts
bmuddha Nov 14, 2025
82f4248
chore: fix lint
thlorenz Nov 14, 2025
ae73b6d
chore: add extra check that unescrowed payer cannot pay/write for tx
thlorenz Nov 14, 2025
e1e6b15
Merge branch 'master' into thlorenz/subscription-metrics
thlorenz Nov 14, 2025
37d6839
Revert "fix: use the latest SVM with gasless feepayer check"
thlorenz Nov 14, 2025
e7c8a03
fix: post execution check for feepayer mutation
bmuddha Nov 14, 2025
c6f75fd
fix: perform feepayer modification check with privileged exception
bmuddha Nov 15, 2025
7162b3b
fix: cleanup print logs
bmuddha Nov 15, 2025
5e66fa2
fix: allow feepayer mutations if delegated
bmuddha Nov 15, 2025
a7a1eaa
Allow not existing feepayer in gasless mode (#631)
GabrielePicco Nov 16, 2025
9118600
feat: add eviction metric to the chainlink LRU
bmuddha Nov 17, 2025
308c1ff
feat: re-add auto airdrop (#632)
GabrielePicco Nov 17, 2025
91683be
feat: add metrics for multiple account requests investigation
taco-paco Nov 17, 2025
13b048e
fix: promotion of accounts that are already subscribed
thlorenz Nov 18, 2025
54a434e
fix: allow paying for rent with auto-airdrop enabled
GabrielePicco Nov 18, 2025
36157fe
chore: refactor
GabrielePicco Nov 18, 2025
1f11b83
hotfix: remove all accounts owned by delegation program on restart
thlorenz Nov 18, 2025
3b9496a
chore: ignore tests that depend on transfers from escrow accounts
thlorenz Nov 18, 2025
968e82d
chore: fix sub with overflow issue in test
thlorenz Nov 18, 2025
2cdfe4f
chore: fix bump in test
thlorenz Nov 18, 2025
e80ffc3
chore: log level fix
thlorenz Nov 18, 2025
4c29cd3
chore: ensure sub before adding to LRU cache
thlorenz Nov 18, 2025
4d433fb
chore: skip rare case undeleg/redeleg test for now
thlorenz Nov 18, 2025
66432ad
chore: remove stray debug print
thlorenz Nov 18, 2025
87a00bb
chore: fix minor nits
thlorenz Nov 18, 2025
30262bf
Merge branch 'master' into thlorenz/subscription-metrics
thlorenz Nov 18, 2025
ee3a330
chore: hoping to unflake failing chainlink test
thlorenz Nov 18, 2025
451effe
chore: retrying during failing test
thlorenz Nov 18, 2025
7a33748
chore: remove print
GabrielePicco Nov 18, 2025
33d984e
chore: lint
GabrielePicco Nov 18, 2025
4ca66b9
chore: improve error lohs
GabrielePicco Nov 18, 2025
7f027b4
chore: add metrics for unborking process
thlorenz Nov 18, 2025
a2694c9
chore: upgrade to latest solana-account
thlorenz Nov 19, 2025
ec29722
chore: initial unborking impl
thlorenz Nov 18, 2025
fabbc8a
fix: simulateTransaction only set innerInstructions if requested
bmuddha Nov 19, 2025
c4f5dbd
chore: fix typo in metrics method name
thlorenz Nov 19, 2025
9fc5335
fix: clippy
thlorenz Nov 19, 2025
46ee1ed
chore: proper evict counter
thlorenz Nov 19, 2025
14228a6
chore: increase sleep to hopefully pass redelegation tests
thlorenz Nov 19, 2025
c288d2b
Update magicblock-processor/src/executor/processing.rs
thlorenz Nov 19, 2025
006676a
chore: fmt
thlorenz Nov 19, 2025
f9f737c
Merge branch 'master' into thlorenz/subscription-metrics
thlorenz Nov 19, 2025
bce0a82
chore: merge thlorenz/subscription-metrics
GabrielePicco Nov 19, 2025
89194b8
chore: lint
GabrielePicco Nov 19, 2025
b627817
Merge branch 'master' into thlorenz/subscription-metrics
thlorenz Nov 19, 2025
521fbc1
chore: harden chainlink test
thlorenz Nov 19, 2025
9d2ba69
chore: fix dropped receiver in tests
thlorenz Nov 19, 2025
ceb9bab
Merge branch 'thlorenz/subscription-metrics' into picco/allow-paying-…
GabrielePicco Nov 19, 2025
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

6 changes: 3 additions & 3 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@ jsonrpc-pubsub = "18.0.0"
jsonrpc-ws-server = "18.0.0"
lazy_static = "1.4.0"
libc = "0.2.153"
log = { version = "0.4.20", features = ["release_max_level_info"] }
log = { version = "0.4.20" }
lru = "0.16.0"
macrotest = "1"
magic-domain-program = { git = "https://github.com/magicblock-labs/magic-domain-program.git", rev = "ea04d46", default-features = false }
Expand Down Expand Up @@ -151,7 +151,7 @@ serde = "1.0.217"
serde_derive = "1.0"
serde_json = "1.0"
sha3 = "0.10.8"
solana-account = { git = "https://github.com/magicblock-labs/solana-account.git", rev = "f454d4a" }
solana-account = { git = "https://github.com/magicblock-labs/solana-account.git", rev = "731fa50" }
solana-account-decoder = { version = "2.2" }
solana-accounts-db = { version = "2.2" }
solana-account-decoder-client-types = { version = "2.2" }
Expand Down Expand Up @@ -227,6 +227,6 @@ features = ["dev-context-only-utils"]
# some solana dependencies have solana-storage-proto as dependency
# we need to patch them with our version, because they use protobuf-src v1.1.0
# and we use protobuf-src v2.1.1. Otherwise compilation fails
solana-account = { git = "https://github.com/magicblock-labs/solana-account.git", rev = "f454d4a" }
solana-account = { git = "https://github.com/magicblock-labs/solana-account.git", rev = "731fa50" }
solana-storage-proto = { path = "./storage-proto" }
solana-svm = { git = "https://github.com/magicblock-labs/magicblock-svm.git", rev = "11bbaf2" }
7 changes: 5 additions & 2 deletions magicblock-account-cloner/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -173,12 +173,15 @@ impl ChainlinkCloner {
// Create and initialize the program account in retracted state
// and then deploy it and finally set the authority to match the
// one on chain
let slot = self.accounts_db.slot();
let DeployableV4Program {
pre_deploy_loader_state,
deploy_instruction,
post_deploy_loader_state,
} = program
.try_into_deploy_data_and_ixs_v4(validator_kp.pubkey())?;
} = program.try_into_deploy_data_and_ixs_v4(
slot,
validator_kp.pubkey(),
)?;

let lamports = Rent::default()
.minimum_balance(pre_deploy_loader_state.len());
Expand Down
4 changes: 2 additions & 2 deletions magicblock-accounts-db/src/lib.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
use std::{path::Path, sync::Arc};
use std::{collections::HashSet, path::Path, sync::Arc};

use error::AccountsDbError;
use index::{
Expand Down Expand Up @@ -356,7 +356,7 @@ impl AccountsBank for AccountsDb {
.iter_all()
.filter(|(pk, acc)| predicate(pk, acc))
.map(|(pk, _)| pk)
.collect::<Vec<_>>();
.collect::<HashSet<_>>();
let removed = to_remove.len();
for pk in to_remove {
self.remove_account(&pk);
Expand Down
2 changes: 1 addition & 1 deletion magicblock-aperture/src/requests/http/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,7 @@ impl HttpDispatcher {
.inspect_err(|e| {
// There is nothing we can do if fetching the account fails
// Log the error and return whatever is in the accounts db
warn!("Failed to ensure account {pubkey}: {e}");
debug!("Failed to ensure account {pubkey}: {e}");

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🟡 Minor

🧩 Analysis chain

Verify inconsistent log levels for similar error conditions.

The log level for account ensure failure was downgraded from warn to debug, but Line 137 in read_accounts_with_ensure still uses warn! for the same type of error. Both functions have identical error handling patterns and comments ("There is nothing we can do if fetching the account fails").

Is this inconsistency intentional? If single-account failures warrant only debug-level logging, should multi-account failures also be downgraded? Alternatively, if ensure failures indicate operational issues (network problems, chain unavailability, etc.) that could lead to stale data being served, should both remain at warn level?

Please clarify:

  1. Why do these semantically equivalent errors have different log levels?
  2. Should they be consistent?
  3. Will downgrading to debug hide important operational issues that need monitoring?

🏁 Script executed:

#!/bin/bash
# Inspect the file to verify log levels and context
cat magicblock-aperture/src/requests/http/mod.rs | sed -n '100,150p'

Length of output: 1968


Ensure consistent log levels for semantically equivalent errors.

Verification confirms the log level inconsistency between read_account_with_ensure (line 115: debug!) and read_accounts_with_ensure (line 137: warn!). Both functions have identical error handling logic and the same recovery comment ("There is nothing we can do if fetching the account(s) fails"), yet they log at different levels.

Either:

  1. Make both consistent—apply the same log level to both single and multi-account failures
  2. Document the intentional difference if single-account failures are genuinely less critical than multi-account failures

Without explicit documentation, this inconsistency suggests an oversight and could unintentionally suppress observability into chain/network issues.

🤖 Prompt for AI Agents
In magicblock-aperture/src/requests/http/mod.rs around lines 115 and 137, the
error logging for semantically equivalent failures is inconsistent (line 115
uses debug! while line 137 uses warn!); make them consistent by using the same
log level for both read_account_with_ensure and read_accounts_with_ensure
(preferably change the debug! at line 115 to warn! to match line 137), and keep
the existing error message and recovery comment unchanged (or add a brief
comment if you intentionally want a different level).

});
self.accountsdb.get_account(pubkey)
}
Expand Down
2 changes: 1 addition & 1 deletion magicblock-aperture/src/requests/http/send_transaction.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
use log::{debug, trace};
use log::*;

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🧹 Nitpick | 🔵 Trivial

Consider explicit imports for clarity.

While wildcard imports (use log::*) are commonly accepted for the log crate, explicit imports (use log::{debug, trace}) improve code clarity by making dependencies visible. This is an optional style preference.

🤖 Prompt for AI Agents
In magicblock-aperture/src/requests/http/send_transaction.rs around line 1, the
file currently uses a wildcard import `use log::*`; replace this with explicit
imports for the logging macros actually used in the file (for example `use
log::{debug, trace, info, warn, error}` only including the ones referenced) so
dependencies are visible and clearer; update the use statement accordingly and
remove the wildcard import.

use magicblock_metrics::metrics::{
TRANSACTION_PROCESSING_TIME, TRANSACTION_SKIP_PREFLIGHT,
};
Expand Down
18 changes: 10 additions & 8 deletions magicblock-aperture/src/requests/http/simulate_transaction.rs
Original file line number Diff line number Diff line change
Expand Up @@ -70,20 +70,22 @@ impl HttpDispatcher {
}
.into()
};
let result = RpcSimulateTransactionResult {
err: result.result.err(),
logs: result.logs,
accounts: None,
units_consumed: Some(result.units_consumed),
return_data: result.return_data.map(Into::into),
inner_instructions: result
let inner_instructions = config.inner_instructions.then(|| {
result
.inner_instructions
.into_iter()
.flatten()
.enumerate()
.map(converter)
.collect::<Vec<_>>()
.into(),
});
let result = RpcSimulateTransactionResult {
err: result.result.err(),
logs: result.logs,
accounts: None,
units_consumed: Some(result.units_consumed),
return_data: result.return_data.map(Into::into),
inner_instructions,
replacement_blockhash,
};

Expand Down
1 change: 1 addition & 0 deletions magicblock-aperture/src/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ fn chainlink(accounts_db: &Arc<AccountsDb>) -> ChainlinkImpl {
None,
Pubkey::new_unique(),
Pubkey::new_unique(),
0,
)
.expect("Failed to create Chainlink")
}
Expand Down
1 change: 1 addition & 0 deletions magicblock-aperture/tests/setup.rs
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,7 @@ fn chainlink(accounts_db: &Arc<AccountsDb>) -> Arc<ChainlinkImpl> {
None,
Pubkey::new_unique(),
Pubkey::new_unique(),
0,
)
.expect("Failed to create Chainlink"),
)
Expand Down
6 changes: 6 additions & 0 deletions magicblock-api/src/magic_validator.rs
Original file line number Diff line number Diff line change
Expand Up @@ -276,6 +276,11 @@ impl MagicValidator {
validator::init_validator_authority(identity_keypair);

let txn_scheduler_state = TransactionSchedulerState {
is_auto_airdrop_lamports_enabled: config
.accounts
.clone
.auto_airdrop_lamports
> 0,
accountsdb: accountsdb.clone(),
ledger: ledger.clone(),
transaction_status_tx: validator_channels.transaction_status,
Expand Down Expand Up @@ -441,6 +446,7 @@ impl MagicValidator {
validator_pubkey,
faucet_pubkey,
chainlink_config,
config.accounts.clone.auto_airdrop_lamports,
)
.await?;

Expand Down
1 change: 0 additions & 1 deletion magicblock-api/src/tickers.rs
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,6 @@ async fn handle_scheduled_commits<C: ScheduledCommitsProcessor>(
error!("Failed to process scheduled commits: {:?}", err);
}
}

#[allow(unused_variables)]
pub fn init_system_metrics_ticker(
tick_duration: Duration,
Expand Down
4 changes: 3 additions & 1 deletion magicblock-chainlink/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ version.workspace = true
edition.workspace = true

[dependencies]
arc-swap = "1.7"
async-trait = { workspace = true }
bincode = { workspace = true }
env_logger = { workspace = true }
Expand All @@ -12,7 +13,8 @@ log = { workspace = true }
lru = { workspace = true }
magicblock-core = { workspace = true }
magicblock-magic-program-api = { workspace = true }
magicblock-delegation-program = { workspace = true }
magicblock-metrics = { workspace = true }
magicblock-delegation-program = { workspace = true }
serde_json = { workspace = true }
solana-account = { workspace = true }
solana-account-decoder = { workspace = true }
Expand Down
5 changes: 4 additions & 1 deletion magicblock-chainlink/src/chainlink/blacklisted_accounts.rs
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@ pub fn blacklisted_accounts(
// want to take a dependency on that crate just for this ID which won't change
const NATIVE_SOL_ID: Pubkey =
solana_sdk::pubkey!("So11111111111111111111111111111111111111112");

let mut blacklisted_accounts = sysvar_accounts()
.into_iter()
.chain(native_program_accounts())
Expand Down Expand Up @@ -48,6 +47,9 @@ pub fn sysvar_accounts() -> HashSet<Pubkey> {
}

pub fn native_program_accounts() -> HashSet<Pubkey> {
const NATIVE_TOKEN_PROGRAM_ID: Pubkey =
solana_sdk::pubkey!("TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA");

let mut blacklisted_programs = HashSet::new();
blacklisted_programs.insert(solana_sdk::address_lookup_table::program::ID);
blacklisted_programs.insert(solana_sdk::bpf_loader::ID);
Expand All @@ -63,5 +65,6 @@ pub fn native_program_accounts() -> HashSet<Pubkey> {
blacklisted_programs.insert(solana_sdk::stake::program::ID);
blacklisted_programs.insert(solana_sdk::system_program::ID);
blacklisted_programs.insert(solana_sdk::vote::program::ID);
blacklisted_programs.insert(NATIVE_TOKEN_PROGRAM_ID);
blacklisted_programs
}
2 changes: 1 addition & 1 deletion magicblock-chainlink/src/chainlink/errors.rs
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ pub enum ChainlinkError {
#[error("Cloner error: {0}")]
ClonerError(#[from] crate::cloner::errors::ClonerError),

#[error("Delegation could not be decoded: {0} ({1:?})")]
#[error("Delegation record could not be decoded: {0} ({1:?})")]
InvalidDelegationRecord(Pubkey, ProgramError),

#[error("Failed to resolve one or more accounts {0} when getting delegation records")]
Expand Down
Loading
Loading