diff --git a/Cargo.lock b/Cargo.lock index 17ec2a16e36..5ff49e16641 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2245,6 +2245,7 @@ dependencies = [ "papyrus_base_layer", "pretty_assertions", "rstest", + "serde_json", "tokio", "tokio-util", "tracing", diff --git a/crates/apollo_deployments/resources/app_configs/replacer_state_sync_config.json b/crates/apollo_deployments/resources/app_configs/replacer_state_sync_config.json index 88e899e899a..e67f8ed27a8 100644 --- a/crates/apollo_deployments/resources/app_configs/replacer_state_sync_config.json +++ b/crates/apollo_deployments/resources/app_configs/replacer_state_sync_config.json @@ -10,6 +10,7 @@ "state_sync_config.static_config.central_sync_client_config.central_source_config.retry_config.retry_base_millis": 30, "state_sync_config.static_config.central_sync_client_config.central_source_config.retry_config.retry_max_delay_millis": 30000, "state_sync_config.static_config.central_sync_client_config.sync_config.base_layer_propagation_sleep_duration": 10, + "state_sync_config.static_config.central_sync_client_config.sync_config.blocks_before_tip_to_disable_batching": 100, "state_sync_config.static_config.central_sync_client_config.sync_config.blocks_max_stream_size": 1000, "state_sync_config.static_config.central_sync_client_config.sync_config.collect_pending_data": false, "state_sync_config.static_config.central_sync_client_config.sync_config.latest_block_poll_interval_millis": 500, diff --git a/crates/apollo_deployments/resources/app_configs/state_sync_config.json b/crates/apollo_deployments/resources/app_configs/state_sync_config.json index 59b79ea9836..51a7924384e 100644 --- a/crates/apollo_deployments/resources/app_configs/state_sync_config.json +++ b/crates/apollo_deployments/resources/app_configs/state_sync_config.json @@ -9,6 +9,7 @@ "state_sync_config.static_config.central_sync_client_config.central_source_config.retry_config.retry_base_millis": 30, "state_sync_config.static_config.central_sync_client_config.central_source_config.retry_config.retry_max_delay_millis": 30000, "state_sync_config.static_config.central_sync_client_config.sync_config.base_layer_propagation_sleep_duration": 10, + "state_sync_config.static_config.central_sync_client_config.sync_config.blocks_before_tip_to_disable_batching": 100, "state_sync_config.static_config.central_sync_client_config.sync_config.latest_block_poll_interval_millis": 500, "state_sync_config.static_config.central_sync_client_config.sync_config.blocks_max_stream_size": 1000, "state_sync_config.static_config.central_sync_client_config.sync_config.collect_pending_data": false, diff --git a/crates/apollo_node/Cargo.toml b/crates/apollo_node/Cargo.toml index b2e4e59e9e9..dce5c09a22f 100644 --- a/crates/apollo_node/Cargo.toml +++ b/crates/apollo_node/Cargo.toml @@ -58,6 +58,7 @@ apollo_state_sync_types.workspace = true futures.workspace = true metrics-exporter-prometheus.workspace = true papyrus_base_layer.workspace = true +serde_json.workspace = true tokio.workspace = true tokio-util = { workspace = true, optional = true, features = ["rt"] } tracing.workspace = true diff --git a/crates/apollo_node/src/bin/config_parity.rs b/crates/apollo_node/src/bin/config_parity.rs new file mode 100644 index 00000000000..7e9fd22879d --- /dev/null +++ b/crates/apollo_node/src/bin/config_parity.rs @@ -0,0 +1,164 @@ +//! Local preset/native config-load parity check (uncommitted dev tool). +//! +//! Loads two config artifacts produced by the deployment pipeline for the same node: +//! - a `preset` artifact (flat dotted-key) via `--config_format preset` +//! - a `native` artifact (nested) via `--config_format native` +//! Both are loaded with the SAME secrets file as a second `--config_file` so secret +//! values match by construction, then the deserialized `SequencerNodeConfig`s are +//! compared via the derived `PartialEq`. +//! +//! NOTE: this uses `SequencerNodeConfig::load_and_process` (deserialize only), NOT +//! `load_and_validate_config`. `validate_node_config` additionally checks runtime +//! environment invariants (e.g. that `/data/*` paths exist and that component URLs / k8s +//! service DNS resolve) that only hold inside the pod and are irrelevant to deserialization +//! parity; running them locally fails on every config. We compare what the node deserializes. +//! +//! Exit codes: 0 = PARITY: PASS, 1 = configs differ (prints a JSON diff), 2 = a load/ +//! deserialize error occurred. + +use std::process::exit; + +use apollo_node_config::node_config::SequencerNodeConfig; +use serde_json::Value; + +struct Args { + preset_file: String, + native_file: String, + secrets_file: String, +} + +fn parse_args() -> Args { + let mut preset_file = None; + let mut native_file = None; + let mut secrets_file = None; + + let mut raw_args = std::env::args().skip(1); + while let Some(flag) = raw_args.next() { + let value = match raw_args.next() { + Some(value) => value, + None => { + eprintln!("Missing value for argument {flag}"); + exit(2); + } + }; + match flag.as_str() { + "--preset-file" => preset_file = Some(value), + "--native-file" => native_file = Some(value), + "--secrets-file" => secrets_file = Some(value), + other => { + eprintln!("Unknown argument: {other}"); + eprintln!( + "Usage: config_parity --preset-file P --native-file N --secrets-file S" + ); + exit(2); + } + } + } + + match (preset_file, native_file, secrets_file) { + (Some(preset_file), Some(native_file), Some(secrets_file)) => { + Args { preset_file, native_file, secrets_file } + } + _ => { + eprintln!( + "Usage: config_parity --preset-file P --native-file N --secrets-file S" + ); + exit(2); + } + } +} + +fn load_preset(preset_file: &str, secrets_file: &str) -> SequencerNodeConfig { + let args = vec![ + "config_parity", + "--config_format", + "preset", + "--config_file", + preset_file, + "--config_file", + secrets_file, + ] + .into_iter() + .map(String::from) + .collect(); + match SequencerNodeConfig::load_and_process(args) { + Ok(config) => config, + Err(error) => { + eprintln!("Failed to load PRESET config from {preset_file}: {error}"); + exit(2); + } + } +} + +fn load_native(native_file: &str, secrets_file: &str) -> SequencerNodeConfig { + // Native requires the first `--config_file` to be the nested base; later files are + // flat secret overrides. + let args = vec![ + "config_parity", + "--config_format", + "native", + "--config_file", + native_file, + "--config_file", + secrets_file, + ] + .into_iter() + .map(String::from) + .collect(); + match SequencerNodeConfig::load_and_process(args) { + Ok(config) => config, + Err(error) => { + eprintln!("Failed to load NATIVE config from {native_file}: {error}"); + exit(2); + } + } +} + +/// Prints the top-level keys whose JSON values differ between the two configs. +fn print_diff(preset_config: &SequencerNodeConfig, native_config: &SequencerNodeConfig) { + let preset_value = serde_json::to_value(preset_config) + .expect("SequencerNodeConfig should serialize to JSON"); + let native_value = serde_json::to_value(native_config) + .expect("SequencerNodeConfig should serialize to JSON"); + + let empty = serde_json::Map::new(); + let preset_object = preset_value.as_object().unwrap_or(&empty); + let native_object = native_value.as_object().unwrap_or(&empty); + + let mut all_keys: Vec<&String> = + preset_object.keys().chain(native_object.keys()).collect(); + all_keys.sort_unstable(); + all_keys.dedup(); + + let null = Value::Null; + for key in all_keys { + let preset_field = preset_object.get(key).unwrap_or(&null); + let native_field = native_object.get(key).unwrap_or(&null); + if preset_field != native_field { + eprintln!("--- DIFF in top-level field `{key}` ---"); + eprintln!( + " preset: {}", + serde_json::to_string(preset_field).unwrap_or_default() + ); + eprintln!( + " native: {}", + serde_json::to_string(native_field).unwrap_or_default() + ); + } + } +} + +fn main() { + let args = parse_args(); + let preset_config = load_preset(&args.preset_file, &args.secrets_file); + let native_config = load_native(&args.native_file, &args.secrets_file); + + if preset_config == native_config { + println!("PARITY: PASS"); + exit(0); + } + + eprintln!("PARITY: FAIL - loaded SequencerNodeConfig values differ"); + print_diff(&preset_config, &native_config); + exit(1); +} diff --git a/crates/apollo_reverts/src/lib.rs b/crates/apollo_reverts/src/lib.rs index c9935fc2474..87d7c10ea9f 100644 --- a/crates/apollo_reverts/src/lib.rs +++ b/crates/apollo_reverts/src/lib.rs @@ -19,17 +19,70 @@ use apollo_storage::state_commitment_infos::StateCommitmentInfosStorageWriter; use apollo_storage::StorageWriter; use futures::future::pending; use futures::never::Never; -use serde::{Deserialize, Serialize}; +use std::fmt; + +use serde::de::{self, Visitor}; +use serde::{Deserialize, Deserializer, Serialize}; use starknet_api::block::BlockNumber; use tracing::info; use validator::Validate; #[derive(Clone, Debug, Serialize, Deserialize, Validate, PartialEq)] pub struct RevertConfig { + #[serde(deserialize_with = "deserialize_revert_block_number")] pub revert_up_to_and_including: BlockNumber, pub should_revert: bool, } +/// Deserialize `revert_up_to_and_including`, tolerating a floating-point representation of the +/// value. +/// +/// The default is the `u64::MAX` "never revert" sentinel. Configs assembled via jsonnet (the +/// `native` config format) render every number as an IEEE-754 double, so `u64::MAX` +/// (18446744073709551615) is emitted as the nearest double, `2^64` (18446744073709551616), which +/// overflows `u64` and would otherwise fail to deserialize. We accept a float and saturating-cast +/// it back to `u64`, so a config produced by jsonnet deserializes to the same `BlockNumber` as the +/// legacy flat (`preset`) path, which carries the exact `u64::MAX` integer. Plain integer values +/// (the common case, including real revert heights) take the `u64` branch unchanged. +fn deserialize_revert_block_number<'de, D>(deserializer: D) -> Result +where + D: Deserializer<'de>, +{ + struct RevertBlockNumberVisitor; + + impl<'de> Visitor<'de> for RevertBlockNumberVisitor { + type Value = BlockNumber; + + fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str( + "a block number; jsonnet renders the u64::MAX sentinel as the 2^64 value, which \ + deserializers may present as u128 or f64", + ) + } + + fn visit_u64(self, value: u64) -> Result { + Ok(BlockNumber(value)) + } + + // jsonnet's f64 rounding turns u64::MAX into 2^64, which serde_json surfaces as a u128. + // Saturate anything past u64::MAX back to the sentinel. + fn visit_u128(self, value: u128) -> Result { + Ok(BlockNumber(u64::try_from(value).unwrap_or(u64::MAX))) + } + + fn visit_i64(self, value: i64) -> Result { + Ok(BlockNumber(u64::try_from(value).unwrap_or(0))) + } + + // Saturating float-to-int cast: 2^64 (and larger) maps to u64::MAX. + fn visit_f64(self, value: f64) -> Result { + Ok(BlockNumber(value as u64)) + } + } + + deserializer.deserialize_any(RevertBlockNumberVisitor) +} + impl Default for RevertConfig { fn default() -> Self { Self {