apollo_deployments,apollo_integration_tests: source harness component configs from jsonnet

crates/apollo_deployments/src/deployments/consolidated.rs

@@ -7,13 +7,15 @@ use apollo_node_config::component_execution_config::{
     ReactiveComponentExecutionConfig,
 };
 use serde::Serialize;
-use strum::{AsRefStr, Display, EnumIter, IntoEnumIterator};
+use strum::{AsRefStr, Display, EnumIter, EnumString, IntoEnumIterator};
 
 use crate::deployment_definitions::ComponentConfigInService;
 use crate::scale_policy::ScalePolicy;
 use crate::service::{GetComponentConfigs, NodeService, ServiceNameInner};
 
-#[derive(Clone, Copy, Debug, Display, PartialEq, Eq, Hash, Serialize, AsRefStr, EnumIter)]
+#[derive(
+    Clone, Copy, Debug, Display, EnumString, PartialEq, Eq, Hash, Serialize, AsRefStr, EnumIter,
+)]
 #[strum(serialize_all = "snake_case")]
 pub enum ConsolidatedNodeServiceName {
     Node,

crates/apollo_deployments/src/deployments/distributed.rs

@@ -7,7 +7,7 @@ use apollo_node_config::component_execution_config::{
     ReactiveComponentExecutionConfig,
 };
 use serde::Serialize;
-use strum::{AsRefStr, Display, EnumIter, IntoEnumIterator};
+use strum::{AsRefStr, Display, EnumIter, EnumString, IntoEnumIterator};
 
 use crate::deployment_definitions::{ComponentConfigInService, RETRIES_FOR_L1_SERVICES};
 use crate::scale_policy::ScalePolicy;
@@ -19,7 +19,9 @@ pub const DISTRIBUTED_NODE_REQUIRED_PORTS_NUM: usize = 11;
 
 // TODO(Tsabary): define consts and functions whenever relevant.
 
-#[derive(Clone, Copy, Debug, Display, PartialEq, Eq, Hash, Serialize, AsRefStr, EnumIter)]
+#[derive(
+    Clone, Copy, Debug, Display, EnumString, PartialEq, Eq, Hash, Serialize, AsRefStr, EnumIter,
+)]
 #[strum(serialize_all = "snake_case")]
 pub enum DistributedNodeServiceName {
     Batcher,

crates/apollo_deployments/src/deployments/hybrid.rs

@@ -7,7 +7,7 @@ use apollo_node_config::component_execution_config::{
     ReactiveComponentExecutionConfig,
 };
 use serde::Serialize;
-use strum::{AsRefStr, Display, EnumIter, IntoEnumIterator};
+use strum::{AsRefStr, Display, EnumIter, EnumString, IntoEnumIterator};
 
 use crate::deployment_definitions::{ComponentConfigInService, RETRIES_FOR_L1_SERVICES};
 use crate::scale_policy::ScalePolicy;
@@ -17,7 +17,9 @@ use crate::utils::InfraPortAllocator;
 // Number of infra-required ports for a hybrid node service distribution.
 pub const HYBRID_NODE_REQUIRED_PORTS_NUM: usize = 11;
 
-#[derive(Clone, Copy, Debug, Display, PartialEq, Eq, Hash, Serialize, AsRefStr, EnumIter)]
+#[derive(
+    Clone, Copy, Debug, Display, EnumString, PartialEq, Eq, Hash, Serialize, AsRefStr, EnumIter,
+)]
 #[strum(serialize_all = "snake_case")]
 pub enum HybridNodeServiceName {
     Committer,

crates/apollo_deployments/src/jsonnet_eval.rs

@@ -3,13 +3,23 @@
 //! default/production dependency graph. The test-only parity/applicative helpers live in the
 //! sibling `jsonnet_tests` module (compiled only under `test`).
 
+use std::collections::{BTreeSet, HashMap};
 use std::path::PathBuf;
+use std::str::FromStr;
 
+use apollo_node_config::component_config::ComponentConfig;
 use jrsonnet_evaluator::trace::PathResolver;
 use jrsonnet_evaluator::{FileImportResolver, State};
 use serde_json::Value;
 
+use crate::deployments::consolidated::ConsolidatedNodeServiceName;
+use crate::deployments::distributed::DistributedNodeServiceName;
+use crate::deployments::hybrid::HybridNodeServiceName;
+use crate::service::{NodeService, NodeType};
+
 const JSONNET_DIR: &str = "crates/apollo_deployments/jsonnet";
+const TOPOLOGY_PARAMS: &str = "{ chain_params: import 'testing/chain_params.libsonnet', \
+                               node_params: import 'testing/node_params.libsonnet' }";
 
 /// Evaluates `build(<params>)` with the `<layout>` layout folded into
 /// `chain_params.mandatory.topology`, and returns the per-service config map: service name → that
@@ -34,6 +44,66 @@ pub fn build_service_configs(layout: &str, params: &str) -> serde_json::Map<Stri
     }
 }
 
+/// Returns each service's topology as a typed `ComponentConfig` for `node_type`, keyed by
+/// `NodeService`.
+pub fn build_component_configs(
+    node_type: NodeType,
+    ports: Option<Vec<u16>>,
+) -> HashMap<NodeService, ComponentConfig> {
+    let mut services = build_service_configs(&node_type.to_string(), TOPOLOGY_PARAMS);
+    if let Some(ports) = ports {
+        remap_component_ports(&mut services, ports);
+    }
+    services
+        .into_iter()
+        .map(|(service_name, mut service_config)| {
+            let node_service = node_service_from_name(node_type, &service_name);
+            let components = service_config.get_mut("components").map(Value::take).unwrap();
+            let component_config = serde_json::from_value::<ComponentConfig>(components).unwrap();
+            (node_service, component_config)
+        })
+        .collect()
+}
+
+/// Replaces `build()`'s baked deploy-time component ports with the runtime-allocated `ports`.
+fn remap_component_ports(services: &mut serde_json::Map<String, Value>, ports: Vec<u16>) {
+    let component_port = |component: &Value| component.get("port").and_then(Value::as_u64);
+    let fixed_ports: BTreeSet<u64> = services
+        .values()
+        .filter_map(|config| config.get("components").and_then(Value::as_object))
+        .flat_map(|components| components.values())
+        .filter_map(component_port)
+        .filter(|&port| port != 0)
+        .collect();
+    assert_eq!(
+        ports.len(),
+        fixed_ports.len(),
+        "runtime port count ({}) must equal the distinct baked component ports ({})",
+        ports.len(),
+        fixed_ports.len()
+    );
+    let port_map: HashMap<u64, u64> =
+        fixed_ports.into_iter().zip(ports.into_iter().map(u64::from)).collect();
+    for config in services.values_mut() {
+        let Some(components) = config.get_mut("components").and_then(Value::as_object_mut) else {
+            continue;
+        };
+        for component in components.values_mut() {
+            if let Some(runtime_port) = component_port(component).and_then(|p| port_map.get(&p)) {
+                component["port"] = Value::from(*runtime_port);
+            }
+        }
+    }
+}
+
+fn node_service_from_name(node_type: NodeType, name: &str) -> NodeService {
+    match node_type {
+        NodeType::Consolidated => ConsolidatedNodeServiceName::from_str(name).unwrap().into(),
+        NodeType::Hybrid => HybridNodeServiceName::from_str(name).unwrap().into(),
+        NodeType::Distributed => DistributedNodeServiceName::from_str(name).unwrap().into(),
+    }
+}
+
 /// Evaluates a jsonnet `snippet` against a fresh evaluator (stdlib installed, imports resolved
 /// relative to the jsonnet dir) and converts the result to a serde `Value`. `context` labels the
 /// evaluation in panic messages.

crates/apollo_integration_tests/Cargo.toml

@@ -35,7 +35,7 @@ apollo_consensus_config.workspace = true
 apollo_consensus_manager_config.workspace = true
 apollo_consensus_orchestrator.workspace = true
 apollo_consensus_orchestrator_config.workspace = true
-apollo_deployments.workspace = true
+apollo_deployments = { workspace = true, features = ["testing"] }
 apollo_gateway_config = { workspace = true, features = ["testing"] }
 apollo_http_server = { workspace = true, features = ["testing"] }
 apollo_infra = { workspace = true, features = ["testing"] }

crates/apollo_integration_tests/src/node_component_configs.rs

@@ -2,14 +2,16 @@ use std::collections::HashMap;
 
 use apollo_deployments::deployments::distributed::DISTRIBUTED_NODE_REQUIRED_PORTS_NUM;
 use apollo_deployments::deployments::hybrid::HYBRID_NODE_REQUIRED_PORTS_NUM;
+use apollo_deployments::jsonnet_eval::build_component_configs;
 use apollo_deployments::service::{NodeService, NodeType};
 use apollo_infra_utils::test_utils::AvailablePortsGenerator;
 use apollo_node_config::component_config::{set_urls_to_localhost, ComponentConfig};
 
 pub type NodeComponentConfigs = HashMap<NodeService, ComponentConfig>;
 
 pub fn create_consolidated_component_configs() -> NodeComponentConfigs {
-    NodeType::Consolidated.get_component_configs(None)
+    // Consolidated runs every component locally, so there are no runtime ports to allocate.
+    build_component_configs(NodeType::Consolidated, None)
 }
 
 pub fn create_distributed_component_configs(
@@ -20,7 +22,7 @@ pub fn create_distributed_component_configs(
         .expect("Failed to get an AvailablePorts instance for distributed node configs");
 
     let ports = available_ports.get_next_ports(DISTRIBUTED_NODE_REQUIRED_PORTS_NUM);
-    let mut services_component_config = NodeType::Distributed.get_component_configs(Some(ports));
+    let mut services_component_config = build_component_configs(NodeType::Distributed, Some(ports));
 
     set_urls_to_localhost(services_component_config.values_mut());
 
@@ -32,10 +34,10 @@ pub fn create_hybrid_component_configs(
 ) -> NodeComponentConfigs {
     let mut available_ports = available_ports_generator
         .next()
-        .expect("Failed to get an AvailablePorts instance for distributed node configs");
+        .expect("Failed to get an AvailablePorts instance for hybrid node configs");
 
     let ports = available_ports.get_next_ports(HYBRID_NODE_REQUIRED_PORTS_NUM);
-    let mut services_component_config = NodeType::Hybrid.get_component_configs(Some(ports));
+    let mut services_component_config = build_component_configs(NodeType::Hybrid, Some(ports));
 
     set_urls_to_localhost(services_component_config.values_mut());