Add measurement logic to reconfigurator

Author: labbott

dev-tools/reconfigurator-cli/src/lib.rs

@@ -34,6 +34,7 @@ use nexus_reconfigurator_simulation::{
 };
 use nexus_reconfigurator_simulation::{SimStateBuilder, SimTufRepoSource};
 use nexus_reconfigurator_simulation::{SimTufRepoDescription, Simulator};
+use nexus_types::deployment::BlueprintMeasurements;
 use nexus_types::deployment::ExpectedVersion;
 use nexus_types::deployment::execution::blueprint_external_dns_config;
 use nexus_types::deployment::execution::blueprint_internal_dns_config;
@@ -720,9 +721,13 @@ struct SledMupdateSource {
     mupdate_id: Option<MupdateUuid>,
 
     /// simulate an error reading the zone manifest
-    #[clap(long, conflicts_with = "sled-mupdate-valid-source")]
+    #[clap(long, conflicts_with_all = ["sled-mupdate-valid-source", "with_measurement_manifest_error"])]
     with_manifest_error: bool,
 
+    /// simulate an error reading the measurement manifest
+    #[clap(long, conflicts_with = "with_manifest_error")]
+    with_measurement_manifest_error: bool,
+
     /// simulate an error validating zones by this artifact ID name
     ///
     /// This uses the `artifact_id_name` representation of a zone kind.
@@ -732,6 +737,13 @@ struct SledMupdateSource {
         requires = "sled-mupdate-valid-source"
     )]
     with_zone_error: Vec<String>,
+    /// simulator a measurement corpus error
+    #[clap(
+        long,
+        value_name = "ARTIFACT_ID_NAME",
+        requires = "sled-mupdate-valid-source"
+    )]
+    with_corpus_error: Vec<String>,
 }
 
 #[derive(Debug, Args)]
@@ -1014,6 +1026,12 @@ enum BlueprintEditCommands {
     /// This initiates a handoff from the current generation of Nexus zones to
     /// the next generation of Nexus zones.
     BumpNexusGeneration,
+    /// Set a sleds measurements to the `unknown` state indicating we can't
+    /// rely on the install dataset
+    SetSledMeasurementsUnknown {
+        /// sled to set the field on
+        sled_id: SledOpt,
+    },
 }
 
 #[derive(Debug, Subcommand)]
@@ -2726,6 +2744,15 @@ fn cmd_blueprint_edit(
             builder.pending_mgs_update_delete(baseboard_id);
             format!("deleted configured update for serial {serial}")
         }
+        BlueprintEditCommands::SetSledMeasurementsUnknown { sled_id } => {
+            let sled_id = sled_id.to_sled_id(system.description())?;
+            builder.sled_set_measurements(
+                sled_id,
+                BlueprintMeasurements::Unknown,
+            )?;
+            format!("set sled measurements to 'unknown' {}", sled_id)
+        }
+
         BlueprintEditCommands::Debug {
             command: BlueprintEditDebugCommands::RemoveSled { sled },
         } => {
@@ -3475,7 +3502,14 @@ fn mupdate_source_to_description(
             format!("from repo at {repo_path}"),
         )?;
         sim_source.simulate_zone_errors(&source.with_zone_error)?;
-        Ok(SimTufRepoDescription::new(sim_source))
+        if source.with_measurement_manifest_error {
+            Ok(SimTufRepoDescription::new_measurement_error(
+                sim_source,
+                "simulated error with measurement manifest".to_owned(),
+            ))
+        } else {
+            Ok(SimTufRepoDescription::new(sim_source))
+        }
     } else if source.valid.to_target_release {
         let description = sim
             .current_state()
@@ -3498,7 +3532,14 @@ fn mupdate_source_to_description(
                     "to target release".to_owned(),
                 )?;
                 sim_source.simulate_zone_errors(&source.with_zone_error)?;
-                Ok(SimTufRepoDescription::new(sim_source))
+                if source.with_measurement_manifest_error {
+                    Ok(SimTufRepoDescription::new_measurement_error(
+                        sim_source,
+                        "simulated error with measurement manifest".to_owned(),
+                    ))
+                } else {
+                    Ok(SimTufRepoDescription::new(sim_source))
+                }
             }
         }
     } else if source.with_manifest_error {

dev-tools/reconfigurator-cli/tests/input/cmds-missing-measurement-manifest.txt

@@ -0,0 +1,29 @@
+# Load an example system.
+load-example --nsleds 4 --ndisks-per-sled 1
+
+sled-list
+
+# Create a TUF repository from a fake manifest. (The output TUF repo is
+# written to a temporary directory that this invocation of `reconfigurator-cli`
+# is running out of as its working directory.)
+tuf-assemble ../../update-common/manifests/fake.toml
+# Create a second TUF repository from a different fake manifest.
+tuf-assemble ../../update-common/manifests/fake-non-semver.toml --allow-non-semver
+
+# Load the target release from the first TUF repository.
+set target-release repo-1.0.0.zip
+
+# On 3 sleds, update the install dataset.
+sled-update-install-dataset serial0 --to-target-release
+sled-update-install-dataset serial1 --to-target-release
+sled-update-install-dataset serial2 --to-target-release
+
+# On a fourth sled, simulate an error with the measurement manifest 
+sled-update-install-dataset serial3 --to-target-release --with-measurement-manifest-error
+
+# Generate an inventory and run a blueprint planning step.
+inventory-generate
+blueprint-plan latest latest
+
+# This diff should show expected changes to the blueprint.
+blueprint-diff latest

dev-tools/reconfigurator-cli/tests/input/cmds-mupdate-update-flow.txt

@@ -130,6 +130,11 @@ blueprint-diff latest
 sled-update-install-dataset serial1 --to-target-release
 inventory-generate
 
+# First get our new measurements
+blueprint-plan latest latest
+blueprint-show latest
+blueprint-diff latest
+
 # This will attempt to update the RoT bootloader on the first sled.
 blueprint-plan latest latest
 blueprint-show latest

dev-tools/reconfigurator-cli/tests/input/cmds-unknown-measurements.txt

@@ -0,0 +1,30 @@
+# Load an example system.
+load-example --nsleds 4 --ndisks-per-sled 1
+
+sled-list
+
+# Create a TUF repository from a fake manifest. (The output TUF repo is
+# written to a temporary directory that this invocation of `reconfigurator-cli`
+# is running out of as its working directory.)
+tuf-assemble ../../update-common/manifests/fake.toml
+# Create a second TUF repository from a different fake manifest.
+tuf-assemble ../../update-common/manifests/fake-non-semver.toml --allow-non-semver
+
+# Load the target release from the first TUF repository.
+set target-release repo-1.0.0.zip
+
+# On 4 sleds, update the install dataset.
+sled-update-install-dataset serial0 --to-target-release
+sled-update-install-dataset serial1 --to-target-release
+sled-update-install-dataset serial2 --to-target-release
+sled-update-install-dataset serial3 --to-target-release
+
+# mark sled 2 with being 'unknown'
+blueprint-edit latest set-sled-measurements-unknown serial2
+
+# Generate an inventory and run a blueprint planning step.
+inventory-generate
+blueprint-plan latest latest
+
+# This diff should show sled2 still picking up the measurements
+blueprint-diff latest

dev-tools/reconfigurator-cli/tests/output/cmds-add-sled-no-disks-stdout

@@ -50,6 +50,8 @@ planning report:
 * no zpools in service for NTP zones on sleds: 00320471-945d-413c-85e7-03e091a70b3c
 * discretionary zone placement waiting for NTP zones on sleds: 00320471-945d-413c-85e7-03e091a70b3c
 * zone updates waiting on zone add blockers
+Measurement updates:
+Blocked on add/updates
 
 
 
@@ -358,6 +360,8 @@ planning report:
 * no zpools in service for NTP zones on sleds: 00320471-945d-413c-85e7-03e091a70b3c
 * discretionary zone placement waiting for NTP zones on sleds: 00320471-945d-413c-85e7-03e091a70b3c
 * zone updates waiting on zone add blockers
+Measurement updates:
+Blocked on add/updates
 
 
 

dev-tools/reconfigurator-cli/tests/output/cmds-add-zones-with-mupdate-override-stdout

@@ -83,16 +83,18 @@ INFO blueprint mupdate override updated to match inventory, phase: do_plan_mupda
 , host_phase_2: 
   - host phase 2 slot A: current contents (unchanged)
   - host phase 2 slot B: current contents (unchanged)
-
+, measurements: (unknown)
 INFO no previous MGS update found as part of updating blueprint mupdate override to match inventory, phase: do_plan_mupdate_override, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6
 INFO updating target release minimum generation based on new set-override actions, phase: do_plan_mupdate_override, current_generation: 1, new_generation: 3
 INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 9, num_already_artifact: 0, num_eligible: 0, num_ineligible: 9
 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a
 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, slot: b, expected_hash: 0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b
+INFO current sled measurements are in an unknown state, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c
 INFO skipped noop image source check on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, reason: remove_mupdate_override is set in the blueprint (2d0f6cbc-addc-47a2-962a-6a01e13376bf)
 INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8
 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a
 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: b, expected_hash: 0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b
+INFO current sled measurements are in an unknown state, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763
 generated blueprint 8da82a8e-bf97-4fbd-8ddd-9f6462732cf1 based on parent blueprint dbcbd3d6-41ff-48ae-ac0b-1becc9b2fd21
 blueprint source: planner with report:
 planning report:
@@ -107,6 +109,8 @@ planning report:
 
 * zone updates waiting on zone add blockers
 * waiting to update top-level nexus_generation: some non-Nexus zone are not yet updated
+Measurement updates:
+Blocked on add/updates
 
 
 
@@ -123,10 +127,11 @@ to:   blueprint 8da82a8e-bf97-4fbd-8ddd-9f6462732cf1
 +   will remove mupdate override:   (none) -> 2d0f6cbc-addc-47a2-962a-6a01e13376bf
 
     measurements:
-    ---------------------------
-    hash      version          
-    ---------------------------
-    unknown   (unknown version)
+    -----------------------------------
+    hash              version          
+    -----------------------------------
+-   unknown           (unknown version)
++   install dataset   (no version)     
 
 
     host phase 2 contents:
@@ -227,10 +232,12 @@ target number of Nexus zones: None -> 4
 INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 9, num_already_artifact: 0, num_eligible: 0, num_ineligible: 9
 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a
 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, slot: b, expected_hash: 0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b
+INFO current sled measurements are in an unknown state, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c
 INFO skipped noop image source check on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, reason: remove_mupdate_override is set in the blueprint (2d0f6cbc-addc-47a2-962a-6a01e13376bf)
 INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8
 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a
 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: b, expected_hash: 0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b
+INFO current sled measurements are in an unknown state, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763
 generated blueprint 58d5e830-0884-47d8-a7cd-b2b3751adeb4 based on parent blueprint 8da82a8e-bf97-4fbd-8ddd-9f6462732cf1
 blueprint source: planner with report:
 planning report:
@@ -245,6 +252,8 @@ planning report:
 
 * zone updates waiting on zone add blockers
 * waiting to update top-level nexus_generation: some non-Nexus zone are not yet updated
+Measurement updates:
+Blocked on add/updates
 
 
 
@@ -289,10 +298,12 @@ planner config updated:
 INFO performed noop zone image source checks on sled, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, num_total: 9, num_already_artifact: 0, num_eligible: 0, num_ineligible: 9
 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a
 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, slot: b, expected_hash: 0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b
+INFO current sled measurements are in an unknown state, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c
 INFO skipped noop image source check on sled, sled_id: 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6, reason: remove_mupdate_override is set in the blueprint (2d0f6cbc-addc-47a2-962a-6a01e13376bf)
 INFO performed noop zone image source checks on sled, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, num_total: 8, num_already_artifact: 0, num_eligible: 0, num_ineligible: 8
 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: a, expected_hash: 0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a
 INFO BootPartitionDetails inventory hash not found in TUF repo, ignoring for noop checks, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763, slot: b, expected_hash: 0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b
+INFO current sled measurements are in an unknown state, sled_id: d81c6a84-79b8-4958-ae41-ea46c9b19763
 generated blueprint af934083-59b5-4bf6-8966-6fb5292c29e1 based on parent blueprint 58d5e830-0884-47d8-a7cd-b2b3751adeb4
 blueprint source: planner with report:
 planning report:
@@ -312,6 +323,8 @@ planner config:
   * nexus zone on sled d81c6a84-79b8-4958-ae41-ea46c9b19763 from source install dataset
 * zone updates waiting on discretionary zones
 * waiting to update top-level nexus_generation: some non-Nexus zone are not yet updated
+Measurement updates:
+Blocked on add/updates
 
 
 

dev-tools/reconfigurator-cli/tests/output/cmds-example-stdout

@@ -722,6 +722,8 @@ planning report:
 
 * adding zones despite being blocked, because: target release generation is 1
 * zone updates waiting on zone add blockers
+Measurement updates:
+Blocked on add/updates
 
 
 
@@ -2155,6 +2157,8 @@ planning report:
 
 * adding zones despite being blocked, because: target release generation is 1
 * zone updates waiting on zone add blockers
+Measurement updates:
+Blocked on add/updates
 
 
 

dev-tools/reconfigurator-cli/tests/output/cmds-expunge-newly-added-external-dns-stdout

@@ -1078,6 +1078,8 @@ planner config:
 * discretionary zones placed:
   * external_dns zone on sled 711ac7f8-d19e-4572-bdb9-e9b50f6e362a from source install dataset
 * zone updates waiting on discretionary zones
+Measurement updates:
+Blocked on add/updates
 
 
 
@@ -1707,6 +1709,8 @@ planner config:
 * discretionary zones placed:
   * external_dns zone on sled 711ac7f8-d19e-4572-bdb9-e9b50f6e362a from source install dataset
 * zone updates waiting on discretionary zones
+Measurement updates:
+Blocked on add/updates
 
 
 

dev-tools/reconfigurator-cli/tests/output/cmds-expunge-newly-added-internal-dns-stdout

@@ -808,6 +808,8 @@ planning report:
 * discretionary zones placed:
   * internal_dns zone on sled 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c from source install dataset
 * zone updates waiting on discretionary zones
+Measurement updates:
+Blocked on add/updates