diff --git a/Cargo.lock b/Cargo.lock index 6a44c7af3d..44ef4eccc5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6071,6 +6071,7 @@ dependencies = [ "indexmap 2.6.0", "internal-dns-resolver", "ipnet", + "itertools 0.13.0", "maplit", "nexus-config", "nexus-inventory", diff --git a/nexus/reconfigurator/planning/Cargo.toml b/nexus/reconfigurator/planning/Cargo.toml index 43a65ad085..efed173a71 100644 --- a/nexus/reconfigurator/planning/Cargo.toml +++ b/nexus/reconfigurator/planning/Cargo.toml @@ -16,6 +16,7 @@ illumos-utils.workspace = true indexmap.workspace = true internal-dns-resolver.workspace = true ipnet.workspace = true +itertools.workspace = true nexus-config.workspace = true nexus-inventory.workspace = true nexus-sled-agent-shared.workspace = true diff --git a/nexus/reconfigurator/planning/src/blueprint_builder/builder.rs b/nexus/reconfigurator/planning/src/blueprint_builder/builder.rs index 394133132b..74e91ff943 100644 --- a/nexus/reconfigurator/planning/src/blueprint_builder/builder.rs +++ b/nexus/reconfigurator/planning/src/blueprint_builder/builder.rs @@ -228,6 +228,14 @@ pub struct SledEditCounts { } impl SledEditCounts { + pub fn zeroes() -> Self { + Self { + disks: EditCounts::zeroes(), + datasets: EditCounts::zeroes(), + zones: EditCounts::zeroes(), + } + } + fn has_nonzero_counts(&self) -> bool { let Self { disks, datasets, zones } = self; disks.has_nonzero_counts() @@ -548,7 +556,7 @@ impl<'a> BlueprintBuilder<'a> { generation: Generation::new(), datasets: BTreeMap::new(), }); - let editor = SledEditor::new( + let editor = SledEditor::for_existing( state, zones.clone(), disks, @@ -565,8 +573,7 @@ impl<'a> BlueprintBuilder<'a> { // that weren't in the parent blueprint. (These are newly-added sleds.) for sled_id in input.all_sled_ids(SledFilter::Commissioned) { if let Entry::Vacant(slot) = sled_editors.entry(sled_id) { - slot.insert(SledEditor::new_empty( - SledState::Active, + slot.insert(SledEditor::for_new_active( build_preexisting_dataset_ids(sled_id)?, )); } @@ -735,8 +742,8 @@ impl<'a> BlueprintBuilder<'a> { .retain(|sled_id, _| in_service_sled_ids.contains(sled_id)); // If we have the clickhouse cluster setup enabled via policy and we - // don't yet have a `ClickhouseClusterConfiguration`, then we must create - // one and feed it to our `ClickhouseAllocator`. + // don't yet have a `ClickhouseClusterConfiguration`, then we must + // create one and feed it to our `ClickhouseAllocator`. let clickhouse_allocator = if self.input.clickhouse_cluster_enabled() { let parent_config = self .parent_blueprint @@ -815,18 +822,18 @@ impl<'a> BlueprintBuilder<'a> { } /// Set the desired state of the given sled. - pub fn set_sled_state( + pub fn set_sled_decommissioned( &mut self, sled_id: SledUuid, - desired_state: SledState, ) -> Result<(), Error> { let editor = self.sled_editors.get_mut(&sled_id).ok_or_else(|| { Error::Planner(anyhow!( "tried to set sled state for unknown sled {sled_id}" )) })?; - editor.set_state(desired_state); - Ok(()) + editor + .decommission() + .map_err(|err| Error::SledEditError { sled_id, err }) } /// Within tests, set an RNG for deterministic results. @@ -1046,15 +1053,18 @@ impl<'a> BlueprintBuilder<'a> { // blueprint for (disk_id, (zpool, disk)) in database_disks { database_disk_ids.insert(disk_id); - editor.ensure_disk( - BlueprintPhysicalDiskConfig { - disposition: BlueprintPhysicalDiskDisposition::InService, - identity: disk.disk_identity.clone(), - id: disk_id, - pool_id: *zpool, - }, - &mut self.rng, - ); + editor + .ensure_disk( + BlueprintPhysicalDiskConfig { + disposition: + BlueprintPhysicalDiskDisposition::InService, + identity: disk.disk_identity.clone(), + id: disk_id, + pool_id: *zpool, + }, + &mut self.rng, + ) + .map_err(|err| Error::SledEditError { sled_id, err })?; } // Remove any disks that appear in the blueprint, but not the database diff --git a/nexus/reconfigurator/planning/src/blueprint_editor/sled_editor.rs b/nexus/reconfigurator/planning/src/blueprint_editor/sled_editor.rs index 13094b97a4..995cc306a5 100644 --- a/nexus/reconfigurator/planning/src/blueprint_editor/sled_editor.rs +++ b/nexus/reconfigurator/planning/src/blueprint_editor/sled_editor.rs @@ -7,18 +7,25 @@ use crate::blueprint_builder::SledEditCounts; use crate::planner::PlannerRng; use illumos_utils::zpool::ZpoolName; +use itertools::Either; +use nexus_sled_agent_shared::inventory::ZoneKind; use nexus_types::deployment::blueprint_zone_type; use nexus_types::deployment::BlueprintDatasetsConfig; use nexus_types::deployment::BlueprintPhysicalDiskConfig; use nexus_types::deployment::BlueprintPhysicalDisksConfig; use nexus_types::deployment::BlueprintZoneConfig; +use nexus_types::deployment::BlueprintZoneDisposition; use nexus_types::deployment::BlueprintZoneFilter; use nexus_types::deployment::BlueprintZoneType; use nexus_types::deployment::BlueprintZonesConfig; use nexus_types::deployment::DiskFilter; use nexus_types::external_api::views::SledState; +use omicron_common::disk::DatasetKind; +use omicron_uuid_kinds::DatasetUuid; use omicron_uuid_kinds::OmicronZoneUuid; use omicron_uuid_kinds::PhysicalDiskUuid; +use omicron_uuid_kinds::ZpoolUuid; +use std::mem; mod datasets; mod disks; @@ -50,6 +57,32 @@ pub enum SledInputError { #[derive(Debug, thiserror::Error)] pub enum SledEditError { + #[error("editing a decommissioned sled is not allowed")] + EditDecommissioned, + #[error( + "sled is not decommissionable: \ + disk {disk_id} (zpool {zpool_id}) is in service" + )] + NonDecommissionableDiskInService { + disk_id: PhysicalDiskUuid, + zpool_id: ZpoolUuid, + }, + #[error( + "sled is not decommissionable: \ + dataset {dataset_id} (kind {kind:?}) is in service" + )] + NonDecommissionableDatasetInService { + dataset_id: DatasetUuid, + kind: DatasetKind, + }, + #[error( + "sled is not decommissionable: \ + zone {zone_id} (kind {kind:?}) is not expunged" + )] + NonDecommissionableZoneNotExpunged { + zone_id: OmicronZoneUuid, + kind: ZoneKind, + }, #[error("failed to edit disks")] EditDisks(#[from] DisksEditError), #[error("failed to edit datasets")] @@ -74,8 +107,196 @@ pub enum SledEditError { } #[derive(Debug)] -pub(crate) struct SledEditor { - state: SledState, +pub(crate) struct SledEditor(InnerSledEditor); + +#[derive(Debug)] +enum InnerSledEditor { + // Internally, `SledEditor` has a variant for each variant of `SledState`, + // as the operations allowed in different states are substantially different + // (i.e., an active sled allows any edit; a decommissioned sled allows + // none). + Active(ActiveSledEditor), + Decommissioned(EditedSled), +} + +impl SledEditor { + pub fn for_existing( + state: SledState, + zones: BlueprintZonesConfig, + disks: BlueprintPhysicalDisksConfig, + datasets: BlueprintDatasetsConfig, + preexisting_dataset_ids: DatasetIdsBackfillFromDb, + ) -> Result { + match state { + SledState::Active => { + let inner = ActiveSledEditor::new( + zones, + disks, + datasets, + preexisting_dataset_ids, + )?; + Ok(Self(InnerSledEditor::Active(inner))) + } + SledState::Decommissioned => { + let inner = EditedSled { + state, + zones, + disks, + datasets, + edit_counts: SledEditCounts::zeroes(), + }; + Ok(Self(InnerSledEditor::Decommissioned(inner))) + } + } + } + + pub fn for_new_active( + preexisting_dataset_ids: DatasetIdsBackfillFromDb, + ) -> Self { + Self(InnerSledEditor::Active(ActiveSledEditor::new_empty( + preexisting_dataset_ids, + ))) + } + + pub fn finalize(self) -> EditedSled { + match self.0 { + InnerSledEditor::Active(editor) => editor.finalize(), + InnerSledEditor::Decommissioned(edited) => edited, + } + } + + pub fn edit_counts(&self) -> SledEditCounts { + match &self.0 { + InnerSledEditor::Active(editor) => editor.edit_counts(), + InnerSledEditor::Decommissioned(edited) => edited.edit_counts, + } + } + + pub fn decommission(&mut self) -> Result<(), SledEditError> { + match &mut self.0 { + InnerSledEditor::Active(editor) => { + // Decommissioning a sled is a one-way trip that has many + // preconditions. We can't check all of them here (e.g., we + // should kick the sled out of trust quorum before + // decommissioning, which is entirely outside the realm of + // `SledEditor`. But we can do some basic checks: all of the + // disks, datasets, and zones for this sled should be expunged. + editor.validate_decommisionable()?; + + // We can't take ownership of `editor` from the `&mut self` + // reference we have, and we need ownership to call + // `finalize()`. Steal the contents via `mem::swap()` with an + // empty editor. This isn't panic safe (i.e., if we panic + // between the `mem::swap()` and the reassignment to `self.0` + // below, we'll be left in the active state with an empty sled + // editor), but omicron in general is not panic safe and aborts + // on panic. Plus `finalize()` should never panic. + let mut stolen = ActiveSledEditor::new_empty( + DatasetIdsBackfillFromDb::empty(), + ); + mem::swap(editor, &mut stolen); + + let mut finalized = stolen.finalize(); + finalized.state = SledState::Decommissioned; + self.0 = InnerSledEditor::Decommissioned(finalized); + } + // If we're already decommissioned, there's nothing to do. + InnerSledEditor::Decommissioned(_) => (), + } + Ok(()) + } + + pub fn disks( + &self, + filter: DiskFilter, + ) -> impl Iterator { + match &self.0 { + InnerSledEditor::Active(editor) => { + Either::Left(editor.disks(filter)) + } + InnerSledEditor::Decommissioned(edited) => Either::Right( + edited + .disks + .disks + .iter() + .filter(move |disk| disk.disposition.matches(filter)), + ), + } + } + + pub fn zones( + &self, + filter: BlueprintZoneFilter, + ) -> impl Iterator { + match &self.0 { + InnerSledEditor::Active(editor) => { + Either::Left(editor.zones(filter)) + } + InnerSledEditor::Decommissioned(edited) => Either::Right( + edited + .zones + .zones + .iter() + .filter(move |zone| zone.disposition.matches(filter)), + ), + } + } + + fn as_active_mut( + &mut self, + ) -> Result<&mut ActiveSledEditor, SledEditError> { + match &mut self.0 { + InnerSledEditor::Active(editor) => Ok(editor), + InnerSledEditor::Decommissioned(_) => { + Err(SledEditError::EditDecommissioned) + } + } + } + + pub fn ensure_disk( + &mut self, + disk: BlueprintPhysicalDiskConfig, + rng: &mut PlannerRng, + ) -> Result<(), SledEditError> { + self.as_active_mut()?.ensure_disk(disk, rng); + Ok(()) + } + + pub fn expunge_disk( + &mut self, + disk_id: &PhysicalDiskUuid, + ) -> Result<(), SledEditError> { + self.as_active_mut()?.expunge_disk(disk_id) + } + + pub fn add_zone( + &mut self, + zone: BlueprintZoneConfig, + rng: &mut PlannerRng, + ) -> Result<(), SledEditError> { + self.as_active_mut()?.add_zone(zone, rng) + } + + pub fn expunge_zone( + &mut self, + zone_id: &OmicronZoneUuid, + ) -> Result<(), SledEditError> { + self.as_active_mut()?.expunge_zone(zone_id) + } + + /// Backwards compatibility / test helper: If we're given a blueprint that + /// has zones but wasn't created via `SledEditor`, it might not have + /// datasets for all its zones. This method backfills them. + pub fn ensure_datasets_for_running_zones( + &mut self, + rng: &mut PlannerRng, + ) -> Result<(), SledEditError> { + self.as_active_mut()?.ensure_datasets_for_running_zones(rng) + } +} + +#[derive(Debug)] +struct ActiveSledEditor { zones: ZonesEditor, disks: DisksEditor, datasets: DatasetsEditor, @@ -90,16 +311,14 @@ pub(crate) struct EditedSled { pub edit_counts: SledEditCounts, } -impl SledEditor { +impl ActiveSledEditor { pub fn new( - state: SledState, zones: BlueprintZonesConfig, disks: BlueprintPhysicalDisksConfig, datasets: BlueprintDatasetsConfig, preexisting_dataset_ids: DatasetIdsBackfillFromDb, ) -> Result { Ok(Self { - state, zones: zones.try_into()?, disks: disks.try_into()?, datasets: DatasetsEditor::new(datasets, preexisting_dataset_ids)?, @@ -107,11 +326,9 @@ impl SledEditor { } pub fn new_empty( - state: SledState, preexisting_dataset_ids: DatasetIdsBackfillFromDb, ) -> Self { Self { - state, zones: ZonesEditor::empty(), disks: DisksEditor::empty(), datasets: DatasetsEditor::empty(preexisting_dataset_ids), @@ -123,7 +340,7 @@ impl SledEditor { let (datasets, datasets_counts) = self.datasets.finalize(); let (zones, zones_counts) = self.zones.finalize(); EditedSled { - state: self.state, + state: SledState::Active, zones, disks, datasets, @@ -135,6 +352,24 @@ impl SledEditor { } } + fn validate_decommisionable(&self) -> Result<(), SledEditError> { + // ... and all zones are expunged. + if let Some(zone) = self.zones(BlueprintZoneFilter::All).find(|zone| { + match zone.disposition { + BlueprintZoneDisposition::InService + | BlueprintZoneDisposition::Quiesced => true, + BlueprintZoneDisposition::Expunged => false, + } + }) { + return Err(SledEditError::NonDecommissionableZoneNotExpunged { + zone_id: zone.id, + kind: zone.zone_type.kind(), + }); + } + + Ok(()) + } + pub fn edit_counts(&self) -> SledEditCounts { SledEditCounts { disks: self.disks.edit_counts(), @@ -143,10 +378,6 @@ impl SledEditor { } } - pub fn set_state(&mut self, new_state: SledState) { - self.state = new_state; - } - pub fn disks( &self, filter: DiskFilter, diff --git a/nexus/reconfigurator/planning/src/blueprint_editor/sled_editor/datasets.rs b/nexus/reconfigurator/planning/src/blueprint_editor/sled_editor/datasets.rs index de397b9caa..211af59aab 100644 --- a/nexus/reconfigurator/planning/src/blueprint_editor/sled_editor/datasets.rs +++ b/nexus/reconfigurator/planning/src/blueprint_editor/sled_editor/datasets.rs @@ -7,7 +7,6 @@ use crate::planner::PlannerRng; use illumos_utils::zpool::ZpoolName; use nexus_types::deployment::BlueprintDatasetConfig; use nexus_types::deployment::BlueprintDatasetDisposition; -use nexus_types::deployment::BlueprintDatasetFilter; use nexus_types::deployment::BlueprintDatasetsConfig; use nexus_types::deployment::SledResources; use nexus_types::deployment::ZpoolFilter; @@ -24,6 +23,9 @@ use std::collections::BTreeMap; use std::collections::BTreeSet; use std::net::SocketAddrV6; +#[cfg(test)] +use nexus_types::deployment::BlueprintDatasetFilter; + #[derive(Debug, thiserror::Error)] #[error( "invalid blueprint input: multiple datasets with kind {kind:?} \ @@ -269,7 +271,7 @@ impl DatasetsEditor { self.counts } - #[allow(dead_code)] // currently only used by tests; this will change soon + #[cfg(test)] pub fn datasets( &self, filter: BlueprintDatasetFilter, diff --git a/nexus/reconfigurator/planning/src/planner.rs b/nexus/reconfigurator/planning/src/planner.rs index 56fc671667..24ef80b253 100644 --- a/nexus/reconfigurator/planning/src/planner.rs +++ b/nexus/reconfigurator/planning/src/planner.rs @@ -159,8 +159,7 @@ impl<'a> Planner<'a> { let num_instances_assigned = 0; if all_zones_expunged && num_instances_assigned == 0 { - self.blueprint - .set_sled_state(sled_id, SledState::Decommissioned)?; + self.blueprint.set_sled_decommissioned(sled_id)?; } }