diff --git a/cmd/machine-api-migration/main.go b/cmd/machine-api-migration/main.go index c550ad43a..1f9369996 100644 --- a/cmd/machine-api-migration/main.go +++ b/cmd/machine-api-migration/main.go @@ -191,10 +191,6 @@ func getControllers(opts commoncmdoptions.OperatorConfig, platform configv1.Plat CAPINamespace: *opts.CAPINamespace, }, "machineset migration": &machinesetmigration.MachineSetMigrationReconciler{ - Platform: platform, - Infra: infra, - InfraTypes: infraTypes, - MAPINamespace: *opts.MAPINamespace, CAPINamespace: *opts.CAPINamespace, }, diff --git a/e2e/go.mod b/e2e/go.mod index 817db5658..a63877d9b 100644 --- a/e2e/go.mod +++ b/e2e/go.mod @@ -24,6 +24,7 @@ require ( github.com/openshift/cluster-api-actuator-pkg v0.0.0-20260310144400-bec013a007a8 github.com/openshift/cluster-api-provider-baremetal v0.0.0-20250619124612-fb678fec5f7e github.com/openshift/cluster-capi-operator v0.0.0-20260425200736-89a8af46df2a + github.com/openshift/library-go v0.0.0-20260413093329-d2db42c961e1 k8s.io/api v0.35.3 k8s.io/apimachinery v0.35.3 k8s.io/client-go v0.35.3 @@ -103,7 +104,6 @@ require ( github.com/openshift/client-go v0.0.0-20260416131737-a19e91702ab5 // indirect github.com/openshift/cluster-api-actuator-pkg/testutils v0.0.0 // indirect github.com/openshift/cluster-autoscaler-operator v0.0.1-0.20250702183526-4eb64d553940 // indirect - github.com/openshift/library-go v0.0.0-20260413093329-d2db42c961e1 // indirect github.com/openshift/machine-api-operator v0.2.1-0.20260226113419-88465550a74b // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect diff --git a/e2e/machine_migration_capi_authoritative_test.go b/e2e/machine_migration_capi_authoritative_test.go index 86c881b89..e21fc6cab 100644 --- a/e2e/machine_migration_capi_authoritative_test.go +++ b/e2e/machine_migration_capi_authoritative_test.go @@ -226,6 +226,7 @@ var _ = Describe("[sig-cluster-lifecycle][OCPFeatureGate:MachineAPIMigration] Ma verifyMachineAuthoritative(newMapiMachine, mapiv1beta1.MachineAuthorityClusterAPI) verifyMAPIMachineSynchronizedCondition(newMapiMachine, mapiv1beta1.MachineAuthorityClusterAPI) verifyMachineSynchronizedGeneration(newMapiMachine, mapiv1beta1.MachineAuthorityClusterAPI) + verifyMachineSynchronizedAPI(newMapiMachine, mapiv1beta1.ClusterAPISynchronized) verifyMachinePausedCondition(newMapiMachine, mapiv1beta1.MachineAuthorityClusterAPI) verifyMachinePausedCondition(newCapiMachine, mapiv1beta1.MachineAuthorityClusterAPI) @@ -235,6 +236,7 @@ var _ = Describe("[sig-cluster-lifecycle][OCPFeatureGate:MachineAPIMigration] Ma verifyMachineAuthoritative(newMapiMachine, mapiv1beta1.MachineAuthorityMachineAPI) verifyMAPIMachineSynchronizedCondition(newMapiMachine, mapiv1beta1.MachineAuthorityMachineAPI) verifyMachineSynchronizedGeneration(newMapiMachine, mapiv1beta1.MachineAuthorityMachineAPI) + verifyMachineSynchronizedAPI(newMapiMachine, mapiv1beta1.MachineAPISynchronized) verifyMachinePausedCondition(newMapiMachine, mapiv1beta1.MachineAuthorityMachineAPI) verifyMachinePausedCondition(newCapiMachine, mapiv1beta1.MachineAuthorityMachineAPI) @@ -244,6 +246,7 @@ var _ = Describe("[sig-cluster-lifecycle][OCPFeatureGate:MachineAPIMigration] Ma verifyMachineAuthoritative(newMapiMachine, mapiv1beta1.MachineAuthorityClusterAPI) verifyMAPIMachineSynchronizedCondition(newMapiMachine, mapiv1beta1.MachineAuthorityClusterAPI) verifyMachineSynchronizedGeneration(newMapiMachine, mapiv1beta1.MachineAuthorityClusterAPI) + verifyMachineSynchronizedAPI(newMapiMachine, mapiv1beta1.ClusterAPISynchronized) verifyMachinePausedCondition(newMapiMachine, mapiv1beta1.MachineAuthorityClusterAPI) verifyMachinePausedCondition(newCapiMachine, mapiv1beta1.MachineAuthorityClusterAPI) diff --git a/e2e/machine_migration_helpers.go b/e2e/machine_migration_helpers.go index 24e898e77..6db95d556 100644 --- a/e2e/machine_migration_helpers.go +++ b/e2e/machine_migration_helpers.go @@ -478,3 +478,14 @@ func verifyMachineSynchronizedGeneration(mapiMachine *mapiv1beta1.Machine, autho Fail(fmt.Sprintf("unknown authoritativeAPI type: %v", authority)) } } + +// verifyMachineSynchronizedAPI verifies that the MAPI Machine's status.synchronizedAPI matches the expected value. +func verifyMachineSynchronizedAPI(mapiMachine *mapiv1beta1.Machine, expectedSynchronizedAPI mapiv1beta1.SynchronizedAPI) { + GinkgoHelper() + + By(fmt.Sprintf("Verifying MAPI Machine SynchronizedAPI is %s", expectedSynchronizedAPI)) + Eventually(komega.Object(mapiMachine), capiframework.WaitMedium, capiframework.RetryMedium).Should( + HaveField("Status.SynchronizedAPI", Equal(expectedSynchronizedAPI)), + fmt.Sprintf("MAPI Machine SynchronizedAPI should be %s", expectedSynchronizedAPI), + ) +} diff --git a/e2e/machine_migration_mapi_authoritative_test.go b/e2e/machine_migration_mapi_authoritative_test.go index 8380bbe3e..0fc5f6a8f 100644 --- a/e2e/machine_migration_mapi_authoritative_test.go +++ b/e2e/machine_migration_mapi_authoritative_test.go @@ -186,6 +186,7 @@ var _ = Describe("[sig-cluster-lifecycle][OCPFeatureGate:MachineAPIMigration] Ma verifyMachineAuthoritative(newMapiMachine, mapiv1beta1.MachineAuthorityMachineAPI) verifyMAPIMachineSynchronizedCondition(newMapiMachine, mapiv1beta1.MachineAuthorityMachineAPI) verifyMachineSynchronizedGeneration(newMapiMachine, mapiv1beta1.MachineAuthorityMachineAPI) + verifyMachineSynchronizedAPI(newMapiMachine, mapiv1beta1.MachineAPISynchronized) verifyMachinePausedCondition(newMapiMachine, mapiv1beta1.MachineAuthorityMachineAPI) verifyMachinePausedCondition(newCapiMachine, mapiv1beta1.MachineAuthorityMachineAPI) @@ -195,6 +196,7 @@ var _ = Describe("[sig-cluster-lifecycle][OCPFeatureGate:MachineAPIMigration] Ma verifyMachineAuthoritative(newMapiMachine, mapiv1beta1.MachineAuthorityClusterAPI) verifyMAPIMachineSynchronizedCondition(newMapiMachine, mapiv1beta1.MachineAuthorityClusterAPI) verifyMachineSynchronizedGeneration(newMapiMachine, mapiv1beta1.MachineAuthorityClusterAPI) + verifyMachineSynchronizedAPI(newMapiMachine, mapiv1beta1.ClusterAPISynchronized) verifyMachinePausedCondition(newMapiMachine, mapiv1beta1.MachineAuthorityClusterAPI) verifyMachinePausedCondition(newCapiMachine, mapiv1beta1.MachineAuthorityClusterAPI) @@ -204,6 +206,7 @@ var _ = Describe("[sig-cluster-lifecycle][OCPFeatureGate:MachineAPIMigration] Ma verifyMachineAuthoritative(newMapiMachine, mapiv1beta1.MachineAuthorityMachineAPI) verifyMAPIMachineSynchronizedCondition(newMapiMachine, mapiv1beta1.MachineAuthorityMachineAPI) verifyMachineSynchronizedGeneration(newMapiMachine, mapiv1beta1.MachineAuthorityMachineAPI) + verifyMachineSynchronizedAPI(newMapiMachine, mapiv1beta1.MachineAPISynchronized) verifyMachinePausedCondition(newMapiMachine, mapiv1beta1.MachineAuthorityMachineAPI) verifyMachinePausedCondition(newCapiMachine, mapiv1beta1.MachineAuthorityMachineAPI) diff --git a/e2e/machineset_migration_capi_authoritative_test.go b/e2e/machineset_migration_capi_authoritative_test.go index 2ab34cc48..18801ed30 100644 --- a/e2e/machineset_migration_capi_authoritative_test.go +++ b/e2e/machineset_migration_capi_authoritative_test.go @@ -182,6 +182,7 @@ var _ = Describe("[sig-cluster-lifecycle][OCPFeatureGate:MachineAPIMigration] Ma verifyMachineSetPausedCondition(mapiMachineSet, mapiv1beta1.MachineAuthorityMachineAPI) verifyMachineSetPausedCondition(capiMachineSet, mapiv1beta1.MachineAuthorityMachineAPI) verifyMAPIMachineSetSynchronizedCondition(mapiMachineSet, mapiv1beta1.MachineAuthorityMachineAPI) + verifyMachineSetSynchronizedAPI(mapiMachineSet, mapiv1beta1.MachineAPISynchronized) By("Scaling up MAPI MachineSet to 3 replicas") Expect(mapiframework.ScaleMachineSet(mapiMSAuthCAPIName, 3)).To(Succeed(), "should be able to scale up MAPI MachineSet") @@ -216,6 +217,7 @@ var _ = Describe("[sig-cluster-lifecycle][OCPFeatureGate:MachineAPIMigration] Ma verifyMachineSetPausedCondition(mapiMachineSet, mapiv1beta1.MachineAuthorityClusterAPI) verifyMachineSetPausedCondition(capiMachineSet, mapiv1beta1.MachineAuthorityClusterAPI) verifyMAPIMachineSetSynchronizedCondition(mapiMachineSet, mapiv1beta1.MachineAuthorityClusterAPI) + verifyMachineSetSynchronizedAPI(mapiMachineSet, mapiv1beta1.ClusterAPISynchronized) By("Deleting CAPI MachineSet and verifying mirrors are removed") capiframework.DeleteMachineSets(ctx, cl, capiMachineSet) diff --git a/e2e/machineset_migration_disruptive_test.go b/e2e/machineset_migration_disruptive_test.go new file mode 100644 index 000000000..086dc3887 --- /dev/null +++ b/e2e/machineset_migration_disruptive_test.go @@ -0,0 +1,245 @@ +// Copyright 2026 Red Hat, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "fmt" + "time" + + . "github.com/onsi/ginkgo/v2" + + configv1 "github.com/openshift/api/config/v1" + "github.com/openshift/api/features" + mapiv1beta1 "github.com/openshift/api/machine/v1beta1" + capiframework "github.com/openshift/cluster-capi-operator/e2e/framework" + awsv1 "sigs.k8s.io/cluster-api-provider-aws/v2/api/v1beta2" + clusterv1 "sigs.k8s.io/cluster-api/api/core/v1beta2" +) + +type machineSetMigrationDisruptiveFixture struct { + awsMachineTemplate *awsv1.AWSMachineTemplate + capiMachineSet *clusterv1.MachineSet + mapiMachineSet *mapiv1beta1.MachineSet +} + +func createZeroReplicaMachineSetMigrationDisruptiveFixture(machineSetNamePrefix string) machineSetMigrationDisruptiveFixture { + GinkgoHelper() + + machineSetName := generateName(machineSetNamePrefix) + mapiMachineSet := createMAPIMachineSetWithAuthoritativeAPI( + ctx, + cl, + 0, + machineSetName, + mapiv1beta1.MachineAuthorityMachineAPI, + mapiv1beta1.MachineAuthorityMachineAPI, + ) + capiMachineSet, awsMachineTemplate := waitForMAPIMachineSetMirrors(machineSetName) + trackResource(awsMachineTemplate) + + return machineSetMigrationDisruptiveFixture{ + awsMachineTemplate: awsMachineTemplate, + capiMachineSet: capiMachineSet, + mapiMachineSet: mapiMachineSet, + } +} + +var _ = Describe("[sig-cluster-lifecycle][OCPFeatureGate:MachineAPIMigration][Disruptive] MachineSet Migration Outage Tests", Ordered, Serial, func() { + var ( + disruptionState *machineSetMigrationDisruptionState + disruptionStateRestored bool + fixtureA machineSetMigrationDisruptiveFixture + fixtureB machineSetMigrationDisruptiveFixture + ) + + BeforeAll(func() { + if platform != configv1.AWSPlatformType { + Skip(fmt.Sprintf("Skipping tests on %s, this is only supported on AWS", platform)) + } + + if !capiframework.IsFeatureGateEnabled(ctx, cl, features.FeatureGateMachineAPIMigration) { + Skip("Skipping, this feature is only supported on MachineAPIMigration enabled clusters") + } + + DeferCleanup(func() { + By("Cleaning up MachineSet outage test resources") + cleanupMachineSetTestResources( + ctx, + cl, + []*clusterv1.MachineSet{fixtureA.capiMachineSet, fixtureB.capiMachineSet}, + []*awsv1.AWSMachineTemplate{fixtureA.awsMachineTemplate, fixtureB.awsMachineTemplate}, + []*mapiv1beta1.MachineSet{fixtureA.mapiMachineSet, fixtureB.mapiMachineSet}, + ) + }) + + DeferCleanup(func() { + if disruptionState == nil || disruptionStateRestored { + return + } + + By("Restoring Deployment/openshift-cluster-api/capi-controller-manager to its original replica count") + scaleDeploymentAndWaitForAvailableReplicas( + ctx, + cl, + capiframework.CAPINamespace, + capiControllerManagerDeploymentName, + disruptionState.capiControllerManagerReplicas, + ) + + By("Restoring Deployment/openshift-cluster-api-operator/capi-operator to its original replica count") + scaleDeploymentAndWaitForAvailableReplicas( + ctx, + cl, + capiframework.CAPIOperatorNamespace, + capiOperatorDeploymentName, + disruptionState.capiOperatorReplicas, + ) + + if disruptionState.capiOperatorOverrideExpectedAbsent { + By("Removing the targeted ClusterVersion unmanaged override for Deployment/openshift-cluster-api-operator/capi-operator") + setMachineSetMigrationCAPIOperatorOverride(ctx, cl, false) + } + + waitForClusterAPIOperatorHealthy(ctx, cl) + }) + }) + + It("should reuse one outage to verify paused-target and unpaused-target rollback behavior", func() { + By("Creating the paused-target rollback fixture before the outage") + fixtureA = createZeroReplicaMachineSetMigrationDisruptiveFixture("ms-disruptive-paused-target-") + verifyMAPIMachineSetSynchronizedState( + fixtureA.mapiMachineSet, + mapiv1beta1.MachineAuthorityMachineAPI, + mapiv1beta1.MachineAPISynchronized, + ) + verifyCAPIMachineSetPausedState(fixtureA.capiMachineSet, mapiv1beta1.MachineAuthorityMachineAPI) + + By("Creating the unpaused-target rollback fixture before the outage") + fixtureB = createZeroReplicaMachineSetMigrationDisruptiveFixture("ms-disruptive-unpaused-target-") + verifyMAPIMachineSetSynchronizedState( + fixtureB.mapiMachineSet, + mapiv1beta1.MachineAuthorityMachineAPI, + mapiv1beta1.MachineAPISynchronized, + ) + verifyCAPIMachineSetPausedState(fixtureB.capiMachineSet, mapiv1beta1.MachineAuthorityMachineAPI) + + By("Migrating the unpaused-target rollback fixture to a healthy ClusterAPI steady state before the outage") + switchMachineSetAuthoritativeAPI(fixtureB.mapiMachineSet, mapiv1beta1.MachineAuthorityClusterAPI) + verifyMAPIMachineSetSynchronizedState( + fixtureB.mapiMachineSet, + mapiv1beta1.MachineAuthorityClusterAPI, + mapiv1beta1.ClusterAPISynchronized, + ) + verifyMachineSetPausedCondition(fixtureB.mapiMachineSet, mapiv1beta1.MachineAuthorityClusterAPI) + verifyCAPIMachineSetPausedState(fixtureB.capiMachineSet, mapiv1beta1.MachineAuthorityClusterAPI) + + By("Reading and validating the outage baseline after both fixtures are prepared") + disruptionStateValue := readAndValidateMachineSetMigrationDisruptionBaseline(ctx, cl) + disruptionState = &disruptionStateValue + + By("Marking Deployment/openshift-cluster-api-operator/capi-operator unmanaged through ClusterVersion overrides") + setMachineSetMigrationCAPIOperatorOverride(ctx, cl, true) + + By("Creating the shared outage by scaling Deployment/openshift-cluster-api-operator/capi-operator to zero") + scaleDeploymentAndWaitForAvailableReplicas( + ctx, + cl, + capiframework.CAPIOperatorNamespace, + capiOperatorDeploymentName, + 0, + ) + + By("Scaling Deployment/openshift-cluster-api/capi-controller-manager to zero in the same outage window") + scaleDeploymentAndWaitForAvailableReplicas( + ctx, + cl, + capiframework.CAPINamespace, + capiControllerManagerDeploymentName, + 0, + ) + + By("Verifying rollback succeeds during the outage when the target CAPI MachineSet was never observed unpaused") + switchMachineSetAuthoritativeAPI(fixtureA.mapiMachineSet, mapiv1beta1.MachineAuthorityClusterAPI) + verifyMachineSetAuthoritative(fixtureA.mapiMachineSet, mapiv1beta1.MachineAuthorityClusterAPI) + switchMachineSetAuthoritativeAPI(fixtureA.mapiMachineSet, mapiv1beta1.MachineAuthorityMachineAPI) + verifyMAPIMachineSetSynchronizedState( + fixtureA.mapiMachineSet, + mapiv1beta1.MachineAuthorityMachineAPI, + mapiv1beta1.MachineAPISynchronized, + ) + verifyCAPIMachineSetPausedState(fixtureA.capiMachineSet, mapiv1beta1.MachineAuthorityMachineAPI) + + By("Requesting rollback during the outage after the target CAPI MachineSet was observed unpaused") + switchMachineSetAuthoritativeAPI(fixtureB.mapiMachineSet, mapiv1beta1.MachineAuthorityMachineAPI) + + By("Verifying rollback stays pinned at ClusterAPI while the outage persists and the target CAPI MachineSet remains unpaused") + consistentlyVerifyMachineSetRollbackPinnedAtClusterAPI( + fixtureB.mapiMachineSet, + fixtureB.capiMachineSet, + 10*time.Second, + ) + + By("Starting recovery for Deployment/openshift-cluster-api/capi-controller-manager and Deployment/openshift-cluster-api-operator/capi-operator together") + scaleDeployment( + ctx, + cl, + capiframework.CAPINamespace, + capiControllerManagerDeploymentName, + disruptionState.capiControllerManagerReplicas, + ) + scaleDeployment( + ctx, + cl, + capiframework.CAPIOperatorNamespace, + capiOperatorDeploymentName, + disruptionState.capiOperatorReplicas, + ) + + By("Waiting for Deployment/openshift-cluster-api/capi-controller-manager to become healthy after recovery starts") + waitForDeploymentAvailableReplicas( + ctx, + cl, + capiframework.CAPINamespace, + capiControllerManagerDeploymentName, + disruptionState.capiControllerManagerReplicas, + ) + + By("Waiting for Deployment/openshift-cluster-api-operator/capi-operator to become healthy after recovery starts") + waitForDeploymentAvailableReplicas( + ctx, + cl, + capiframework.CAPIOperatorNamespace, + capiOperatorDeploymentName, + disruptionState.capiOperatorReplicas, + ) + + if disruptionState.capiOperatorOverrideExpectedAbsent { + By("Removing the targeted ClusterVersion unmanaged override for Deployment/openshift-cluster-api-operator/capi-operator") + setMachineSetMigrationCAPIOperatorOverride(ctx, cl, false) + } + + By("Waiting for the Cluster API operator to return to a healthy state after the outage") + waitForClusterAPIOperatorHealthy(ctx, cl) + disruptionStateRestored = true + + By("Verifying the already-requested rollback resumes automatically after recovery") + verifyMAPIMachineSetSynchronizedState( + fixtureB.mapiMachineSet, + mapiv1beta1.MachineAuthorityMachineAPI, + mapiv1beta1.MachineAPISynchronized, + ) + verifyCAPIMachineSetPausedState(fixtureB.capiMachineSet, mapiv1beta1.MachineAuthorityMachineAPI) + }) +}) diff --git a/e2e/machineset_migration_helpers.go b/e2e/machineset_migration_helpers.go index d8de79377..844867340 100644 --- a/e2e/machineset_migration_helpers.go +++ b/e2e/machineset_migration_helpers.go @@ -24,13 +24,17 @@ import ( . "github.com/onsi/gomega" "github.com/onsi/gomega/types" + configv1 "github.com/openshift/api/config/v1" mapiv1beta1 "github.com/openshift/api/machine/v1beta1" mapiframework "github.com/openshift/cluster-api-actuator-pkg/pkg/framework" capiframework "github.com/openshift/cluster-capi-operator/e2e/framework" "github.com/openshift/cluster-capi-operator/pkg/conversion/mapi2capi" + "github.com/openshift/library-go/pkg/config/clusteroperator/v1helpers" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" awsv1 "sigs.k8s.io/cluster-api-provider-aws/v2/api/v1beta2" clusterv1 "sigs.k8s.io/cluster-api/api/core/v1beta2" "sigs.k8s.io/controller-runtime/pkg/client" @@ -38,6 +42,19 @@ import ( yaml "sigs.k8s.io/yaml" ) +const ( + capiOperatorDeploymentName = "capi-operator" + capiControllerManagerDeploymentName = "capi-controller-manager" + clusterOperatorName = "cluster-api" + clusterVersionName = "version" +) + +type machineSetMigrationDisruptionState struct { + capiOperatorReplicas int32 + capiControllerManagerReplicas int32 + capiOperatorOverrideExpectedAbsent bool +} + // createCAPIMachineSet creates a CAPI MachineSet with an AWSMachineTemplate and waits for it to be ready. func createCAPIMachineSet(ctx context.Context, cl client.Client, replicas int32, machineSetName string, instanceType string) *clusterv1.MachineSet { GinkgoHelper() @@ -87,11 +104,15 @@ func createCAPIMachineSet(ctx context.Context, cl client.Client, replicas int32, return machineSet } -// createMAPIMachineSetWithAuthoritativeAPI creates a MAPI MachineSet with specified authoritativeAPI and waits for the CAPI mirror to be created. +// createMAPIMachineSetWithAuthoritativeAPI creates a MAPI MachineSet with the requested authority and waits for the CAPI mirror. func createMAPIMachineSetWithAuthoritativeAPI(ctx context.Context, cl client.Client, replicas int, machineSetName string, machineSetAuthority mapiv1beta1.MachineAuthority, machineAuthority mapiv1beta1.MachineAuthority) *mapiv1beta1.MachineSet { GinkgoHelper() - By(fmt.Sprintf("Creating MAPI MachineSet with spec.authoritativeAPI: %s, spec.template.spec.authoritativeAPI: %s, replicas=%d", machineSetAuthority, machineAuthority, replicas)) + if machineSetAuthority == machineAuthority { + By(fmt.Sprintf("Creating MAPI MachineSet with spec.authoritativeAPI=%s, replicas=%d", machineSetAuthority, replicas)) + } else { + By(fmt.Sprintf("Creating MAPI MachineSet with spec.authoritativeAPI=%s, spec.template.spec.authoritativeAPI=%s, replicas=%d", machineSetAuthority, machineAuthority, replicas)) + } machineSetParams := mapiframework.BuildMachineSetParams(ctx, cl, replicas) machineSetParams.Name = machineSetName machineSetParams.Labels[mapiframework.MachineSetKey] = machineSetName @@ -151,12 +172,64 @@ func verifyMachineSetAuthoritative(mapiMachineSet *mapiv1beta1.MachineSet, autho By(fmt.Sprintf("Verifying the MachineSet authoritative is %s", authority)) - Eventually(komega.Object(mapiMachineSet), capiframework.WaitMedium, capiframework.RetryMedium).Should( + Eventually(komega.Object(mapiMachineSet), capiframework.WaitMedium, capiframework.RetryShort).Should( HaveField("Status.AuthoritativeAPI", Equal(authority)), "MachineSet %s: wanted AuthoritativeAPI %s", mapiMachineSet.Name, authority, ) } +func verifyMAPIMachineSetSynchronizedState(mapiMachineSet *mapiv1beta1.MachineSet, authority mapiv1beta1.MachineAuthority, expectedSynchronizedAPI mapiv1beta1.SynchronizedAPI) { + GinkgoHelper() + + Expect(mapiMachineSet).NotTo(BeNil(), "MAPI MachineSet parameter cannot be nil") + Expect(mapiMachineSet.GetName()).NotTo(BeEmpty(), "MAPI MachineSet name cannot be empty") + + var expectedMessage string + switch authority { + case mapiv1beta1.MachineAuthorityMachineAPI: + expectedMessage = "Successfully synchronized MAPI MachineSet to CAPI" + case mapiv1beta1.MachineAuthorityClusterAPI: + expectedMessage = "Successfully synchronized CAPI MachineSet to MAPI" + default: + Fail(fmt.Sprintf("unknown authoritativeAPI type: %v", authority)) + } + + By(fmt.Sprintf( + "Verifying MAPI MachineSet %s state is authoritative=%s, synchronizedAPI=%s, and Synchronized=True", + mapiMachineSet.Name, + authority, + expectedSynchronizedAPI, + )) + + Eventually(func(g Gomega) { + g.Expect(komega.Get(mapiMachineSet)()).To(Succeed()) + g.Expect(mapiMachineSet.Status.AuthoritativeAPI).To( + Equal(authority), + "MachineSet %s: wanted AuthoritativeAPI %s", + mapiMachineSet.Name, + authority, + ) + g.Expect(mapiMachineSet.Status.SynchronizedAPI).To( + Equal(expectedSynchronizedAPI), + "MachineSet %s: wanted SynchronizedAPI %s", + mapiMachineSet.Name, + expectedSynchronizedAPI, + ) + g.Expect(mapiMachineSet.Status.Conditions).To( + ContainElement(SatisfyAll( + HaveField("Type", Equal(SynchronizedCondition)), + HaveField("Status", Equal(corev1.ConditionTrue)), + HaveField("Reason", Equal("ResourceSynchronized")), + HaveField("Message", Equal(expectedMessage)), + )), + "MachineSet %s: wanted Synchronized condition for authority %s, got conditions: %s", + mapiMachineSet.Name, + authority, + summarizeMAPIConditions(mapiMachineSet.Status.Conditions), + ) + }, capiframework.WaitMedium, capiframework.RetryShort).Should(Succeed()) +} + // verifyMachineSetPausedCondition verifies the Paused condition of a MachineSet (MAPI or CAPI) based on its authoritative API. func verifyMachineSetPausedCondition(machineSet client.Object, authority mapiv1beta1.MachineAuthority) { GinkgoHelper() @@ -197,7 +270,7 @@ func verifyMachineSetPausedCondition(machineSet client.Object, authority mapiv1b g.Expect(ms.Status.Conditions).To(ContainElement(conditionMatcher), "MAPI MachineSet %s: wanted Paused condition for authority %s, got conditions: %s", ms.Name, authority, summarizeMAPIConditions(ms.Status.Conditions)) - }, capiframework.WaitMedium, capiframework.RetryMedium).Should(Succeed()) + }, capiframework.WaitMedium, capiframework.RetryShort).Should(Succeed()) case *clusterv1.MachineSet: // This is a CAPI MachineSet @@ -227,13 +300,135 @@ func verifyMachineSetPausedCondition(machineSet client.Object, authority mapiv1b g.Expect(ms.Status.Conditions).To(ContainElement(conditionMatcher), "CAPI MachineSet %s: wanted Paused condition for authority %s, got conditions: %s", ms.Name, authority, summarizeV1Beta2Conditions(ms.Status.Conditions)) - }, capiframework.WaitMedium, capiframework.RetryMedium).Should(Succeed()) + }, capiframework.WaitMedium, capiframework.RetryShort).Should(Succeed()) default: Fail(fmt.Sprintf("unsupported MachineSet type: %T", machineSet)) } } +func verifyCAPIMachineSetPausedState(capiMachineSet *clusterv1.MachineSet, authority mapiv1beta1.MachineAuthority) { + GinkgoHelper() + + Expect(capiMachineSet).NotTo(BeNil(), "CAPI MachineSet parameter cannot be nil") + Expect(capiMachineSet.GetName()).NotTo(BeEmpty(), "CAPI MachineSet name cannot be empty") + + var annotationState string + var annotationMatcher types.GomegaMatcher + var conditionMatcher types.GomegaMatcher + + switch authority { + case mapiv1beta1.MachineAuthorityMachineAPI: + annotationState = "present" + annotationMatcher = HaveKey(clusterv1.PausedAnnotation) + conditionMatcher = SatisfyAll( + HaveField("Type", Equal(CAPIPausedCondition)), + HaveField("Status", Equal(metav1.ConditionTrue)), + HaveField("Reason", Equal("Paused")), + ) + case mapiv1beta1.MachineAuthorityClusterAPI: + annotationState = "absent" + annotationMatcher = SatisfyAny( + BeNil(), + Not(HaveKey(clusterv1.PausedAnnotation)), + ) + conditionMatcher = SatisfyAll( + HaveField("Type", Equal(CAPIPausedCondition)), + HaveField("Status", Equal(metav1.ConditionFalse)), + HaveField("Reason", Equal("NotPaused")), + ) + default: + Fail(fmt.Sprintf("unknown authoritativeAPI type: %v", authority)) + } + + By(fmt.Sprintf("Verifying CAPI MachineSet %s paused state matches authority %s", capiMachineSet.Name, authority)) + + Eventually(func(g Gomega) { + g.Expect(komega.Get(capiMachineSet)()).To(Succeed()) + g.Expect(capiMachineSet.Annotations).To( + annotationMatcher, + "CAPI MachineSet %s paused annotation should be %s", + capiMachineSet.Name, + annotationState, + ) + g.Expect(capiMachineSet.Status.Conditions).To( + ContainElement(conditionMatcher), + "CAPI MachineSet %s: wanted Paused condition for authority %s, got conditions: %s", + capiMachineSet.Name, + authority, + summarizeV1Beta2Conditions(capiMachineSet.Status.Conditions), + ) + }, capiframework.WaitMedium, capiframework.RetryShort).Should(Succeed()) +} + +func verifyCAPIMachineSetPausedAnnotation(capiMachineSet *clusterv1.MachineSet, shouldBePresent bool) { + GinkgoHelper() + + Expect(capiMachineSet).NotTo(BeNil(), "CAPI MachineSet parameter cannot be nil") + Expect(capiMachineSet.GetName()).NotTo(BeEmpty(), "CAPI MachineSet name cannot be empty") + + state := "present" + matcher := HaveField("ObjectMeta.Annotations", HaveKey(clusterv1.PausedAnnotation)) + if !shouldBePresent { + state = "absent" + matcher = HaveField("ObjectMeta.Annotations", SatisfyAny( + BeNil(), + Not(HaveKey(clusterv1.PausedAnnotation)), + )) + } + + By(fmt.Sprintf("Verifying CAPI MachineSet %s paused annotation is %s", capiMachineSet.Name, state)) + Eventually(komega.Object(capiMachineSet), capiframework.WaitMedium, capiframework.RetryMedium).Should( + matcher, + "CAPI MachineSet %s paused annotation should be %s", + capiMachineSet.Name, + state, + ) +} + +func consistentlyVerifyMachineSetRollbackPinnedAtClusterAPI(mapiMachineSet *mapiv1beta1.MachineSet, capiMachineSet *clusterv1.MachineSet, duration time.Duration) { + GinkgoHelper() + + Expect(mapiMachineSet).NotTo(BeNil(), "MAPI MachineSet parameter cannot be nil") + Expect(mapiMachineSet.GetName()).NotTo(BeEmpty(), "MAPI MachineSet name cannot be empty") + Expect(capiMachineSet).NotTo(BeNil(), "CAPI MachineSet parameter cannot be nil") + Expect(capiMachineSet.GetName()).NotTo(BeEmpty(), "CAPI MachineSet name cannot be empty") + + By(fmt.Sprintf( + "Verifying rollback for MAPI MachineSet %s remains pinned at ClusterAPI while CAPI MachineSet %s stays unpaused", + mapiMachineSet.Name, + capiMachineSet.Name, + )) + Consistently(func(g Gomega) { + g.Expect(komega.Get(mapiMachineSet)()).To(Succeed()) + g.Expect(mapiMachineSet.Status.AuthoritativeAPI).To( + Equal(mapiv1beta1.MachineAuthorityClusterAPI), + "MAPI MachineSet %s should remain at ClusterAPI while rollback is blocked", + mapiMachineSet.Name, + ) + + g.Expect(komega.Get(capiMachineSet)()).To(Succeed()) + g.Expect(capiMachineSet.Annotations).To( + SatisfyAny( + BeNil(), + Not(HaveKey(clusterv1.PausedAnnotation)), + ), + "CAPI MachineSet %s should remain unpaused while rollback is blocked", + capiMachineSet.Name, + ) + g.Expect(capiMachineSet.Status.Conditions).To( + ContainElement(SatisfyAll( + HaveField("Type", Equal(CAPIPausedCondition)), + HaveField("Status", Equal(metav1.ConditionFalse)), + HaveField("Reason", Equal("NotPaused")), + )), + "CAPI MachineSet %s should keep Paused=False while rollback is blocked, got conditions: %s", + capiMachineSet.Name, + summarizeV1Beta2Conditions(capiMachineSet.Status.Conditions), + ) + }, duration, capiframework.RetryShort).Should(Succeed()) +} + // verifyMachinesetReplicas verifies that a MachineSet (MAPI or CAPI) has the expected number of replicas in its status. func verifyMachinesetReplicas(machineSet client.Object, replicas int) { GinkgoHelper() @@ -289,6 +484,17 @@ func verifyMAPIMachineSetSynchronizedCondition(mapiMachineSet *mapiv1beta1.Machi }, capiframework.WaitMedium, capiframework.RetryMedium).Should(Succeed()) } +// verifyMachineSetSynchronizedAPI verifies that the MAPI MachineSet's status.synchronizedAPI matches the expected value. +func verifyMachineSetSynchronizedAPI(mapiMachineSet *mapiv1beta1.MachineSet, expectedSynchronizedAPI mapiv1beta1.SynchronizedAPI) { + GinkgoHelper() + + By(fmt.Sprintf("Verifying MAPI MachineSet SynchronizedAPI is %s", expectedSynchronizedAPI)) + Eventually(komega.Object(mapiMachineSet), capiframework.WaitMedium, capiframework.RetryMedium).Should( + HaveField("Status.SynchronizedAPI", Equal(expectedSynchronizedAPI)), + fmt.Sprintf("MAPI MachineSet SynchronizedAPI should be %s", expectedSynchronizedAPI), + ) +} + // verifyMAPIMachineSetProviderSpec verifies that a MAPI MachineSet's providerSpec matches the given Gomega matcher. func verifyMAPIMachineSetProviderSpec(mapiMachineSet *mapiv1beta1.MachineSet, matcher types.GomegaMatcher) { GinkgoHelper() @@ -494,3 +700,311 @@ func cleanupMachineSetTestResources(ctx context.Context, cl client.Client, capiM deleteAWSMachineTemplates(cleanupCtx, cl, template) } } + +func readAndValidateMachineSetMigrationDisruptionBaseline(ctx context.Context, cl client.Client) machineSetMigrationDisruptionState { + GinkgoHelper() + + By("Reading and validating the rollback test baseline") + + clusterOperator := &configv1.ClusterOperator{} + Expect(cl.Get(ctx, client.ObjectKey{Name: clusterOperatorName}, clusterOperator)).To( + Succeed(), + "should read ClusterOperator/%s before the outage", + clusterOperatorName, + ) + Expect(isClusterAPIOperatorHealthy(clusterOperator)).To( + BeTrue(), + "ClusterOperator/%s must be healthy before the outage, got conditions: %v", + clusterOperatorName, + clusterOperator.Status.Conditions, + ) + + clusterVersion := &configv1.ClusterVersion{} + Expect(cl.Get(ctx, client.ObjectKey{Name: clusterVersionName}, clusterVersion)).To( + Succeed(), + "should read ClusterVersion/%s before the outage", + clusterVersionName, + ) + Expect(countMachineSetMigrationDeploymentOverrides( + clusterVersion.Spec.Overrides, + capiframework.CAPIOperatorNamespace, + capiOperatorDeploymentName, + )).To( + Equal(0), + "ClusterVersion/%s must not have pre-existing overrides for Deployment %s/%s", + clusterVersionName, + capiframework.CAPIOperatorNamespace, + capiOperatorDeploymentName, + ) + + return machineSetMigrationDisruptionState{ + capiOperatorReplicas: readAndValidateOutageDeploymentBaseline( + ctx, + cl, + capiframework.CAPIOperatorNamespace, + capiOperatorDeploymentName, + ), + capiControllerManagerReplicas: readAndValidateOutageDeploymentBaseline( + ctx, + cl, + capiframework.CAPINamespace, + capiControllerManagerDeploymentName, + ), + capiOperatorOverrideExpectedAbsent: true, + } +} + +func readAndValidateOutageDeploymentBaseline(ctx context.Context, cl client.Client, namespace, name string) int32 { + GinkgoHelper() + + deployment := &appsv1.Deployment{} + Expect(cl.Get(ctx, client.ObjectKey{Namespace: namespace, Name: name}, deployment)).To( + Succeed(), + "should read Deployment/%s/%s before the outage", + namespace, + name, + ) + + desiredReplicas := ptr.Deref(deployment.Spec.Replicas, int32(1)) + Expect(desiredReplicas).To( + BeNumerically(">", 0), + "Deployment/%s/%s must have nonzero desired replicas before the outage", + namespace, + name, + ) + Expect(deployment.Status.ObservedGeneration).To( + Equal(deployment.Generation), + "Deployment/%s/%s must be observed before the outage", + namespace, + name, + ) + Expect(deployment.Status.AvailableReplicas).To( + Equal(desiredReplicas), + "Deployment/%s/%s must have all desired replicas available before the outage", + namespace, + name, + ) + + return desiredReplicas +} + +func setMachineSetMigrationCAPIOperatorOverride(ctx context.Context, cl client.Client, shouldBePresent bool) { + GinkgoHelper() + setMachineSetMigrationDeploymentOverride(ctx, cl, capiframework.CAPIOperatorNamespace, capiOperatorDeploymentName, shouldBePresent) +} + +func setMachineSetMigrationDeploymentOverride(ctx context.Context, cl client.Client, namespace, name string, shouldBePresent bool) { + GinkgoHelper() + + state := "present" + if !shouldBePresent { + state = "absent" + } + + By(fmt.Sprintf( + "Ensuring ClusterVersion/%s override for Deployment %s/%s is %s", + clusterVersionName, + namespace, + name, + state, + )) + + Eventually(func() error { + clusterVersion := &configv1.ClusterVersion{} + if err := cl.Get(ctx, client.ObjectKey{Name: clusterVersionName}, clusterVersion); err != nil { + return err + } + + currentCount := countMachineSetMigrationDeploymentOverrides(clusterVersion.Spec.Overrides, namespace, name) + if shouldBePresent && currentCount == 1 && hasMachineSetMigrationDeploymentUnmanagedOverride(clusterVersion.Spec.Overrides, namespace, name) { + return nil + } + if !shouldBePresent && currentCount == 0 { + return nil + } + if shouldBePresent && currentCount != 0 { + return fmt.Errorf( + "expected no existing ClusterVersion overrides for Deployment %s/%s, found %d", + namespace, + name, + currentCount, + ) + } + + clusterVersionCopy := clusterVersion.DeepCopy() + clusterVersion.Spec.Overrides = desiredMachineSetMigrationDeploymentOverrides( + clusterVersion.Spec.Overrides, + namespace, + name, + shouldBePresent, + ) + + return cl.Patch(ctx, clusterVersion, client.MergeFrom(clusterVersionCopy)) + }, capiframework.WaitMedium, capiframework.RetryMedium).Should( + Succeed(), + "ClusterVersion/%s override for Deployment %s/%s should become %s", + clusterVersionName, + namespace, + name, + state, + ) + + expectedCount := 0 + if shouldBePresent { + expectedCount = 1 + } + + Eventually(func(g Gomega) { + clusterVersion := &configv1.ClusterVersion{} + g.Expect(cl.Get(ctx, client.ObjectKey{Name: clusterVersionName}, clusterVersion)).To(Succeed()) + g.Expect(clusterVersion.Status.ObservedGeneration).To(Equal(clusterVersion.Generation)) + g.Expect(countMachineSetMigrationDeploymentOverrides(clusterVersion.Spec.Overrides, namespace, name)).To(Equal(expectedCount)) + if shouldBePresent { + g.Expect(hasMachineSetMigrationDeploymentUnmanagedOverride(clusterVersion.Spec.Overrides, namespace, name)).To(BeTrue()) + } + }, capiframework.WaitMedium, capiframework.RetryMedium).Should( + Succeed(), + "ClusterVersion/%s observedGeneration should catch up after ensuring the override for Deployment %s/%s is %s", + clusterVersionName, + namespace, + name, + state, + ) +} + +func desiredMachineSetMigrationDeploymentOverrides(current []configv1.ComponentOverride, namespace, name string, shouldBePresent bool) []configv1.ComponentOverride { + updatedOverrides := make([]configv1.ComponentOverride, 0, len(current)+1) + replacedOrRemoved := false + + for _, override := range current { + if !replacedOrRemoved && isMachineSetMigrationDeploymentOverride(override, namespace, name) { + replacedOrRemoved = true + if shouldBePresent { + updatedOverrides = append(updatedOverrides, machineSetMigrationDeploymentOverride(namespace, name)) + } + + continue + } + + updatedOverrides = append(updatedOverrides, override) + } + + if shouldBePresent && !replacedOrRemoved { + updatedOverrides = append(updatedOverrides, machineSetMigrationDeploymentOverride(namespace, name)) + } + + return updatedOverrides +} + +func machineSetMigrationDeploymentOverride(namespace, name string) configv1.ComponentOverride { + return configv1.ComponentOverride{ + Group: "apps", + Kind: "Deployment", + Namespace: namespace, + Name: name, + Unmanaged: true, + } +} + +func hasMachineSetMigrationDeploymentUnmanagedOverride(overrides []configv1.ComponentOverride, namespace, name string) bool { + for _, override := range overrides { + if override == machineSetMigrationDeploymentOverride(namespace, name) { + return true + } + } + + return false +} + +func countMachineSetMigrationDeploymentOverrides(overrides []configv1.ComponentOverride, namespace, name string) int { + count := 0 + for _, override := range overrides { + if isMachineSetMigrationDeploymentOverride(override, namespace, name) { + count++ + } + } + + return count +} + +func isMachineSetMigrationDeploymentOverride(override configv1.ComponentOverride, namespace, name string) bool { + return override.Group == "apps" && + override.Kind == "Deployment" && + override.Namespace == namespace && + override.Name == name +} + +func scaleDeploymentAndWaitForAvailableReplicas(ctx context.Context, cl client.Client, namespace, name string, replicas int32) { + GinkgoHelper() + + scaleDeployment(ctx, cl, namespace, name, replicas) + waitForDeploymentAvailableReplicas(ctx, cl, namespace, name, replicas) +} + +func scaleDeployment(ctx context.Context, cl client.Client, namespace, name string, replicas int32) { + GinkgoHelper() + + By(fmt.Sprintf("Scaling Deployment %s/%s to %d replicas", namespace, name, replicas)) + Eventually(func() error { + deployment := &appsv1.Deployment{} + if err := cl.Get(ctx, client.ObjectKey{Namespace: namespace, Name: name}, deployment); err != nil { + return err + } + + if ptr.Deref(deployment.Spec.Replicas, int32(1)) == replicas { + return nil + } + + deploymentCopy := deployment.DeepCopy() + deployment.Spec.Replicas = ptr.To(replicas) + + return cl.Patch(ctx, deployment, client.MergeFrom(deploymentCopy)) + }, capiframework.WaitMedium, capiframework.RetryShort).Should( + Succeed(), + "Deployment %s/%s should scale to %d replicas", + namespace, + name, + replicas, + ) +} + +func waitForDeploymentAvailableReplicas(ctx context.Context, cl client.Client, namespace, name string, replicas int32) { + GinkgoHelper() + + By(fmt.Sprintf("Waiting for Deployment %s/%s to report %d available replicas", namespace, name, replicas)) + Eventually(func(g Gomega) { + deployment := &appsv1.Deployment{} + g.Expect(cl.Get(ctx, client.ObjectKey{Namespace: namespace, Name: name}, deployment)).To(Succeed()) + g.Expect(ptr.Deref(deployment.Spec.Replicas, int32(1))).To(Equal(replicas)) + g.Expect(deployment.Status.ObservedGeneration).To(Equal(deployment.Generation)) + g.Expect(deployment.Status.AvailableReplicas).To(Equal(replicas)) + }, capiframework.WaitLong, capiframework.RetryMedium).Should( + Succeed(), + "Deployment %s/%s should report %d available replicas", + namespace, + name, + replicas, + ) +} + +func waitForClusterAPIOperatorHealthy(ctx context.Context, cl client.Client) { + GinkgoHelper() + + By("Waiting for ClusterOperator/cluster-api to return to Available=True, Progressing=False, Degraded=False") + Eventually(func(g Gomega) { + clusterOperator := &configv1.ClusterOperator{} + g.Expect(cl.Get(ctx, client.ObjectKey{Name: clusterOperatorName}, clusterOperator)).To(Succeed()) + g.Expect(isClusterAPIOperatorHealthy(clusterOperator)).To( + BeTrue(), + "ClusterOperator/%s conditions: %v", + clusterOperatorName, + clusterOperator.Status.Conditions, + ) + }, capiframework.WaitOverLong, capiframework.RetryMedium).Should(Succeed()) +} + +func isClusterAPIOperatorHealthy(clusterOperator *configv1.ClusterOperator) bool { + return v1helpers.IsStatusConditionTrue(clusterOperator.Status.Conditions, configv1.OperatorAvailable) && + v1helpers.IsStatusConditionFalse(clusterOperator.Status.Conditions, configv1.OperatorProgressing) && + v1helpers.IsStatusConditionFalse(clusterOperator.Status.Conditions, configv1.OperatorDegraded) +} diff --git a/e2e/machineset_migration_mapi_authoritative_test.go b/e2e/machineset_migration_mapi_authoritative_test.go index 6e40e24de..32e631d31 100644 --- a/e2e/machineset_migration_mapi_authoritative_test.go +++ b/e2e/machineset_migration_mapi_authoritative_test.go @@ -181,6 +181,8 @@ var _ = Describe("[sig-cluster-lifecycle][OCPFeatureGate:MachineAPIMigration] Ma verifyMachineSetPausedCondition(mapiMachineSet, mapiv1beta1.MachineAuthorityClusterAPI) verifyMachineSetPausedCondition(capiMachineSet, mapiv1beta1.MachineAuthorityClusterAPI) verifyMAPIMachineSetSynchronizedCondition(mapiMachineSet, mapiv1beta1.MachineAuthorityClusterAPI) + verifyMachineSetAuthoritative(mapiMachineSet, mapiv1beta1.MachineAuthorityClusterAPI) + verifyMachineSetSynchronizedAPI(mapiMachineSet, mapiv1beta1.ClusterAPISynchronized) By("Scaling up CAPI MachineSet to 3") capiframework.ScaleCAPIMachineSet(mapiMSAuthMAPIName, 3, capiframework.CAPINamespace) @@ -218,6 +220,7 @@ var _ = Describe("[sig-cluster-lifecycle][OCPFeatureGate:MachineAPIMigration] Ma verifyMachineSetPausedCondition(mapiMachineSet, mapiv1beta1.MachineAuthorityMachineAPI) verifyMachineSetPausedCondition(capiMachineSet, mapiv1beta1.MachineAuthorityMachineAPI) verifyMAPIMachineSetSynchronizedCondition(mapiMachineSet, mapiv1beta1.MachineAuthorityMachineAPI) + verifyMachineSetSynchronizedAPI(mapiMachineSet, mapiv1beta1.MachineAPISynchronized) By("Deleting MAPI MachineSet and verifying mirrors are removed") Expect(mapiframework.DeleteMachineSets(cl, mapiMachineSet)).To(Succeed(), "Should be able to delete test MachineSet") @@ -345,6 +348,8 @@ var _ = Describe("[sig-cluster-lifecycle][OCPFeatureGate:MachineAPIMigration] Ma switchMachineSetTemplateAuthoritativeAPI(mapiMachineSet, mapiv1beta1.MachineAuthorityClusterAPI) switchMachineSetAuthoritativeAPI(mapiMachineSet, mapiv1beta1.MachineAuthorityClusterAPI) verifyMAPIMachineSetSynchronizedCondition(mapiMachineSet, mapiv1beta1.MachineAuthorityClusterAPI) + verifyMachineSetAuthoritative(mapiMachineSet, mapiv1beta1.MachineAuthorityClusterAPI) + verifyMachineSetSynchronizedAPI(mapiMachineSet, mapiv1beta1.ClusterAPISynchronized) }) It("should reject MAPI updates and allow CAPI InfraTemplate updates", func() { diff --git a/pkg/controllers/machinemigration/machine_migration_controller.go b/pkg/controllers/machinemigration/machine_migration_controller.go index 686bc84c6..fc5a58c2c 100644 --- a/pkg/controllers/machinemigration/machine_migration_controller.go +++ b/pkg/controllers/machinemigration/machine_migration_controller.go @@ -19,17 +19,30 @@ package machinemigration import ( "context" "fmt" - "time" + "slices" "github.com/go-logr/logr" + metal3v1 "github.com/metal3-io/cluster-api-provider-metal3/api/v1beta1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/tools/record" + azurev1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" + gcpv1 "sigs.k8s.io/cluster-api-provider-gcp/api/v1beta1" + ibmpowervsv1 "sigs.k8s.io/cluster-api-provider-ibmcloud/api/v1beta2" + openstackv1 "sigs.k8s.io/cluster-api-provider-openstack/api/v1beta1" + vspherev1 "sigs.k8s.io/cluster-api-provider-vsphere/apis/v1beta1" + + configv1 "github.com/openshift/api/config/v1" + mapiv1beta1 "github.com/openshift/api/machine/v1beta1" + machinev1applyconfigs "github.com/openshift/client-go/machine/applyconfigurations/machine/v1beta1" + "github.com/openshift/cluster-capi-operator/pkg/controllers" + "github.com/openshift/cluster-capi-operator/pkg/controllers/migrationcommon" + "github.com/openshift/cluster-capi-operator/pkg/util" + awsv1 "sigs.k8s.io/cluster-api-provider-aws/v2/api/v1beta2" clusterv1 "sigs.k8s.io/cluster-api/api/core/v1beta2" "sigs.k8s.io/cluster-api/controllers/external" - "sigs.k8s.io/cluster-api/util/annotations" "sigs.k8s.io/cluster-api/util/conditions" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" @@ -37,13 +50,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/handler" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/reconcile" - - configv1 "github.com/openshift/api/config/v1" - mapiv1beta1 "github.com/openshift/api/machine/v1beta1" - machinev1applyconfigs "github.com/openshift/client-go/machine/applyconfigurations/machine/v1beta1" - "github.com/openshift/cluster-capi-operator/pkg/controllers" - "github.com/openshift/cluster-capi-operator/pkg/controllers/synccommon" - "github.com/openshift/cluster-capi-operator/pkg/util" ) const controllerName = "MachineMigrationController" @@ -61,6 +67,51 @@ type MachineMigrationReconciler struct { MAPINamespace string } +type machineMigratable struct { + reconciler *MachineMigrationReconciler + mapiMachine *mapiv1beta1.Machine +} + +// MAPIObject returns the backing Machine API machine. +func (m *machineMigratable) MAPIObject() client.Object { + return m.mapiMachine +} + +// DesiredAuthority returns the requested authoritative API from spec. +func (m *machineMigratable) DesiredAuthority() mapiv1beta1.MachineAuthority { + return m.mapiMachine.Spec.AuthoritativeAPI +} + +// CurrentAuthority returns the observed authoritative API from status. +func (m *machineMigratable) CurrentAuthority() mapiv1beta1.MachineAuthority { + return m.mapiMachine.Status.AuthoritativeAPI +} + +// SynchronizedAPI returns the last synchronized API recorded in status. +func (m *machineMigratable) SynchronizedAPI() mapiv1beta1.SynchronizedAPI { + return m.mapiMachine.Status.SynchronizedAPI +} + +// SynchronizedGeneration returns the generation recorded by the sync controller. +func (m *machineMigratable) SynchronizedGeneration() int64 { + return m.mapiMachine.Status.SynchronizedGeneration +} + +// MAPIConditions returns the Machine API conditions used by migration logic. +func (m *machineMigratable) MAPIConditions() []mapiv1beta1.Condition { + return m.mapiMachine.Status.Conditions +} + +// EnsureCAPIPaused pauses the primary Cluster API machine and its infra object. +func (m *machineMigratable) EnsureCAPIPaused(ctx context.Context, capiMachine *clusterv1.Machine) (bool, error) { + return m.reconciler.ensureCAPIPaused(ctx, capiMachine) +} + +// EnsureCAPIUnpaused removes pause from the primary Cluster API machine and its infra object. +func (m *machineMigratable) EnsureCAPIUnpaused(ctx context.Context, capiMachine *clusterv1.Machine) (bool, error) { + return m.reconciler.ensureCAPIUnpaused(ctx, capiMachine) +} + // SetupWithManager sets up the MachineMigration controller. func (r *MachineMigrationReconciler) SetupWithManager(mgr ctrl.Manager) error { // Allow the namespaces to be set externally for test purposes, when not set, @@ -98,8 +149,6 @@ func (r *MachineMigrationReconciler) SetupWithManager(mgr ctrl.Manager) error { } // Reconcile performs the reconciliation for a Machine. -// -//nolint:funlen func (r *MachineMigrationReconciler) Reconcile(ctx context.Context, req reconcile.Request) (ctrl.Result, error) { logger := logf.FromContext(ctx).WithValues("namespace", req.Namespace, "name", req.Name) ctx = logr.NewContext(ctx, logger) @@ -107,117 +156,99 @@ func (r *MachineMigrationReconciler) Reconcile(ctx context.Context, req reconcil logger.V(1).Info("Reconciling machine") defer logger.V(1).Info("Finished reconciling machine") - mapiMachine := &mapiv1beta1.Machine{} - if err := r.Get(ctx, client.ObjectKey{Namespace: req.Namespace, Name: req.Name}, mapiMachine); err != nil && !apierrors.IsNotFound(err) { - return ctrl.Result{}, fmt.Errorf("failed to get MAPI machine: %w", err) - } else if apierrors.IsNotFound(err) { + mapiMachine, found, err := r.getMAPIMachine(ctx, req) + if err != nil { + return ctrl.Result{}, err + } + + if !found { logger.Info("Machine has been deleted. Migration not required") return ctrl.Result{}, nil } - if mapiMachine.Spec.AuthoritativeAPI == mapiMachine.Status.AuthoritativeAPI { - // No migration is being requested for this resource, nothing to do. - return ctrl.Result{}, nil + result, err := migrationcommon.Reconcile( + ctx, + r.Client, + controllerName, + r.CAPINamespace, + machinev1applyconfigs.Machine, + &machineMigratable{ + reconciler: r, + mapiMachine: mapiMachine, + }, + ) + if err != nil { + return ctrl.Result{}, fmt.Errorf("failed to reconcile machine migration state: %w", err) } - // If authoritativeAPI status is empty, it means it is the first time we see this resource. - // Set the status.authoritativeAPI to match the spec.authoritativeAPI. - // - // N.B. Very similar logic is also present in the Machine API machine/machineset controllers - // to cover for the cases when the migration controller is not running (e.g. on not yet supported platforms), - // as such if any change is done to this logic, please consider changing it also there. See: - // https://github.com/openshift/machine-api-operator/pull/1386/files#diff-8a4a734efbb8fef769f9f6ba5d30d94f19433a0b1eaeb1be4f2a55aa226c3b3dR180-R197 - if mapiMachine.Status.AuthoritativeAPI == "" { - if err := r.applyStatusAuthoritativeAPIWithPatch(ctx, mapiMachine, mapiMachine.Spec.AuthoritativeAPI); err != nil { - return ctrl.Result{}, fmt.Errorf("unable to apply authoritativeAPI to status with patch: %w", err) + return result, nil +} + +func (r *MachineMigrationReconciler) getMAPIMachine(ctx context.Context, req reconcile.Request) (*mapiv1beta1.Machine, bool, error) { + mapiMachine := &mapiv1beta1.Machine{} + if err := r.Get(ctx, client.ObjectKey{Namespace: req.Namespace, Name: req.Name}, mapiMachine); err != nil { + if apierrors.IsNotFound(err) { + return nil, false, nil } - // Wait for the patching to take effect. - return ctrl.Result{}, nil + return nil, false, fmt.Errorf("failed to get MAPI machine: %w", err) } - // Check that the resource is synchronized and up-to-date. - // - // This MUST be checked BEFORE setting status.authoritativeAPI to Migrating, - // because after that the sync controller will not run to update it and we - // will deadlock. - if isSynchronized, err := r.isSynchronized(ctx, mapiMachine); err != nil { - return ctrl.Result{}, fmt.Errorf("unable to check the resource is synchronized and up-to-date with its authority: %w", err) - } else if !isSynchronized { - // The to-be Authoritative API resource is not fully synced up yet, requeue to check later. - logger.Info("Authoritative machine and its copy are not synchronized yet, will retry later") + return mapiMachine, true, nil +} - return ctrl.Result{}, nil +func (r *MachineMigrationReconciler) getCAPIInfraMachine(ctx context.Context, capiMachine *clusterv1.Machine) (client.Object, bool, error) { + if capiMachine.Spec.InfrastructureRef.Name == "" { + return nil, false, nil } - // Make sure the authoritativeAPI resource status is set to migrating. - if mapiMachine.Status.AuthoritativeAPI != mapiv1beta1.MachineAuthorityMigrating { - logger.Info("Detected migration request for machine") - - if err := r.applyStatusAuthoritativeAPIWithPatch(ctx, mapiMachine, mapiv1beta1.MachineAuthorityMigrating); err != nil { - return ctrl.Result{}, fmt.Errorf("unable to set authoritativeAPI %q to status: %w", mapiv1beta1.MachineAuthorityMigrating, err) + infraMachine, err := external.GetObjectFromContractVersionedRef(ctx, r.Client, capiMachine.Spec.InfrastructureRef, capiMachine.Namespace) + if err != nil { + if apierrors.IsNotFound(err) { + return nil, false, nil } - logger.Info("Acknowledged migration request for machine") - - // Wait for the change to propagate. - return ctrl.Result{}, nil + return nil, false, fmt.Errorf("failed to get Cluster API infra machine: %w", err) } - // Request pausing on the authoritative resource. - if updated, err := r.requestOldAuthoritativeResourcePaused(ctx, mapiMachine); err != nil { - return ctrl.Result{}, fmt.Errorf("failed to request pause on authoritative machine: %w", err) - } else if updated { - logger.Info("Requested pausing for authoritative machine") - - // Wait for the change to propagate. - // Since there is not a watch for CAPI infra machines here - // we will not get an event for the CAPI infra machine's paused condition change, - // as such, manually requeue. - return ctrl.Result{RequeueAfter: time.Second}, nil - } - - // Check that the authoritative resource is paused. - if paused, err := r.isOldAuthoritativeResourcePaused(ctx, mapiMachine); err != nil { - return ctrl.Result{}, fmt.Errorf("failed to check paused on authoritative machine: %w", err) - } else if !paused { - // The Authoritative API resource is not paused yet, requeue to check later. - logger.Info("Authoritative machine is not paused yet, will retry later") + return infraMachine, true, nil +} - return ctrl.Result{}, nil +func (r *MachineMigrationReconciler) ensureCAPIPaused(ctx context.Context, capiMachine *clusterv1.Machine) (bool, error) { + paused, err := r.ensureCAPIMachinePaused(ctx, capiMachine) + if err != nil { + return false, err } - // Make sure the new authoritative resource has been requested to unpause. - if err := r.ensureUnpauseRequestedOnNewAuthoritativeResource(ctx, mapiMachine); err != nil { - return ctrl.Result{}, fmt.Errorf("unable to ensure the new AuthoritativeAPI has been un-paused: %w", err) + if !paused { + return false, nil } - // Set the actual AuthoritativeAPI to the desired one, reset the synchronized generation and condition. - if err := synccommon.ApplyAuthoritativeAPIAndResetSyncStatus[*machinev1applyconfigs.MachineStatusApplyConfiguration](ctx, r.Client, controllerName, machinev1applyconfigs.Machine, mapiMachine, mapiMachine.Spec.AuthoritativeAPI); err != nil { - return ctrl.Result{}, fmt.Errorf("failed to apply authoritativeAPI and reset sync status: %w", err) + infraMachine, found, err := r.getCAPIInfraMachine(ctx, capiMachine) + if err != nil { + return false, err } - logger.Info("Machine authority switch has now been completed and the resource unpaused") - logger.Info("Machine migrated successfully") + if !found { + return true, nil + } - return ctrl.Result{}, nil + return r.ensureCAPIInfraMachinePaused(ctx, infraMachine) } -// isOldAuthoritativeResourcePaused checks whether the old authoritative resource is paused. -func (r *MachineMigrationReconciler) isOldAuthoritativeResourcePaused(ctx context.Context, m *mapiv1beta1.Machine) (bool, error) { - if m.Spec.AuthoritativeAPI == mapiv1beta1.MachineAuthorityClusterAPI { - cond, err := util.GetConditionStatus(m, "Paused") - if err != nil { - return false, fmt.Errorf("unable to get paused condition for %s/%s: %w", m.Namespace, m.Name, err) - } +func (r *MachineMigrationReconciler) ensureCAPIMachinePaused(ctx context.Context, capiMachine *clusterv1.Machine) (bool, error) { + changed, err := migrationcommon.AddPausedAnnotation(ctx, r.Client, capiMachine) + if err != nil { + return false, fmt.Errorf("failed to request pause on Cluster API machine: %w", err) + } - return cond == corev1.ConditionTrue, nil + if changed { + return false, nil } - // For MachineAuthorityMachineAPI, check the corresponding CAPI resource. - capiMachine := &clusterv1.Machine{} - if err := r.Get(ctx, client.ObjectKey{Namespace: r.CAPINamespace, Name: m.Name}, capiMachine); err != nil { - return false, fmt.Errorf("failed to get Cluster API machine: %w", err) + // Same reasoning as below in ensureCAPIInfraMachinePaused + if !slices.Contains(capiMachine.Finalizers, clusterv1.MachineFinalizer) { + return true, nil } machinePausedCondition := conditions.Get(capiMachine, clusterv1.PausedCondition) @@ -225,172 +256,100 @@ func (r *MachineMigrationReconciler) isOldAuthoritativeResourcePaused(ctx contex return false, nil } - infraMachineRef := capiMachine.Spec.InfrastructureRef + return machinePausedCondition.Status == metav1.ConditionTrue, nil +} - infraMachine, err := external.GetObjectFromContractVersionedRef(ctx, r.Client, infraMachineRef, capiMachine.Namespace) +func (r *MachineMigrationReconciler) ensureCAPIInfraMachinePaused(ctx context.Context, infraMachine client.Object) (bool, error) { + changed, err := migrationcommon.AddPausedAnnotation(ctx, r.Client, infraMachine) if err != nil { - return false, fmt.Errorf("failed to get Cluster API infra machine: %w", err) + return false, fmt.Errorf("failed to request pause on Cluster API infra machine: %w", err) } - infraMachinePausedConditionStatus, err := util.GetConditionStatus(infraMachine, clusterv1.PausedCondition) - if err != nil { - return false, fmt.Errorf("unable to get paused condition for %s/%s: %w", infraMachine.GetNamespace(), infraMachine.GetName(), err) + if changed { + return false, nil } - return (machinePausedCondition.Status == metav1.ConditionTrue) && (infraMachinePausedConditionStatus == corev1.ConditionTrue), nil -} - -func (r *MachineMigrationReconciler) ensureUnpauseRequestedOnNewAuthoritativeResource(ctx context.Context, mapiMachine *mapiv1beta1.Machine) error { - // Request that the new authoritative resource reconciliation is un-paused. - //nolint:wsl - switch mapiMachine.Spec.AuthoritativeAPI { - case mapiv1beta1.MachineAuthorityClusterAPI: - // For requesting unpausing of a CAPI resource, remove the paused annotation on it. - // So check if the ClusterAPI resource has the paused annotation and if so remove it. - capiMachine := &clusterv1.Machine{} - if err := r.Get(ctx, client.ObjectKey{Namespace: r.CAPINamespace, Name: mapiMachine.Name}, capiMachine); err != nil { - return fmt.Errorf("failed to get Cluster API machine: %w", err) - } - - infraMachineRef := capiMachine.Spec.InfrastructureRef - - infraMachine, err := external.GetObjectFromContractVersionedRef(ctx, r.Client, infraMachineRef, capiMachine.Namespace) - if err != nil { - return fmt.Errorf("failed to get Cluster API infra machine: %w", err) - } - - if annotations.HasPaused(capiMachine) { - capiMachineCopy := capiMachine.DeepCopy() - delete(capiMachine.Annotations, clusterv1.PausedAnnotation) - - if err := r.Patch(ctx, capiMachine, client.MergeFrom(capiMachineCopy)); err != nil { - return fmt.Errorf("failed to patch Cluster API machine: %w", err) - } - } - - if annotations.HasPaused(infraMachine) { - infraMachineCopy, ok := infraMachine.DeepCopyObject().(client.Object) - if !ok { - return fmt.Errorf("unable to assert Cluster API infra machine as client.Object: %w", err) - } + finalizer, err := capiInfraMachineFinalizerForPlatform(r.Platform) + if err != nil { + return false, err + } - util.RemoveAnnotation(infraMachine, clusterv1.PausedAnnotation) + // If the finalizer is present we know that the controller is running. It will observe the paused annotation and will eventually set the paused condition. We must wait for the paused condition because it may be actively reconciling. + // If the finalizer is not present then either: + // The controller is not running, in which it is safe to continue. + // The controller has not yet observed the object, in which case (guaranteed by optimistic locking) it will observe our paused annotation before taking any action, so it is safe to continue. + if !slices.Contains(infraMachine.GetFinalizers(), finalizer) { + return true, nil + } - if err := r.Patch(ctx, infraMachine, client.MergeFrom(infraMachineCopy)); err != nil { - return fmt.Errorf("failed to patch Cluster API infra machine: %w", err) - } - } - case mapiv1beta1.MachineAuthorityMachineAPI: - // For requesting unpausing of a MAPI resource, it is sufficient to switch the spec.AuthoritativeAPI field on the MAPI resource. - // which is already done before this code runs in this controller. Nothing to do here. - case mapiv1beta1.MachineAuthorityMigrating: - // Value is disallowed by the openAPI schema validation. + pausedStatus, err := util.GetConditionStatus(infraMachine, clusterv1.PausedCondition) + if err != nil { + return false, fmt.Errorf("unable to get paused condition for %s/%s: %w", infraMachine.GetNamespace(), infraMachine.GetName(), err) } - return nil + return pausedStatus == corev1.ConditionTrue, nil } -// requestOldAuthoritativeResourcePaused requests the old authoritative resource is paused. -func (r *MachineMigrationReconciler) requestOldAuthoritativeResourcePaused(ctx context.Context, m *mapiv1beta1.Machine) (bool, error) { - // Request that the old authoritative resource reconciliation is paused. - updated := false - //nolint:wsl - switch m.Spec.AuthoritativeAPI { - case mapiv1beta1.MachineAuthorityClusterAPI: - // For requesting pausing of a MAPI resource, it is sufficient to switch the spec.AuthoritativeAPI field on the MAPI resource. - // which is already done before this code runs in this controller. - case mapiv1beta1.MachineAuthorityMachineAPI: - // For requesting pausing of a CAPI resource, set the paused annotation on it. - // The spec.AuthoritativeAPI is set to MachineAPI, meaning that the old authoritativeAPI was ClusterAPI. - // So Check if the ClusterAPI resource has the paused annotation, otherwise set it. - capiMachine := &clusterv1.Machine{} - if err := r.Get(ctx, client.ObjectKey{Namespace: r.CAPINamespace, Name: m.Name}, capiMachine); err != nil { - return false, fmt.Errorf("failed to get Cluster API machine: %w", err) - } - - infraMachineRef := capiMachine.Spec.InfrastructureRef - - infraMachine, err := external.GetObjectFromContractVersionedRef(ctx, r.Client, infraMachineRef, capiMachine.Namespace) - if err != nil { - return false, fmt.Errorf("failed to get Cluster API infra machine: %w", err) - } +func (r *MachineMigrationReconciler) ensureCAPIUnpaused(ctx context.Context, capiMachine *clusterv1.Machine) (bool, error) { + changed, err := migrationcommon.RemovePausedAnnotation(ctx, r.Client, capiMachine) + if err != nil { + return false, fmt.Errorf("failed to remove paused annotation from Cluster API machine: %w", err) + } - if !annotations.HasPaused(capiMachine) { - capiMachineCopy := capiMachine.DeepCopy() - annotations.AddAnnotations(capiMachine, map[string]string{clusterv1.PausedAnnotation: ""}) + if changed { + return false, nil + } - if err := r.Patch(ctx, capiMachine, client.MergeFrom(capiMachineCopy)); err != nil { - return false, fmt.Errorf("failed to patch Cluster API machine: %w", err) - } + infraMachine, found, err := r.getCAPIInfraMachine(ctx, capiMachine) + if err != nil { + return false, err + } - updated = true + if found { + changed, err = migrationcommon.RemovePausedAnnotation(ctx, r.Client, infraMachine) + if err != nil { + return false, fmt.Errorf("failed to remove paused annotation from Cluster API infra machine: %w", err) } - if !annotations.HasPaused(infraMachine) { - infraMachineCopy, ok := infraMachine.DeepCopyObject().(client.Object) - if !ok { - return false, fmt.Errorf("unable to assert Cluster API infra machine as client.Object: %w", err) - } - - annotations.AddAnnotations(infraMachine, map[string]string{clusterv1.PausedAnnotation: ""}) - - if err := r.Patch(ctx, infraMachine, client.MergeFrom(infraMachineCopy)); err != nil { - return false, fmt.Errorf("failed to patch Cluster API infra machine: %w", err) - } - - updated = true + if changed { + return false, nil } - case mapiv1beta1.MachineAuthorityMigrating: - // Value is disallowed by the openAPI schema validation. } - return updated, nil -} - -func (r *MachineMigrationReconciler) isSynchronized(ctx context.Context, mapiMachine *mapiv1beta1.Machine) (bool, error) { - // Check if the Synchronized condition is set to True. - // If it is not, this indicates an unmigratable resource and therefore should take no action. - if cond, err := util.GetConditionStatus(mapiMachine, string(controllers.SynchronizedCondition)); err != nil { - return false, fmt.Errorf("unable to get synchronized condition for %s/%s: %w", mapiMachine.Namespace, mapiMachine.Name, err) - } else if cond != corev1.ConditionTrue { + machinePausedCondition := conditions.Get(capiMachine, clusterv1.PausedCondition) + if machinePausedCondition != nil && machinePausedCondition.Status == metav1.ConditionTrue { return false, nil } - // Because we are in a migration (spec.authoritativeAPI != - // status.authoritativeAPI), we assume that spec.authoritativeAPI is - // currently the migration target, not the migration source. So: - // - // target: spec.authoritativeAPI - // source: opposite of target - // - // We want to assert that source has been synched to target, so we need to - // treat spec.AuthoritativeAPI as the opposite of the direction we want to - // check. - // - // We may revisit this, as this assumption is not safe if a user aborts an - // in-progress migration by resetting spec.authoritativeAPI to its original - // value. - - switch mapiMachine.Spec.AuthoritativeAPI { - case mapiv1beta1.MachineAuthorityClusterAPI: - return mapiMachine.Status.SynchronizedGeneration == mapiMachine.Generation, nil - case mapiv1beta1.MachineAuthorityMachineAPI: - capiMachine := &clusterv1.Machine{} - if err := r.Get(ctx, client.ObjectKey{Namespace: r.CAPINamespace, Name: mapiMachine.Name}, capiMachine); err != nil { - return false, fmt.Errorf("failed to get Cluster API machine: %w", err) - } + if !found { + return true, nil + } - // Given the CAPI infra machine template is immutable - // we do not check for its generation to be synced up with the generation of the MAPI machine set. - return (mapiMachine.Status.SynchronizedGeneration == capiMachine.Generation), nil - case mapiv1beta1.MachineAuthorityMigrating: + infraPausedStatus, err := util.GetConditionStatus(infraMachine, clusterv1.PausedCondition) + if err != nil { + return false, fmt.Errorf("unable to get paused condition for %s/%s: %w", infraMachine.GetNamespace(), infraMachine.GetName(), err) } - // Should have been prevented by validation - return false, fmt.Errorf("%w: %s", controllers.ErrInvalidSpecAuthoritativeAPI, mapiMachine.Spec.AuthoritativeAPI) + return infraPausedStatus != corev1.ConditionTrue, nil } -// applyStatusAuthoritativeAPIWithPatch updates the resource status.authoritativeAPI using a server-side apply patch. -func (r *MachineMigrationReconciler) applyStatusAuthoritativeAPIWithPatch(ctx context.Context, m *mapiv1beta1.Machine, authority mapiv1beta1.MachineAuthority) error { - return synccommon.ApplyAuthoritativeAPI[*machinev1applyconfigs.MachineStatusApplyConfiguration](ctx, r.Client, controllerName, machinev1applyconfigs.Machine, m, authority) +func capiInfraMachineFinalizerForPlatform(platform configv1.PlatformType) (string, error) { + switch platform { + case configv1.AWSPlatformType: + return awsv1.MachineFinalizer, nil + case configv1.AzurePlatformType: + return azurev1.MachineFinalizer, nil + case configv1.GCPPlatformType: + return gcpv1.MachineFinalizer, nil + case configv1.PowerVSPlatformType: + return ibmpowervsv1.IBMPowerVSMachineFinalizer, nil + case configv1.VSpherePlatformType: + return vspherev1.MachineFinalizer, nil + case configv1.OpenStackPlatformType: + return openstackv1.MachineFinalizer, nil + case configv1.BareMetalPlatformType: + return metal3v1.MachineFinalizer, nil + default: + return "", fmt.Errorf("%w: %s", util.ErrUnsupportedPlatform, platform) + } } diff --git a/pkg/controllers/machinemigration/machine_migration_controller_test.go b/pkg/controllers/machinemigration/machine_migration_controller_test.go index 02cc71954..0896e20e7 100644 --- a/pkg/controllers/machinemigration/machine_migration_controller_test.go +++ b/pkg/controllers/machinemigration/machine_migration_controller_test.go @@ -19,6 +19,7 @@ package machinemigration import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + ctrl "sigs.k8s.io/controller-runtime" mapiv1beta1 "github.com/openshift/api/machine/v1beta1" capiv1resourcebuilder "github.com/openshift/cluster-api-actuator-pkg/testutils/resourcebuilder/cluster-api/core/v1beta2" @@ -26,6 +27,8 @@ import ( corev1resourcebuilder "github.com/openshift/cluster-api-actuator-pkg/testutils/resourcebuilder/core/v1" machinev1resourcebuilder "github.com/openshift/cluster-api-actuator-pkg/testutils/resourcebuilder/machine/v1beta1" corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" awsv1 "sigs.k8s.io/cluster-api-provider-aws/v2/api/v1beta2" clusterv1beta1 "sigs.k8s.io/cluster-api/api/core/v1beta1" @@ -33,16 +36,14 @@ import ( configv1 "github.com/openshift/api/config/v1" "github.com/openshift/cluster-api-actuator-pkg/testutils" - consts "github.com/openshift/cluster-capi-operator/pkg/controllers" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "github.com/openshift/cluster-capi-operator/pkg/controllers/migrationcommon" + migrationcontrollertest "github.com/openshift/cluster-capi-operator/pkg/controllers/migrationcommon/controllertest" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/envtest/komega" "sigs.k8s.io/controller-runtime/pkg/reconcile" ) -var _ = Describe("With a running MachineMigration controller", func() { +var _ = Describe("MachineMigration controller", func() { var ( k komega.Komega reconciler *MachineMigrationReconciler @@ -62,6 +63,69 @@ var _ = Describe("With a running MachineMigration controller", func() { capiCluster *clusterv1.Cluster ) + capaPausedCondition := func(status corev1.ConditionStatus) clusterv1beta1.Condition { + return clusterv1beta1.Condition{ + Type: clusterv1beta1.PausedV1Beta2Condition, + Status: status, + LastTransitionTime: metav1.Now(), + } + } + + createCAPIMachinePair := func() { + GinkgoHelper() + + capiMachine = capiMachineBuilder.Build() + Eventually(k8sClient.Create(ctx, capiMachine)).Should(Succeed(), "CAPI machine should be able to be created") + + capaMachine = capaMachineBuilder.Build() + Eventually(k8sClient.Create(ctx, capaMachine)).Should(Succeed(), "CAPI infra machine should be able to be created") + } + + updateMAPIMachineStatus := func(authority mapiv1beta1.MachineAuthority, synchronizedAPI mapiv1beta1.SynchronizedAPI, synchronizedGeneration int64, conditions ...mapiv1beta1.Condition) { + GinkgoHelper() + + Eventually(k.UpdateStatus(mapiMachine, func() { + mapiMachine.Status.AuthoritativeAPI = authority + mapiMachine.Status.SynchronizedAPI = synchronizedAPI + mapiMachine.Status.SynchronizedGeneration = synchronizedGeneration + mapiMachine.Status.Conditions = conditions + })).Should(Succeed()) + } + + updateCAPIMachineStatus := func(conditions ...metav1.Condition) { + GinkgoHelper() + + Eventually(k.UpdateStatus(capiMachine, func() { + capiMachine.Status.Conditions = conditions + })).Should(Succeed()) + } + + updateCAPAStatus := func(conditions ...clusterv1beta1.Condition) { + GinkgoHelper() + + Eventually(k.UpdateStatus(capaMachine, func() { + capaMachine.Status.Conditions = conditions + })).Should(Succeed()) + } + + reconcileOnce := func() (ctrl.Result, error) { + GinkgoHelper() + + return reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(mapiMachine)}) + } + + expectSyncStatusReset := func(authority mapiv1beta1.MachineAuthority) { + GinkgoHelper() + + migrationcontrollertest.ExpectSyncStatusReset(k, mapiMachine, authority) + } + + expectSuccessfulReconcile := func() { + GinkgoHelper() + + migrationcontrollertest.ExpectSuccessfulReconcile(reconcileOnce) + } + BeforeEach(func() { By("Setting up namespaces for the test") @@ -100,12 +164,10 @@ var _ = Describe("With a running MachineMigration controller", func() { WithNamespace(capiNamespace.GetName()). WithName("machine-template") - capaMachine = capaMachineBuilder.Build() - capaMachineRef := clusterv1.ContractVersionedObjectReference{ APIGroup: awsv1.GroupVersion.Group, - Kind: capaMachine.Kind, - Name: capaMachine.GetName(), + Kind: "AWSMachine", + Name: "machine-template", } capiMachineBuilder = capiv1resourcebuilder.Machine(). @@ -117,6 +179,7 @@ var _ = Describe("With a running MachineMigration controller", func() { reconciler = &MachineMigrationReconciler{ Client: k8sClient, Scheme: testEnv.Scheme, + Platform: configv1.AWSPlatformType, CAPINamespace: capiNamespace.GetName(), MAPINamespace: mapiNamespace.GetName(), } @@ -141,640 +204,530 @@ var _ = Describe("With a running MachineMigration controller", func() { }) Describe("Reconcile", func() { - var req reconcile.Request - - Context("when no migration is requested (status equals spec)", func() { + Context("when no migration is requested and MachineAPI is already authoritative", func() { BeforeEach(func() { - By("Setting the MAPI machine spec AuthoritativeAPI to MachineAPI") - mapiMachine = mapiMachineBuilder. WithAuthoritativeAPI(mapiv1beta1.MachineAuthorityMachineAPI). Build() Eventually(k8sClient.Create(ctx, mapiMachine)).Should(Succeed()) - By("Setting the MAPI machine status AuthoritativeAPI to MachineAPI") - Eventually(k.UpdateStatus(mapiMachine, func() { - mapiMachine.Status.AuthoritativeAPI = mapiv1beta1.MachineAuthorityMachineAPI - })).Should(Succeed()) - - req = reconcile.Request{NamespacedName: client.ObjectKeyFromObject(mapiMachine)} + updateMAPIMachineStatus( + mapiv1beta1.MachineAuthorityMachineAPI, + mapiv1beta1.MachineAPISynchronized, + mapiMachine.Generation, + migrationcontrollertest.SynchronizedCondition(corev1.ConditionTrue), + ) }) It("should do nothing", func() { - initialMAPIMachineRV := mapiMachine.ResourceVersion - _, err := reconciler.Reconcile(ctx, req) - Expect(err).NotTo(HaveOccurred(), "reconciler should not have errored") - Eventually(k.Object(mapiMachine)).Should(HaveField("ObjectMeta.ResourceVersion", Equal(initialMAPIMachineRV)), "should not have modified the machine") + current := &mapiv1beta1.Machine{} + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(mapiMachine), current)).To(Succeed()) + initialResourceVersion := current.ResourceVersion + + expectSuccessfulReconcile() + + Eventually(k.Object(mapiMachine)).Should( + HaveField("ObjectMeta.ResourceVersion", Equal(initialResourceVersion)), + ) }) }) - Context("when status.AuthoritativeAPI is empty (first observation)", func() { + Context("when status.AuthoritativeAPI is empty", func() { BeforeEach(func() { - By("Setting the MAPI machine spec AuthoritativeAPI to MachineAPI") + mapiMachine = mapiMachineBuilder. + WithAuthoritativeAPI(mapiv1beta1.MachineAuthorityClusterAPI). + Build() + Eventually(k8sClient.Create(ctx, mapiMachine)).Should(Succeed()) + }) + + It("should patch the status to match spec", func() { + expectSuccessfulReconcile() + + Eventually(k.Object(mapiMachine)).Should( + HaveField("Status.AuthoritativeAPI", Equal(mapiv1beta1.MachineAuthorityClusterAPI)), + ) + }) + }) + Context("when migrating from MachineAPI to ClusterAPI and the stable sync gate is not satisfied", func() { + BeforeEach(func() { mapiMachine = mapiMachineBuilder. - WithAuthoritativeAPI(mapiv1beta1.MachineAuthorityMachineAPI). + WithAuthoritativeAPI(mapiv1beta1.MachineAuthorityClusterAPI). Build() Eventually(k8sClient.Create(ctx, mapiMachine)).Should(Succeed()) - By("Leaving the MAPI machine status AuthoritativeAPI empty") + updateMAPIMachineStatus( + mapiv1beta1.MachineAuthorityMachineAPI, + mapiv1beta1.MachineAPISynchronized, + mapiMachine.Generation, + migrationcontrollertest.SynchronizedCondition(corev1.ConditionFalse), + ) + }) + + It("should wait without acknowledging the migration", func() { + expectSuccessfulReconcile() + + Eventually(k.Object(mapiMachine)).Should( + HaveField("Status.AuthoritativeAPI", Equal(mapiv1beta1.MachineAuthorityMachineAPI)), + ) + }) + }) + + Context("when migrating from MachineAPI to ClusterAPI and status.SynchronizedAPI is empty", func() { + BeforeEach(func() { + mapiMachine = mapiMachineBuilder. + WithAuthoritativeAPI(mapiv1beta1.MachineAuthorityClusterAPI). + Build() + Eventually(k8sClient.Create(ctx, mapiMachine)).Should(Succeed()) - req = reconcile.Request{NamespacedName: client.ObjectKeyFromObject(mapiMachine)} + updateMAPIMachineStatus( + mapiv1beta1.MachineAuthorityMachineAPI, + "", + mapiMachine.Generation, + migrationcontrollertest.SynchronizedCondition(corev1.ConditionTrue), + ) }) - It("should patch the status to match spec and requeue", func() { - By("Running one reconciliation") + It("should wait without acknowledging the migration", func() { + current := &mapiv1beta1.Machine{} + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(mapiMachine), current)).To(Succeed()) + initialResourceVersion := current.ResourceVersion - _, err := reconciler.Reconcile(ctx, req) - Expect(err).NotTo(HaveOccurred(), "reconciler should not have errored") + expectSuccessfulReconcile() - updatedM := &mapiv1beta1.Machine{} - Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(mapiMachine), updatedM)).To(Succeed()) - Expect(updatedM.Status.AuthoritativeAPI).To(Equal(updatedM.Spec.AuthoritativeAPI)) + Eventually(k.Object(mapiMachine)).Should(SatisfyAll( + HaveField("ObjectMeta.ResourceVersion", Equal(initialResourceVersion)), + HaveField("Status.AuthoritativeAPI", Equal(mapiv1beta1.MachineAuthorityMachineAPI)), + HaveField("Status.SynchronizedAPI", BeEmpty()), + )) }) }) - Context("when the Synchronized condition is not True", func() { + Context("when migrating from MachineAPI to ClusterAPI and the stable sync gate is satisfied", func() { BeforeEach(func() { - By("Setting the MAPI machine spec AuthoritativeAPI to ClusterAPI") + mapiMachine = mapiMachineBuilder. + WithAuthoritativeAPI(mapiv1beta1.MachineAuthorityClusterAPI). + Build() + Eventually(k8sClient.Create(ctx, mapiMachine)).Should(Succeed()) + + updateMAPIMachineStatus( + mapiv1beta1.MachineAuthorityMachineAPI, + mapiv1beta1.MachineAPISynchronized, + mapiMachine.Generation, + migrationcontrollertest.SynchronizedCondition(corev1.ConditionTrue), + ) + }) + It("should patch status to Migrating", func() { + expectSuccessfulReconcile() + + Eventually(k.Object(mapiMachine)).Should( + HaveField("Status.AuthoritativeAPI", Equal(mapiv1beta1.MachineAuthorityMigrating)), + ) + }) + }) + + Context("when spec.AuthoritativeAPI is ClusterAPI and status.AuthoritativeAPI is Migrating", func() { + BeforeEach(func() { mapiMachine = mapiMachineBuilder. WithAuthoritativeAPI(mapiv1beta1.MachineAuthorityClusterAPI). Build() Eventually(k8sClient.Create(ctx, mapiMachine)).Should(Succeed()) + }) - By("Setting the MAPI machine status AuthoritativeAPI to MachineAPI") - Eventually(k.UpdateStatus(mapiMachine, func() { - updatedMAPIMachine := mapiMachineBuilder. - WithAuthoritativeAPIStatus(mapiv1beta1.MachineAuthorityMachineAPI). - WithConditions([]mapiv1beta1.Condition{{ - Type: consts.SynchronizedCondition, - LastTransitionTime: metav1.Now(), - Status: corev1.ConditionFalse}}). - Build() - mapiMachine.Status.AuthoritativeAPI = updatedMAPIMachine.Status.AuthoritativeAPI - mapiMachine.Status.Conditions = updatedMAPIMachine.Status.Conditions - })).Should(Succeed()) + Context("when Machine API is not paused yet", func() { + BeforeEach(func() { + updateMAPIMachineStatus( + mapiv1beta1.MachineAuthorityMigrating, + mapiv1beta1.MachineAPISynchronized, + mapiMachine.Generation, + migrationcontrollertest.SynchronizedCondition(corev1.ConditionTrue), + migrationcontrollertest.MAPIPausedCondition(corev1.ConditionFalse), + ) + }) - req = reconcile.Request{NamespacedName: client.ObjectKeyFromObject(mapiMachine)} + It("should keep waiting in Migrating", func() { + expectSuccessfulReconcile() + + Eventually(k.Object(mapiMachine)).Should(SatisfyAll( + HaveField("Status.AuthoritativeAPI", Equal(mapiv1beta1.MachineAuthorityMigrating)), + HaveField("Status.SynchronizedGeneration", Equal(mapiMachine.Generation)), + )) + }) }) - It("should do nothing", func() { - initialMAPIMachineRV := mapiMachine.ResourceVersion - _, err := reconciler.Reconcile(ctx, req) - Expect(err).NotTo(HaveOccurred(), "reconciler should not have errored") - Eventually(k.Object(mapiMachine)).Should(HaveField("ObjectMeta.ResourceVersion", Equal(initialMAPIMachineRV)), "should not have modified the machine") + Context("when Machine API is paused", func() { + BeforeEach(func() { + updateMAPIMachineStatus( + mapiv1beta1.MachineAuthorityMigrating, + mapiv1beta1.MachineAPISynchronized, + mapiMachine.Generation, + migrationcontrollertest.SynchronizedCondition(corev1.ConditionTrue), + migrationcontrollertest.MAPIPausedCondition(corev1.ConditionTrue), + ) + }) + + It("should complete the switch to ClusterAPI and reset sync status", func() { + expectSuccessfulReconcile() + + expectSyncStatusReset(mapiv1beta1.MachineAuthorityClusterAPI) + }) }) }) - Context("when a migration request is first detected", func() { + Context("when ClusterAPI is authoritative but the CAPI infra machine is still paused", func() { BeforeEach(func() { - By("Setting the MAPI machine spec AuthoritativeAPI to ClusterAPI") - mapiMachine = mapiMachineBuilder. WithAuthoritativeAPI(mapiv1beta1.MachineAuthorityClusterAPI). Build() Eventually(k8sClient.Create(ctx, mapiMachine)).Should(Succeed()) - By("Creating a mirror CAPI machine") - capiMachine = capiMachineBuilder.Build() Eventually(k8sClient.Create(ctx, capiMachine)).Should(Succeed()) - capaMachine = capaMachineBuilder.Build() + capaMachine = capaMachineBuilder. + WithAnnotations(map[string]string{clusterv1.PausedAnnotation: ""}). + Build() Eventually(k8sClient.Create(ctx, capaMachine)).Should(Succeed()) - By("Setting the MAPI machine status AuthoritativeAPI to MachineAPI") - Eventually(k.UpdateStatus(mapiMachine, func() { - updatedMAPIMachine := mapiMachineBuilder. - WithAuthoritativeAPIStatus(mapiv1beta1.MachineAuthorityMachineAPI). - WithConditions([]mapiv1beta1.Condition{{ - Type: consts.SynchronizedCondition, - LastTransitionTime: metav1.Now(), - Status: corev1.ConditionTrue}}). - Build() - mapiMachine.Status.AuthoritativeAPI = updatedMAPIMachine.Status.AuthoritativeAPI - mapiMachine.Status.Conditions = updatedMAPIMachine.Status.Conditions - mapiMachine.Status.SynchronizedGeneration = capiMachine.Generation - })).Should(Succeed()) + updateMAPIMachineStatus( + mapiv1beta1.MachineAuthorityClusterAPI, + mapiv1beta1.ClusterAPISynchronized, + capiMachine.Generation, + migrationcontrollertest.SynchronizedCondition(corev1.ConditionTrue), + ) + }) + + It("should remove the paused annotation from the CAPI infra machine", func() { + expectSuccessfulReconcile() - req = reconcile.Request{NamespacedName: client.ObjectKeyFromObject(mapiMachine)} + Eventually(k.Object(capaMachine)).ShouldNot( + HaveField("ObjectMeta.Annotations", HaveKey(clusterv1.PausedAnnotation)), + ) + Eventually(k.Object(mapiMachine)).Should( + HaveField("Status.AuthoritativeAPI", Equal(mapiv1beta1.MachineAuthorityClusterAPI)), + ) }) + }) - It("should acknowledge the migration by updating status to 'Migrating' and requeuing", func() { - _, err := reconciler.Reconcile(ctx, req) - Expect(err).NotTo(HaveOccurred()) + Context("when migrating from ClusterAPI to MachineAPI and the CAPI pause request has not been written yet", func() { + BeforeEach(func() { + mapiMachine = mapiMachineBuilder. + WithAuthoritativeAPI(mapiv1beta1.MachineAuthorityMachineAPI). + Build() + Eventually(k8sClient.Create(ctx, mapiMachine)).Should(Succeed()) + + createCAPIMachinePair() + + updateMAPIMachineStatus( + mapiv1beta1.MachineAuthorityClusterAPI, + mapiv1beta1.ClusterAPISynchronized, + capiMachine.Generation, + migrationcontrollertest.SynchronizedCondition(corev1.ConditionTrue), + ) + }) - updatedM := &mapiv1beta1.Machine{} - Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(mapiMachine), updatedM)).To(Succeed()) - Expect(updatedM.Status.AuthoritativeAPI).To(Equal(mapiv1beta1.MachineAuthorityMigrating)) + It("should pause the CAPI machine before entering Migrating", func() { + expectSuccessfulReconcile() + + Eventually(k.Object(capiMachine)).Should( + HaveField("ObjectMeta.Annotations", HaveKeyWithValue(clusterv1.PausedAnnotation, "")), + ) + Eventually(k.Object(capaMachine)).ShouldNot( + HaveField("ObjectMeta.Annotations", HaveKey(clusterv1.PausedAnnotation)), + ) + Eventually(k.Object(mapiMachine)).Should( + HaveField("Status.AuthoritativeAPI", Equal(mapiv1beta1.MachineAuthorityClusterAPI)), + ) }) }) - Context("when the resource migration has been acknowledged (resource status migrating)", func() { - Context("when migrating from MachineAPI to ClusterAPI", func() { - BeforeEach(func() { - By("Setting the MAPI machine spec AuthoritativeAPI to ClusterAPI") - - mapiMachine = mapiMachineBuilder. - WithAuthoritativeAPI(mapiv1beta1.MachineAuthorityClusterAPI). - Build() - Eventually(k8sClient.Create(ctx, mapiMachine)).Should(Succeed()) - - By("Creating a mirror CAPI machine") - - capiMachine = capiMachineBuilder.Build() - Eventually(k8sClient.Create(ctx, capiMachine)).Should(Succeed()) - - capaMachine = capaMachineBuilder.Build() - Eventually(k8sClient.Create(ctx, capaMachine)).Should(Succeed()) - - By("Setting the MAPI machine status AuthoritativeAPI to 'Migrating'") - Eventually(k.UpdateStatus(mapiMachine, func() { - updatedMAPIMachine := mapiMachineBuilder. - WithAuthoritativeAPIStatus(mapiv1beta1.MachineAuthorityMigrating). - WithConditions([]mapiv1beta1.Condition{{ - Type: consts.SynchronizedCondition, - LastTransitionTime: metav1.Now(), - Status: corev1.ConditionTrue}}). - Build() - mapiMachine.Status.AuthoritativeAPI = updatedMAPIMachine.Status.AuthoritativeAPI - mapiMachine.Status.Conditions = updatedMAPIMachine.Status.Conditions - mapiMachine.Status.SynchronizedGeneration = capiMachine.Generation - })).Should(Succeed()) - - req = reconcile.Request{NamespacedName: client.ObjectKeyFromObject(mapiMachine)} - }) + Context("when migrating from ClusterAPI to MachineAPI and status.SynchronizedAPI points at MachineAPI", func() { + BeforeEach(func() { + mapiMachine = mapiMachineBuilder. + WithAuthoritativeAPI(mapiv1beta1.MachineAuthorityMachineAPI). + Build() + Eventually(k8sClient.Create(ctx, mapiMachine)).Should(Succeed()) - It("should request pausing for the old authoritative resource (MAPI) and stay in Migrating status", func() { - _, err := reconciler.Reconcile(ctx, req) - Expect(err).NotTo(HaveOccurred()) - - updatedM := &mapiv1beta1.Machine{} - Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(mapiMachine), updatedM)).To(Succeed()) - // To check for requesting pausing on the the MAPI resource it is sufficient - // see that the spec.AuthoritativeAPI field is set to ClusterAPI, - // which is already done anyway on the requester side. - Expect(updatedM.Spec.AuthoritativeAPI).To(Equal(mapiv1beta1.MachineAuthorityClusterAPI)) - Expect(updatedM.Status.AuthoritativeAPI).To(Equal(mapiv1beta1.MachineAuthorityMigrating)) - }) + capiMachine = capiMachineBuilder. + WithAnnotations(map[string]string{clusterv1.PausedAnnotation: ""}). + Build() + Eventually(k8sClient.Create(ctx, capiMachine)).Should(Succeed()) + + capaMachine = capaMachineBuilder. + WithAnnotations(map[string]string{clusterv1.PausedAnnotation: ""}). + Build() + Eventually(k8sClient.Create(ctx, capaMachine)).Should(Succeed()) + + updateMAPIMachineStatus( + mapiv1beta1.MachineAuthorityClusterAPI, + mapiv1beta1.MachineAPISynchronized, + capiMachine.Generation, + migrationcontrollertest.SynchronizedCondition(corev1.ConditionTrue), + ) }) - Context("when migrating from ClusterAPI to MachineAPI", func() { - BeforeEach(func() { - By("Setting the MAPI machine spec AuthoritativeAPI to MachineAPI") - - mapiMachine = mapiMachineBuilder.WithAuthoritativeAPI(mapiv1beta1.MachineAuthorityMachineAPI).Build() - Eventually(k8sClient.Create(ctx, mapiMachine)).Should(Succeed()) - - By("Creating a mirror CAPI machine") - - capiMachine = capiMachineBuilder.Build() - Eventually(k8sClient.Create(ctx, capiMachine)).Should(Succeed()) - - capaMachine = capaMachineBuilder.Build() - Eventually(k8sClient.Create(ctx, capaMachine)).Should(Succeed()) - - By("Setting the MAPI machine status AuthoritativeAPI to 'Migrating'") - Eventually(k.UpdateStatus(mapiMachine, func() { - updatedMAPIMachine := mapiMachineBuilder. - WithAuthoritativeAPIStatus(mapiv1beta1.MachineAuthorityMigrating). - WithConditions([]mapiv1beta1.Condition{{ - Type: consts.SynchronizedCondition, - LastTransitionTime: metav1.Now(), - Status: corev1.ConditionTrue}}). - Build() - mapiMachine.Status.AuthoritativeAPI = updatedMAPIMachine.Status.AuthoritativeAPI - mapiMachine.Status.Conditions = updatedMAPIMachine.Status.Conditions - mapiMachine.Status.SynchronizedGeneration = mapiMachine.Generation - })).Should(Succeed()) - - req = reconcile.Request{NamespacedName: client.ObjectKeyFromObject(mapiMachine)} - }) - It("should request pausing for the old authoritative resource (CAPI) and stay in Migrating status", func() { - _, err := reconciler.Reconcile(ctx, req) - Expect(err).NotTo(HaveOccurred()) + It("should wait without entering Migrating", func() { + current := &mapiv1beta1.Machine{} + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(mapiMachine), current)).To(Succeed()) + initialResourceVersion := current.ResourceVersion - updatedM := &mapiv1beta1.Machine{} - Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(mapiMachine), updatedM)).To(Succeed()) - Expect(updatedM.Status.AuthoritativeAPI).To(Equal(mapiv1beta1.MachineAuthorityMigrating)) + expectSuccessfulReconcile() - updatedCAPIM := &clusterv1.Machine{} - Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(capiMachine), updatedCAPIM)).To(Succeed()) - Expect(updatedCAPIM.Annotations).To(HaveKeyWithValue(clusterv1.PausedAnnotation, "")) + Eventually(k.Object(mapiMachine)).Should(SatisfyAll( + HaveField("ObjectMeta.ResourceVersion", Equal(initialResourceVersion)), + HaveField("Status.AuthoritativeAPI", Equal(mapiv1beta1.MachineAuthorityClusterAPI)), + HaveField("Status.SynchronizedAPI", Equal(mapiv1beta1.MachineAPISynchronized)), + )) + }) + }) - updatedCAPIInfraMachine := &awsv1.AWSMachine{} - Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(capaMachine), updatedCAPIInfraMachine)).To(Succeed()) - Expect(updatedCAPIInfraMachine.Annotations).To(HaveKeyWithValue(clusterv1.PausedAnnotation, "")) - }) + Context("when MachineAPI is authoritative but the CAPI infra machine is not paused", func() { + BeforeEach(func() { + mapiMachine = mapiMachineBuilder. + WithAuthoritativeAPI(mapiv1beta1.MachineAuthorityMachineAPI). + Build() + Eventually(k8sClient.Create(ctx, mapiMachine)).Should(Succeed()) + + capiMachine = capiMachineBuilder. + WithAnnotations(map[string]string{clusterv1.PausedAnnotation: ""}). + Build() + Eventually(k8sClient.Create(ctx, capiMachine)).Should(Succeed()) + + capaMachine = capaMachineBuilder.Build() + Eventually(k8sClient.Create(ctx, capaMachine)).Should(Succeed()) + + updateMAPIMachineStatus( + mapiv1beta1.MachineAuthorityMachineAPI, + mapiv1beta1.MachineAPISynchronized, + mapiMachine.Generation, + migrationcontrollertest.SynchronizedCondition(corev1.ConditionTrue), + ) + }) + + It("should add the paused annotation to the CAPI infra machine", func() { + expectSuccessfulReconcile() + + Eventually(k.Object(capaMachine)).Should( + HaveField("ObjectMeta.Annotations", HaveKeyWithValue(clusterv1.PausedAnnotation, "")), + ) + Eventually(k.Object(mapiMachine)).Should( + HaveField("Status.AuthoritativeAPI", Equal(mapiv1beta1.MachineAuthorityMachineAPI)), + ) }) }) - Context("when the old authoritative resource pausing has been requested", func() { - Context("when migrating from MachineAPI to ClusterAPI", func() { - Context("when status is not paused for the old authoritative resource (MAPI)", func() { - BeforeEach(func() { - By("Setting the MAPI machine spec AuthoritativeAPI to ClusterAPI") - - mapiMachine = mapiMachineBuilder. - // Set desired authoritative API in spec to ClusterAPI. - // To check for requesting pausing on the the MAPI resource it is sufficient - // see that the spec.AuthoritativeAPI field is set to ClusterAPI. - WithAuthoritativeAPI(mapiv1beta1.MachineAuthorityClusterAPI). - Build() - Eventually(k8sClient.Create(ctx, mapiMachine)).Should(Succeed()) - - By("Creating a mirror CAPI machine") - - capiMachine = capiMachineBuilder.Build() - Eventually(k8sClient.Create(ctx, capiMachine)).Should(Succeed()) - - capaMachine = capaMachineBuilder.Build() - Eventually(k8sClient.Create(ctx, capaMachine)).Should(Succeed()) - - By("Setting the MAPI machine status AuthoritativeAPI to 'Migrating'") - Eventually(k.UpdateStatus(mapiMachine, func() { - updatedMAPIMachine := mapiMachineBuilder. - WithAuthoritativeAPIStatus(mapiv1beta1.MachineAuthorityMigrating). - WithConditions([]mapiv1beta1.Condition{ - { - Type: consts.SynchronizedCondition, - LastTransitionTime: metav1.Now(), - Status: corev1.ConditionTrue, - }, - { - Type: "Paused", - LastTransitionTime: metav1.Now(), - Status: corev1.ConditionTrue, - }, - }). - Build() - mapiMachine.Status.AuthoritativeAPI = updatedMAPIMachine.Status.AuthoritativeAPI - mapiMachine.Status.Conditions = updatedMAPIMachine.Status.Conditions - mapiMachine.Status.SynchronizedGeneration = capiMachine.Generation - })).Should(Succeed()) - - req = reconcile.Request{NamespacedName: client.ObjectKeyFromObject(mapiMachine)} - }) - - It("should set old authoritative API (MAPI) status to paused and requeue", func() { - _, err := reconciler.Reconcile(ctx, req) - Expect(err).NotTo(HaveOccurred()) - - Eventually(komega.Object(mapiMachine)).Should( - HaveField("Status.Conditions", SatisfyAll( - Not(BeEmpty()), - ContainElement(SatisfyAll( - HaveField("Type", BeEquivalentTo("Paused")), - HaveField("Status", Equal(corev1.ConditionTrue)), - )), - )), - ) - }) - }) + Context("when MachineAPI is authoritative and the CAPI machine is missing", func() { + BeforeEach(func() { + mapiMachine = mapiMachineBuilder. + WithAuthoritativeAPI(mapiv1beta1.MachineAuthorityMachineAPI). + Build() + Eventually(k8sClient.Create(ctx, mapiMachine)).Should(Succeed()) + + updateMAPIMachineStatus( + mapiv1beta1.MachineAuthorityMachineAPI, + mapiv1beta1.MachineAPISynchronized, + mapiMachine.Generation, + migrationcontrollertest.SynchronizedCondition(corev1.ConditionTrue), + ) }) - Context("when migrating from ClusterAPI to MachineAPI", func() { - Context("when status is not paused for the old authoritative resource (CAPI)", func() { - BeforeEach(func() { - By("Setting the MAPI machine spec AuthoritativeAPI to MachineAPI") - - mapiMachine = mapiMachineBuilder. - WithAuthoritativeAPI(mapiv1beta1.MachineAuthorityMachineAPI). - Build() - Eventually(k8sClient.Create(ctx, mapiMachine)).Should(Succeed()) - - By("Creating a mirror CAPI machine") - - capiMachine = capiMachineBuilder.Build() - Eventually(k8sClient.Create(ctx, capiMachine)).Should(Succeed()) - - capaMachine = capaMachineBuilder.Build() - Eventually(k8sClient.Create(ctx, capaMachine)).Should(Succeed()) - - By("Setting the MAPI machine status AuthoritativeAPI to 'Migrating'") - Eventually(k.UpdateStatus(mapiMachine, func() { - updatedMAPIMachine := mapiMachineBuilder. - WithAuthoritativeAPIStatus(mapiv1beta1.MachineAuthorityMigrating). - WithConditions([]mapiv1beta1.Condition{{ - Type: consts.SynchronizedCondition, - LastTransitionTime: metav1.Now(), - Status: corev1.ConditionTrue}}). - Build() - mapiMachine.Status.AuthoritativeAPI = updatedMAPIMachine.Status.AuthoritativeAPI - mapiMachine.Status.Conditions = updatedMAPIMachine.Status.Conditions - })).Should(Succeed()) - - By("Setting the CAPI machine status condition to 'Paused'") - Eventually(k.UpdateStatus(capiMachine, func() { - updatedCAPIMachine := capiMachineBuilder.Build() - updatedCAPIMachine.Status.Conditions = []metav1.Condition{{ - Type: clusterv1.PausedCondition, - Status: metav1.ConditionTrue, - LastTransitionTime: metav1.Now(), - }} - capiMachine.Status = updatedCAPIMachine.Status - })).Should(Succeed()) - - By("Setting the CAPI infra machine status condition to 'Paused'") - Eventually(k.UpdateStatus(capaMachine, func() { - updatedCAPIInfraMachine := capaMachineBuilder.Build() - updatedCAPIInfraMachine.Status.Conditions = clusterv1beta1.Conditions{ - { - Type: clusterv1beta1.PausedV1Beta2Condition, - Status: corev1.ConditionTrue, - LastTransitionTime: metav1.Now(), - }, - } - capaMachine.Status = updatedCAPIInfraMachine.Status - })).Should(Succeed()) - - req = reconcile.Request{NamespacedName: client.ObjectKeyFromObject(mapiMachine)} - }) - - It("should set old authoritative API (CAPI) status to paused and requeue", func() { - _, err := reconciler.Reconcile(ctx, req) - Expect(err).NotTo(HaveOccurred()) - - Eventually(komega.Object(capiMachine)).Should( - HaveField("Status.Conditions", SatisfyAll( - Not(BeEmpty()), - ContainElement(SatisfyAll( - HaveField("Type", Equal(clusterv1.PausedCondition)), - HaveField("Status", Equal(metav1.ConditionTrue)), - )), - )), - ) - Eventually(komega.Object(capaMachine)).Should( - HaveField("Status.Conditions", SatisfyAll( - Not(BeEmpty()), - ContainElement(SatisfyAll( - HaveField("Type", BeEquivalentTo(clusterv1beta1.PausedV1Beta2Condition)), - HaveField("Status", Equal(corev1.ConditionTrue)), - )), - )), - ) - }) - }) + + It("should treat the missing CAPI machine as already safely paused", func() { + current := &mapiv1beta1.Machine{} + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(mapiMachine), current)).To(Succeed()) + initialResourceVersion := current.ResourceVersion + + expectSuccessfulReconcile() + + Eventually(k.Object(mapiMachine)).Should(SatisfyAll( + HaveField("ObjectMeta.ResourceVersion", Equal(initialResourceVersion)), + HaveField("Status.AuthoritativeAPI", Equal(mapiv1beta1.MachineAuthorityMachineAPI)), + )) }) }) - Context("when the old authoritative resource has been paused", func() { - Context("when migrating from MachineAPI to ClusterAPI", func() { - Context("when status synchronizedGeneration is not matching the old authoritativeAPI generation (MAPI)", func() { - BeforeEach(func() { - By("Setting the MAPI machine spec AuthoritativeAPI to ClusterAPI") - - mapiMachine = mapiMachineBuilder. - // Set desired authoritative API in spec to ClusterAPI. - // To check for requesting pausing on the the MAPI resource it is sufficient - // see that the spec.AuthoritativeAPI field is set to ClusterAPI. - WithAuthoritativeAPI(mapiv1beta1.MachineAuthorityClusterAPI). - Build() - Eventually(k8sClient.Create(ctx, mapiMachine)).Should(Succeed()) - - By("Creating a mirror CAPI machine") - - capiMachine = capiMachineBuilder.Build() - Eventually(k8sClient.Create(ctx, capiMachine)).Should(Succeed()) - - capaMachine = capaMachineBuilder.Build() - Eventually(k8sClient.Create(ctx, capaMachine)).Should(Succeed()) - - By("Setting the MAPI machine status AuthoritativeAPI to 'Migrating'") - Eventually(k.UpdateStatus(mapiMachine, func() { - updatedMAPIMachine := mapiMachineBuilder. - WithAuthoritativeAPIStatus(mapiv1beta1.MachineAuthorityMigrating). - WithConditions([]mapiv1beta1.Condition{{ - Type: consts.SynchronizedCondition, - LastTransitionTime: metav1.Now(), - Status: corev1.ConditionTrue}}). - Build() - mapiMachine.Status.AuthoritativeAPI = updatedMAPIMachine.Status.AuthoritativeAPI - mapiMachine.Status.SynchronizedGeneration = 9999 // Do not match .metadata.generation field. - mapiMachine.Status.Conditions = updatedMAPIMachine.Status.Conditions - })).Should(Succeed()) - - req = reconcile.Request{NamespacedName: client.ObjectKeyFromObject(mapiMachine)} - }) - - It("should do nothing", func() { - initialMAPIMachineRV := mapiMachine.ResourceVersion - _, err := reconciler.Reconcile(ctx, req) - Expect(err).NotTo(HaveOccurred(), "reconciler should not have errored") - Eventually(k.Object(mapiMachine)).Should(HaveField("ObjectMeta.ResourceVersion", Equal(initialMAPIMachineRV)), "should not have modified the machine") - }) - }) + Context("when migrating from ClusterAPI to MachineAPI and only unrelated finalizers remain", func() { + BeforeEach(func() { + mapiMachine = mapiMachineBuilder. + WithAuthoritativeAPI(mapiv1beta1.MachineAuthorityMachineAPI). + Build() + Eventually(k8sClient.Create(ctx, mapiMachine)).Should(Succeed()) + + capiMachine = capiMachineBuilder. + WithAnnotations(map[string]string{clusterv1.PausedAnnotation: ""}). + Build() + capiMachine.Finalizers = append(capiMachine.Finalizers, "example.com/other-machine-finalizer") + Eventually(k8sClient.Create(ctx, capiMachine)).Should(Succeed()) + + capaMachine = capaMachineBuilder. + WithAnnotations(map[string]string{clusterv1.PausedAnnotation: ""}). + Build() + capaMachine.Finalizers = append(capaMachine.Finalizers, "example.com/other-infra-finalizer") + Eventually(k8sClient.Create(ctx, capaMachine)).Should(Succeed()) + + updateCAPIMachineStatus(migrationcontrollertest.CAPIPausedCondition(metav1.ConditionFalse)) + updateCAPAStatus(capaPausedCondition(corev1.ConditionFalse)) + + updateMAPIMachineStatus( + mapiv1beta1.MachineAuthorityClusterAPI, + mapiv1beta1.ClusterAPISynchronized, + capiMachine.Generation, + migrationcontrollertest.SynchronizedCondition(corev1.ConditionTrue), + ) }) - Context("when migrating from ClusterAPI to MachineAPI", func() { - Context("when status synchronizedGeneration is not matching the old authoritativeAPI generation (CAPI)", func() { - BeforeEach(func() { - By("Setting the MAPI machine spec AuthoritativeAPI to MachineAPI") - - mapiMachine = mapiMachineBuilder. - WithAuthoritativeAPI(mapiv1beta1.MachineAuthorityMachineAPI). - Build() - Eventually(k8sClient.Create(ctx, mapiMachine)).Should(Succeed()) - - By("Creating a mirror CAPI machine") - - capiMachine = capiMachineBuilder.Build() - Eventually(k8sClient.Create(ctx, capiMachine)).Should(Succeed()) - - capaMachine = capaMachineBuilder.Build() - Eventually(k8sClient.Create(ctx, capaMachine)).Should(Succeed()) - - By("Setting the MAPI machine status AuthoritativeAPI to 'Migrating'") - Eventually(k.UpdateStatus(mapiMachine, func() { - updatedMAPIMachine := mapiMachineBuilder. - WithAuthoritativeAPIStatus(mapiv1beta1.MachineAuthorityMigrating). - WithConditions([]mapiv1beta1.Condition{{ - Type: consts.SynchronizedCondition, - LastTransitionTime: metav1.Now(), - Status: corev1.ConditionTrue}}). - Build() - mapiMachine.Status.AuthoritativeAPI = updatedMAPIMachine.Status.AuthoritativeAPI - mapiMachine.Status.SynchronizedGeneration = 9999 // Do not match .metadata.generation field. - mapiMachine.Status.Conditions = updatedMAPIMachine.Status.Conditions - })).Should(Succeed()) - - req = reconcile.Request{NamespacedName: client.ObjectKeyFromObject(mapiMachine)} - }) - - It("should do nothing", func() { - initialMAPIMachineRV := mapiMachine.ResourceVersion - _, err := reconciler.Reconcile(ctx, req) - Expect(err).NotTo(HaveOccurred(), "reconciler should not have errored") - Eventually(k.Object(mapiMachine)).Should(HaveField("ObjectMeta.ResourceVersion", Equal(initialMAPIMachineRV)), "should not have modified the machine") - }) - }) + + It("should treat the CAPI side as safely paused and enter Migrating", func() { + expectSuccessfulReconcile() + + Eventually(k.Object(mapiMachine)).Should( + HaveField("Status.AuthoritativeAPI", Equal(mapiv1beta1.MachineAuthorityMigrating)), + ) + }) + }) + + Context("when migrating from ClusterAPI to MachineAPI and controller finalizers are still present", func() { + BeforeEach(func() { + mapiMachine = mapiMachineBuilder. + WithAuthoritativeAPI(mapiv1beta1.MachineAuthorityMachineAPI). + Build() + Eventually(k8sClient.Create(ctx, mapiMachine)).Should(Succeed()) + + capiMachine = capiMachineBuilder. + WithAnnotations(map[string]string{clusterv1.PausedAnnotation: ""}). + Build() + capiMachine.Finalizers = append(capiMachine.Finalizers, clusterv1.MachineFinalizer) + Eventually(k8sClient.Create(ctx, capiMachine)).Should(Succeed()) + + capaMachine = capaMachineBuilder. + WithAnnotations(map[string]string{clusterv1.PausedAnnotation: ""}). + Build() + capaMachine.Finalizers = append(capaMachine.Finalizers, awsv1.MachineFinalizer) + Eventually(k8sClient.Create(ctx, capaMachine)).Should(Succeed()) + + updateCAPIMachineStatus(migrationcontrollertest.CAPIPausedCondition(metav1.ConditionFalse)) + updateCAPAStatus(capaPausedCondition(corev1.ConditionFalse)) + + updateMAPIMachineStatus( + mapiv1beta1.MachineAuthorityClusterAPI, + mapiv1beta1.ClusterAPISynchronized, + capiMachine.Generation, + migrationcontrollertest.SynchronizedCondition(corev1.ConditionTrue), + ) + }) + + It("should wait for paused observation before entering Migrating", func() { + expectSuccessfulReconcile() + + Eventually(k.Object(mapiMachine)).Should(SatisfyAll( + HaveField("Status.AuthoritativeAPI", Equal(mapiv1beta1.MachineAuthorityClusterAPI)), + HaveField("Status.SynchronizedGeneration", Equal(capiMachine.Generation)), + )) }) }) - Context("when all the prerequisites for switching the authoritative API are satisfied", func() { - Context("when migrating from MachineAPI to ClusterAPI", func() { + Context("when spec.AuthoritativeAPI is MachineAPI and status.AuthoritativeAPI is Migrating", func() { + BeforeEach(func() { + mapiMachine = mapiMachineBuilder. + WithAuthoritativeAPI(mapiv1beta1.MachineAuthorityMachineAPI). + Build() + Eventually(k8sClient.Create(ctx, mapiMachine)).Should(Succeed()) + }) + + Context("when Cluster API objects exist and are not paused", func() { BeforeEach(func() { - By("Setting the MAPI machine spec AuthoritativeAPI to ClusterAPI") - - mapiMachine = mapiMachineBuilder. - // Set desired authoritative API in spec to ClusterAPI. - // To check for requesting pausing on the the MAPI resource it is sufficient - // see that the spec.AuthoritativeAPI field is set to ClusterAPI. - WithAuthoritativeAPI(mapiv1beta1.MachineAuthorityClusterAPI). - Build() - Eventually(k8sClient.Create(ctx, mapiMachine)).Should(Succeed()) - - By("Setting the MAPI machine status AuthoritativeAPI to 'Migrating'") - Eventually(k.UpdateStatus(mapiMachine, func() { - updatedMAPIMachine := mapiMachineBuilder. - WithAuthoritativeAPIStatus(mapiv1beta1.MachineAuthorityMigrating). - WithConditions([]mapiv1beta1.Condition{ - { - Type: consts.SynchronizedCondition, - LastTransitionTime: metav1.Now(), - Status: corev1.ConditionTrue, - }, - { - Type: "Paused", - LastTransitionTime: metav1.Now(), - Status: corev1.ConditionTrue, - }, - }). - Build() - mapiMachine.Status.AuthoritativeAPI = updatedMAPIMachine.Status.AuthoritativeAPI - mapiMachine.Status.SynchronizedGeneration = mapiMachine.Generation // Match the MAPI .metadata.generation field. - mapiMachine.Status.Conditions = updatedMAPIMachine.Status.Conditions - })).Should(Succeed()) - - By("Creating a mirror CAPI machine") - - capiMachine = capiMachineBuilder. - WithAnnotations(map[string]string{ - clusterv1.PausedAnnotation: "", - }). - Build() - capiMachine.Finalizers = append(capiMachine.Finalizers, clusterv1.MachineFinalizer) - Eventually(k8sClient.Create(ctx, capiMachine)).Should(Succeed()) - - capaMachine = capaMachineBuilder. - WithAnnotations(map[string]string{ - clusterv1.PausedAnnotation: "", - }). - Build() - Eventually(k8sClient.Create(ctx, capaMachine)).Should(Succeed()) - - By("Setting the CAPI machine status condition to 'Paused'") - Eventually(k.UpdateStatus(capiMachine, func() { - updatedCAPIMachine := capiMachineBuilder.Build() - updatedCAPIMachine.Status.Conditions = []metav1.Condition{{ - Type: clusterv1.PausedCondition, - Status: metav1.ConditionTrue, - LastTransitionTime: metav1.Now(), - }} - capiMachine.Status = updatedCAPIMachine.Status - })).Should(Succeed()) - - req = reconcile.Request{NamespacedName: client.ObjectKeyFromObject(mapiMachine)} + createCAPIMachinePair() + + updateMAPIMachineStatus( + mapiv1beta1.MachineAuthorityMigrating, + mapiv1beta1.MachineAPISynchronized, + mapiMachine.Generation, + migrationcontrollertest.SynchronizedCondition(corev1.ConditionTrue), + ) }) - It("should set the new to-be authoritative resource (CAPI) to actually be authoritative and unpause it", func() { - result, err := reconciler.Reconcile(ctx, req) - Expect(err).NotTo(HaveOccurred()) - Expect(result.Requeue).To(BeFalse()) + It("should converge to MachineAPI without pausing the Cluster API objects", func() { + expectSuccessfulReconcile() - Eventually(komega.Object(mapiMachine)).Should(SatisfyAll( - HaveField("Status.AuthoritativeAPI", Equal(mapiv1beta1.MachineAuthorityClusterAPI)), - HaveField("Status.SynchronizedGeneration", BeZero()), - )) - - Eventually(komega.Object(capiMachine)).ShouldNot( - HaveField("ObjectMeta.Annotations", ContainElement(HaveKeyWithValue(clusterv1.PausedAnnotation, "")))) + expectSyncStatusReset(mapiv1beta1.MachineAuthorityMachineAPI) + Eventually(k.Object(capiMachine)).ShouldNot( + HaveField("ObjectMeta.Annotations", HaveKey(clusterv1.PausedAnnotation)), + ) + Eventually(k.Object(capaMachine)).ShouldNot( + HaveField("ObjectMeta.Annotations", HaveKey(clusterv1.PausedAnnotation)), + ) }) }) - Context("when migrating from ClusterAPI to MachineAPI", func() { + + Context("when status.SynchronizedAPI is empty", func() { BeforeEach(func() { - By("Setting the MAPI machine spec AuthoritativeAPI to MachineAPI") - - mapiMachine = mapiMachineBuilder. - WithAuthoritativeAPI(mapiv1beta1.MachineAuthorityMachineAPI). - Build() - Eventually(k8sClient.Create(ctx, mapiMachine)).Should(Succeed()) - - By("Creating a mirror CAPI machine") - - capiMachine = capiMachineBuilder. - WithAnnotations(map[string]string{ - clusterv1.PausedAnnotation: "", - }). - Build() - Eventually(k8sClient.Create(ctx, capiMachine)).Should(Succeed()) - - capaMachine = capaMachineBuilder. - WithAnnotations(map[string]string{ - clusterv1.PausedAnnotation: "", - }). - Build() - Eventually(k8sClient.Create(ctx, capaMachine)).Should(Succeed()) - - By("Setting the MAPI machine status AuthoritativeAPI to 'Migrating'") - Eventually(k.UpdateStatus(mapiMachine, func() { - updatedMAPIMachine := mapiMachineBuilder. - WithAuthoritativeAPIStatus(mapiv1beta1.MachineAuthorityMigrating). - WithConditions([]mapiv1beta1.Condition{ - { - Type: consts.SynchronizedCondition, - LastTransitionTime: metav1.Now(), - Status: corev1.ConditionTrue, - }, - { - Type: "Paused", - LastTransitionTime: metav1.Now(), - Status: corev1.ConditionTrue, - }, - }). - Build() - mapiMachine.Status.AuthoritativeAPI = updatedMAPIMachine.Status.AuthoritativeAPI - mapiMachine.Status.SynchronizedGeneration = capiMachine.Generation // Match the CAPI .metadata.generation field. - mapiMachine.Status.Conditions = updatedMAPIMachine.Status.Conditions - })).Should(Succeed()) - - By("Setting the CAPI machine status condition to 'Paused'") - Eventually(k.UpdateStatus(capiMachine, func() { - updatedCAPIMachine := capiMachineBuilder.Build() - updatedCAPIMachine.Status.Conditions = []metav1.Condition{{ - Type: clusterv1.PausedCondition, - Status: metav1.ConditionTrue, - LastTransitionTime: metav1.Now(), - }} - capiMachine.Status = updatedCAPIMachine.Status - })).Should(Succeed()) - - By("Setting the CAPI infra machine status condition to 'Paused'") - Eventually(k.UpdateStatus(capaMachine, func() { - updatedCAPIInfraMachine := capaMachineBuilder.WithAnnotations(map[string]string{clusterv1.PausedAnnotation: ""}).Build() - updatedCAPIInfraMachine.Status.Conditions = clusterv1beta1.Conditions{ - { - Type: clusterv1beta1.PausedV1Beta2Condition, - Status: corev1.ConditionTrue, - LastTransitionTime: metav1.Now(), - }, - } - capaMachine.Status = updatedCAPIInfraMachine.Status - })).Should(Succeed()) - - req = reconcile.Request{NamespacedName: client.ObjectKeyFromObject(mapiMachine)} + updateMAPIMachineStatus( + mapiv1beta1.MachineAuthorityMigrating, + "", + 1, + migrationcontrollertest.SynchronizedCondition(corev1.ConditionTrue), + ) }) - It("should set the new to-be authoritative resource (MAPI) to actually be authoritative and requeue to unpause it", func() { - result, err := reconciler.Reconcile(ctx, req) - Expect(err).NotTo(HaveOccurred()) - Expect(result.Requeue).To(BeFalse()) + It("should still converge to MachineAPI", func() { + expectSuccessfulReconcile() - Eventually(komega.Object(mapiMachine)).Should(SatisfyAll( - HaveField("Status.AuthoritativeAPI", Equal(mapiv1beta1.MachineAuthorityMachineAPI)), - HaveField("Status.SynchronizedGeneration", BeZero()), - )) + expectSyncStatusReset(mapiv1beta1.MachineAuthorityMachineAPI) }) }) }) + + Context("when ClusterAPI is authoritative but the CAPI machine is missing", func() { + BeforeEach(func() { + mapiMachine = mapiMachineBuilder. + WithAuthoritativeAPI(mapiv1beta1.MachineAuthorityMachineAPI). + Build() + Eventually(k8sClient.Create(ctx, mapiMachine)).Should(Succeed()) + + updateMAPIMachineStatus( + mapiv1beta1.MachineAuthorityClusterAPI, + mapiv1beta1.ClusterAPISynchronized, + 1, + migrationcontrollertest.SynchronizedCondition(corev1.ConditionTrue), + ) + }) + + It("should wait for the sync controller to restore the authoritative CAPI copy", func() { + expectSuccessfulReconcile() + + Eventually(k.Object(mapiMachine)).Should(SatisfyAll( + HaveField("Status.AuthoritativeAPI", Equal(mapiv1beta1.MachineAuthorityClusterAPI)), + HaveField("Status.SynchronizedGeneration", Equal(int64(1))), + )) + }) + }) + }) + + Describe("addPausedAnnotation", func() { + Context("when the object has changed since it was read", func() { + It("should fail with a conflict", func() { + staleInfraMachine := capaMachineBuilder. + WithName("stale-infra-machine"). + Build() + Expect(k8sClient.Create(ctx, staleInfraMachine)).To(Succeed(), "infra machine should be created") + + staleCopy := &awsv1.AWSMachine{} + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(staleInfraMachine), staleCopy)).To(Succeed(), "stale copy should be fetched") + + liveInfraMachine := &awsv1.AWSMachine{} + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(staleInfraMachine), liveInfraMachine)).To(Succeed(), "live copy should be fetched") + + if liveInfraMachine.Annotations == nil { + liveInfraMachine.Annotations = map[string]string{} + } + + liveInfraMachine.Annotations["test.openshift.io/stale"] = "true" + Expect(k8sClient.Update(ctx, liveInfraMachine)).To(Succeed(), "live infra machine should be updated to make the stale copy outdated") + + changed, err := migrationcommon.AddPausedAnnotation(ctx, k8sClient, staleCopy) + Expect(changed).To(BeFalse(), "stale writes should not report a successful change") + Expect(err).To(HaveOccurred(), "stale writes should fail") + Expect(apierrors.IsConflict(err)).To(BeTrue(), "expected stale patch to fail with a conflict") + }) + }) }) }) diff --git a/pkg/controllers/machinesetmigration/machineset_migration_controller.go b/pkg/controllers/machinesetmigration/machineset_migration_controller.go index 4d67c5fba..77821dc4f 100644 --- a/pkg/controllers/machinesetmigration/machineset_migration_controller.go +++ b/pkg/controllers/machinesetmigration/machineset_migration_controller.go @@ -19,16 +19,20 @@ package machinesetmigration import ( "context" "fmt" + "slices" "github.com/go-logr/logr" - corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/tools/record" clusterv1 "sigs.k8s.io/cluster-api/api/core/v1beta2" - "sigs.k8s.io/cluster-api/util/annotations" + mapiv1beta1 "github.com/openshift/api/machine/v1beta1" + machinev1applyconfigs "github.com/openshift/client-go/machine/applyconfigurations/machine/v1beta1" + "github.com/openshift/cluster-capi-operator/pkg/controllers" + "github.com/openshift/cluster-capi-operator/pkg/controllers/migrationcommon" + "github.com/openshift/cluster-capi-operator/pkg/util" "sigs.k8s.io/cluster-api/util/conditions" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" @@ -36,13 +40,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/handler" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/reconcile" - - configv1 "github.com/openshift/api/config/v1" - mapiv1beta1 "github.com/openshift/api/machine/v1beta1" - machinev1applyconfigs "github.com/openshift/client-go/machine/applyconfigurations/machine/v1beta1" - "github.com/openshift/cluster-capi-operator/pkg/controllers" - "github.com/openshift/cluster-capi-operator/pkg/controllers/synccommon" - "github.com/openshift/cluster-capi-operator/pkg/util" ) const controllerName = "MachineSetMigrationController" @@ -53,13 +50,55 @@ type MachineSetMigrationReconciler struct { Scheme *runtime.Scheme Recorder record.EventRecorder - Infra *configv1.Infrastructure - Platform configv1.PlatformType - InfraTypes util.InfraTypes CAPINamespace string MAPINamespace string } +type machineSetMigratable struct { + reconciler *MachineSetMigrationReconciler + mapiMachineSet *mapiv1beta1.MachineSet +} + +// MAPIObject returns the backing Machine API machine set. +func (m *machineSetMigratable) MAPIObject() client.Object { + return m.mapiMachineSet +} + +// DesiredAuthority returns the requested authoritative API from spec. +func (m *machineSetMigratable) DesiredAuthority() mapiv1beta1.MachineAuthority { + return m.mapiMachineSet.Spec.AuthoritativeAPI +} + +// CurrentAuthority returns the observed authoritative API from status. +func (m *machineSetMigratable) CurrentAuthority() mapiv1beta1.MachineAuthority { + return m.mapiMachineSet.Status.AuthoritativeAPI +} + +// SynchronizedAPI returns the last synchronized API recorded in status. +func (m *machineSetMigratable) SynchronizedAPI() mapiv1beta1.SynchronizedAPI { + return m.mapiMachineSet.Status.SynchronizedAPI +} + +// SynchronizedGeneration returns the generation recorded by the sync controller. +func (m *machineSetMigratable) SynchronizedGeneration() int64 { + return m.mapiMachineSet.Status.SynchronizedGeneration +} + +// MAPIConditions returns the Machine API conditions used by migration logic. +func (m *machineSetMigratable) MAPIConditions() []mapiv1beta1.Condition { + return m.mapiMachineSet.Status.Conditions +} + +// EnsureCAPIPaused pauses the primary Cluster API machine set. +func (m *machineSetMigratable) EnsureCAPIPaused(ctx context.Context, capiMachineSet *clusterv1.MachineSet) (bool, error) { + return m.reconciler.ensureCAPIPaused(ctx, capiMachineSet) +} + +// EnsureCAPIUnpaused removes pause from the primary Cluster API machine set. +func (m *machineSetMigratable) EnsureCAPIUnpaused(ctx context.Context, capiMachineSet *clusterv1.MachineSet) (bool, error) { + return m.reconciler.ensureCAPIUnpaused(ctx, capiMachineSet) +} + // SetupWithManager sets up the MachineSetMigration controller. func (r *MachineSetMigrationReconciler) SetupWithManager(mgr ctrl.Manager) error { // Allow the namespaces to be set externally for test purposes, when not set, @@ -80,11 +119,6 @@ func (r *MachineSetMigrationReconciler) SetupWithManager(mgr ctrl.Manager) error handler.EnqueueRequestsFromMapFunc(util.RewriteNamespace(r.MAPINamespace)), builder.WithPredicates(util.FilterNamespace(r.CAPINamespace)), ). - Watches( - r.InfraTypes.Template(), - handler.EnqueueRequestsFromMapFunc(util.RewriteNamespace(r.MAPINamespace)), - builder.WithPredicates(util.FilterNamespace(r.CAPINamespace)), - ). Complete(r); err != nil { return fmt.Errorf("failed to create controller: %w", err) } @@ -97,8 +131,6 @@ func (r *MachineSetMigrationReconciler) SetupWithManager(mgr ctrl.Manager) error } // Reconcile performs the reconciliation for a MachineSet. -// -//nolint:funlen func (r *MachineSetMigrationReconciler) Reconcile(ctx context.Context, req reconcile.Request) (ctrl.Result, error) { logger := logf.FromContext(ctx).WithValues("namespace", req.Namespace, "name", req.Name) ctx = logr.NewContext(ctx, logger) @@ -106,247 +138,91 @@ func (r *MachineSetMigrationReconciler) Reconcile(ctx context.Context, req recon logger.V(1).Info("Reconciling machine set") defer logger.V(1).Info("Finished reconciling machine set") - mapiMachineSet := &mapiv1beta1.MachineSet{} - if err := r.Get(ctx, client.ObjectKey{Namespace: req.Namespace, Name: req.Name}, mapiMachineSet); err != nil && !apierrors.IsNotFound(err) { - return ctrl.Result{}, fmt.Errorf("failed to get MAPI machine set: %w", err) - } else if apierrors.IsNotFound(err) { - logger.Info("MachineSet has been deleted. Migration not required") - return ctrl.Result{}, nil + mapiMachineSet, found, err := r.getMAPIMachineSet(ctx, req) + if err != nil { + return ctrl.Result{}, err } - if mapiMachineSet.Spec.AuthoritativeAPI == mapiMachineSet.Status.AuthoritativeAPI { - // No migration is being requested for this resource, nothing to do. - return ctrl.Result{}, nil - } - - // If authoritativeAPI status is empty, it means it is the first time we see this resource. - // Set the status.authoritativeAPI to match the spec.authoritativeAPI. - // - // N.B. Very similar logic is also present in the Machine API machine/machineset controllers - // to cover for the cases when the migration controller is not running (e.g. on not yet supported platforms), - // as such if any change is done to this logic, please consider changing it also there. See: - // https://github.com/openshift/machine-api-operator/pull/1386/files#diff-3a93acbdaa255c0afa7f52535fc7df9c3890d6403035dd4c3bd47b0092eb3a37R177-R194 - if mapiMachineSet.Status.AuthoritativeAPI == "" { - if err := r.applyStatusAuthoritativeAPIWithPatch(ctx, mapiMachineSet, mapiMachineSet.Spec.AuthoritativeAPI); err != nil { - return ctrl.Result{}, fmt.Errorf("unable to apply authoritativeAPI to status with patch: %w", err) - } - - // Wait for the patching to take effect. + if !found { + logger.Info("MachineSet has been deleted. Migration not required") return ctrl.Result{}, nil } - // Check that the resource is synchronized and up-to-date. - // - // This MUST be checked BEFORE setting status.authoritativeAPI to Migrating, - // because after that the sync controller will not run to update it and we - // will deadlock. - if isSynchronized, err := r.isSynchronized(ctx, mapiMachineSet); err != nil { - return ctrl.Result{}, fmt.Errorf("unable to check the resource is synchronized and up-to-date with its authority: %w", err) - } else if !isSynchronized { - // The Authoritative API resource is not fully synced up yet, requeue to check later. - logger.Info("Authoritative machine set and its copy are not synchronized yet, will retry later") - - return ctrl.Result{}, nil + result, err := migrationcommon.Reconcile( + ctx, + r.Client, + controllerName, + r.CAPINamespace, + machinev1applyconfigs.MachineSet, + &machineSetMigratable{ + reconciler: r, + mapiMachineSet: mapiMachineSet, + }, + ) + if err != nil { + return ctrl.Result{}, fmt.Errorf("failed to reconcile machine set migration state: %w", err) } - // Make sure the authoritativeAPI resource status is set to migrating. - if mapiMachineSet.Status.AuthoritativeAPI != mapiv1beta1.MachineAuthorityMigrating { - logger.Info("Detected migration request for machine set") + return result, nil +} - if err := r.applyStatusAuthoritativeAPIWithPatch(ctx, mapiMachineSet, mapiv1beta1.MachineAuthorityMigrating); err != nil { - return ctrl.Result{}, fmt.Errorf("unable to set authoritativeAPI %q to status: %w", mapiv1beta1.MachineAuthorityMigrating, err) +func (r *MachineSetMigrationReconciler) getMAPIMachineSet(ctx context.Context, req reconcile.Request) (*mapiv1beta1.MachineSet, bool, error) { + mapiMachineSet := &mapiv1beta1.MachineSet{} + if err := r.Get(ctx, client.ObjectKey{Namespace: req.Namespace, Name: req.Name}, mapiMachineSet); err != nil { + if apierrors.IsNotFound(err) { + return nil, false, nil } - logger.Info("Acknowledged migration request for machine set") - - // Wait for the change to propagate. - return ctrl.Result{}, nil - } - - // Request pausing on the authoritative resource. - if updated, err := r.requestOldAuthoritativeResourcePaused(ctx, mapiMachineSet); err != nil { - return ctrl.Result{}, fmt.Errorf("failed to request pause on authoritative machine set: %w", err) - } else if updated { - logger.Info("Requested pausing for authoritative machine set") - - // Wait for the change to propagate. - return ctrl.Result{}, nil - } - - // Check that the old authoritative resource is paused. - if paused, err := r.isOldAuthoritativeResourcePaused(ctx, mapiMachineSet); err != nil { - return ctrl.Result{}, fmt.Errorf("failed to check paused on old authoritative machine set: %w", err) - } else if !paused { - // The Authoritative API resource is not paused yet, requeue to check later. - logger.Info("Authoritative machine set is not paused yet, will retry later") - - return ctrl.Result{}, nil + return nil, false, fmt.Errorf("failed to get MAPI machine set: %w", err) } - // Make sure the new authoritative resource has been requested to unpause. - if err := r.ensureUnpauseRequestedOnNewAuthoritativeResource(ctx, mapiMachineSet); err != nil { - return ctrl.Result{}, fmt.Errorf("unable to ensure the new AuthoritativeAPI has been un-paused: %w", err) - } - - // Set the actual AuthoritativeAPI to the desired one, reset the synchronized generation and condition. - if err := synccommon.ApplyAuthoritativeAPIAndResetSyncStatus[*machinev1applyconfigs.MachineSetStatusApplyConfiguration](ctx, r.Client, controllerName, machinev1applyconfigs.MachineSet, mapiMachineSet, mapiMachineSet.Spec.AuthoritativeAPI); err != nil { - return ctrl.Result{}, fmt.Errorf("failed to apply authoritativeAPI and reset sync status: %w", err) - } - - logger.Info("Machine set authority switch has now been completed and the resource unpaused") - logger.Info("Machine set migrated successfully") - - return ctrl.Result{}, nil + return mapiMachineSet, true, nil } -// isOldAuthoritativeResourcePaused checks whether the old authoritative resource is paused. -func (r *MachineSetMigrationReconciler) isOldAuthoritativeResourcePaused(ctx context.Context, ms *mapiv1beta1.MachineSet) (bool, error) { - if ms.Spec.AuthoritativeAPI == mapiv1beta1.MachineAuthorityClusterAPI { - cond, err := util.GetConditionStatus(ms, "Paused") - if err != nil { - return false, fmt.Errorf("unable to get paused condition for %s/%s: %w", ms.Namespace, ms.Name, err) - } - - return cond == corev1.ConditionTrue, nil - } +func (r *MachineSetMigrationReconciler) ensureCAPIPaused(ctx context.Context, capiMachineSet *clusterv1.MachineSet) (bool, error) { + return r.ensureCAPIMachineSetPaused(ctx, capiMachineSet) +} - // For MachineAuthorityMachineAPI, check the corresponding CAPI resource. - capiMachineSet := &clusterv1.MachineSet{} - if err := r.Get(ctx, client.ObjectKey{Namespace: r.CAPINamespace, Name: ms.Name}, capiMachineSet); err != nil { - return false, fmt.Errorf("failed to get Cluster API machine set: %w", err) +func (r *MachineSetMigrationReconciler) ensureCAPIMachineSetPaused(ctx context.Context, capiMachineSet *clusterv1.MachineSet) (bool, error) { + changed, err := migrationcommon.AddPausedAnnotation(ctx, r.Client, capiMachineSet) + if err != nil { + return false, fmt.Errorf("failed to request pause on Cluster API machine set: %w", err) } - machinePausedCondition := conditions.Get(capiMachineSet, clusterv1.PausedCondition) - if machinePausedCondition == nil { + if changed { return false, nil } - // InfraMachineTemplate doesn't have a reconciler and thus it doesn't need pausing. - // The only provider we are interested in, that reconciles the inframachinetemplate, is ibmcloud/powervs - // which only updates its status - // see: https://github.com/kubernetes-sigs/cluster-api-provider-ibmcloud/blob/main/controllers/ibmpowervsmachinetemplate_controller.go - return (machinePausedCondition.Status == metav1.ConditionTrue), nil -} - -func (r *MachineSetMigrationReconciler) ensureUnpauseRequestedOnNewAuthoritativeResource(ctx context.Context, mapiMachineSet *mapiv1beta1.MachineSet) error { - // Request that the new authoritative resource reconciliation is un-paused. - //nolint:wsl - switch mapiMachineSet.Spec.AuthoritativeAPI { - case mapiv1beta1.MachineAuthorityClusterAPI: - // For requesting unpausing of a CAPI resource, remove the paused annotation on it. - // So check if the ClusterAPI resource has the paused annotation and if so remove it. - capiMachineSet := &clusterv1.MachineSet{} - if err := r.Get(ctx, client.ObjectKey{Namespace: r.CAPINamespace, Name: mapiMachineSet.Name}, capiMachineSet); err != nil { - return fmt.Errorf("failed to get Cluster API machine set: %w", err) - } - - if annotations.HasPaused(capiMachineSet) { - capiMachineSetCopy := capiMachineSet.DeepCopy() - delete(capiMachineSet.Annotations, clusterv1.PausedAnnotation) - - if err := r.Patch(ctx, capiMachineSet, client.MergeFrom(capiMachineSetCopy)); err != nil { - return fmt.Errorf("failed to patch Cluster API machine set: %w", err) - } - } + // If the finalizer is present we know that the controller is running. It will observe the paused annotation and will eventually set the paused condition. We must wait for the paused condition because it may be actively reconciling. + // If the finalizer is not present then either: + // The controller is not running, in which it is safe to continue. + // The controller has not yet observed the object, in which case (guaranteed by optimistic locking) it will observe our paused annotation before taking any action, so it is safe to continue. + if !slices.Contains(capiMachineSet.Finalizers, clusterv1.MachineSetFinalizer) { + return true, nil + } - // InfraMachineTemplate doesn't have a reconciler and thus it doesn't need pausing. - // The only provider we are interested in, that reconciles the inframachinetemplate, is ibmcloud/powervs - // which only updates its status - // see: https://github.com/kubernetes-sigs/cluster-api-provider-ibmcloud/blob/main/controllers/ibmpowervsmachinetemplate_controller.go - case mapiv1beta1.MachineAuthorityMachineAPI: - // For requesting unpausing of a MAPI resource, it is sufficient to switch the spec.AuthoritativeAPI field on the MAPI resource. - // which is already done before this code runs in this controller. Nothing to do here. - case mapiv1beta1.MachineAuthorityMigrating: - // Value is disallowed by the openAPI schema validation. + machineSetPausedCondition := conditions.Get(capiMachineSet, clusterv1.PausedCondition) + if machineSetPausedCondition == nil { + return false, nil } - return nil + return machineSetPausedCondition.Status == metav1.ConditionTrue, nil } -// requestOldAuthoritativeResourcePaused requests the old authoritative resource is paused. -func (r *MachineSetMigrationReconciler) requestOldAuthoritativeResourcePaused(ctx context.Context, ms *mapiv1beta1.MachineSet) (bool, error) { - // Request that the old authoritative resource reconciliation is paused. - updated := false - //nolint:wsl - switch ms.Spec.AuthoritativeAPI { - case mapiv1beta1.MachineAuthorityClusterAPI: - // For requesting pausing of a MAPI resource, it is sufficient to switch the spec.AuthoritativeAPI field on the MAPI resource. - // which is already done before this code runs in this controller. - case mapiv1beta1.MachineAuthorityMachineAPI: - // For requesting pausing of a CAPI resource, set the paused annotation on it. - // The spec.AuthoritativeAPI is set to MachineAPI, meaning that the old authoritativeAPI was ClusterAPI. - // So Check if the ClusterAPI resource has the paused annotation, otherwise set it. - capiMachineSet := &clusterv1.MachineSet{} - if err := r.Get(ctx, client.ObjectKey{Namespace: r.CAPINamespace, Name: ms.Name}, capiMachineSet); err != nil { - return false, fmt.Errorf("failed to get Cluster API machine set: %w", err) - } - - if !annotations.HasPaused(capiMachineSet) { - capiMachineSetCopy := capiMachineSet.DeepCopy() - annotations.AddAnnotations(capiMachineSet, map[string]string{clusterv1.PausedAnnotation: ""}) - - if err := r.Patch(ctx, capiMachineSet, client.MergeFrom(capiMachineSetCopy)); err != nil { - return false, fmt.Errorf("failed to patch Cluster API machine set: %w", err) - } - - updated = true - } - - // InfraMachineTemplate doesn't have a reconciler and thus it doesn't need pausing. - // The only provider we are interested in, that reconciles the inframachinetemplate, is ibmcloud/powervs - // which only updates its status - // see: https://github.com/kubernetes-sigs/cluster-api-provider-ibmcloud/blob/main/controllers/ibmpowervsmachinetemplate_controller.go - case mapiv1beta1.MachineAuthorityMigrating: - // Value is disallowed by the openAPI schema validation. +func (r *MachineSetMigrationReconciler) ensureCAPIUnpaused(ctx context.Context, capiMachineSet *clusterv1.MachineSet) (bool, error) { + changed, err := migrationcommon.RemovePausedAnnotation(ctx, r.Client, capiMachineSet) + if err != nil { + return false, fmt.Errorf("failed to remove paused annotation from Cluster API machine set: %w", err) } - return updated, nil -} - -func (r *MachineSetMigrationReconciler) isSynchronized(ctx context.Context, mapiMachineSet *mapiv1beta1.MachineSet) (bool, error) { - // Check if the Synchronized condition is set to True. - // If it is not, this indicates an unmigratable resource and therefore should take no action. - if cond, err := util.GetConditionStatus(mapiMachineSet, string(controllers.SynchronizedCondition)); err != nil { - return false, fmt.Errorf("unable to get synchronized condition for %s/%s: %w", mapiMachineSet.Namespace, mapiMachineSet.Name, err) - } else if cond != corev1.ConditionTrue { + if changed { return false, nil } - // Because we are in a migration (spec.authoritativeAPI != - // status.authoritativeAPI), we assume that spec.authoritativeAPI is - // currently the migration target, not the migration source. So: - // - // target: spec.authoritativeAPI - // source: opposite of target - // - // We want to assert that source has been synched to target, so we need to - // treat spec.AuthoritativeAPI as the opposite of the direction we want to - // check. - // - // We may revisit this, as this assumption is not safe if a user aborts an - // in-progress migration by resetting spec.authoritativeAPI to its original - // value. - - switch mapiMachineSet.Spec.AuthoritativeAPI { - case mapiv1beta1.MachineAuthorityClusterAPI: - return mapiMachineSet.Status.SynchronizedGeneration == mapiMachineSet.Generation, nil - case mapiv1beta1.MachineAuthorityMachineAPI: - capiMachineSet := &clusterv1.MachineSet{} - if err := r.Get(ctx, client.ObjectKey{Namespace: r.CAPINamespace, Name: mapiMachineSet.Name}, capiMachineSet); err != nil { - return false, fmt.Errorf("failed to get Cluster API machine set: %w", err) - } - - // Given the CAPI infra machine template is immutable - // we do not check for its generation to be synced up with the generation of the MAPI machine set. - return (mapiMachineSet.Status.SynchronizedGeneration == capiMachineSet.Generation), nil - case mapiv1beta1.MachineAuthorityMigrating: + machineSetPausedCondition := conditions.Get(capiMachineSet, clusterv1.PausedCondition) + if machineSetPausedCondition != nil && machineSetPausedCondition.Status == metav1.ConditionTrue { + return false, nil } - // Should have been prevented by validation - return false, fmt.Errorf("%w: %s", controllers.ErrInvalidSpecAuthoritativeAPI, mapiMachineSet.Spec.AuthoritativeAPI) -} - -// applyStatusAuthoritativeAPIWithPatch updates the resource status.authoritativeAPI using a server-side apply patch. -func (r *MachineSetMigrationReconciler) applyStatusAuthoritativeAPIWithPatch(ctx context.Context, ms *mapiv1beta1.MachineSet, authority mapiv1beta1.MachineAuthority) error { - return synccommon.ApplyAuthoritativeAPI[*machinev1applyconfigs.MachineSetStatusApplyConfiguration](ctx, r.Client, controllerName, machinev1applyconfigs.MachineSet, ms, authority) + return true, nil } diff --git a/pkg/controllers/machinesetmigration/machineset_migration_controller_test.go b/pkg/controllers/machinesetmigration/machineset_migration_controller_test.go index 99756b691..a303b7290 100644 --- a/pkg/controllers/machinesetmigration/machineset_migration_controller_test.go +++ b/pkg/controllers/machinesetmigration/machineset_migration_controller_test.go @@ -19,6 +19,7 @@ package machinesetmigration import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + ctrl "sigs.k8s.io/controller-runtime" mapiv1beta1 "github.com/openshift/api/machine/v1beta1" clusterv1resourcebuilder "github.com/openshift/cluster-api-actuator-pkg/testutils/resourcebuilder/cluster-api/core/v1beta2" @@ -26,22 +27,21 @@ import ( corev1resourcebuilder "github.com/openshift/cluster-api-actuator-pkg/testutils/resourcebuilder/core/v1" machinev1resourcebuilder "github.com/openshift/cluster-api-actuator-pkg/testutils/resourcebuilder/machine/v1beta1" corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" awsv1 "sigs.k8s.io/cluster-api-provider-aws/v2/api/v1beta2" clusterv1 "sigs.k8s.io/cluster-api/api/core/v1beta2" - configv1 "github.com/openshift/api/config/v1" "github.com/openshift/cluster-api-actuator-pkg/testutils" - consts "github.com/openshift/cluster-capi-operator/pkg/controllers" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "github.com/openshift/cluster-capi-operator/pkg/controllers/migrationcommon" + migrationcontrollertest "github.com/openshift/cluster-capi-operator/pkg/controllers/migrationcommon/controllertest" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/envtest/komega" "sigs.k8s.io/controller-runtime/pkg/reconcile" ) -var _ = Describe("With a running MachineSetMigration controller", func() { +var _ = Describe("MachineSetMigration controller", func() { var ( k komega.Komega reconciler *MachineSetMigrationReconciler @@ -50,17 +50,58 @@ var _ = Describe("With a running MachineSetMigration controller", func() { capiNamespace *corev1.Namespace mapiNamespace *corev1.Namespace - mapiMachineSetBuilder machinev1resourcebuilder.MachineSetBuilder - mapiMachineSet *mapiv1beta1.MachineSet - capiMachineSetBuilder clusterv1resourcebuilder.MachineSetBuilder - capiMachineSet *clusterv1.MachineSet - capaMachineTemplateBuilder awsv1resourcebuilder.AWSMachineTemplateBuilder - capaMachineTemplate *awsv1.AWSMachineTemplate - capaClusterBuilder awsv1resourcebuilder.AWSClusterBuilder - capiClusterBuilder clusterv1resourcebuilder.ClusterBuilder - capiCluster *clusterv1.Cluster + mapiMachineSetBuilder machinev1resourcebuilder.MachineSetBuilder + mapiMachineSet *mapiv1beta1.MachineSet + capiMachineSetBuilder clusterv1resourcebuilder.MachineSetBuilder + capiMachineSet *clusterv1.MachineSet + capaClusterBuilder awsv1resourcebuilder.AWSClusterBuilder + capiClusterBuilder clusterv1resourcebuilder.ClusterBuilder ) + createCAPIMachineSet := func() { + GinkgoHelper() + + capiMachineSet = capiMachineSetBuilder.Build() + Eventually(k8sClient.Create(ctx, capiMachineSet)).Should(Succeed(), "CAPI machine set should be able to be created") + } + + updateMAPIMachineSetStatus := func(authority mapiv1beta1.MachineAuthority, synchronizedAPI mapiv1beta1.SynchronizedAPI, synchronizedGeneration int64, conditions ...mapiv1beta1.Condition) { + GinkgoHelper() + + Eventually(k.UpdateStatus(mapiMachineSet, func() { + mapiMachineSet.Status.AuthoritativeAPI = authority + mapiMachineSet.Status.SynchronizedAPI = synchronizedAPI + mapiMachineSet.Status.SynchronizedGeneration = synchronizedGeneration + mapiMachineSet.Status.Conditions = conditions + })).Should(Succeed()) + } + + updateCAPIMachineSetStatus := func(conditions ...metav1.Condition) { + GinkgoHelper() + + Eventually(k.UpdateStatus(capiMachineSet, func() { + capiMachineSet.Status.Conditions = conditions + })).Should(Succeed()) + } + + reconcileOnce := func() (ctrl.Result, error) { + GinkgoHelper() + + return reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(mapiMachineSet)}) + } + + expectSyncStatusReset := func(authority mapiv1beta1.MachineAuthority) { + GinkgoHelper() + + migrationcontrollertest.ExpectSyncStatusReset(k, mapiMachineSet, authority) + } + + expectSuccessfulReconcile := func() { + GinkgoHelper() + + migrationcontrollertest.ExpectSuccessfulReconcile(reconcileOnce) + } + BeforeEach(func() { By("Setting up namespaces for the test") @@ -92,23 +133,16 @@ var _ = Describe("With a running MachineSetMigration controller", func() { WithName(infrastructureName) Expect(k8sClient.Create(ctx, capiClusterBuilder.Build())).To(Succeed(), "CAPI cluster should be able to be created") - capiCluster = &clusterv1.Cluster{} - Expect(k8sClient.Get(ctx, client.ObjectKey{Name: infrastructureName, Namespace: capiNamespace.GetName()}, capiCluster)).To(Succeed()) - - capaMachineTemplateBuilder = awsv1resourcebuilder.AWSMachineTemplate(). - WithNamespace(capiNamespace.GetName()). - WithName("machine-template") - capaMachineTemplate = capaMachineTemplateBuilder.Build() - capiMachineTemplate := clusterv1.MachineTemplateSpec{ Spec: clusterv1.MachineSpec{ InfrastructureRef: clusterv1.ContractVersionedObjectReference{ APIGroup: awsv1.GroupVersion.Group, - Kind: capaMachineTemplate.Kind, - Name: capaMachineTemplate.GetName(), + Kind: "AWSMachineTemplate", + Name: "machine-template", }, }, } + capiMachineSetBuilder = clusterv1resourcebuilder.MachineSet(). WithNamespace(capiNamespace.GetName()). WithName("foo"). @@ -125,14 +159,12 @@ var _ = Describe("With a running MachineSetMigration controller", func() { }) AfterEach(func() { - By("Cleaning up MAPI test resources") + By("Cleaning up test resources") testutils.CleanupResources(Default, ctx, cfg, k8sClient, mapiNamespace.GetName(), - &mapiv1beta1.Machine{}, &mapiv1beta1.MachineSet{}, - &configv1.Infrastructure{}, ) testutils.CleanupResources(Default, ctx, cfg, k8sClient, capiNamespace.GetName(), - &clusterv1.Machine{}, + &clusterv1.Cluster{}, &clusterv1.MachineSet{}, &awsv1.AWSCluster{}, &awsv1.AWSMachineTemplate{}, @@ -140,569 +172,462 @@ var _ = Describe("With a running MachineSetMigration controller", func() { }) Describe("Reconcile", func() { - var req reconcile.Request - - Context("when no migration is requested (status equals spec)", func() { + Context("when no migration is requested and MachineAPI is authoritative", func() { BeforeEach(func() { - By("Setting the MAPI machine set spec AuthoritativeAPI to MachineAPI") - mapiMachineSet = mapiMachineSetBuilder. WithAuthoritativeAPI(mapiv1beta1.MachineAuthorityMachineAPI). Build() Eventually(k8sClient.Create(ctx, mapiMachineSet)).Should(Succeed()) - By("Setting the MAPI machine set status AuthoritativeAPI to MachineAPI") - Eventually(k.UpdateStatus(mapiMachineSet, func() { - mapiMachineSet.Status.AuthoritativeAPI = mapiv1beta1.MachineAuthorityMachineAPI - })).Should(Succeed()) + capiMachineSet = capiMachineSetBuilder.Build() + Eventually(k8sClient.Create(ctx, capiMachineSet)).Should(Succeed()) - req = reconcile.Request{NamespacedName: client.ObjectKeyFromObject(mapiMachineSet)} + updateMAPIMachineSetStatus( + mapiv1beta1.MachineAuthorityMachineAPI, + mapiv1beta1.MachineAPISynchronized, + mapiMachineSet.Generation, + migrationcontrollertest.SynchronizedCondition(corev1.ConditionTrue), + ) }) - It("should do nothing", func() { - initialMAPIMachineSetRV := mapiMachineSet.ResourceVersion - _, err := reconciler.Reconcile(ctx, req) - Expect(err).NotTo(HaveOccurred(), "reconciler should not have errored") - Eventually(k.Object(mapiMachineSet)).Should(HaveField("ObjectMeta.ResourceVersion", Equal(initialMAPIMachineSetRV)), "should not have modified the machine set") + It("should still repair pause drift on the CAPI MachineSet", func() { + expectSuccessfulReconcile() + + Eventually(k.Object(capiMachineSet)).Should( + HaveField("ObjectMeta.Annotations", HaveKeyWithValue(clusterv1.PausedAnnotation, "")), + ) + Eventually(k.Object(mapiMachineSet)).Should( + HaveField("Status.AuthoritativeAPI", Equal(mapiv1beta1.MachineAuthorityMachineAPI)), + ) }) }) - Context("when status.AuthoritativeAPI is empty (first observation)", func() { + Context("when no migration is requested and ClusterAPI is authoritative", func() { BeforeEach(func() { - By("Setting the MAPI machine set spec AuthoritativeAPI to MachineAPI") - mapiMachineSet = mapiMachineSetBuilder. - WithAuthoritativeAPI(mapiv1beta1.MachineAuthorityMachineAPI). + WithAuthoritativeAPI(mapiv1beta1.MachineAuthorityClusterAPI). Build() Eventually(k8sClient.Create(ctx, mapiMachineSet)).Should(Succeed()) - By("Leaving the MAPI machine set status AuthoritativeAPI empty") + capiMachineSet = capiMachineSetBuilder. + WithAnnotations(map[string]string{clusterv1.PausedAnnotation: ""}). + Build() + Eventually(k8sClient.Create(ctx, capiMachineSet)).Should(Succeed()) - req = reconcile.Request{NamespacedName: client.ObjectKeyFromObject(mapiMachineSet)} + updateMAPIMachineSetStatus( + mapiv1beta1.MachineAuthorityClusterAPI, + mapiv1beta1.ClusterAPISynchronized, + capiMachineSet.Generation, + migrationcontrollertest.SynchronizedCondition(corev1.ConditionTrue), + ) }) - It("should patch the status to match spec and requeue", func() { - By("Running one reconciliation") - - _, err := reconciler.Reconcile(ctx, req) - Expect(err).NotTo(HaveOccurred(), "reconciler should not have errored") + It("should still repair unpause drift on the CAPI MachineSet", func() { + expectSuccessfulReconcile() - updatedMS := &mapiv1beta1.MachineSet{} - Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(mapiMachineSet), updatedMS)).To(Succeed()) - Expect(updatedMS.Status.AuthoritativeAPI).To(Equal(updatedMS.Spec.AuthoritativeAPI)) + Eventually(k.Object(capiMachineSet)).ShouldNot( + HaveField("ObjectMeta.Annotations", HaveKey(clusterv1.PausedAnnotation)), + ) + Eventually(k.Object(mapiMachineSet)).Should( + HaveField("Status.AuthoritativeAPI", Equal(mapiv1beta1.MachineAuthorityClusterAPI)), + ) }) }) - Context("when the Synchronized condition is not True", func() { + Context("when status.AuthoritativeAPI is empty", func() { BeforeEach(func() { - By("Setting the MAPI machine set spec AuthoritativeAPI to ClusterAPI") - mapiMachineSet = mapiMachineSetBuilder. WithAuthoritativeAPI(mapiv1beta1.MachineAuthorityClusterAPI). Build() Eventually(k8sClient.Create(ctx, mapiMachineSet)).Should(Succeed()) - - By("Setting the MAPI machine set status AuthoritativeAPI to MachineAPI") - Eventually(k.UpdateStatus(mapiMachineSet, func() { - updatedMAPIMachineSet := mapiMachineSetBuilder. - WithAuthoritativeAPIStatus(mapiv1beta1.MachineAuthorityMachineAPI). - WithConditions([]mapiv1beta1.Condition{{ - Type: consts.SynchronizedCondition, - LastTransitionTime: metav1.Now(), - Status: corev1.ConditionFalse}}). - Build() - mapiMachineSet.Status.AuthoritativeAPI = updatedMAPIMachineSet.Status.AuthoritativeAPI - mapiMachineSet.Status.Conditions = updatedMAPIMachineSet.Status.Conditions - })).Should(Succeed()) - - req = reconcile.Request{NamespacedName: client.ObjectKeyFromObject(mapiMachineSet)} }) - It("should do nothing", func() { - initialMAPIMachineSetRV := mapiMachineSet.ResourceVersion - _, err := reconciler.Reconcile(ctx, req) - Expect(err).NotTo(HaveOccurred(), "reconciler should not have errored") - Eventually(k.Object(mapiMachineSet)).Should(HaveField("ObjectMeta.ResourceVersion", Equal(initialMAPIMachineSetRV)), "should not have modified the machine set") + It("should patch the status to match spec", func() { + expectSuccessfulReconcile() + + Eventually(k.Object(mapiMachineSet)).Should( + HaveField("Status.AuthoritativeAPI", Equal(mapiv1beta1.MachineAuthorityClusterAPI)), + ) }) }) - Context("when a migration request is first detected", func() { + Context("when migrating from MachineAPI to ClusterAPI and status.SynchronizedAPI is empty", func() { BeforeEach(func() { - By("Setting the MAPI machine set spec AuthoritativeAPI to ClusterAPI") - mapiMachineSet = mapiMachineSetBuilder. WithAuthoritativeAPI(mapiv1beta1.MachineAuthorityClusterAPI). Build() Eventually(k8sClient.Create(ctx, mapiMachineSet)).Should(Succeed()) - By("Creating a mirror CAPI machine set") + updateMAPIMachineSetStatus( + mapiv1beta1.MachineAuthorityMachineAPI, + "", + mapiMachineSet.Generation, + migrationcontrollertest.SynchronizedCondition(corev1.ConditionTrue), + ) + }) - capiMachineSet = capiMachineSetBuilder.Build() + It("should wait without acknowledging the migration", func() { + current := &mapiv1beta1.MachineSet{} + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(mapiMachineSet), current)).To(Succeed()) + initialResourceVersion := current.ResourceVersion + + expectSuccessfulReconcile() + + Eventually(k.Object(mapiMachineSet)).Should(SatisfyAll( + HaveField("ObjectMeta.ResourceVersion", Equal(initialResourceVersion)), + HaveField("Status.AuthoritativeAPI", Equal(mapiv1beta1.MachineAuthorityMachineAPI)), + HaveField("Status.SynchronizedAPI", BeEmpty()), + )) + }) + }) + + Context("when migrating from ClusterAPI to MachineAPI and status.SynchronizedAPI points at MachineAPI", func() { + BeforeEach(func() { + mapiMachineSet = mapiMachineSetBuilder. + WithAuthoritativeAPI(mapiv1beta1.MachineAuthorityMachineAPI). + Build() + Eventually(k8sClient.Create(ctx, mapiMachineSet)).Should(Succeed()) + + capiMachineSet = capiMachineSetBuilder. + WithAnnotations(map[string]string{clusterv1.PausedAnnotation: ""}). + Build() Eventually(k8sClient.Create(ctx, capiMachineSet)).Should(Succeed()) - By("Setting the MAPI machine set status AuthoritativeAPI to MachineAPI") - Eventually(k.UpdateStatus(mapiMachineSet, func() { - updatedMAPIMachineSet := mapiMachineSetBuilder. - WithAuthoritativeAPIStatus(mapiv1beta1.MachineAuthorityMachineAPI). - WithConditions([]mapiv1beta1.Condition{{ - Type: consts.SynchronizedCondition, - LastTransitionTime: metav1.Now(), - Status: corev1.ConditionTrue}}). - Build() - mapiMachineSet.Status.AuthoritativeAPI = updatedMAPIMachineSet.Status.AuthoritativeAPI - mapiMachineSet.Status.Conditions = updatedMAPIMachineSet.Status.Conditions - mapiMachineSet.Status.SynchronizedGeneration = capiMachineSet.Generation - })).Should(Succeed()) + updateMAPIMachineSetStatus( + mapiv1beta1.MachineAuthorityClusterAPI, + mapiv1beta1.MachineAPISynchronized, + capiMachineSet.Generation, + migrationcontrollertest.SynchronizedCondition(corev1.ConditionTrue), + ) + }) + + It("should wait without entering Migrating", func() { + current := &mapiv1beta1.MachineSet{} + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(mapiMachineSet), current)).To(Succeed()) + initialResourceVersion := current.ResourceVersion - req = reconcile.Request{NamespacedName: client.ObjectKeyFromObject(mapiMachineSet)} + expectSuccessfulReconcile() + + Eventually(k.Object(mapiMachineSet)).Should(SatisfyAll( + HaveField("ObjectMeta.ResourceVersion", Equal(initialResourceVersion)), + HaveField("Status.AuthoritativeAPI", Equal(mapiv1beta1.MachineAuthorityClusterAPI)), + HaveField("Status.SynchronizedAPI", Equal(mapiv1beta1.MachineAPISynchronized)), + )) }) + }) - It("should acknowledge the migration by updating status to 'Migrating' and requeuing", func() { - _, err := reconciler.Reconcile(ctx, req) - Expect(err).NotTo(HaveOccurred()) + Context("when migrating from MachineAPI to ClusterAPI and the stable sync gate is satisfied", func() { + BeforeEach(func() { + mapiMachineSet = mapiMachineSetBuilder. + WithAuthoritativeAPI(mapiv1beta1.MachineAuthorityClusterAPI). + Build() + Eventually(k8sClient.Create(ctx, mapiMachineSet)).Should(Succeed()) - updatedMS := &mapiv1beta1.MachineSet{} - Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(mapiMachineSet), updatedMS)).To(Succeed()) - Expect(updatedMS.Status.AuthoritativeAPI).To(Equal(mapiv1beta1.MachineAuthorityMigrating)) + updateMAPIMachineSetStatus( + mapiv1beta1.MachineAuthorityMachineAPI, + mapiv1beta1.MachineAPISynchronized, + mapiMachineSet.Generation, + migrationcontrollertest.SynchronizedCondition(corev1.ConditionTrue), + ) + }) + + It("should patch status to Migrating", func() { + expectSuccessfulReconcile() + + Eventually(k.Object(mapiMachineSet)).Should( + HaveField("Status.AuthoritativeAPI", Equal(mapiv1beta1.MachineAuthorityMigrating)), + ) }) }) - Context("when the resource migration has been acknowledged (resource status migrating)", func() { - Context("when migrating from MachineAPI to ClusterAPI", func() { + Context("when spec.AuthoritativeAPI is ClusterAPI and status.AuthoritativeAPI is Migrating", func() { + BeforeEach(func() { + mapiMachineSet = mapiMachineSetBuilder. + WithAuthoritativeAPI(mapiv1beta1.MachineAuthorityClusterAPI). + Build() + Eventually(k8sClient.Create(ctx, mapiMachineSet)).Should(Succeed()) + }) + + Context("when Machine API is not paused yet", func() { BeforeEach(func() { - By("Setting the MAPI machine set spec AuthoritativeAPI to ClusterAPI") + updateMAPIMachineSetStatus( + mapiv1beta1.MachineAuthorityMigrating, + mapiv1beta1.MachineAPISynchronized, + mapiMachineSet.Generation, + migrationcontrollertest.SynchronizedCondition(corev1.ConditionTrue), + migrationcontrollertest.MAPIPausedCondition(corev1.ConditionFalse), + ) + }) - mapiMachineSet = mapiMachineSetBuilder. - WithAuthoritativeAPI(mapiv1beta1.MachineAuthorityClusterAPI). - Build() - Eventually(k8sClient.Create(ctx, mapiMachineSet)).Should(Succeed()) + It("should keep waiting in Migrating", func() { + expectSuccessfulReconcile() - By("Creating a mirror CAPI machine set") + Eventually(k.Object(mapiMachineSet)).Should(SatisfyAll( + HaveField("Status.AuthoritativeAPI", Equal(mapiv1beta1.MachineAuthorityMigrating)), + HaveField("Status.SynchronizedGeneration", Equal(mapiMachineSet.Generation)), + )) + }) + }) - capiMachineSet = capiMachineSetBuilder.Build() + Context("when Machine API is paused", func() { + BeforeEach(func() { + capiMachineSet = capiMachineSetBuilder. + WithAnnotations(map[string]string{clusterv1.PausedAnnotation: ""}). + Build() Eventually(k8sClient.Create(ctx, capiMachineSet)).Should(Succeed()) - By("Setting the MAPI machine set status AuthoritativeAPI to 'Migrating'") - Eventually(k.UpdateStatus(mapiMachineSet, func() { - updatedMAPIMachineSet := mapiMachineSetBuilder. - WithAuthoritativeAPIStatus(mapiv1beta1.MachineAuthorityMigrating). - WithConditions([]mapiv1beta1.Condition{{ - Type: consts.SynchronizedCondition, - LastTransitionTime: metav1.Now(), - Status: corev1.ConditionTrue}}). - Build() - mapiMachineSet.Status.AuthoritativeAPI = updatedMAPIMachineSet.Status.AuthoritativeAPI - mapiMachineSet.Status.Conditions = updatedMAPIMachineSet.Status.Conditions - mapiMachineSet.Status.SynchronizedGeneration = capiMachineSet.Generation - })).Should(Succeed()) - - req = reconcile.Request{NamespacedName: client.ObjectKeyFromObject(mapiMachineSet)} + updateCAPIMachineSetStatus(migrationcontrollertest.CAPIPausedCondition(metav1.ConditionTrue)) + + updateMAPIMachineSetStatus( + mapiv1beta1.MachineAuthorityMigrating, + mapiv1beta1.MachineAPISynchronized, + mapiMachineSet.Generation, + migrationcontrollertest.SynchronizedCondition(corev1.ConditionTrue), + migrationcontrollertest.MAPIPausedCondition(corev1.ConditionTrue), + ) }) - It("should request pausing for the old authoritative resource (MAPI) and stay in Migrating status", func() { - _, err := reconciler.Reconcile(ctx, req) - Expect(err).NotTo(HaveOccurred()) - - updatedMS := &mapiv1beta1.MachineSet{} - Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(mapiMachineSet), updatedMS)).To(Succeed()) - // To check for requesting pausing on the the MAPI resource it is sufficient - // see that the spec.AuthoritativeAPI field is set to ClusterAPI, - // which is already done anyway on the requester side. - Expect(updatedMS.Spec.AuthoritativeAPI).To(Equal(mapiv1beta1.MachineAuthorityClusterAPI)) - Expect(updatedMS.Status.AuthoritativeAPI).To(Equal(mapiv1beta1.MachineAuthorityMigrating)) + It("should complete the switch to ClusterAPI and reset sync status", func() { + expectSuccessfulReconcile() + + expectSyncStatusReset(mapiv1beta1.MachineAuthorityClusterAPI) + Eventually(k.Object(capiMachineSet)).Should( + HaveField("ObjectMeta.Annotations", HaveKeyWithValue(clusterv1.PausedAnnotation, "")), + ) }) }) - Context("when migrating from ClusterAPI to MachineAPI", func() { - BeforeEach(func() { - By("Setting the MAPI machine set spec AuthoritativeAPI to MachineAPI") + }) - mapiMachineSet = mapiMachineSetBuilder.WithAuthoritativeAPI(mapiv1beta1.MachineAuthorityMachineAPI).Build() - Eventually(k8sClient.Create(ctx, mapiMachineSet)).Should(Succeed()) + Context("when status.AuthoritativeAPI is Migrating with empty status.SynchronizedAPI and spec.AuthoritativeAPI is MachineAPI", func() { + BeforeEach(func() { + mapiMachineSet = mapiMachineSetBuilder. + WithAuthoritativeAPI(mapiv1beta1.MachineAuthorityMachineAPI). + Build() + Eventually(k8sClient.Create(ctx, mapiMachineSet)).Should(Succeed()) - By("Creating a mirror CAPI machine set") + updateMAPIMachineSetStatus( + mapiv1beta1.MachineAuthorityMigrating, + "", + 1, + migrationcontrollertest.SynchronizedCondition(corev1.ConditionTrue), + ) + }) - capiMachineSet = capiMachineSetBuilder.Build() - Eventually(k8sClient.Create(ctx, capiMachineSet)).Should(Succeed()) + It("should converge to MachineAPI without error", func() { + expectSuccessfulReconcile() - By("Setting the MAPI machine set status AuthoritativeAPI to 'Migrating'") - Eventually(k.UpdateStatus(mapiMachineSet, func() { - updatedMAPIMachineSet := mapiMachineSetBuilder. - WithAuthoritativeAPIStatus(mapiv1beta1.MachineAuthorityMigrating). - WithConditions([]mapiv1beta1.Condition{{ - Type: consts.SynchronizedCondition, - LastTransitionTime: metav1.Now(), - Status: corev1.ConditionTrue}}). - Build() - mapiMachineSet.Status.AuthoritativeAPI = updatedMAPIMachineSet.Status.AuthoritativeAPI - mapiMachineSet.Status.Conditions = updatedMAPIMachineSet.Status.Conditions - mapiMachineSet.Status.SynchronizedGeneration = mapiMachineSet.Generation - })).Should(Succeed()) - - req = reconcile.Request{NamespacedName: client.ObjectKeyFromObject(mapiMachineSet)} - }) + expectSyncStatusReset(mapiv1beta1.MachineAuthorityMachineAPI) + }) + }) + + Context("when migrating from ClusterAPI to MachineAPI and the CAPI MachineSet is not paused yet", func() { + BeforeEach(func() { + mapiMachineSet = mapiMachineSetBuilder. + WithAuthoritativeAPI(mapiv1beta1.MachineAuthorityMachineAPI). + Build() + Eventually(k8sClient.Create(ctx, mapiMachineSet)).Should(Succeed()) - It("should request pausing for the old authoritative resource (CAPI) and stay in Migrating status", func() { - _, err := reconciler.Reconcile(ctx, req) - Expect(err).NotTo(HaveOccurred()) + createCAPIMachineSet() - updatedMS := &mapiv1beta1.MachineSet{} - Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(mapiMachineSet), updatedMS)).To(Succeed()) - Expect(updatedMS.Status.AuthoritativeAPI).To(Equal(mapiv1beta1.MachineAuthorityMigrating)) + updateMAPIMachineSetStatus( + mapiv1beta1.MachineAuthorityClusterAPI, + mapiv1beta1.ClusterAPISynchronized, + capiMachineSet.Generation, + migrationcontrollertest.SynchronizedCondition(corev1.ConditionTrue), + ) + }) - updatedCAPIMS := &clusterv1.MachineSet{} - Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(capiMachineSet), updatedCAPIMS)).To(Succeed()) - Expect(updatedCAPIMS.Annotations).To(HaveKeyWithValue(clusterv1.PausedAnnotation, "")) - }) + It("should pause the CAPI MachineSet before entering Migrating", func() { + expectSuccessfulReconcile() + + Eventually(k.Object(capiMachineSet)).Should( + HaveField("ObjectMeta.Annotations", HaveKeyWithValue(clusterv1.PausedAnnotation, "")), + ) + Eventually(k.Object(mapiMachineSet)).Should( + HaveField("Status.AuthoritativeAPI", Equal(mapiv1beta1.MachineAuthorityClusterAPI)), + ) }) }) - Context("when the old authoritative resource pausing has been requested", func() { - Context("when migrating from MachineAPI to ClusterAPI", func() { - Context("when status is not paused for the old authoritative resource (MAPI)", func() { - BeforeEach(func() { - By("Setting the MAPI machine set spec AuthoritativeAPI to ClusterAPI") - - mapiMachineSet = mapiMachineSetBuilder. - // Set desired authoritative API in spec to ClusterAPI. - // To check for requesting pausing on the the MAPI resource it is sufficient - // see that the spec.AuthoritativeAPI field is set to ClusterAPI. - WithAuthoritativeAPI(mapiv1beta1.MachineAuthorityClusterAPI). - Build() - Eventually(k8sClient.Create(ctx, mapiMachineSet)).Should(Succeed()) - - By("Creating a mirror CAPI machine set") - - capiMachineSet = capiMachineSetBuilder.Build() - Eventually(k8sClient.Create(ctx, capiMachineSet)).Should(Succeed()) - - By("Setting the MAPI machine set status AuthoritativeAPI to 'Migrating'") - Eventually(k.UpdateStatus(mapiMachineSet, func() { - updatedMAPIMachineSet := mapiMachineSetBuilder. - WithAuthoritativeAPIStatus(mapiv1beta1.MachineAuthorityMigrating). - WithConditions([]mapiv1beta1.Condition{ - { - Type: consts.SynchronizedCondition, - LastTransitionTime: metav1.Now(), - Status: corev1.ConditionTrue, - }, - { - Type: "Paused", - LastTransitionTime: metav1.Now(), - Status: corev1.ConditionTrue, - }, - }). - Build() - mapiMachineSet.Status.AuthoritativeAPI = updatedMAPIMachineSet.Status.AuthoritativeAPI - mapiMachineSet.Status.Conditions = updatedMAPIMachineSet.Status.Conditions - mapiMachineSet.Status.SynchronizedGeneration = capiMachineSet.Generation - })).Should(Succeed()) - - req = reconcile.Request{NamespacedName: client.ObjectKeyFromObject(mapiMachineSet)} - }) - - It("should set old authoritative API (MAPI) status to paused and requeue", func() { - _, err := reconciler.Reconcile(ctx, req) - Expect(err).NotTo(HaveOccurred()) - - Eventually(komega.Object(mapiMachineSet)).Should( - HaveField("Status.Conditions", SatisfyAll( - Not(BeEmpty()), - ContainElement(SatisfyAll( - HaveField("Type", BeEquivalentTo("Paused")), - HaveField("Status", BeEquivalentTo(corev1.ConditionTrue)), - )), - )), - ) - }) - }) + Context("when migrating from ClusterAPI to MachineAPI and the CAPI MachineSet is missing", func() { + BeforeEach(func() { + mapiMachineSet = mapiMachineSetBuilder. + WithAuthoritativeAPI(mapiv1beta1.MachineAuthorityMachineAPI). + Build() + Eventually(k8sClient.Create(ctx, mapiMachineSet)).Should(Succeed()) + + updateMAPIMachineSetStatus( + mapiv1beta1.MachineAuthorityClusterAPI, + mapiv1beta1.ClusterAPISynchronized, + 1, + migrationcontrollertest.SynchronizedCondition(corev1.ConditionTrue), + ) }) - Context("when migrating from ClusterAPI to MachineAPI", func() { - Context("when status is not paused for the old authoritative resource (CAPI)", func() { - BeforeEach(func() { - By("Setting the MAPI machine set spec AuthoritativeAPI to MachineAPI") - - mapiMachineSet = mapiMachineSetBuilder. - WithAuthoritativeAPI(mapiv1beta1.MachineAuthorityMachineAPI). - Build() - Eventually(k8sClient.Create(ctx, mapiMachineSet)).Should(Succeed()) - - By("Creating a mirror CAPI machine set") - - capiMachineSet = capiMachineSetBuilder.Build() - Eventually(k8sClient.Create(ctx, capiMachineSet)).Should(Succeed()) - - By("Setting the MAPI machine set status AuthoritativeAPI to 'Migrating'") - Eventually(k.UpdateStatus(mapiMachineSet, func() { - updatedMAPIMachineSet := mapiMachineSetBuilder. - WithAuthoritativeAPIStatus(mapiv1beta1.MachineAuthorityMigrating). - WithConditions([]mapiv1beta1.Condition{{ - Type: consts.SynchronizedCondition, - LastTransitionTime: metav1.Now(), - Status: corev1.ConditionTrue}}). - Build() - mapiMachineSet.Status.AuthoritativeAPI = updatedMAPIMachineSet.Status.AuthoritativeAPI - mapiMachineSet.Status.Conditions = updatedMAPIMachineSet.Status.Conditions - })).Should(Succeed()) - - By("Setting the CAPI machine set status condition to 'Paused'") - Eventually(k.UpdateStatus(capiMachineSet, func() { - updatedCAPIMachineSet := capiMachineSetBuilder.Build() - updatedCAPIMachineSet.Status.Conditions = []metav1.Condition{{ - Type: clusterv1.PausedCondition, - Status: metav1.ConditionTrue, - LastTransitionTime: metav1.Now(), - }} - capiMachineSet.Status = updatedCAPIMachineSet.Status - })).Should(Succeed()) - - req = reconcile.Request{NamespacedName: client.ObjectKeyFromObject(mapiMachineSet)} - }) - - It("should set old authoritative API (CAPI) status to paused and requeue", func() { - _, err := reconciler.Reconcile(ctx, req) - Expect(err).NotTo(HaveOccurred()) - - Eventually(komega.Object(capiMachineSet)).Should( - HaveField("Status.Conditions", SatisfyAll( - Not(BeEmpty()), - ContainElement(SatisfyAll( - HaveField("Type", Equal(clusterv1.PausedCondition)), - HaveField("Status", Equal(metav1.ConditionTrue)), - )), - )), - ) - }) - }) + + It("should wait for the sync controller to restore the authoritative CAPI copy", func() { + current := &mapiv1beta1.MachineSet{} + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(mapiMachineSet), current)).To(Succeed()) + initialResourceVersion := current.ResourceVersion + + expectSuccessfulReconcile() + + Eventually(k.Object(mapiMachineSet)).Should(SatisfyAll( + HaveField("ObjectMeta.ResourceVersion", Equal(initialResourceVersion)), + HaveField("Status.AuthoritativeAPI", Equal(mapiv1beta1.MachineAuthorityClusterAPI)), + HaveField("Status.SynchronizedGeneration", Equal(int64(1))), + )) }) }) - Context("when the old authoritative resource has been paused", func() { - Context("when migrating from MachineAPI to ClusterAPI", func() { - Context("when status synchronizedGeneration is not matching the old authoritativeAPI generation (MAPI)", func() { - BeforeEach(func() { - By("Setting the MAPI machine set spec AuthoritativeAPI to ClusterAPI") - - mapiMachineSet = mapiMachineSetBuilder. - // Set desired authoritative API in spec to ClusterAPI. - // To check for requesting pausing on the the MAPI resource it is sufficient - // see that the spec.AuthoritativeAPI field is set to ClusterAPI. - WithAuthoritativeAPI(mapiv1beta1.MachineAuthorityClusterAPI). - Build() - Eventually(k8sClient.Create(ctx, mapiMachineSet)).Should(Succeed()) - - By("Creating a mirror CAPI machine set") - - capiMachineSet = capiMachineSetBuilder.Build() - Eventually(k8sClient.Create(ctx, capiMachineSet)).Should(Succeed()) - - By("Setting the MAPI machine set status AuthoritativeAPI to 'Migrating'") - Eventually(k.UpdateStatus(mapiMachineSet, func() { - updatedMAPIMachineSet := mapiMachineSetBuilder. - WithAuthoritativeAPIStatus(mapiv1beta1.MachineAuthorityMigrating). - WithSynchronizedGeneration(9999). // Do not match .metadata.generation field. - WithConditions([]mapiv1beta1.Condition{{ - Type: consts.SynchronizedCondition, - LastTransitionTime: metav1.Now(), - Status: corev1.ConditionTrue}}). - Build() - mapiMachineSet.Status.AuthoritativeAPI = updatedMAPIMachineSet.Status.AuthoritativeAPI - mapiMachineSet.Status.SynchronizedGeneration = updatedMAPIMachineSet.Status.SynchronizedGeneration - mapiMachineSet.Status.Conditions = updatedMAPIMachineSet.Status.Conditions - })).Should(Succeed()) - - req = reconcile.Request{NamespacedName: client.ObjectKeyFromObject(mapiMachineSet)} - }) - - It("should do nothing", func() { - initialMAPIMachineSetRV := mapiMachineSet.ResourceVersion - _, err := reconciler.Reconcile(ctx, req) - Expect(err).NotTo(HaveOccurred(), "reconciler should not have errored") - Eventually(k.Object(mapiMachineSet)).Should(HaveField("ObjectMeta.ResourceVersion", Equal(initialMAPIMachineSetRV)), "should not have modified the machine set") - }) - }) + Context("when migrating from ClusterAPI to MachineAPI and only unrelated finalizers remain", func() { + BeforeEach(func() { + mapiMachineSet = mapiMachineSetBuilder. + WithAuthoritativeAPI(mapiv1beta1.MachineAuthorityMachineAPI). + Build() + Eventually(k8sClient.Create(ctx, mapiMachineSet)).Should(Succeed()) + + capiMachineSet = capiMachineSetBuilder. + WithAnnotations(map[string]string{clusterv1.PausedAnnotation: ""}). + Build() + capiMachineSet.Finalizers = append(capiMachineSet.Finalizers, "example.com/other-machineset-finalizer") + Eventually(k8sClient.Create(ctx, capiMachineSet)).Should(Succeed()) + + updateCAPIMachineSetStatus(migrationcontrollertest.CAPIPausedCondition(metav1.ConditionFalse)) + + updateMAPIMachineSetStatus( + mapiv1beta1.MachineAuthorityClusterAPI, + mapiv1beta1.ClusterAPISynchronized, + capiMachineSet.Generation, + migrationcontrollertest.SynchronizedCondition(corev1.ConditionTrue), + ) }) - Context("when migrating from ClusterAPI to MachineAPI", func() { - Context("when status synchronizedGeneration is not matching the old authoritativeAPI generation (CAPI)", func() { - BeforeEach(func() { - By("Setting the MAPI machine set spec AuthoritativeAPI to MachineAPI") - - mapiMachineSet = mapiMachineSetBuilder. - WithAuthoritativeAPI(mapiv1beta1.MachineAuthorityMachineAPI). - Build() - Eventually(k8sClient.Create(ctx, mapiMachineSet)).Should(Succeed()) - - By("Creating a mirror CAPI machine set") - - capiMachineSet = capiMachineSetBuilder.Build() - Eventually(k8sClient.Create(ctx, capiMachineSet)).Should(Succeed()) - - By("Setting the MAPI machine set status AuthoritativeAPI to 'Migrating'") - Eventually(k.UpdateStatus(mapiMachineSet, func() { - updatedMAPIMachineSet := mapiMachineSetBuilder. - WithAuthoritativeAPIStatus(mapiv1beta1.MachineAuthorityMigrating). - WithSynchronizedGeneration(9999). // Do not match .metadata.generation field. - WithConditions([]mapiv1beta1.Condition{{ - Type: consts.SynchronizedCondition, - LastTransitionTime: metav1.Now(), - Status: corev1.ConditionTrue}}). - Build() - mapiMachineSet.Status.AuthoritativeAPI = updatedMAPIMachineSet.Status.AuthoritativeAPI - mapiMachineSet.Status.SynchronizedGeneration = updatedMAPIMachineSet.Status.SynchronizedGeneration - mapiMachineSet.Status.Conditions = updatedMAPIMachineSet.Status.Conditions - })).Should(Succeed()) - - req = reconcile.Request{NamespacedName: client.ObjectKeyFromObject(mapiMachineSet)} - }) - - It("should do nothing", func() { - initialMAPIMachineSetRV := mapiMachineSet.ResourceVersion - _, err := reconciler.Reconcile(ctx, req) - Expect(err).NotTo(HaveOccurred(), "reconciler should not have errored") - Eventually(k.Object(mapiMachineSet)).Should(HaveField("ObjectMeta.ResourceVersion", Equal(initialMAPIMachineSetRV)), "should not have modified the machine set") - }) - }) + + It("should treat the CAPI side as safely paused and enter Migrating", func() { + expectSuccessfulReconcile() + + Eventually(k.Object(mapiMachineSet)).Should( + HaveField("Status.AuthoritativeAPI", Equal(mapiv1beta1.MachineAuthorityMigrating)), + ) }) }) - Context("when all the prerequisites for switching the authoritative API are satisfied", func() { - Context("when migrating from MachineAPI to ClusterAPI", func() { - BeforeEach(func() { - By("Setting the MAPI machine set spec AuthoritativeAPI to ClusterAPI") + Context("when migrating from ClusterAPI to MachineAPI and the MachineSet finalizer is still present", func() { + BeforeEach(func() { + mapiMachineSet = mapiMachineSetBuilder. + WithAuthoritativeAPI(mapiv1beta1.MachineAuthorityMachineAPI). + Build() + Eventually(k8sClient.Create(ctx, mapiMachineSet)).Should(Succeed()) - mapiMachineSet = mapiMachineSetBuilder. - // Set desired authoritative API in spec to ClusterAPI. - // To check for requesting pausing on the the MAPI resource it is sufficient - // see that the spec.AuthoritativeAPI field is set to ClusterAPI. - WithAuthoritativeAPI(mapiv1beta1.MachineAuthorityClusterAPI). - Build() - Eventually(k8sClient.Create(ctx, mapiMachineSet)).Should(Succeed()) - - By("Setting the MAPI machine set status AuthoritativeAPI to 'Migrating'") - Eventually(k.UpdateStatus(mapiMachineSet, func() { - updatedMAPIMachineSet := mapiMachineSetBuilder. - WithAuthoritativeAPIStatus(mapiv1beta1.MachineAuthorityMigrating). - WithSynchronizedGeneration(mapiMachineSet.Generation). // Match the MAPI .metadata.generation field. - WithConditions([]mapiv1beta1.Condition{ - { - Type: consts.SynchronizedCondition, - LastTransitionTime: metav1.Now(), - Status: corev1.ConditionTrue, - }, - { - Type: "Paused", - LastTransitionTime: metav1.Now(), - Status: corev1.ConditionTrue, - }, - }). - Build() - mapiMachineSet.Status.AuthoritativeAPI = updatedMAPIMachineSet.Status.AuthoritativeAPI - mapiMachineSet.Status.SynchronizedGeneration = updatedMAPIMachineSet.Status.SynchronizedGeneration - mapiMachineSet.Status.Conditions = updatedMAPIMachineSet.Status.Conditions - })).Should(Succeed()) - - By("Creating a mirror CAPI machine set") + capiMachineSet = capiMachineSetBuilder. + WithAnnotations(map[string]string{clusterv1.PausedAnnotation: ""}). + Build() + capiMachineSet.Finalizers = append(capiMachineSet.Finalizers, clusterv1.MachineSetFinalizer) + Eventually(k8sClient.Create(ctx, capiMachineSet)).Should(Succeed()) - capiMachineSet = capiMachineSetBuilder. - WithAnnotations(map[string]string{ - clusterv1.PausedAnnotation: "", - }). - Build() - capiMachineSet.Finalizers = append(capiMachineSet.Finalizers, clusterv1.MachineSetFinalizer) - Eventually(k8sClient.Create(ctx, capiMachineSet)).Should(Succeed()) + updateCAPIMachineSetStatus(migrationcontrollertest.CAPIPausedCondition(metav1.ConditionFalse)) - By("Setting the CAPI machine set status condition to 'Paused'") - Eventually(k.UpdateStatus(capiMachineSet, func() { - updatedCAPIMachineSet := capiMachineSetBuilder.Build() - updatedCAPIMachineSet.Status.Conditions = []metav1.Condition{{ - Type: clusterv1.PausedCondition, - Status: metav1.ConditionTrue, - LastTransitionTime: metav1.Now(), - }} - capiMachineSet.Status = updatedCAPIMachineSet.Status - })).Should(Succeed()) - - req = reconcile.Request{NamespacedName: client.ObjectKeyFromObject(mapiMachineSet)} - }) + updateMAPIMachineSetStatus( + mapiv1beta1.MachineAuthorityClusterAPI, + mapiv1beta1.ClusterAPISynchronized, + capiMachineSet.Generation, + migrationcontrollertest.SynchronizedCondition(corev1.ConditionTrue), + ) + }) - It("should set the new to-be authoritative resource (CAPI) to actually be authoritative and unpause it", func() { - result, err := reconciler.Reconcile(ctx, req) - Expect(err).NotTo(HaveOccurred()) - Expect(result.Requeue).To(BeFalse()) + It("should wait for paused observation before entering Migrating", func() { + expectSuccessfulReconcile() - Eventually(komega.Object(mapiMachineSet)).Should(SatisfyAll( - HaveField("Status.AuthoritativeAPI", Equal(mapiv1beta1.MachineAuthorityClusterAPI)), - HaveField("Status.SynchronizedGeneration", BeZero()), - )) + Eventually(k.Object(mapiMachineSet)).Should(SatisfyAll( + HaveField("Status.AuthoritativeAPI", Equal(mapiv1beta1.MachineAuthorityClusterAPI)), + HaveField("Status.SynchronizedGeneration", Equal(capiMachineSet.Generation)), + )) + }) + }) - Eventually(komega.Object(capiMachineSet)).ShouldNot( - HaveField("ObjectMeta.Annotations", ContainElement(HaveKeyWithValue(clusterv1.PausedAnnotation, "")))) - }) + Context("when MachineAPI is authoritative and the CAPI MachineSet is missing", func() { + BeforeEach(func() { + mapiMachineSet = mapiMachineSetBuilder. + WithAuthoritativeAPI(mapiv1beta1.MachineAuthorityMachineAPI). + Build() + Eventually(k8sClient.Create(ctx, mapiMachineSet)).Should(Succeed()) + + updateMAPIMachineSetStatus( + mapiv1beta1.MachineAuthorityMachineAPI, + mapiv1beta1.MachineAPISynchronized, + mapiMachineSet.Generation, + migrationcontrollertest.SynchronizedCondition(corev1.ConditionTrue), + ) }) - Context("when migrating from ClusterAPI to MachineAPI", func() { - BeforeEach(func() { - By("Setting the MAPI machine set spec AuthoritativeAPI to MachineAPI") - mapiMachineSet = mapiMachineSetBuilder. - WithAuthoritativeAPI(mapiv1beta1.MachineAuthorityMachineAPI). - Build() - Eventually(k8sClient.Create(ctx, mapiMachineSet)).Should(Succeed()) + It("should treat the missing CAPI MachineSet as already safely paused", func() { + current := &mapiv1beta1.MachineSet{} + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(mapiMachineSet), current)).To(Succeed()) + initialResourceVersion := current.ResourceVersion - By("Creating a mirror CAPI machine set") + expectSuccessfulReconcile() - capiMachineSet = capiMachineSetBuilder. - WithAnnotations(map[string]string{ - clusterv1.PausedAnnotation: "", - }). - Build() - Eventually(k8sClient.Create(ctx, capiMachineSet)).Should(Succeed()) + Eventually(k.Object(mapiMachineSet)).Should(SatisfyAll( + HaveField("ObjectMeta.ResourceVersion", Equal(initialResourceVersion)), + HaveField("Status.AuthoritativeAPI", Equal(mapiv1beta1.MachineAuthorityMachineAPI)), + )) + }) + }) - By("Setting the MAPI machine set status AuthoritativeAPI to 'Migrating'") - Eventually(k.UpdateStatus(mapiMachineSet, func() { - updatedMAPIMachineSet := mapiMachineSetBuilder. - WithAuthoritativeAPIStatus(mapiv1beta1.MachineAuthorityMigrating). - WithSynchronizedGeneration(capiMachineSet.Generation). // Match the CAPI .metadata.generation field. - WithConditions([]mapiv1beta1.Condition{ - { - Type: consts.SynchronizedCondition, - LastTransitionTime: metav1.Now(), - Status: corev1.ConditionTrue, - }, - { - Type: "Paused", - LastTransitionTime: metav1.Now(), - Status: corev1.ConditionTrue, - }, - }). - Build() - mapiMachineSet.Status.AuthoritativeAPI = updatedMAPIMachineSet.Status.AuthoritativeAPI - mapiMachineSet.Status.SynchronizedGeneration = updatedMAPIMachineSet.Status.SynchronizedGeneration - mapiMachineSet.Status.Conditions = updatedMAPIMachineSet.Status.Conditions - })).Should(Succeed()) - - By("Setting the CAPI machine set status condition to 'Paused'") - Eventually(k.UpdateStatus(capiMachineSet, func() { - updatedCAPIMachineSet := capiMachineSetBuilder.Build() - updatedCAPIMachineSet.Status.Conditions = []metav1.Condition{{ - Type: clusterv1.PausedCondition, - Status: metav1.ConditionTrue, - LastTransitionTime: metav1.Now(), - }} - capiMachineSet.Status = updatedCAPIMachineSet.Status - })).Should(Succeed()) - - req = reconcile.Request{NamespacedName: client.ObjectKeyFromObject(mapiMachineSet)} - }) + Context("when ClusterAPI is authoritative and the CAPI MachineSet is missing", func() { + BeforeEach(func() { + mapiMachineSet = mapiMachineSetBuilder. + WithAuthoritativeAPI(mapiv1beta1.MachineAuthorityClusterAPI). + Build() + Eventually(k8sClient.Create(ctx, mapiMachineSet)).Should(Succeed()) - It("should set the new to-be authoritative resource (MAPI) to actually be authoritative and requeue to unpause it", func() { - result, err := reconciler.Reconcile(ctx, req) - Expect(err).NotTo(HaveOccurred()) - Expect(result.Requeue).To(BeFalse()) + updateMAPIMachineSetStatus( + mapiv1beta1.MachineAuthorityClusterAPI, + mapiv1beta1.ClusterAPISynchronized, + 1, + migrationcontrollertest.SynchronizedCondition(corev1.ConditionTrue), + ) + }) - Eventually(komega.Object(mapiMachineSet)).Should(SatisfyAll( - HaveField("Status.AuthoritativeAPI", Equal(mapiv1beta1.MachineAuthorityMachineAPI)), - HaveField("Status.SynchronizedGeneration", BeZero()), - )) - }) + It("should treat the missing CAPI MachineSet as already unpaused", func() { + current := &mapiv1beta1.MachineSet{} + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(mapiMachineSet), current)).To(Succeed()) + initialResourceVersion := current.ResourceVersion + + expectSuccessfulReconcile() + + Eventually(k.Object(mapiMachineSet)).Should(SatisfyAll( + HaveField("ObjectMeta.ResourceVersion", Equal(initialResourceVersion)), + HaveField("Status.AuthoritativeAPI", Equal(mapiv1beta1.MachineAuthorityClusterAPI)), + )) + }) + }) + }) + + Describe("addPausedAnnotation", func() { + Context("when the object has changed since it was read", func() { + It("should fail with a conflict", func() { + staleMachineSet := capiMachineSetBuilder. + WithName("stale-machineset"). + Build() + Expect(k8sClient.Create(ctx, staleMachineSet)).To(Succeed(), "machine set should be created") + + staleCopy := &clusterv1.MachineSet{} + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(staleMachineSet), staleCopy)).To(Succeed(), "stale copy should be fetched") + + liveMachineSet := &clusterv1.MachineSet{} + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(staleMachineSet), liveMachineSet)).To(Succeed(), "live copy should be fetched") + + if liveMachineSet.Annotations == nil { + liveMachineSet.Annotations = map[string]string{} + } + + liveMachineSet.Annotations["test.openshift.io/stale"] = "true" + Expect(k8sClient.Update(ctx, liveMachineSet)).To(Succeed(), "live machine set should be updated to make the stale copy outdated") + + changed, err := migrationcommon.AddPausedAnnotation(ctx, k8sClient, staleCopy) + Expect(changed).To(BeFalse(), "stale writes should not report a successful change") + Expect(err).To(HaveOccurred(), "stale writes should fail") + Expect(apierrors.IsConflict(err)).To(BeTrue(), "expected stale patch to fail with a conflict") }) }) }) diff --git a/pkg/controllers/machinesetsync/machineset_sync_controller.go b/pkg/controllers/machinesetsync/machineset_sync_controller.go index 97bc6185e..0940f0e30 100644 --- a/pkg/controllers/machinesetsync/machineset_sync_controller.go +++ b/pkg/controllers/machinesetsync/machineset_sync_controller.go @@ -724,14 +724,12 @@ func (r *MachineSetSyncReconciler) convertMAPIToCAPIMachineSet(mapiMachineSet *m } } -// applySynchronizedConditionWithPatch updates the synchronized condition -// using a server side apply patch. We do this to force ownership of the -// 'Synchronized' condition and 'SynchronizedGeneration'. func (r *MachineSetSyncReconciler) applySynchronizedConditionWithPatch(ctx context.Context, mapiMachineSet *mapiv1beta1.MachineSet, status corev1.ConditionStatus, reason, message string, generation *int64) error { return synccommon.ApplySyncStatus[*machinev1applyconfigs.MachineSetStatusApplyConfiguration]( ctx, r.Client, controllerName, machinev1applyconfigs.MachineSet, mapiMachineSet, - status, reason, message, generation) + status, reason, message, generation, + synccommon.AuthoritativeAPIToSynchronizedAPI(mapiMachineSet.Status.AuthoritativeAPI)) } // createOrUpdateCAPIInfraMachineTemplate creates a CAPI infra machine template from a MAPI machine set, or updates if it exists and it is out of date. @@ -1078,9 +1076,10 @@ func setChangedMAPIMachineSetStatusFields(existingMAPIMachineSet, convertedMAPIM // Copy them back to the convertedMAPIMachineSet. convertedMAPIMachineSet.Status.Conditions = existingMAPIMachineSet.Status.Conditions - // Keep the current SynchronizedGeneration and AuthorativeAPI. They get handled separately in `applySynchronizedConditionWithPatch` + // Keep the current SynchronizedGeneration, AuthoritativeAPI, and SynchronizedAPI. They get handled separately in `applySynchronizedConditionWithPatch` convertedMAPIMachineSet.Status.SynchronizedGeneration = existingMAPIMachineSet.Status.SynchronizedGeneration convertedMAPIMachineSet.Status.AuthoritativeAPI = existingMAPIMachineSet.Status.AuthoritativeAPI + convertedMAPIMachineSet.Status.SynchronizedAPI = existingMAPIMachineSet.Status.SynchronizedAPI // Finally overwrite the entire existingMAPIMachineSet status with the convertedMAPIMachineSet status. existingMAPIMachineSet.Status = convertedMAPIMachineSet.Status diff --git a/pkg/controllers/machinesetsync/machineset_sync_controller_test.go b/pkg/controllers/machinesetsync/machineset_sync_controller_test.go index b17df5822..39ff1b883 100644 --- a/pkg/controllers/machinesetsync/machineset_sync_controller_test.go +++ b/pkg/controllers/machinesetsync/machineset_sync_controller_test.go @@ -1370,6 +1370,12 @@ var _ = Describe("applySynchronizedConditionWithPatch", func() { HaveField("Status.SynchronizedGeneration", Equal(int64(22))), ) }) + + It("should set SynchronizedAPI to MachineAPISynchronized", func() { + Eventually(k.Object(mapiMachineSet), timeout).Should( + HaveField("Status.SynchronizedAPI", Equal(mapiv1beta1.MachineAPISynchronized)), + ) + }) }) Context("when condition status is Unknown", func() { @@ -1396,10 +1402,16 @@ var _ = Describe("applySynchronizedConditionWithPatch", func() { HaveField("Status.SynchronizedGeneration", Equal(int64(22))), ) }) + + It("should set SynchronizedAPI to MachineAPISynchronized", func() { + Eventually(k.Object(mapiMachineSet), timeout).Should( + HaveField("Status.SynchronizedAPI", Equal(mapiv1beta1.MachineAPISynchronized)), + ) + }) }) Context("when condition status is True", func() { - BeforeEach(func() { + JustBeforeEach(func() { err := reconciler.applySynchronizedConditionWithPatch(ctx, mapiMachineSet, corev1.ConditionTrue, consts.ReasonResourceSynchronized, messageSuccessfullySynchronizedMAPItoCAPI, &mapiMachineSet.Generation) Expect(err).NotTo(HaveOccurred()) }) @@ -1422,6 +1434,34 @@ var _ = Describe("applySynchronizedConditionWithPatch", func() { HaveField("Status.SynchronizedGeneration", Equal(int64(23))), ) }) + + Context("when AuthoritativeAPI is MachineAPI", func() { + It("should set SynchronizedAPI to MachineAPISynchronized", func() { + Eventually(k.Object(mapiMachineSet), timeout).Should( + HaveField("Status.SynchronizedAPI", Equal(mapiv1beta1.MachineAPISynchronized)), + ) + }) + }) + + Context("when AuthoritativeAPI is ClusterAPI", func() { + BeforeEach(func() { + By("Set the status of the MAPI MachineSet with ClusterAPI authority") + Eventually(k.UpdateStatus(mapiMachineSet, func() { + mapiMachineSet.Status.AuthoritativeAPI = mapiv1beta1.MachineAuthorityMigrating + })).Should(Succeed()) + Eventually(k.UpdateStatus(mapiMachineSet, func() { + mapiMachineSet.Status.AuthoritativeAPI = mapiv1beta1.MachineAuthorityClusterAPI + })).Should(Succeed()) + // Restore the artificial generation after UpdateStatus refreshes the object. + mapiMachineSet.Generation = int64(23) + }) + + It("should set SynchronizedAPI to ClusterAPISynchronized", func() { + Eventually(k.Object(mapiMachineSet), timeout).Should( + HaveField("Status.SynchronizedAPI", Equal(mapiv1beta1.ClusterAPISynchronized)), + ) + }) + }) }) }) diff --git a/pkg/controllers/machinesetsync/machineset_sync_controller_unit_test.go b/pkg/controllers/machinesetsync/machineset_sync_controller_unit_test.go index 5d38b6ed2..8ff176a19 100644 --- a/pkg/controllers/machinesetsync/machineset_sync_controller_unit_test.go +++ b/pkg/controllers/machinesetsync/machineset_sync_controller_unit_test.go @@ -16,17 +16,97 @@ limitations under the License. package machinesetsync import ( + "errors" "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + configv1 "github.com/openshift/api/config/v1" + mapiv1beta1 "github.com/openshift/api/machine/v1beta1" + capiv1resourcebuilder "github.com/openshift/cluster-api-actuator-pkg/testutils/resourcebuilder/cluster-api/core/v1beta2" + capav1builder "github.com/openshift/cluster-api-actuator-pkg/testutils/resourcebuilder/cluster-api/infrastructure/v1beta2" + machinev1resourcebuilder "github.com/openshift/cluster-api-actuator-pkg/testutils/resourcebuilder/machine/v1beta1" + controllers "github.com/openshift/cluster-capi-operator/pkg/controllers" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/validation/field" "k8s.io/utils/ptr" + awsv1 "sigs.k8s.io/cluster-api-provider-aws/v2/api/v1beta2" + ibmpowervsv1 "sigs.k8s.io/cluster-api-provider-ibmcloud/api/v1beta2" + openstackv1 "sigs.k8s.io/cluster-api-provider-openstack/api/v1beta1" clusterv1 "sigs.k8s.io/cluster-api/api/core/v1beta2" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" ) +type stubDiffResult struct { + metadata bool + spec bool + providerSpec bool + status bool +} + +func (d stubDiffResult) HasChanges() bool { + return d.metadata || d.spec || d.providerSpec || d.status +} + +func (d stubDiffResult) String() string { + return "stub diff" +} + +func (d stubDiffResult) HasMetadataChanges() bool { + return d.metadata +} + +func (d stubDiffResult) HasSpecChanges() bool { + return d.spec +} + +func (d stubDiffResult) HasProviderSpecChanges() bool { + return d.providerSpec +} + +func (d stubDiffResult) HasStatusChanges() bool { + return d.status +} + +func newMachineSetSyncUnitReconciler(objs []client.Object) *MachineSetSyncReconciler { + builder := fake.NewClientBuilder(). + WithScheme(testEnv.Scheme). + WithStatusSubresource( + &mapiv1beta1.MachineSet{}, + &clusterv1.MachineSet{}, + ) + + if len(objs) > 0 { + builder = builder.WithObjects(objs...) + } + + return &MachineSetSyncReconciler{ + Client: builder.Build(), + Scheme: testEnv.Scheme, + Platform: configv1.AWSPlatformType, + MAPINamespace: "openshift-machine-api", + CAPINamespace: "openshift-cluster-api", + } +} + +func expectFieldError(err error, expectedType field.ErrorType, expectedField, expectedDetail string) { + GinkgoHelper() + + var fieldErr *field.Error + Expect(errors.As(err, &fieldErr)).To(BeTrue(), "expected a Kubernetes field error") + Expect(fieldErr).To(SatisfyAll( + HaveField("Type", Equal(expectedType)), + HaveField("Field", Equal(expectedField)), + )) + + if expectedDetail != "" { + Expect(fieldErr).To(HaveField("Detail", Equal(expectedDetail))) + } +} + var _ = Describe("Unit tests for CAPIMachineSetStatusEqual", func() { now := metav1.Now() later := metav1.NewTime(now.Add(1 * time.Hour)) @@ -348,3 +428,299 @@ var _ = Describe("Unit tests for CAPIMachineSetStatusEqual", func() { }), ) }) + +var _ = Describe("Unit tests for MachineSetSync owner reference validation", func() { + var reconciler *MachineSetSyncReconciler + + BeforeEach(func() { + reconciler = newMachineSetSyncUnitReconciler(nil) + }) + + Context("when validating a Machine API machine set", func() { + It("should allow machine sets without owner references", func() { + mapiMachineSet := machinev1resourcebuilder.MachineSet(). + WithNamespace("openshift-machine-api"). + WithName("foo"). + Build() + + Expect(reconciler.validateMAPIMachineSetOwnerReferences(mapiMachineSet)).To(Succeed()) + }) + + It("should reject machine sets with owner references", func() { + mapiMachineSet := machinev1resourcebuilder.MachineSet(). + WithNamespace("openshift-machine-api"). + WithName("foo"). + Build() + mapiMachineSet.OwnerReferences = []metav1.OwnerReference{{ + APIVersion: "healthchecking.openshift.io/v1beta1", + Kind: "MachineHealthCheck", + Name: "foo", + }} + + err := reconciler.validateMAPIMachineSetOwnerReferences(mapiMachineSet) + expectFieldError(err, field.ErrorTypeInvalid, "metadata.ownerReferences", errMachineAPIMachineSetOwnerReferenceConversionUnsupported.Error()) + }) + }) + + Context("when validating a Cluster API machine set", func() { + clusterOwnerReference := metav1.OwnerReference{ + APIVersion: clusterv1.GroupVersion.String(), + Kind: clusterv1.ClusterKind, + Name: "cluster", + } + + It("should allow machine sets without owner references", func() { + capiMachineSet := capiv1resourcebuilder.MachineSet(). + WithNamespace("openshift-cluster-api"). + WithName("foo"). + Build() + + Expect(reconciler.validateCAPIMachineSetOwnerReferences(capiMachineSet)).To(Succeed()) + }) + + It("should allow a single Cluster owner reference", func() { + capiMachineSet := capiv1resourcebuilder.MachineSet(). + WithNamespace("openshift-cluster-api"). + WithName("foo"). + WithOwnerReferences([]metav1.OwnerReference{clusterOwnerReference}). + Build() + + Expect(reconciler.validateCAPIMachineSetOwnerReferences(capiMachineSet)).To(Succeed()) + }) + + It("should reject non-Cluster owner references", func() { + capiMachineSet := capiv1resourcebuilder.MachineSet(). + WithNamespace("openshift-cluster-api"). + WithName("foo"). + WithOwnerReferences([]metav1.OwnerReference{{ + APIVersion: clusterv1.GroupVersion.String(), + Kind: "MachineDeployment", + Name: "foo", + }}). + Build() + + err := reconciler.validateCAPIMachineSetOwnerReferences(capiMachineSet) + expectFieldError(err, field.ErrorTypeInvalid, "metadata.ownerReferences", errUnsuportedOwnerKindForConversion.Error()) + }) + + It("should reject more than one owner reference", func() { + capiMachineSet := capiv1resourcebuilder.MachineSet(). + WithNamespace("openshift-cluster-api"). + WithName("foo"). + WithOwnerReferences([]metav1.OwnerReference{ + clusterOwnerReference, + { + APIVersion: clusterv1.GroupVersion.String(), + Kind: clusterv1.ClusterKind, + Name: "another-cluster", + }, + }). + Build() + + err := reconciler.validateCAPIMachineSetOwnerReferences(capiMachineSet) + expectFieldError(err, field.ErrorTypeTooMany, "metadata.ownerReferences", "") + }) + }) +}) + +var _ = Describe("Unit tests for infrastructure machine template cleanup helpers", func() { + DescribeTable("should filter outdated infrastructure machine templates for supported providers", + func(list client.ObjectList, currentTemplateName string, expectedNames []string) { + outdatedTemplates, err := filterOutdatedInfraMachineTemplates(list, currentTemplateName) + Expect(err).NotTo(HaveOccurred()) + + outdatedTemplateNames := make([]string, 0, len(outdatedTemplates)) + for _, template := range outdatedTemplates { + outdatedTemplateNames = append(outdatedTemplateNames, template.GetName()) + } + + Expect(outdatedTemplateNames).To(ConsistOf(expectedNames)) + }, + Entry("AWS machine templates", + &awsv1.AWSMachineTemplateList{ + Items: []awsv1.AWSMachineTemplate{ + {ObjectMeta: metav1.ObjectMeta{Name: "current"}}, + {ObjectMeta: metav1.ObjectMeta{Name: "old-1"}}, + {ObjectMeta: metav1.ObjectMeta{Name: "old-2"}}, + }, + }, + "current", + []string{"old-1", "old-2"}, + ), + Entry("PowerVS machine templates", + &ibmpowervsv1.IBMPowerVSMachineTemplateList{ + Items: []ibmpowervsv1.IBMPowerVSMachineTemplate{ + {ObjectMeta: metav1.ObjectMeta{Name: "current"}}, + {ObjectMeta: metav1.ObjectMeta{Name: "old"}}, + }, + }, + "current", + []string{"old"}, + ), + Entry("OpenStack machine templates", + &openstackv1.OpenStackMachineTemplateList{ + Items: []openstackv1.OpenStackMachineTemplate{ + {ObjectMeta: metav1.ObjectMeta{Name: "current"}}, + {ObjectMeta: metav1.ObjectMeta{Name: "old"}}, + }, + }, + "current", + []string{"old"}, + ), + ) + + It("should reject unexpected infrastructure machine template lists", func() { + outdatedTemplates, err := filterOutdatedInfraMachineTemplates(&corev1.PodList{}, "current") + Expect(outdatedTemplates).To(BeNil(), "expected no templates to be returned for unsupported list types") + Expect(err).To(MatchError(ContainSubstring(errUnexpectedInfraMachineTemplateListType.Error()))) + }) + + It("should separate active and deleting outdated templates", func() { + now := metav1.Now() + templatesToDelete, deletingTemplates := categorizeInfraMachineTemplates([]client.Object{ + &awsv1.AWSMachineTemplate{ObjectMeta: metav1.ObjectMeta{Name: "old-active"}}, + &awsv1.AWSMachineTemplate{ObjectMeta: metav1.ObjectMeta{ + Name: "old-deleting", + DeletionTimestamp: &now, + }}, + }) + + Expect(templatesToDelete).To(ConsistOf(HaveField("Name", Equal("old-active")))) + Expect(deletingTemplates).To(ConsistOf(HaveField("Name", Equal("old-deleting")))) + }) +}) + +var _ = Describe("Unit tests for deleteOutdatedCAPIInfraMachineTemplates", func() { + var mapiMachineSet *mapiv1beta1.MachineSet + + BeforeEach(func() { + mapiMachineSet = machinev1resourcebuilder.MachineSet(). + WithNamespace("openshift-machine-api"). + WithName("foo"). + Build() + }) + + It("should do nothing when there are no labeled outdated templates", func() { + currentTemplate := capav1builder.AWSMachineTemplate(). + WithNamespace("openshift-cluster-api"). + WithName("current"). + Build() + currentTemplate.Labels = map[string]string{controllers.MachineSetOpenshiftLabelKey: mapiMachineSet.Name} + + unlabelledOldTemplate := capav1builder.AWSMachineTemplate(). + WithNamespace("openshift-cluster-api"). + WithName("old-unlabelled"). + Build() + + reconciler := newMachineSetSyncUnitReconciler([]client.Object{currentTemplate, unlabelledOldTemplate}) + + shouldRequeue, err := reconciler.deleteOutdatedCAPIInfraMachineTemplates(ctx, mapiMachineSet, currentTemplate.Name) + Expect(err).NotTo(HaveOccurred()) + Expect(shouldRequeue).To(BeFalse(), "expected no requeue when only the current labeled template exists") + }) + + It("should requeue while labeled outdated templates are already deleting", func() { + now := metav1.Now() + currentTemplate := capav1builder.AWSMachineTemplate(). + WithNamespace("openshift-cluster-api"). + WithName("current"). + Build() + currentTemplate.Labels = map[string]string{controllers.MachineSetOpenshiftLabelKey: mapiMachineSet.Name} + + deletingTemplate := capav1builder.AWSMachineTemplate(). + WithNamespace("openshift-cluster-api"). + WithName("old-deleting"). + Build() + deletingTemplate.Labels = map[string]string{controllers.MachineSetOpenshiftLabelKey: mapiMachineSet.Name} + deletingTemplate.Finalizers = []string{"example.com/finalizer"} + deletingTemplate.DeletionTimestamp = &now + + reconciler := newMachineSetSyncUnitReconciler([]client.Object{currentTemplate, deletingTemplate}) + + shouldRequeue, err := reconciler.deleteOutdatedCAPIInfraMachineTemplates(ctx, mapiMachineSet, currentTemplate.Name) + Expect(err).NotTo(HaveOccurred()) + Expect(shouldRequeue).To(BeTrue(), "expected a requeue while outdated templates are already deleting") + + storedDeletingTemplate := &awsv1.AWSMachineTemplate{} + Expect(reconciler.Client.Get(ctx, client.ObjectKeyFromObject(deletingTemplate), storedDeletingTemplate)).To(Succeed()) + Expect(storedDeletingTemplate.GetDeletionTimestamp()).NotTo(BeNil(), "expected the deleting template to remain in deleting state") + }) +}) + +var _ = Describe("Unit tests for MachineSet status generation gating", func() { + It("should wait to update Cluster API status until the Machine API observed generation catches up", func() { + mapiMachineSet := machinev1resourcebuilder.MachineSet(). + WithNamespace("openshift-machine-api"). + WithName("foo"). + Build() + mapiMachineSet.Generation = 2 + mapiMachineSet.Status.ObservedGeneration = 1 + + existingCAPIMachineSet := capiv1resourcebuilder.MachineSet(). + WithNamespace("openshift-cluster-api"). + WithName("foo"). + Build() + existingCAPIMachineSet.Status.Replicas = ptr.To[int32](1) + + convertedCAPIMachineSet := existingCAPIMachineSet.DeepCopy() + updatedOrCreatedCAPIMachineSet := existingCAPIMachineSet.DeepCopy() + updatedOrCreatedCAPIMachineSet.Generation = 3 + + reconciler := newMachineSetSyncUnitReconciler([]client.Object{existingCAPIMachineSet.DeepCopy()}) + + updated, err := reconciler.ensureCAPIMachineSetStatusUpdated( + ctx, + mapiMachineSet, + existingCAPIMachineSet, + convertedCAPIMachineSet, + updatedOrCreatedCAPIMachineSet, + stubDiffResult{}, + true, + ) + Expect(err).NotTo(HaveOccurred()) + Expect(updated).To(BeFalse(), "expected status updates to wait for the Machine API observed generation") + + storedCAPIMachineSet := &clusterv1.MachineSet{} + Expect(reconciler.Client.Get(ctx, client.ObjectKeyFromObject(existingCAPIMachineSet), storedCAPIMachineSet)).To(Succeed()) + Expect(storedCAPIMachineSet.Status.Replicas).To(HaveValue(BeEquivalentTo(1)), "expected the stored Cluster API status to remain unchanged") + Expect(storedCAPIMachineSet.Status.ObservedGeneration).To(BeZero(), "expected the stored Cluster API observed generation to remain unchanged") + }) + + It("should wait to update Machine API status until the Cluster API observed generation catches up", func() { + existingMAPIMachineSet := machinev1resourcebuilder.MachineSet(). + WithNamespace("openshift-machine-api"). + WithName("foo"). + Build() + existingMAPIMachineSet.Status.Replicas = 1 + + convertedMAPIMachineSet := existingMAPIMachineSet.DeepCopy() + updatedMAPIMachineSet := existingMAPIMachineSet.DeepCopy() + updatedMAPIMachineSet.Generation = 4 + + sourceCAPIMachineSet := capiv1resourcebuilder.MachineSet(). + WithNamespace("openshift-cluster-api"). + WithName("foo"). + Build() + sourceCAPIMachineSet.Generation = 3 + sourceCAPIMachineSet.Status.ObservedGeneration = 2 + + reconciler := newMachineSetSyncUnitReconciler([]client.Object{existingMAPIMachineSet.DeepCopy()}) + + updated, err := reconciler.ensureMAPIMachineSetStatusUpdated( + ctx, + existingMAPIMachineSet, + convertedMAPIMachineSet, + updatedMAPIMachineSet, + sourceCAPIMachineSet, + stubDiffResult{}, + true, + ) + Expect(err).NotTo(HaveOccurred()) + Expect(updated).To(BeFalse(), "expected status updates to wait for the Cluster API observed generation") + + storedMAPIMachineSet := &mapiv1beta1.MachineSet{} + Expect(reconciler.Client.Get(ctx, client.ObjectKeyFromObject(existingMAPIMachineSet), storedMAPIMachineSet)).To(Succeed()) + Expect(storedMAPIMachineSet.Status.Replicas).To(BeEquivalentTo(1), "expected the stored Machine API status to remain unchanged") + Expect(storedMAPIMachineSet.Status.ObservedGeneration).To(BeZero(), "expected the stored Machine API observed generation to remain unchanged") + }) +}) diff --git a/pkg/controllers/machinesync/machine_sync_controller.go b/pkg/controllers/machinesync/machine_sync_controller.go index 292409e56..be6f08cda 100644 --- a/pkg/controllers/machinesync/machine_sync_controller.go +++ b/pkg/controllers/machinesync/machine_sync_controller.go @@ -1505,6 +1505,7 @@ func setChangedMAPIMachineStatusFields(existingMAPIMachine, convertedMAPIMachine // Copy the other fields that are not present in the convertedMAPIMachine from the existingMAPIMachine. convertedMAPIMachine.Status.AuthoritativeAPI = existingMAPIMachine.Status.AuthoritativeAPI convertedMAPIMachine.Status.SynchronizedGeneration = existingMAPIMachine.Status.SynchronizedGeneration + convertedMAPIMachine.Status.SynchronizedAPI = existingMAPIMachine.Status.SynchronizedAPI convertedMAPIMachine.Status.LastOperation = existingMAPIMachine.Status.LastOperation // ProviderStatus is handled separately by setChangedMAPIMachineProviderStatusFields @@ -1512,14 +1513,17 @@ func setChangedMAPIMachineStatusFields(existingMAPIMachine, convertedMAPIMachine existingMAPIMachine.Status = convertedMAPIMachine.Status } -// applySynchronizedConditionWithPatch updates the synchronized condition -// using a server side apply patch. We do this to force ownership of the -// 'Synchronized' condition and 'SynchronizedGeneration'. func (r *MachineSyncReconciler) applySynchronizedConditionWithPatch(ctx context.Context, mapiMachine *mapiv1beta1.Machine, status corev1.ConditionStatus, reason, message string, generation *int64) error { + var synchronizedAPI *mapiv1beta1.SynchronizedAPI + if generation != nil { + synchronizedAPI = synccommon.AuthoritativeAPIToSynchronizedAPI(mapiMachine.Status.AuthoritativeAPI) + } + return synccommon.ApplySyncStatus[*machinev1applyconfigs.MachineStatusApplyConfiguration]( ctx, r.Client, controllerName, machinev1applyconfigs.Machine, mapiMachine, - status, reason, message, generation) + status, reason, message, generation, + synchronizedAPI) } // isTerminalConfigurationError returns true if the provided error is diff --git a/pkg/controllers/machinesync/machine_sync_controller_test.go b/pkg/controllers/machinesync/machine_sync_controller_test.go index 5badedc6c..38cac96cb 100644 --- a/pkg/controllers/machinesync/machine_sync_controller_test.go +++ b/pkg/controllers/machinesync/machine_sync_controller_test.go @@ -226,6 +226,27 @@ var _ = Describe("With a running MachineSync Reconciler", func() { <-mgrDone } + appendIfMissing := func(finalizers []string, finalizer string) []string { + for _, existing := range finalizers { + if existing == finalizer { + return finalizers + } + } + + return append(finalizers, finalizer) + } + + removeFinalizer := func(finalizers []string, finalizer string) []string { + filtered := make([]string, 0, len(finalizers)) + for _, existing := range finalizers { + if existing != finalizer { + filtered = append(filtered, existing) + } + } + + return filtered + } + BeforeEach(func() { By("Setting up a namespaces for the test") @@ -1141,6 +1162,100 @@ var _ = Describe("With a running MachineSync Reconciler", func() { }) }) + Context("when a synchronized MAPI-authoritative machine is deleted", func() { + It("should delete the mirrored Cluster API resources only after the Machine API finalizer is cleared", func() { + By("Creating the MAPI machine") + + mapiMachine = mapiMachineBuilder.Build() + Eventually(k8sClient.Create(ctx, mapiMachine)).Should(Succeed(), "mapi machine should be able to be created") + + By("Setting the MAPI machine AuthoritativeAPI to MachineAPI") + Eventually(k.UpdateStatus(mapiMachine, func() { + mapiMachine.Status.AuthoritativeAPI = mapiv1beta1.MachineAuthorityMachineAPI + })).Should(Succeed()) + + By("Waiting for the mirrored Cluster API resources and sync finalizers") + + capiMachine = clusterv1resourcebuilder.Machine().WithName(mapiMachine.Name).WithNamespace(capiNamespace.Name).Build() + capaMachine = awsv1resourcebuilder.AWSMachine().WithName(mapiMachine.Name).WithNamespace(capiNamespace.Name).Build() + + Eventually(k.Object(mapiMachine), timeout).Should( + HaveField("ObjectMeta.Finalizers", ContainElement(SyncFinalizer)), + ) + Eventually(k.Object(capiMachine), timeout).Should( + HaveField("ObjectMeta.Finalizers", ContainElement(SyncFinalizer)), + ) + Eventually(k.Object(capaMachine), timeout).Should( + HaveField("ObjectMeta.Finalizers", ContainElement(SyncFinalizer)), + ) + + By("Adding finalizers that force the deletion choreography through its ordered cleanup path") + Eventually(k.Update(mapiMachine, func() { + mapiMachine.Finalizers = appendIfMissing(mapiMachine.Finalizers, mapiv1beta1.MachineFinalizer) + })).Should(Succeed()) + Eventually(k.Update(capiMachine, func() { + capiMachine.Finalizers = appendIfMissing(capiMachine.Finalizers, clusterv1.MachineFinalizer) + })).Should(Succeed()) + Eventually(k.Update(capaMachine, func() { + capaMachine.Finalizers = appendIfMissing(capaMachine.Finalizers, "awsmachine.infrastructure.cluster.x-k8s.io") + })).Should(Succeed()) + + By("Deleting the authoritative Machine API machine") + Eventually(func() error { + return k8sClient.Delete(ctx, mapiMachine) + }).Should(Succeed(), "mapi machine should be able to be deleted") + + By("Checking that the mirrored Cluster API resources are deleting but still blocked on their finalizers") + Eventually(k.Object(mapiMachine), timeout).Should( + HaveField("ObjectMeta.DeletionTimestamp", Not(BeNil())), + ) + Eventually(k.Object(capiMachine), timeout).Should( + SatisfyAll( + HaveField("ObjectMeta.DeletionTimestamp", Not(BeNil())), + HaveField("ObjectMeta.Finalizers", ContainElement(clusterv1.MachineFinalizer)), + HaveField("ObjectMeta.Finalizers", ContainElement(SyncFinalizer)), + ), + ) + Eventually(k.Object(capaMachine), timeout).Should( + SatisfyAll( + HaveField("ObjectMeta.DeletionTimestamp", Not(BeNil())), + HaveField("ObjectMeta.Finalizers", ContainElement("awsmachine.infrastructure.cluster.x-k8s.io")), + HaveField("ObjectMeta.Finalizers", ContainElement(SyncFinalizer)), + ), + ) + Consistently(k.Object(capiMachine), consistentlyTimeout).Should( + HaveField("ObjectMeta.Finalizers", ContainElement(clusterv1.MachineFinalizer)), + ) + + By("Removing the Machine API finalizer so the controller can finish cleanup") + Eventually(k.Update(mapiMachine, func() { + mapiMachine.Finalizers = removeFinalizer(mapiMachine.Finalizers, mapiv1beta1.MachineFinalizer) + })).Should(Succeed()) + + By("Simulating the CAPI and infrastructure controllers clearing their own finalizers") + Eventually(k.Update(capiMachine, func() { + capiMachine.Finalizers = removeFinalizer(capiMachine.Finalizers, clusterv1.MachineFinalizer) + })).Should(Succeed()) + Eventually(k.Update(capaMachine, func() { + capaMachine.Finalizers = removeFinalizer(capaMachine.Finalizers, "awsmachine.infrastructure.cluster.x-k8s.io") + })).Should(Succeed()) + + By("Checking that all mirrored resources are eventually deleted") + Eventually(func() bool { + err := k8sClient.Get(ctx, client.ObjectKey{Name: mapiMachine.Name, Namespace: mapiNamespace.Name}, &mapiv1beta1.Machine{}) + return apierrors.IsNotFound(err) + }, timeout).Should(BeTrue(), "mapi machine should eventually be deleted") + Eventually(func() bool { + err := k8sClient.Get(ctx, client.ObjectKey{Name: capiMachine.Name, Namespace: capiNamespace.Name}, &clusterv1.Machine{}) + return apierrors.IsNotFound(err) + }, timeout).Should(BeTrue(), "capi machine should eventually be deleted") + Eventually(func() bool { + err := k8sClient.Get(ctx, client.ObjectKey{Name: capaMachine.Name, Namespace: capiNamespace.Name}, &awsv1.AWSMachine{}) + return apierrors.IsNotFound(err) + }, timeout).Should(BeTrue(), "capi infra machine should eventually be deleted") + }) + }) + Context("validating admission policy tests", func() { const ( testLabelValue = "test-value" @@ -2607,6 +2722,12 @@ var _ = Describe("applySynchronizedConditionWithPatch", func() { HaveField("Status.SynchronizedGeneration", Equal(int64(22))), ) }) + + It("should leave SynchronizedAPI unchanged when there is no successful sync generation", func() { + Eventually(k.Object(mapiMachine), timeout).Should( + HaveField("Status.SynchronizedAPI", BeEmpty()), + ) + }) }) Context("when condition status is Unknown", func() { @@ -2633,10 +2754,16 @@ var _ = Describe("applySynchronizedConditionWithPatch", func() { HaveField("Status.SynchronizedGeneration", Equal(int64(22))), ) }) + + It("should leave SynchronizedAPI unchanged when there is no successful sync generation", func() { + Eventually(k.Object(mapiMachine), timeout).Should( + HaveField("Status.SynchronizedAPI", BeEmpty()), + ) + }) }) Context("when condition status is True", func() { - BeforeEach(func() { + JustBeforeEach(func() { err := reconciler.applySynchronizedConditionWithPatch(ctx, mapiMachine, corev1.ConditionTrue, consts.ReasonResourceSynchronized, messageSuccessfullySynchronizedMAPItoCAPI, &mapiMachine.Generation) Expect(err).NotTo(HaveOccurred()) }) @@ -2659,6 +2786,34 @@ var _ = Describe("applySynchronizedConditionWithPatch", func() { HaveField("Status.SynchronizedGeneration", Equal(int64(23))), ) }) + + Context("when AuthoritativeAPI is MachineAPI", func() { + It("should set SynchronizedAPI to MachineAPISynchronized", func() { + Eventually(k.Object(mapiMachine), timeout).Should( + HaveField("Status.SynchronizedAPI", Equal(mapiv1beta1.MachineAPISynchronized)), + ) + }) + }) + + Context("when AuthoritativeAPI is ClusterAPI", func() { + BeforeEach(func() { + By("Set the status of the MAPI Machine with ClusterAPI authority") + Eventually(k.UpdateStatus(mapiMachine, func() { + mapiMachine.Status.AuthoritativeAPI = mapiv1beta1.MachineAuthorityMigrating + })).Should(Succeed()) + Eventually(k.UpdateStatus(mapiMachine, func() { + mapiMachine.Status.AuthoritativeAPI = mapiv1beta1.MachineAuthorityClusterAPI + })).Should(Succeed()) + // Restore the artificial generation after UpdateStatus refreshes the object. + mapiMachine.Generation = int64(23) + }) + + It("should set SynchronizedAPI to ClusterAPISynchronized", func() { + Eventually(k.Object(mapiMachine), timeout).Should( + HaveField("Status.SynchronizedAPI", Equal(mapiv1beta1.ClusterAPISynchronized)), + ) + }) + }) }) }) diff --git a/pkg/controllers/migrationcommon/controller.go b/pkg/controllers/migrationcommon/controller.go new file mode 100644 index 000000000..6e174a682 --- /dev/null +++ b/pkg/controllers/migrationcommon/controller.go @@ -0,0 +1,369 @@ +/* +Copyright 2026 Red Hat, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationcommon + +import ( + "context" + "errors" + "fmt" + + mapiv1beta1 "github.com/openshift/api/machine/v1beta1" + "github.com/openshift/cluster-capi-operator/pkg/controllers" + "github.com/openshift/cluster-capi-operator/pkg/controllers/synccommon" + "github.com/openshift/cluster-capi-operator/pkg/util" + corev1 "k8s.io/api/core/v1" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +var ( + errUnsupportedCurrentAuthority = errors.New("unsupported current authoritativeAPI") + errUnsupportedStableAuthority = errors.New("unsupported stable authority") + errUnrecognizedPausedConditionStatus = errors.New("unrecognized paused condition status") +) + +func unsupportedCurrentAuthorityError(authority mapiv1beta1.MachineAuthority) error { + return fmt.Errorf("%w: %q", errUnsupportedCurrentAuthority, authority) +} + +func unsupportedStableAuthorityError(authority mapiv1beta1.MachineAuthority) error { + return fmt.Errorf("%w: %q", errUnsupportedStableAuthority, authority) +} + +func unrecognizedPausedConditionStatusError(obj client.Object, status corev1.ConditionStatus) error { + return fmt.Errorf("%w for %s/%s: %q", errUnrecognizedPausedConditionStatus, obj.GetNamespace(), obj.GetName(), status) +} + +type primaryCAPIObjectP[objT any] interface { + *objT + client.Object +} + +// Migratable exposes the data and side effects needed to reconcile migration +// status for a Machine API object and its primary Cluster API counterpart. +type Migratable[ + capiT any, + capiPT primaryCAPIObjectP[capiT], +] interface { + MAPIObject() client.Object + + DesiredAuthority() mapiv1beta1.MachineAuthority + CurrentAuthority() mapiv1beta1.MachineAuthority + SynchronizedAPI() mapiv1beta1.SynchronizedAPI + SynchronizedGeneration() int64 + MAPIConditions() []mapiv1beta1.Condition + + EnsureCAPIPaused(ctx context.Context, capi capiPT) (bool, error) + EnsureCAPIUnpaused(ctx context.Context, capi capiPT) (bool, error) +} + +// Reconcile advances migration state for a Machine API object toward its +// desired authoritative API. +func Reconcile[ + statusPT synccommon.StatusApplyConfigurationP[statusT, statusPT], + objPT synccommon.ObjApplyConfigurationP[objT, objPT, statusPT], + statusT, objT, capiT any, + capiPT primaryCAPIObjectP[capiT], +]( + ctx context.Context, + k8sClient client.Client, + controllerName string, + capiNamespace string, + newApplyConfig synccommon.ObjApplyConfigurationConstructor[objPT, statusPT], + migratable Migratable[capiT, capiPT], +) (ctrl.Result, error) { + desiredAuthority := migratable.DesiredAuthority() + + if migratable.CurrentAuthority() == "" { + if err := validateDesiredAuthority(desiredAuthority); err != nil { + return ctrl.Result{}, err + } + + if err := synccommon.ApplyMigrationStatus(ctx, k8sClient, controllerName, newApplyConfig, migratable.MAPIObject(), desiredAuthority); err != nil { + return ctrl.Result{}, fmt.Errorf("unable to apply authoritativeAPI to status: %w", err) + } + + return ctrl.Result{}, nil + } + + switch desiredAuthority { + case mapiv1beta1.MachineAuthorityClusterAPI: + return reconcileToCAPI(ctx, k8sClient, controllerName, capiNamespace, newApplyConfig, migratable) + case mapiv1beta1.MachineAuthorityMachineAPI: + return reconcileToMAPI(ctx, k8sClient, controllerName, capiNamespace, newApplyConfig, migratable) + case mapiv1beta1.MachineAuthorityMigrating: + fallthrough + default: + return ctrl.Result{}, validateDesiredAuthority(desiredAuthority) + } +} + +func validateDesiredAuthority(desiredAuthority mapiv1beta1.MachineAuthority) error { + switch desiredAuthority { + case mapiv1beta1.MachineAuthorityClusterAPI, mapiv1beta1.MachineAuthorityMachineAPI: + return nil + case mapiv1beta1.MachineAuthorityMigrating: + fallthrough + default: + return fmt.Errorf("unable to determine desired migration direction: %w", synccommon.UnsupportedTargetAuthorityError(desiredAuthority)) + } +} + +//nolint:funlen // Keep the full MachineAPI->ClusterAPI transition flow in one function. +func reconcileToCAPI[ + statusPT synccommon.StatusApplyConfigurationP[statusT, statusPT], + objPT synccommon.ObjApplyConfigurationP[objT, objPT, statusPT], + statusT, objT, capiT any, + capiPT primaryCAPIObjectP[capiT], +]( + ctx context.Context, + k8sClient client.Client, + controllerName string, + capiNamespace string, + newApplyConfig synccommon.ObjApplyConfigurationConstructor[objPT, statusPT], + migratable Migratable[capiT, capiPT], +) (ctrl.Result, error) { + switch migratable.CurrentAuthority() { + case mapiv1beta1.MachineAuthorityMachineAPI: + isSynchronized, err := isStableStateSynchronized(ctx, k8sClient, capiNamespace, migratable, mapiv1beta1.MachineAuthorityMachineAPI) + if err != nil { + return ctrl.Result{}, fmt.Errorf("unable to check Machine API stable sync gate: %w", err) + } + + if !isSynchronized { + return ctrl.Result{}, nil + } + + if err := synccommon.ApplyMigrationStatus(ctx, k8sClient, controllerName, newApplyConfig, migratable.MAPIObject(), mapiv1beta1.MachineAuthorityMigrating); err != nil { + return ctrl.Result{}, fmt.Errorf("unable to set authoritativeAPI to Migrating: %w", err) + } + + return ctrl.Result{}, nil + case mapiv1beta1.MachineAuthorityMigrating: + // The migrating state controls the pausing of Machine API controller. + paused, err := isMAPIPaused(migratable) + if err != nil { + return ctrl.Result{}, fmt.Errorf("failed to check Machine API paused condition: %w", err) + } + + if !paused { + return ctrl.Result{}, nil + } + + if err := synccommon.ApplyMigrationStatusAndResetSyncStatus(ctx, k8sClient, controllerName, newApplyConfig, migratable.MAPIObject(), mapiv1beta1.MachineAuthorityClusterAPI); err != nil { + return ctrl.Result{}, fmt.Errorf("failed to apply authoritativeAPI and reset sync status: %w", err) + } + + return ctrl.Result{}, nil + case mapiv1beta1.MachineAuthorityClusterAPI: + // This prevents the user from deliberately pausing the Cluster API object. + // We should look into a way to allow this in the future. + capiObj, found, err := getPrimaryCAPIObject[capiT, capiPT](ctx, k8sClient, capiNamespace, migratable.MAPIObject().GetName()) + if err != nil { + return ctrl.Result{}, err + } + + if !found { + return ctrl.Result{}, nil + } + + unpaused, err := migratable.EnsureCAPIUnpaused(ctx, capiObj) + if err != nil { + return ctrl.Result{}, fmt.Errorf("failed to ensure the Cluster API side is unpaused: %w", err) + } + + if !unpaused { + return ctrl.Result{}, nil + } + + return ctrl.Result{}, nil + default: + return ctrl.Result{}, unsupportedCurrentAuthorityError(migratable.CurrentAuthority()) + } +} + +//nolint:funlen // Keep the full ClusterAPI->MachineAPI transition flow in one function. +func reconcileToMAPI[ + statusPT synccommon.StatusApplyConfigurationP[statusT, statusPT], + objPT synccommon.ObjApplyConfigurationP[objT, objPT, statusPT], + statusT, objT, capiT any, + capiPT primaryCAPIObjectP[capiT], +]( + ctx context.Context, + k8sClient client.Client, + controllerName string, + capiNamespace string, + newApplyConfig synccommon.ObjApplyConfigurationConstructor[objPT, statusPT], + migratable Migratable[capiT, capiPT], +) (ctrl.Result, error) { + switch migratable.CurrentAuthority() { + case mapiv1beta1.MachineAuthorityClusterAPI: + capiObj, found, err := getPrimaryCAPIObject[capiT, capiPT](ctx, k8sClient, capiNamespace, migratable.MAPIObject().GetName()) + if err != nil { + return ctrl.Result{}, err + } + + if !found { + return ctrl.Result{}, nil + } + + paused, err := migratable.EnsureCAPIPaused(ctx, capiObj) + if err != nil { + return ctrl.Result{}, fmt.Errorf("failed to ensure the Cluster API side is paused: %w", err) + } + + if !paused { + return ctrl.Result{}, nil + } + + isSynchronized, err := isStableStateSynchronized(ctx, k8sClient, capiNamespace, migratable, mapiv1beta1.MachineAuthorityClusterAPI) + if err != nil { + return ctrl.Result{}, fmt.Errorf("unable to check Cluster API stable sync gate: %w", err) + } + + if !isSynchronized { + return ctrl.Result{}, nil + } + + if err := synccommon.ApplyMigrationStatus(ctx, k8sClient, controllerName, newApplyConfig, migratable.MAPIObject(), mapiv1beta1.MachineAuthorityMigrating); err != nil { + return ctrl.Result{}, fmt.Errorf("unable to set authoritativeAPI to Migrating: %w", err) + } + + return ctrl.Result{}, nil + case mapiv1beta1.MachineAuthorityMigrating: + // This state is not required by the logic. We go through Migrating to satisfy API validation rule. + if err := synccommon.ApplyMigrationStatusAndResetSyncStatus(ctx, k8sClient, controllerName, newApplyConfig, migratable.MAPIObject(), mapiv1beta1.MachineAuthorityMachineAPI); err != nil { + return ctrl.Result{}, fmt.Errorf("failed to apply authoritativeAPI and reset sync status: %w", err) + } + + return ctrl.Result{}, nil + case mapiv1beta1.MachineAuthorityMachineAPI: + capiObj, found, err := getPrimaryCAPIObject[capiT, capiPT](ctx, k8sClient, capiNamespace, migratable.MAPIObject().GetName()) + if err != nil { + return ctrl.Result{}, err + } + + if !found { + return ctrl.Result{}, nil + } + + paused, err := migratable.EnsureCAPIPaused(ctx, capiObj) + if err != nil { + return ctrl.Result{}, fmt.Errorf("failed to ensure the Cluster API side is paused: %w", err) + } + + if !paused { + return ctrl.Result{}, nil + } + + return ctrl.Result{}, nil + default: + return ctrl.Result{}, unsupportedCurrentAuthorityError(migratable.CurrentAuthority()) + } +} + +func isStableStateSynchronized[ + capiT any, + capiPT primaryCAPIObjectP[capiT], +]( + ctx context.Context, + k8sClient client.Client, + capiNamespace string, + migratable Migratable[capiT, capiPT], + authority mapiv1beta1.MachineAuthority, +) (bool, error) { + cond, err := util.GetConditionStatus(migratable.MAPIObject(), string(controllers.SynchronizedCondition)) + if err != nil { + return false, fmt.Errorf("unable to get synchronized condition for %s/%s: %w", migratable.MAPIObject().GetNamespace(), migratable.MAPIObject().GetName(), err) + } + + if cond != corev1.ConditionTrue { + return false, nil + } + + expectedSynchronizedAPI := synccommon.AuthoritativeAPIToSynchronizedAPI(authority) + if expectedSynchronizedAPI == nil { + return false, unsupportedStableAuthorityError(authority) + } + + if migratable.SynchronizedAPI() != *expectedSynchronizedAPI { + return false, nil + } + + switch authority { + case mapiv1beta1.MachineAuthorityMachineAPI: + return migratable.SynchronizedGeneration() == migratable.MAPIObject().GetGeneration(), nil + case mapiv1beta1.MachineAuthorityClusterAPI: + capiObj, found, err := getPrimaryCAPIObject[capiT, capiPT](ctx, k8sClient, capiNamespace, migratable.MAPIObject().GetName()) + if err != nil { + return false, err + } + + if !found { + return false, nil + } + + return migratable.SynchronizedGeneration() == capiObj.GetGeneration(), nil + case mapiv1beta1.MachineAuthorityMigrating: + return false, unsupportedStableAuthorityError(authority) + default: + return false, unsupportedStableAuthorityError(authority) + } +} + +func isMAPIPaused[ + capiT any, + capiPT primaryCAPIObjectP[capiT], +](migratable Migratable[capiT, capiPT]) (bool, error) { + pausedCondition := util.GetMAPICondition(migratable.MAPIConditions(), "Paused") + if pausedCondition == nil { + return false, nil + } + + switch pausedCondition.Status { + case corev1.ConditionTrue: + return true, nil + case corev1.ConditionFalse, corev1.ConditionUnknown: + return false, nil + default: + return false, unrecognizedPausedConditionStatusError(migratable.MAPIObject(), pausedCondition.Status) + } +} + +func getPrimaryCAPIObject[ + capiT any, + capiPT primaryCAPIObjectP[capiT], +]( + ctx context.Context, + k8sClient client.Client, + namespace string, + name string, +) (capiPT, bool, error) { + var zero capiPT + + capiObj := capiPT(new(capiT)) + + if err := k8sClient.Get(ctx, client.ObjectKey{Namespace: namespace, Name: name}, capiObj); err != nil { + if client.IgnoreNotFound(err) == nil { + return zero, false, nil + } + + return zero, false, fmt.Errorf("failed to get primary Cluster API object %s/%s: %w", namespace, name, err) + } + + return capiObj, true, nil +} diff --git a/pkg/controllers/migrationcommon/controller_test.go b/pkg/controllers/migrationcommon/controller_test.go new file mode 100644 index 000000000..ee5df733e --- /dev/null +++ b/pkg/controllers/migrationcommon/controller_test.go @@ -0,0 +1,447 @@ +/* +Copyright 2026 Red Hat, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationcommon + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + mapiv1beta1 "github.com/openshift/api/machine/v1beta1" + machinev1applyconfigs "github.com/openshift/client-go/machine/applyconfigurations/machine/v1beta1" + "github.com/openshift/cluster-api-actuator-pkg/testutils" + capiv1resourcebuilder "github.com/openshift/cluster-api-actuator-pkg/testutils/resourcebuilder/cluster-api/core/v1beta2" + corev1resourcebuilder "github.com/openshift/cluster-api-actuator-pkg/testutils/resourcebuilder/core/v1" + machinev1resourcebuilder "github.com/openshift/cluster-api-actuator-pkg/testutils/resourcebuilder/machine/v1beta1" + consts "github.com/openshift/cluster-capi-operator/pkg/controllers" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clusterv1 "sigs.k8s.io/cluster-api/api/core/v1beta2" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest/komega" +) + +type fakeMachineMigratable struct { + mapiMachine *mapiv1beta1.Machine + desired *mapiv1beta1.MachineAuthority + + ensurePausedResult bool + ensurePausedErr error + ensurePausedCalls int + ensureUnpausedResult bool + ensureUnpausedErr error + ensureUnpausedCalls int +} + +func (f *fakeMachineMigratable) MAPIObject() client.Object { + return f.mapiMachine +} + +func (f *fakeMachineMigratable) DesiredAuthority() mapiv1beta1.MachineAuthority { + if f.desired != nil { + return *f.desired + } + + return f.mapiMachine.Spec.AuthoritativeAPI +} + +func (f *fakeMachineMigratable) CurrentAuthority() mapiv1beta1.MachineAuthority { + return f.mapiMachine.Status.AuthoritativeAPI +} + +func (f *fakeMachineMigratable) SynchronizedAPI() mapiv1beta1.SynchronizedAPI { + return f.mapiMachine.Status.SynchronizedAPI +} + +func (f *fakeMachineMigratable) SynchronizedGeneration() int64 { + return f.mapiMachine.Status.SynchronizedGeneration +} + +func (f *fakeMachineMigratable) MAPIConditions() []mapiv1beta1.Condition { + return f.mapiMachine.Status.Conditions +} + +func (f *fakeMachineMigratable) EnsureCAPIPaused(_ context.Context, _ *clusterv1.Machine) (bool, error) { + f.ensurePausedCalls++ + return f.ensurePausedResult, f.ensurePausedErr +} + +func (f *fakeMachineMigratable) EnsureCAPIUnpaused(_ context.Context, _ *clusterv1.Machine) (bool, error) { + f.ensureUnpausedCalls++ + return f.ensureUnpausedResult, f.ensureUnpausedErr +} + +var _ = Describe("Reconcile", func() { + var ( + k komega.Komega + mapiNamespace *corev1.Namespace + capiNamespace *corev1.Namespace + mapiMachine *mapiv1beta1.Machine + capiMachine *clusterv1.Machine + mapiBuilder machinev1resourcebuilder.MachineBuilder + capiBuilder capiv1resourcebuilder.MachineBuilder + migratable *fakeMachineMigratable + controllerName string + ) + + synchronizedCondition := func(status corev1.ConditionStatus) mapiv1beta1.Condition { + return mapiv1beta1.Condition{ + Type: consts.SynchronizedCondition, + Status: status, + LastTransitionTime: metav1.Now(), + } + } + + mapiPausedCondition := func(status corev1.ConditionStatus) mapiv1beta1.Condition { + return mapiv1beta1.Condition{ + Type: "Paused", + Status: status, + LastTransitionTime: metav1.Now(), + } + } + + createCAPIMachine := func() { + GinkgoHelper() + + capiMachine = capiBuilder.Build() + Eventually(k8sClient.Create(ctx, capiMachine)).Should(Succeed(), "CAPI machine should be created for the test") + } + + updateMAPIStatus := func(authority mapiv1beta1.MachineAuthority, synchronizedAPI mapiv1beta1.SynchronizedAPI, synchronizedGeneration int64, conditions ...mapiv1beta1.Condition) { + GinkgoHelper() + + Eventually(k.UpdateStatus(mapiMachine, func() { + mapiMachine.Status.AuthoritativeAPI = authority + mapiMachine.Status.SynchronizedAPI = synchronizedAPI + mapiMachine.Status.SynchronizedGeneration = synchronizedGeneration + mapiMachine.Status.Conditions = conditions + })).Should(Succeed(), "MAPI machine status should be updated for the test") + } + + reconcileOnce := func() error { + GinkgoHelper() + + _, err := Reconcile[*machinev1applyconfigs.MachineStatusApplyConfiguration]( + ctx, + k8sClient, + controllerName, + capiNamespace.GetName(), + machinev1applyconfigs.Machine, + migratable, + ) + + return err + } + + expectSyncStatusReset := func(authority mapiv1beta1.MachineAuthority) { + GinkgoHelper() + + Eventually(k.Object(mapiMachine)).Should(SatisfyAll( + HaveField("Status.AuthoritativeAPI", Equal(authority)), + HaveField("Status.SynchronizedGeneration", BeZero()), + HaveField("Status.Conditions", ContainElement(SatisfyAll( + HaveField("Type", Equal(consts.SynchronizedCondition)), + HaveField("Status", Equal(corev1.ConditionUnknown)), + HaveField("Reason", Equal(consts.ReasonAuthoritativeAPIChanged)), + HaveField("Message", Equal("Waiting for resync after change of AuthoritativeAPI")), + HaveField("Severity", Equal(mapiv1beta1.ConditionSeverityInfo)), + ))), + )) + } + + BeforeEach(func() { + mapiNamespace = corev1resourcebuilder.Namespace(). + WithGenerateName("migrationcommon-mapi-").Build() + Expect(k8sClient.Create(ctx, mapiNamespace)).To(Succeed(), "MAPI namespace should be created") + + capiNamespace = corev1resourcebuilder.Namespace(). + WithGenerateName("migrationcommon-capi-").Build() + Expect(k8sClient.Create(ctx, capiNamespace)).To(Succeed(), "CAPI namespace should be created") + + mapiBuilder = machinev1resourcebuilder.Machine(). + WithNamespace(mapiNamespace.GetName()). + WithName("foo") + capiBuilder = capiv1resourcebuilder.Machine(). + WithNamespace(capiNamespace.GetName()). + WithName("foo") + + controllerName = "MigrationCommonTestController" + k = komega.New(k8sClient) + }) + + AfterEach(func() { + testutils.CleanupResources(Default, ctx, cfg, k8sClient, mapiNamespace.GetName(), + &mapiv1beta1.Machine{}, + ) + testutils.CleanupResources(Default, ctx, cfg, k8sClient, capiNamespace.GetName(), + &clusterv1.Machine{}, + ) + }) + + Context("when status.authoritativeAPI is empty", func() { + BeforeEach(func() { + mapiMachine = mapiBuilder. + WithAuthoritativeAPI(mapiv1beta1.MachineAuthorityClusterAPI). + Build() + Expect(k8sClient.Create(ctx, mapiMachine)).To(Succeed(), "MAPI machine should be created") + + migratable = &fakeMachineMigratable{mapiMachine: mapiMachine} + }) + + It("should initialize status.authoritativeAPI from spec and stop", func() { + Expect(reconcileOnce()).To(Succeed()) + + Eventually(k.Object(mapiMachine)).Should(HaveField("Status.AuthoritativeAPI", Equal(mapiv1beta1.MachineAuthorityClusterAPI))) + Expect(migratable.ensurePausedCalls).To(BeZero(), "expected no pause reconciliation during status initialization") + Expect(migratable.ensureUnpausedCalls).To(BeZero(), "expected no unpause reconciliation during status initialization") + }) + + Context("when spec.authoritativeAPI is not a supported stable target", func() { + BeforeEach(func() { + desiredAuthority := mapiv1beta1.MachineAuthorityMigrating + migratable.desired = &desiredAuthority + }) + + It("should return an error without seeding status.authoritativeAPI", func() { + Expect(reconcileOnce()).To(MatchError(ContainSubstring("unable to determine desired migration direction"))) + + Consistently(func(g Gomega) { + current := &mapiv1beta1.Machine{} + g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(mapiMachine), current)).To(Succeed()) + g.Expect(current.Status.AuthoritativeAPI).To(BeEmpty(), "status.authoritativeAPI should remain unset for an unsupported desired authority") + }).Should(Succeed()) + Expect(migratable.ensurePausedCalls).To(BeZero(), "expected no pause reconciliation during status initialization") + Expect(migratable.ensureUnpausedCalls).To(BeZero(), "expected no unpause reconciliation during status initialization") + }) + }) + }) + + Context("when migrating to ClusterAPI", func() { + BeforeEach(func() { + mapiMachine = mapiBuilder. + WithAuthoritativeAPI(mapiv1beta1.MachineAuthorityClusterAPI). + Build() + Expect(k8sClient.Create(ctx, mapiMachine)).To(Succeed(), "MAPI machine should be created") + + migratable = &fakeMachineMigratable{ + mapiMachine: mapiMachine, + ensureUnpausedResult: true, + ensurePausedResult: true, + ensurePausedCalls: 0, + ensureUnpausedCalls: 0, + } + }) + + Context("when the Machine API side is not yet synchronized", func() { + BeforeEach(func() { + updateMAPIStatus( + mapiv1beta1.MachineAuthorityMachineAPI, + mapiv1beta1.MachineAPISynchronized, + mapiMachine.Generation, + synchronizedCondition(corev1.ConditionFalse), + ) + }) + + It("should wait without acknowledging the migration", func() { + Expect(reconcileOnce()).To(Succeed()) + + Eventually(k.Object(mapiMachine)).Should(HaveField("Status.AuthoritativeAPI", Equal(mapiv1beta1.MachineAuthorityMachineAPI))) + }) + }) + + Context("when the Machine API side is synchronized", func() { + BeforeEach(func() { + updateMAPIStatus( + mapiv1beta1.MachineAuthorityMachineAPI, + mapiv1beta1.MachineAPISynchronized, + mapiMachine.Generation, + synchronizedCondition(corev1.ConditionTrue), + ) + }) + + It("should set status.authoritativeAPI to Migrating", func() { + Expect(reconcileOnce()).To(Succeed()) + + Eventually(k.Object(mapiMachine)).Should(HaveField("Status.AuthoritativeAPI", Equal(mapiv1beta1.MachineAuthorityMigrating))) + }) + }) + + Context("when already in Migrating and Machine API is not paused", func() { + BeforeEach(func() { + updateMAPIStatus( + mapiv1beta1.MachineAuthorityMigrating, + mapiv1beta1.MachineAPISynchronized, + mapiMachine.Generation, + synchronizedCondition(corev1.ConditionTrue), + mapiPausedCondition(corev1.ConditionFalse), + ) + }) + + It("should keep waiting in Migrating", func() { + Expect(reconcileOnce()).To(Succeed()) + + Eventually(k.Object(mapiMachine)).Should(HaveField("Status.AuthoritativeAPI", Equal(mapiv1beta1.MachineAuthorityMigrating))) + }) + }) + + Context("when already in Migrating and Machine API is paused", func() { + BeforeEach(func() { + updateMAPIStatus( + mapiv1beta1.MachineAuthorityMigrating, + mapiv1beta1.MachineAPISynchronized, + mapiMachine.Generation, + synchronizedCondition(corev1.ConditionTrue), + mapiPausedCondition(corev1.ConditionTrue), + ) + }) + + It("should acknowledge ClusterAPI and reset sync status", func() { + Expect(reconcileOnce()).To(Succeed()) + + expectSyncStatusReset(mapiv1beta1.MachineAuthorityClusterAPI) + }) + }) + + Context("when ClusterAPI is already authoritative and the primary CAPI object is missing", func() { + BeforeEach(func() { + updateMAPIStatus( + mapiv1beta1.MachineAuthorityClusterAPI, + mapiv1beta1.ClusterAPISynchronized, + 1, + synchronizedCondition(corev1.ConditionTrue), + ) + }) + + It("should treat the missing primary object as already safe", func() { + Expect(reconcileOnce()).To(Succeed()) + + Eventually(k.Object(mapiMachine)).Should(HaveField("Status.AuthoritativeAPI", Equal(mapiv1beta1.MachineAuthorityClusterAPI))) + Expect(migratable.ensureUnpausedCalls).To(BeZero(), "expected no unpause call when the primary CAPI object is missing") + }) + }) + }) + + Context("when migrating to MachineAPI", func() { + BeforeEach(func() { + mapiMachine = mapiBuilder. + WithAuthoritativeAPI(mapiv1beta1.MachineAuthorityMachineAPI). + Build() + Expect(k8sClient.Create(ctx, mapiMachine)).To(Succeed(), "MAPI machine should be created") + + migratable = &fakeMachineMigratable{ + mapiMachine: mapiMachine, + ensurePausedResult: true, + ensureUnpausedResult: true, + } + }) + + Context("when the authoritative CAPI copy is missing", func() { + BeforeEach(func() { + updateMAPIStatus( + mapiv1beta1.MachineAuthorityClusterAPI, + mapiv1beta1.ClusterAPISynchronized, + 1, + synchronizedCondition(corev1.ConditionTrue), + ) + }) + + It("should wait for the sync controller to restore it", func() { + Expect(reconcileOnce()).To(Succeed()) + + Eventually(k.Object(mapiMachine)).Should(HaveField("Status.AuthoritativeAPI", Equal(mapiv1beta1.MachineAuthorityClusterAPI))) + Expect(migratable.ensurePausedCalls).To(BeZero(), "expected no pause call when the primary CAPI object is missing") + }) + }) + + Context("when the primary CAPI object is not yet paused", func() { + BeforeEach(func() { + createCAPIMachine() + updateMAPIStatus( + mapiv1beta1.MachineAuthorityClusterAPI, + mapiv1beta1.ClusterAPISynchronized, + capiMachine.Generation, + synchronizedCondition(corev1.ConditionTrue), + ) + + migratable.ensurePausedResult = false + }) + + It("should wait without entering Migrating", func() { + Expect(reconcileOnce()).To(Succeed()) + + Eventually(k.Object(mapiMachine)).Should(HaveField("Status.AuthoritativeAPI", Equal(mapiv1beta1.MachineAuthorityClusterAPI))) + Expect(migratable.ensurePausedCalls).To(Equal(1), "expected a pause attempt against the primary CAPI object") + }) + }) + + Context("when the primary CAPI object is paused but not yet synchronized", func() { + BeforeEach(func() { + createCAPIMachine() + updateMAPIStatus( + mapiv1beta1.MachineAuthorityClusterAPI, + mapiv1beta1.ClusterAPISynchronized, + capiMachine.Generation+1, + synchronizedCondition(corev1.ConditionTrue), + ) + }) + + It("should wait without entering Migrating", func() { + Expect(reconcileOnce()).To(Succeed()) + + Eventually(k.Object(mapiMachine)).Should(HaveField("Status.AuthoritativeAPI", Equal(mapiv1beta1.MachineAuthorityClusterAPI))) + Expect(migratable.ensurePausedCalls).To(Equal(1), "expected a pause attempt before the stable sync gate") + }) + }) + + Context("when the primary CAPI object is paused and synchronized", func() { + BeforeEach(func() { + createCAPIMachine() + updateMAPIStatus( + mapiv1beta1.MachineAuthorityClusterAPI, + mapiv1beta1.ClusterAPISynchronized, + capiMachine.Generation, + synchronizedCondition(corev1.ConditionTrue), + ) + }) + + It("should set status.authoritativeAPI to Migrating", func() { + Expect(reconcileOnce()).To(Succeed()) + + Eventually(k.Object(mapiMachine)).Should(HaveField("Status.AuthoritativeAPI", Equal(mapiv1beta1.MachineAuthorityMigrating))) + Expect(migratable.ensurePausedCalls).To(Equal(1), "expected a pause attempt before acknowledging the migration") + }) + }) + + Context("when already in Migrating", func() { + BeforeEach(func() { + updateMAPIStatus( + mapiv1beta1.MachineAuthorityMigrating, + mapiv1beta1.ClusterAPISynchronized, + 1, + synchronizedCondition(corev1.ConditionTrue), + ) + }) + + It("should acknowledge MachineAPI and reset sync status", func() { + Expect(reconcileOnce()).To(Succeed()) + + expectSyncStatusReset(mapiv1beta1.MachineAuthorityMachineAPI) + }) + }) + }) +}) diff --git a/pkg/controllers/migrationcommon/controllertest/helpers.go b/pkg/controllers/migrationcommon/controllertest/helpers.go new file mode 100644 index 000000000..e41090b00 --- /dev/null +++ b/pkg/controllers/migrationcommon/controllertest/helpers.go @@ -0,0 +1,92 @@ +/* +Copyright 2026 Red Hat, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllertest + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + mapiv1beta1 "github.com/openshift/api/machine/v1beta1" + consts "github.com/openshift/cluster-capi-operator/pkg/controllers" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clusterv1 "sigs.k8s.io/cluster-api/api/core/v1beta2" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest/komega" +) + +// ReconcileFunc executes a single reconcile invocation for test assertions. +type ReconcileFunc func() (ctrl.Result, error) + +// SynchronizedCondition returns a Machine API synchronized condition with the +// provided status. +func SynchronizedCondition(status corev1.ConditionStatus) mapiv1beta1.Condition { + return mapiv1beta1.Condition{ + Type: consts.SynchronizedCondition, + Status: status, + LastTransitionTime: metav1.Now(), + } +} + +// MAPIPausedCondition returns a Machine API paused condition with the provided +// status. +func MAPIPausedCondition(status corev1.ConditionStatus) mapiv1beta1.Condition { + return mapiv1beta1.Condition{ + Type: "Paused", + Status: status, + LastTransitionTime: metav1.Now(), + } +} + +// CAPIPausedCondition returns a Cluster API paused condition with the provided +// status. +func CAPIPausedCondition(status metav1.ConditionStatus) metav1.Condition { + return metav1.Condition{ + Type: clusterv1.PausedCondition, + Status: status, + LastTransitionTime: metav1.Now(), + } +} + +// ExpectSuccessfulReconcile asserts that a reconcile call succeeds without +// requeueing. +func ExpectSuccessfulReconcile(reconcile ReconcileFunc) { + GinkgoHelper() + + result, err := reconcile() + Expect(err).NotTo(HaveOccurred(), "reconcile should succeed") + Expect(result).To(Equal(ctrl.Result{}), "reconcile should not requeue") +} + +// ExpectSyncStatusReset asserts that migration status has switched authority and +// reset the synchronization bookkeeping. +func ExpectSyncStatusReset(k komega.Komega, obj client.Object, authority mapiv1beta1.MachineAuthority) { + GinkgoHelper() + + Eventually(k.Object(obj)).Should(SatisfyAll( + HaveField("Status.AuthoritativeAPI", Equal(authority)), + HaveField("Status.SynchronizedGeneration", BeZero()), + HaveField("Status.Conditions", ContainElement(SatisfyAll( + HaveField("Type", Equal(consts.SynchronizedCondition)), + HaveField("Status", Equal(corev1.ConditionUnknown)), + HaveField("Reason", Equal(consts.ReasonAuthoritativeAPIChanged)), + HaveField("Message", Equal("Waiting for resync after change of AuthoritativeAPI")), + HaveField("Severity", Equal(mapiv1beta1.ConditionSeverityInfo)), + ))), + )) +} diff --git a/pkg/controllers/migrationcommon/pause.go b/pkg/controllers/migrationcommon/pause.go new file mode 100644 index 000000000..372b535d5 --- /dev/null +++ b/pkg/controllers/migrationcommon/pause.go @@ -0,0 +1,81 @@ +/* +Copyright 2026 Red Hat, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationcommon + +import ( + "context" + "errors" + "fmt" + + "github.com/openshift/cluster-capi-operator/pkg/util" + clusterv1 "sigs.k8s.io/cluster-api/api/core/v1beta2" + "sigs.k8s.io/cluster-api/util/annotations" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +var errDeepCopyDoesNotImplementClientObject = errors.New("deep copy does not implement client.Object") + +// AddPausedAnnotation adds the Cluster API paused annotation and patches the +// object with optimistic locking. +func AddPausedAnnotation(ctx context.Context, k8sClient client.Client, obj client.Object) (bool, error) { + if annotations.HasPaused(obj) { + return false, nil + } + + before, err := deepCopyClientObject(obj) + if err != nil { + return false, err + } + + annotations.AddAnnotations(obj, map[string]string{clusterv1.PausedAnnotation: ""}) + + if err := k8sClient.Patch(ctx, obj, client.MergeFromWithOptions(before, client.MergeFromWithOptimisticLock{})); err != nil { + return false, fmt.Errorf("failed to patch %T %s/%s: %w", obj, obj.GetNamespace(), obj.GetName(), err) + } + + return true, nil +} + +// RemovePausedAnnotation removes the Cluster API paused annotation and patches +// the object. +func RemovePausedAnnotation(ctx context.Context, k8sClient client.Client, obj client.Object) (bool, error) { + if !annotations.HasPaused(obj) { + return false, nil + } + + before, err := deepCopyClientObject(obj) + if err != nil { + return false, err + } + + util.RemoveAnnotation(obj, clusterv1.PausedAnnotation) + + if err := k8sClient.Patch(ctx, obj, client.MergeFromWithOptions(before, client.MergeFromWithOptimisticLock{})); err != nil { + return false, fmt.Errorf("failed to patch %T %s/%s: %w", obj, obj.GetNamespace(), obj.GetName(), err) + } + + return true, nil +} + +func deepCopyClientObject(obj client.Object) (client.Object, error) { + before, ok := obj.DeepCopyObject().(client.Object) + if !ok { + return nil, fmt.Errorf("%w: %T", errDeepCopyDoesNotImplementClientObject, obj) + } + + return before, nil +} diff --git a/pkg/controllers/migrationcommon/pause_test.go b/pkg/controllers/migrationcommon/pause_test.go new file mode 100644 index 000000000..2dee3521a --- /dev/null +++ b/pkg/controllers/migrationcommon/pause_test.go @@ -0,0 +1,121 @@ +/* +Copyright 2026 Red Hat, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationcommon + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/openshift/cluster-api-actuator-pkg/testutils" + capiv1resourcebuilder "github.com/openshift/cluster-api-actuator-pkg/testutils/resourcebuilder/cluster-api/core/v1beta2" + corev1resourcebuilder "github.com/openshift/cluster-api-actuator-pkg/testutils/resourcebuilder/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + clusterv1 "sigs.k8s.io/cluster-api/api/core/v1beta2" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest/komega" +) + +var _ = Describe("Paused annotation helpers", func() { + var ( + k komega.Komega + capiNamespace string + machine *clusterv1.Machine + ) + + BeforeEach(func() { + namespace := corev1resourcebuilder.Namespace(). + WithGenerateName("migrationcommon-pause-").Build() + Expect(k8sClient.Create(ctx, namespace)).To(Succeed(), "CAPI namespace should be created") + capiNamespace = namespace.GetName() + + machine = capiv1resourcebuilder.Machine(). + WithNamespace(capiNamespace). + WithName("foo"). + Build() + Expect(k8sClient.Create(ctx, machine)).To(Succeed(), "CAPI machine should be created") + + k = komega.New(k8sClient) + }) + + AfterEach(func() { + testutils.CleanupResources(Default, ctx, cfg, k8sClient, capiNamespace, + &clusterv1.Machine{}, + ) + }) + + Describe("AddPausedAnnotation", func() { + Context("when the object has changed since it was read", func() { + It("should fail with a conflict", func() { + staleCopy := &clusterv1.Machine{} + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(machine), staleCopy)).To(Succeed(), "stale copy should be fetched") + + liveMachine := &clusterv1.Machine{} + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(machine), liveMachine)).To(Succeed(), "live machine should be fetched") + + if liveMachine.Annotations == nil { + liveMachine.Annotations = map[string]string{} + } + + liveMachine.Annotations["test.openshift.io/stale"] = "true" + Expect(k8sClient.Update(ctx, liveMachine)).To(Succeed(), "live machine should be updated to make the stale copy outdated") + + changed, err := AddPausedAnnotation(ctx, k8sClient, staleCopy) + Expect(changed).To(BeFalse(), "stale writes should not report a successful change") + Expect(err).To(HaveOccurred(), "stale writes should fail") + Expect(apierrors.IsConflict(err)).To(BeTrue(), "expected stale patch to fail with a conflict") + }) + }) + }) + + Describe("RemovePausedAnnotation", func() { + Context("when the paused annotation is present", func() { + BeforeEach(func() { + changed, err := AddPausedAnnotation(ctx, k8sClient, machine) + Expect(err).NotTo(HaveOccurred()) + Expect(changed).To(BeTrue(), "expected setup to add the paused annotation") + }) + + It("should remove the paused annotation", func() { + changed, err := RemovePausedAnnotation(ctx, k8sClient, machine) + Expect(err).NotTo(HaveOccurred()) + Expect(changed).To(BeTrue(), "expected the helper to report that it removed the paused annotation") + + Eventually(k.Object(machine)).ShouldNot(HaveField("Annotations", HaveKey(clusterv1.PausedAnnotation))) + }) + + It("should fail with a conflict when the object has changed since it was read", func() { + staleCopy := &clusterv1.Machine{} + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(machine), staleCopy)).To(Succeed(), "stale copy should be fetched") + + liveMachine := &clusterv1.Machine{} + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(machine), liveMachine)).To(Succeed(), "live machine should be fetched") + + if liveMachine.Annotations == nil { + liveMachine.Annotations = map[string]string{} + } + + liveMachine.Annotations["test.openshift.io/stale"] = "true" + Expect(k8sClient.Update(ctx, liveMachine)).To(Succeed(), "live machine should be updated to make the stale copy outdated") + + changed, err := RemovePausedAnnotation(ctx, k8sClient, staleCopy) + Expect(changed).To(BeFalse(), "stale writes should not report a successful change") + Expect(err).To(HaveOccurred(), "stale writes should fail") + Expect(apierrors.IsConflict(err)).To(BeTrue(), "expected stale patch to fail with a conflict") + }) + }) + }) +}) diff --git a/pkg/controllers/migrationcommon/suite_test.go b/pkg/controllers/migrationcommon/suite_test.go new file mode 100644 index 000000000..541669d54 --- /dev/null +++ b/pkg/controllers/migrationcommon/suite_test.go @@ -0,0 +1,90 @@ +/* +Copyright 2026 Red Hat, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationcommon + +import ( + "context" + "testing" + + "github.com/go-logr/logr" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/openshift/cluster-capi-operator/pkg/test" + + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/client-go/rest" + "k8s.io/klog/v2" + + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/apiutil" + "sigs.k8s.io/controller-runtime/pkg/envtest" + "sigs.k8s.io/controller-runtime/pkg/envtest/komega" + logf "sigs.k8s.io/controller-runtime/pkg/log" +) + +var cfg *rest.Config +var k8sClient client.Client +var testEnv *envtest.Environment +var testRESTMapper meta.RESTMapper +var ctx = context.Background() +var testLogger logr.Logger + +func TestMigrationCommon(t *testing.T) { + RegisterFailHandler(Fail) + + RunSpecs(t, "MigrationCommon Suite") +} + +var _ = BeforeSuite(func() { + klog.SetOutput(GinkgoWriter) + + testLogger = test.NewVerboseGinkgoLogger(0) + logf.SetLogger(testLogger) + ctrl.SetLogger(testLogger) + + ctx = logf.IntoContext(ctx, testLogger) + + By("bootstrapping test environment") + + var err error + + testEnv = &envtest.Environment{} + cfg, k8sClient, err = test.StartEnvTest(testEnv) + + Expect(err).NotTo(HaveOccurred()) + Expect(cfg).NotTo(BeNil()) + Expect(k8sClient).NotTo(BeNil()) + + httpClient, err := rest.HTTPClientFor(cfg) + Expect(err).NotTo(HaveOccurred()) + Expect(httpClient).NotTo(BeNil()) + + testRESTMapper, err = apiutil.NewDynamicRESTMapper(cfg, httpClient) + Expect(err).NotTo(HaveOccurred()) + + komega.SetClient(k8sClient) + komega.SetContext(ctx) +}) + +var _ = AfterSuite(func() { + By("tearing down the test environment") + + err := testEnv.Stop() + Expect(err).NotTo(HaveOccurred()) +}) diff --git a/pkg/controllers/synccommon/applyconfiguration.go b/pkg/controllers/synccommon/applyconfiguration.go index 761a334c7..e2bbedb74 100644 --- a/pkg/controllers/synccommon/applyconfiguration.go +++ b/pkg/controllers/synccommon/applyconfiguration.go @@ -21,36 +21,40 @@ import ( machinev1applyconfigs "github.com/openshift/client-go/machine/applyconfigurations/machine/v1beta1" ) -// syncObjApplyConfiguration is an apply configuration for objects managed by -// the sync controller. This is currently MAPI Machine and MachineSet. -type syncObjApplyConfiguration[objPT any, statusPT syncStatusApplyConfiguration[statusPT]] interface { +// ObjApplyConfiguration is an apply configuration for objects managed by the +// sync and migration controllers. This is currently MAPI Machine and +// MachineSet. +type ObjApplyConfiguration[objPT any, statusPT StatusApplyConfiguration[statusPT]] interface { WithStatus(statusPT) objPT WithResourceVersion(string) objPT } -// syncObjApplyConfigurationP asserts that a syncObjApplyConfiguration is a pointer to a specific concrete type. -type syncObjApplyConfigurationP[objT any, objPT *objT, statusPT syncStatusApplyConfiguration[statusPT]] interface { +// ObjApplyConfigurationP asserts that an ObjApplyConfiguration is a pointer to +// a specific concrete type. +type ObjApplyConfigurationP[objT any, objPT *objT, statusPT StatusApplyConfiguration[statusPT]] interface { *objT - syncObjApplyConfiguration[objPT, statusPT] + ObjApplyConfiguration[objPT, statusPT] } -// syncStatusApplyConfiguration is an apply configuration for the status of -// objects managed by the sync controller. This is currently MAPI Machine and -// MachineSet. -type syncStatusApplyConfiguration[statusPT any] interface { +// StatusApplyConfiguration is an apply configuration for the status of objects +// managed by the sync and migration controllers. This is currently MAPI +// Machine and MachineSet. +type StatusApplyConfiguration[statusPT any] interface { WithConditions(...*machinev1applyconfigs.ConditionApplyConfiguration) statusPT WithSynchronizedGeneration(int64) statusPT WithAuthoritativeAPI(mapiv1beta1.MachineAuthority) statusPT + WithSynchronizedAPI(mapiv1beta1.SynchronizedAPI) statusPT } -// syncStatusApplyConfigurationP asserts that a syncStatusApplyConfiguration is a pointer to a specific concrete type. -type syncStatusApplyConfigurationP[statusT any, statusPT any] interface { +// StatusApplyConfigurationP asserts that a StatusApplyConfiguration is a +// pointer to a specific concrete type. +type StatusApplyConfigurationP[statusT any, statusPT any] interface { *statusT - syncStatusApplyConfiguration[statusPT] + StatusApplyConfiguration[statusPT] } -// syncObjApplyConfigurationConstructor is a constructor for -// SyncObjApplyConfigurations. It takes a name and namespace and returns an -// apply configuration. This constructor will have been generated automatically by +// ObjApplyConfigurationConstructor is a constructor for +// ObjApplyConfigurations. It takes a name and namespace and returns an apply +// configuration. This constructor will have been generated automatically by // the applyconfig generator. -type syncObjApplyConfigurationConstructor[objPT syncObjApplyConfiguration[objPT, statusPT], statusPT syncStatusApplyConfiguration[statusPT]] func(string, string) objPT +type ObjApplyConfigurationConstructor[objPT ObjApplyConfiguration[objPT, statusPT], statusPT StatusApplyConfiguration[statusPT]] func(string, string) objPT diff --git a/pkg/controllers/synccommon/migratestatus.go b/pkg/controllers/synccommon/migratestatus.go index 6abef0a0c..f32d2ee14 100644 --- a/pkg/controllers/synccommon/migratestatus.go +++ b/pkg/controllers/synccommon/migratestatus.go @@ -30,24 +30,27 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) -// ApplyAuthoritativeAPIAndResetSyncStatus writes the status of the migration -// controller, and also resets the status written by the sync controller. It -// does this in a single operation, using the field owner of the migration -// controller. +// ApplyMigrationStatusAndResetSyncStatus writes the migration controller status fields +// and resets the sync controller status (sets Synchronized condition to Unknown and +// synchronizedGeneration to 0). +// +// This is used when completing a migration to signal the sync controller that +// it needs to re-synchronize from the new authoritative API. // // Due to the potential for racing with the sync controller, it sets // ResourceVersion in the operation. -func ApplyAuthoritativeAPIAndResetSyncStatus[ - statusPT syncStatusApplyConfigurationP[statusT, statusPT], - objPT syncObjApplyConfigurationP[objT, objPT, statusPT], +func ApplyMigrationStatusAndResetSyncStatus[ + statusPT StatusApplyConfigurationP[statusT, statusPT], + objPT ObjApplyConfigurationP[objT, objPT, statusPT], statusT, objT any, ]( ctx context.Context, k8sClient client.Client, controllerName string, - newApplyConfig syncObjApplyConfigurationConstructor[objPT, statusPT], mapiObj client.Object, + newApplyConfig ObjApplyConfigurationConstructor[objPT, statusPT], mapiObj client.Object, authority mapiv1beta1.MachineAuthority, ) error { objAC, statusAC, err := newSyncStatusApplyConfiguration(newApplyConfig, mapiObj, - corev1.ConditionUnknown, controllers.ReasonAuthoritativeAPIChanged, "Waiting for resync after change of AuthoritativeAPI", ptr.To(int64(0))) + corev1.ConditionUnknown, controllers.ReasonAuthoritativeAPIChanged, "Waiting for resync after change of AuthoritativeAPI", ptr.To(int64(0)), + nil) if err != nil { return err } @@ -55,15 +58,14 @@ func ApplyAuthoritativeAPIAndResetSyncStatus[ return applyAuthoritativeAPI(ctx, k8sClient, controllerName, mapiObj, authority, objAC, statusAC) } -// ApplyAuthoritativeAPI writes the status of the migration controller to a MAPI -// object. -func ApplyAuthoritativeAPI[ - statusPT syncStatusApplyConfigurationP[statusT, statusPT], - objPT syncObjApplyConfigurationP[objT, objPT, statusPT], +// ApplyMigrationStatus writes the migration controller status fields to a MAPI object. +func ApplyMigrationStatus[ + statusPT StatusApplyConfigurationP[statusT, statusPT], + objPT ObjApplyConfigurationP[objT, objPT, statusPT], statusT, objT any, ]( ctx context.Context, k8sClient client.Client, controllerName string, - newApplyConfig syncObjApplyConfigurationConstructor[objPT, statusPT], mapiObj client.Object, + newApplyConfig ObjApplyConfigurationConstructor[objPT, statusPT], mapiObj client.Object, authority mapiv1beta1.MachineAuthority, ) error { statusAC := statusPT(new(statusT)) @@ -78,8 +80,8 @@ func ApplyAuthoritativeAPI[ // the migration controller. This allows us combine the sync and migration // statuses in a single transaction when required. func applyAuthoritativeAPI[ - statusPT syncStatusApplyConfigurationP[statusT, statusPT], - objPT syncObjApplyConfigurationP[objT, objPT, statusPT], + statusPT StatusApplyConfigurationP[statusT, statusPT], + objPT ObjApplyConfigurationP[objT, objPT, statusPT], statusT, objT any, ]( ctx context.Context, k8sClient client.Client, controllerName string, @@ -95,13 +97,12 @@ func applyAuthoritativeAPI[ // Note that we are writing fields owned by the synchronization controller // and forcing ownership to the AuthoritativeAPI. The synchronization // controller will force ownership of its own fields back again the next - // time it modifies them. We think this is probably going to work out ok. - // Apologies to future self if it didn't. + // time it modifies them. // // We need to do this due to a validation rule which prevents resetting // synchronizedGeneration unless also changing authoritativeAPI. Given that - // these fields are owned by different controllers, some fudging is - // required. + // these fields are owned by different controllers, explicit field ownership + // management is required. if err := k8sClient.Status().Patch(ctx, mapiObj, util.ApplyConfigPatch(objAC), client.ForceOwnership, client.FieldOwner(controllerName+"-AuthoritativeAPI")); err != nil { return fmt.Errorf("failed to patch Machine API object set status with authoritativeAPI %q: %w", authority, err) } diff --git a/pkg/controllers/synccommon/migratestatus_test.go b/pkg/controllers/synccommon/migratestatus_test.go new file mode 100644 index 000000000..741f026c6 --- /dev/null +++ b/pkg/controllers/synccommon/migratestatus_test.go @@ -0,0 +1,174 @@ +/* +Copyright 2025 Red Hat, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package synccommon + +import ( + "errors" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + mapiv1beta1 "github.com/openshift/api/machine/v1beta1" + machinev1applyconfigs "github.com/openshift/client-go/machine/applyconfigurations/machine/v1beta1" + "github.com/openshift/cluster-api-actuator-pkg/testutils" + corev1resourcebuilder "github.com/openshift/cluster-api-actuator-pkg/testutils/resourcebuilder/core/v1" + machinev1resourcebuilder "github.com/openshift/cluster-api-actuator-pkg/testutils/resourcebuilder/machine/v1beta1" + "github.com/openshift/cluster-capi-operator/pkg/controllers" + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +var _ = Describe("Migration status helpers", func() { + var namespaces []string + + createNamespace := func(prefix string) *corev1.Namespace { + GinkgoHelper() + + namespace := corev1resourcebuilder.Namespace(). + WithGenerateName(prefix). + Build() + Expect(k8sClient.Create(ctx, namespace)).To(Succeed()) + namespaces = append(namespaces, namespace.Name) + + return namespace + } + + AfterEach(func() { + for _, namespace := range namespaces { + testutils.CleanupResources(Default, ctx, cfg, k8sClient, namespace, + &mapiv1beta1.Machine{}, + &mapiv1beta1.MachineSet{}, + ) + } + }) + + Describe("ApplyMigrationStatus", func() { + It("should preserve synchronizedAPI when setting a machine to Migrating", func() { + By("Creating a namespace and Machine API machine") + + namespace := createNamespace("synccommon-machine-") + + machine := machinev1resourcebuilder.Machine(). + WithNamespace(namespace.Name). + WithName("machine"). + WithProviderSpecBuilder(machinev1resourcebuilder.AWSProviderSpec().WithLoadBalancers(nil)). + Build() + Expect(k8sClient.Create(ctx, machine)).To(Succeed()) + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(machine), machine)).To(Succeed()) + + By("Recording synchronized status through the sync helper") + + Expect(ApplySyncStatus[*machinev1applyconfigs.MachineStatusApplyConfiguration]( + ctx, + k8sClient, + "machine-sync-controller", + machinev1applyconfigs.Machine, + machine, + corev1.ConditionTrue, + controllers.ReasonResourceSynchronized, + "Successfully synchronized MAPI Machine to CAPI", + &machine.Generation, + AuthoritativeAPIToSynchronizedAPI(mapiv1beta1.MachineAuthorityMachineAPI), + )).To(Succeed()) + + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(machine), machine)).To(Succeed()) + Expect(machine.Status.SynchronizedAPI).To(Equal(mapiv1beta1.MachineAPISynchronized)) + + By("Setting status.AuthoritativeAPI to Migrating through the migration helper") + + Expect(ApplyMigrationStatus[*machinev1applyconfigs.MachineStatusApplyConfiguration]( + ctx, + k8sClient, + "machine-migration-controller", + machinev1applyconfigs.Machine, + machine, + mapiv1beta1.MachineAuthorityMigrating, + )).To(Succeed()) + + updatedMachine := &mapiv1beta1.Machine{} + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(machine), updatedMachine)).To(Succeed()) + Expect(updatedMachine.Status).To(SatisfyAll( + HaveField("AuthoritativeAPI", Equal(mapiv1beta1.MachineAuthorityMigrating)), + HaveField("SynchronizedAPI", Equal(mapiv1beta1.MachineAPISynchronized)), + )) + }) + + It("should preserve synchronizedAPI when setting a machine set to Migrating", func() { + By("Creating a namespace and Machine API machine set") + + namespace := createNamespace("synccommon-machineset-") + + machineSet := machinev1resourcebuilder.MachineSet(). + WithNamespace(namespace.Name). + WithName("machine-set"). + WithProviderSpecBuilder(machinev1resourcebuilder.AWSProviderSpec().WithLoadBalancers(nil)). + Build() + Expect(k8sClient.Create(ctx, machineSet)).To(Succeed()) + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(machineSet), machineSet)).To(Succeed()) + + By("Recording synchronized status through the sync helper") + + Expect(ApplySyncStatus[*machinev1applyconfigs.MachineSetStatusApplyConfiguration]( + ctx, + k8sClient, + "machineset-sync-controller", + machinev1applyconfigs.MachineSet, + machineSet, + corev1.ConditionTrue, + controllers.ReasonResourceSynchronized, + "Successfully synchronized MAPI MachineSet to CAPI", + &machineSet.Generation, + AuthoritativeAPIToSynchronizedAPI(mapiv1beta1.MachineAuthorityMachineAPI), + )).To(Succeed()) + + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(machineSet), machineSet)).To(Succeed()) + Expect(machineSet.Status.SynchronizedAPI).To(Equal(mapiv1beta1.MachineAPISynchronized)) + + By("Setting status.AuthoritativeAPI to Migrating through the migration helper") + + Expect(ApplyMigrationStatus[*machinev1applyconfigs.MachineSetStatusApplyConfiguration]( + ctx, + k8sClient, + "machineset-migration-controller", + machinev1applyconfigs.MachineSet, + machineSet, + mapiv1beta1.MachineAuthorityMigrating, + )).To(Succeed()) + + updatedMachineSet := &mapiv1beta1.MachineSet{} + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(machineSet), updatedMachineSet)).To(Succeed()) + Expect(updatedMachineSet.Status).To(SatisfyAll( + HaveField("AuthoritativeAPI", Equal(mapiv1beta1.MachineAuthorityMigrating)), + HaveField("SynchronizedAPI", Equal(mapiv1beta1.MachineAPISynchronized)), + )) + }) + }) + + Describe("ApplyMigrationStatusAndResetSyncStatus", func() { + It("should reject unsupported Machine API object types before patching", func() { + err := ApplyMigrationStatusAndResetSyncStatus[*machinev1applyconfigs.MachineStatusApplyConfiguration]( + ctx, + nil, + "machine-migration-controller", + machinev1applyconfigs.Machine, + &corev1.ConfigMap{}, + mapiv1beta1.MachineAuthorityClusterAPI, + ) + Expect(err).To(HaveOccurred()) + Expect(errors.Is(err, errUnsupportedSyncStatusType)).To(BeTrue()) + }) + }) +}) diff --git a/pkg/controllers/synccommon/suite_test.go b/pkg/controllers/synccommon/suite_test.go new file mode 100644 index 000000000..59b75836c --- /dev/null +++ b/pkg/controllers/synccommon/suite_test.go @@ -0,0 +1,66 @@ +/* +Copyright 2025 Red Hat, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package synccommon + +import ( + "context" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/openshift/cluster-capi-operator/pkg/test" + "k8s.io/client-go/rest" + "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" + "sigs.k8s.io/controller-runtime/pkg/envtest/komega" + logf "sigs.k8s.io/controller-runtime/pkg/log" +) + +var ( + ctx = context.Background() + cfg *rest.Config + k8sClient client.WithWatch + testEnv *envtest.Environment +) + +func TestSyncCommon(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "SyncCommon Suite") +} + +var _ = BeforeSuite(func() { + logf.SetLogger(klog.Background()) + + By("bootstrapping test environment") + + var err error + + testEnv = &envtest.Environment{} + cfg, k8sClient, err = test.StartEnvTest(testEnv) + Expect(err).NotTo(HaveOccurred()) + Expect(cfg).NotTo(BeNil()) + Expect(k8sClient).NotTo(BeNil()) + + komega.SetClient(k8sClient) + komega.SetContext(ctx) + + DeferCleanup(func() { + By("tearing down the test environment") + Expect(test.StopEnvTest(testEnv)).To(Succeed()) + }) +}) diff --git a/pkg/controllers/synccommon/syncstatus.go b/pkg/controllers/synccommon/syncstatus.go index 403ba7443..fc980cc11 100644 --- a/pkg/controllers/synccommon/syncstatus.go +++ b/pkg/controllers/synccommon/syncstatus.go @@ -25,6 +25,7 @@ import ( "github.com/openshift/cluster-capi-operator/pkg/controllers" "github.com/openshift/cluster-capi-operator/pkg/util" corev1 "k8s.io/api/core/v1" + "k8s.io/utils/ptr" machinev1applyconfigs "github.com/openshift/client-go/machine/applyconfigurations/machine/v1beta1" "sigs.k8s.io/controller-runtime/pkg/client" @@ -36,6 +37,19 @@ var ( // errUnsupportedSyncStatusType is returned when attempting to set sync status on a type which does not support it. errUnsupportedSyncStatusType = errors.New("type does not support setting sync status") + + // ErrInvalidSynchronizedAPI is returned when SynchronizedAPI has an unexpected value. + ErrInvalidSynchronizedAPI = errors.New("invalid synchronizedAPI value") + + // ErrMissingSynchronizedAPI is returned when SynchronizedAPI is required but not set. + ErrMissingSynchronizedAPI = errors.New("missing synchronizedAPI value") + + // ErrUnsupportedTargetAuthority is returned when the target authority is not supported. + ErrUnsupportedTargetAuthority = errors.New("unsupported target authority") + + // ErrUnexpectedCurrentAuthorityDuringCancellation is returned when a migration cancellation + // cannot map the current authority back to a supported source authority. + ErrUnexpectedCurrentAuthorityDuringCancellation = errors.New("unexpected current authority while cancelling migration") ) // ApplySyncStatus updates the MAPI object status for a sync controller, either @@ -45,15 +59,16 @@ var ( // provided explicitly. The remaining type arguments will be inferred, and can // be omitted. func ApplySyncStatus[ - statusPT syncStatusApplyConfigurationP[statusT, statusPT], - objPT syncObjApplyConfigurationP[objT, objPT, statusPT], + statusPT StatusApplyConfigurationP[statusT, statusPT], + objPT ObjApplyConfigurationP[objT, objPT, statusPT], statusT, objT any, ]( ctx context.Context, k8sClient client.Client, controllerName string, - applyConfigConstructor syncObjApplyConfigurationConstructor[objPT, statusPT], mapiObj client.Object, + applyConfigConstructor ObjApplyConfigurationConstructor[objPT, statusPT], mapiObj client.Object, status corev1.ConditionStatus, reason, message string, generation *int64, + synchronizedAPI *mapiv1beta1.SynchronizedAPI, ) error { - objAC, _, err := newSyncStatusApplyConfiguration(applyConfigConstructor, mapiObj, status, reason, message, generation) + objAC, _, err := newSyncStatusApplyConfiguration(applyConfigConstructor, mapiObj, status, reason, message, generation, synchronizedAPI) if err != nil { return err } @@ -75,12 +90,13 @@ func ApplySyncStatus[ // provided explicitly. The remaining type arguments will be inferred, and can // be omitted. func newSyncStatusApplyConfiguration[ - statusPT syncStatusApplyConfigurationP[statusT, statusPT], - objPT syncObjApplyConfigurationP[objT, objPT, statusPT], + statusPT StatusApplyConfigurationP[statusT, statusPT], + objPT ObjApplyConfigurationP[objT, objPT, statusPT], statusT, objT any, ]( - applyConfigConstructor syncObjApplyConfigurationConstructor[objPT, statusPT], mapiObj client.Object, + applyConfigConstructor ObjApplyConfigurationConstructor[objPT, statusPT], mapiObj client.Object, status corev1.ConditionStatus, reason, message string, generation *int64, + synchronizedAPI *mapiv1beta1.SynchronizedAPI, ) (objPT, statusPT, error) { var ( severity mapiv1beta1.ConditionSeverity @@ -90,25 +106,20 @@ func newSyncStatusApplyConfiguration[ err error ) - synchronizedGeneration, oldConditions, err = getPreviousSyncStatus(mapiObj) + var currentSynchronizedAPI mapiv1beta1.SynchronizedAPI + + synchronizedGeneration, oldConditions, currentSynchronizedAPI, err = getPreviousSyncStatus(mapiObj) if err != nil { return nil, nil, err } - // Update synchronizedGeneration if a new value was passed in explicitly. if generation != nil { synchronizedGeneration = *generation } - switch status { - case corev1.ConditionTrue: - severity = mapiv1beta1.ConditionSeverityNone - case corev1.ConditionFalse: - severity = mapiv1beta1.ConditionSeverityError - case corev1.ConditionUnknown: - severity = mapiv1beta1.ConditionSeverityInfo - default: - return nil, nil, fmt.Errorf("%w: %s", errUnrecognizedConditionStatus, status) + severity, err = conditionSeverityForStatus(status) + if err != nil { + return nil, nil, err } conditionAC := machinev1applyconfigs.Condition(). @@ -124,6 +135,15 @@ func newSyncStatusApplyConfiguration[ WithConditions(conditionAC). WithSynchronizedGeneration(synchronizedGeneration) + // Set SynchronizedAPI to define deterministically which object's generation + // the SynchronizedGeneration refers to. When the caller does not supply a + // new value, preserve the current one instead of clearing a field owned by + // the sync controller. + synchronizedAPI = preserveSynchronizedAPI(synchronizedAPI, currentSynchronizedAPI) + if synchronizedAPI != nil { + statusAC.WithSynchronizedAPI(*synchronizedAPI) + } + objAC := applyConfigConstructor(mapiObj.GetName(), mapiObj.GetNamespace()). WithResourceVersion(mapiObj.GetResourceVersion()). WithStatus(statusAC) @@ -131,15 +151,108 @@ func newSyncStatusApplyConfiguration[ return objAC, statusAC, nil } -func getPreviousSyncStatus(mapiObj interface{}) (int64, []mapiv1beta1.Condition, error) { +func conditionSeverityForStatus(status corev1.ConditionStatus) (mapiv1beta1.ConditionSeverity, error) { + switch status { + case corev1.ConditionTrue: + return mapiv1beta1.ConditionSeverityNone, nil + case corev1.ConditionFalse: + return mapiv1beta1.ConditionSeverityError, nil + case corev1.ConditionUnknown: + return mapiv1beta1.ConditionSeverityInfo, nil + default: + return "", fmt.Errorf("%w: %s", errUnrecognizedConditionStatus, status) + } +} + +func preserveSynchronizedAPI(synchronizedAPI *mapiv1beta1.SynchronizedAPI, currentSynchronizedAPI mapiv1beta1.SynchronizedAPI) *mapiv1beta1.SynchronizedAPI { + if synchronizedAPI == nil && currentSynchronizedAPI != "" { + return ptr.To(currentSynchronizedAPI) + } + + return synchronizedAPI +} + +// AuthoritativeAPIToSynchronizedAPI converts a MachineAuthority to its corresponding SynchronizedAPI value. +// Returns nil for values that don't have a direct mapping. +func AuthoritativeAPIToSynchronizedAPI(authority mapiv1beta1.MachineAuthority) *mapiv1beta1.SynchronizedAPI { + switch authority { + case mapiv1beta1.MachineAuthorityMachineAPI: + return ptr.To(mapiv1beta1.MachineAPISynchronized) + case mapiv1beta1.MachineAuthorityClusterAPI: + return ptr.To(mapiv1beta1.ClusterAPISynchronized) + case mapiv1beta1.MachineAuthorityMigrating: + return nil + } + + return nil +} + +// SynchronizedAPIToAuthoritativeAPI converts a SynchronizedAPI to its corresponding MachineAuthority. +// Returns an empty value for values that don't have a direct mapping. +func SynchronizedAPIToAuthoritativeAPI(synchronizedAPI mapiv1beta1.SynchronizedAPI) mapiv1beta1.MachineAuthority { + switch synchronizedAPI { + case mapiv1beta1.MachineAPISynchronized: + return mapiv1beta1.MachineAuthorityMachineAPI + case mapiv1beta1.ClusterAPISynchronized: + return mapiv1beta1.MachineAuthorityClusterAPI + } + + return "" +} + +// MigrationDirection determines the current and desired authorities for a migration. +// When statusAuthority is Migrating, it uses SynchronizedAPI to infer the current authority. +// On success, currentAuthority is guaranteed to be either MachineAPI or ClusterAPI. +// Missing or invalid SynchronizedAPI values are returned as errors instead of +// surfacing as a Migrating or otherwise unsupported current authority. +func MigrationDirection(statusAuthority mapiv1beta1.MachineAuthority, synchronizedAPI mapiv1beta1.SynchronizedAPI, specAuthority mapiv1beta1.MachineAuthority) (currentAuthority, desiredAuthority mapiv1beta1.MachineAuthority, isMigrating bool, err error) { + desiredAuthority = specAuthority + if statusAuthority != mapiv1beta1.MachineAuthorityMigrating { + return statusAuthority, desiredAuthority, false, nil + } + + if synchronizedAPI == "" { + return "", desiredAuthority, true, MissingSynchronizedAPIError() + } + + currentAuthority = SynchronizedAPIToAuthoritativeAPI(synchronizedAPI) + if currentAuthority == "" { + return "", desiredAuthority, true, InvalidSynchronizedAPIError(synchronizedAPI) + } + + return currentAuthority, desiredAuthority, true, nil +} + +// UnsupportedTargetAuthorityError returns a shared error for unsupported migration targets. +func UnsupportedTargetAuthorityError(targetAuthority mapiv1beta1.MachineAuthority) error { + return fmt.Errorf("%w: %s", ErrUnsupportedTargetAuthority, targetAuthority) +} + +// UnexpectedCurrentAuthorityDuringCancellationError returns a shared error for unexpected +// current authorities encountered while cancelling a migration. +func UnexpectedCurrentAuthorityDuringCancellationError(currentAuthority mapiv1beta1.MachineAuthority) error { + return fmt.Errorf("%w: %s", ErrUnexpectedCurrentAuthorityDuringCancellation, currentAuthority) +} + +// InvalidSynchronizedAPIError returns a shared error for unexpected SynchronizedAPI values. +func InvalidSynchronizedAPIError(synchronizedAPI mapiv1beta1.SynchronizedAPI) error { + return fmt.Errorf("%w: %s", ErrInvalidSynchronizedAPI, synchronizedAPI) +} + +// MissingSynchronizedAPIError returns a shared error for missing SynchronizedAPI values. +func MissingSynchronizedAPIError() error { + return fmt.Errorf("%w while authoritativeAPI is Migrating", ErrMissingSynchronizedAPI) +} + +func getPreviousSyncStatus(mapiObj interface{}) (int64, []mapiv1beta1.Condition, mapiv1beta1.SynchronizedAPI, error) { // Unlike the apply configurations, which have method accessors, we can't // define an interface to assert the presence of fields. switch o := mapiObj.(type) { case *mapiv1beta1.Machine: - return o.Status.SynchronizedGeneration, o.Status.Conditions, nil + return o.Status.SynchronizedGeneration, o.Status.Conditions, o.Status.SynchronizedAPI, nil case *mapiv1beta1.MachineSet: - return o.Status.SynchronizedGeneration, o.Status.Conditions, nil + return o.Status.SynchronizedGeneration, o.Status.Conditions, o.Status.SynchronizedAPI, nil default: - return 0, nil, fmt.Errorf("%w: %T", errUnsupportedSyncStatusType, mapiObj) + return 0, nil, "", fmt.Errorf("%w: %T", errUnsupportedSyncStatusType, mapiObj) } } diff --git a/pkg/controllers/synccommon/syncstatus_integration_test.go b/pkg/controllers/synccommon/syncstatus_integration_test.go new file mode 100644 index 000000000..f35349eb8 --- /dev/null +++ b/pkg/controllers/synccommon/syncstatus_integration_test.go @@ -0,0 +1,307 @@ +/* +Copyright 2026 Red Hat, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package synccommon + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + mapiv1beta1 "github.com/openshift/api/machine/v1beta1" + machinev1applyconfigs "github.com/openshift/client-go/machine/applyconfigurations/machine/v1beta1" + "github.com/openshift/cluster-api-actuator-pkg/testutils" + corev1resourcebuilder "github.com/openshift/cluster-api-actuator-pkg/testutils/resourcebuilder/core/v1" + machinev1resourcebuilder "github.com/openshift/cluster-api-actuator-pkg/testutils/resourcebuilder/machine/v1beta1" + "github.com/openshift/cluster-capi-operator/pkg/controllers" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +var _ = Describe("ApplySyncStatus", func() { + const ( + migrationControllerName = "MachineSetMigrationController" + syncControllerName = "MachineSetSyncController" + successMessage = "Successfully synchronized MAPI MachineSet to CAPI" + ) + + var ( + namespace *corev1.Namespace + machineSet *mapiv1beta1.MachineSet + machineSetKey client.ObjectKey + staleMachineSet *mapiv1beta1.MachineSet + ) + + BeforeEach(func() { + By("Creating a namespace and Machine API machine set") + + namespace = corev1resourcebuilder.Namespace(). + WithGenerateName("synccommon-syncstatus-"). + Build() + Expect(k8sClient.Create(ctx, namespace)).To(Succeed()) + + machineSet = machinev1resourcebuilder.MachineSet(). + WithNamespace(namespace.Name). + WithName("machine-set"). + WithProviderSpecBuilder(machinev1resourcebuilder.AWSProviderSpec().WithLoadBalancers(nil)). + Build() + Expect(k8sClient.Create(ctx, machineSet)).To(Succeed()) + + machineSetKey = client.ObjectKeyFromObject(machineSet) + + By("Setting status.AuthoritativeAPI to MachineAPI through the migration helper") + + Expect(ApplyMigrationStatus[*machinev1applyconfigs.MachineSetStatusApplyConfiguration]( + ctx, + k8sClient, + migrationControllerName, + machinev1applyconfigs.MachineSet, + machineSet, + mapiv1beta1.MachineAuthorityMachineAPI, + )).To(Succeed()) + + Expect(k8sClient.Get(ctx, machineSetKey, machineSet)).To(Succeed()) + + By("Recording synchronized status through the sync helper") + + Expect(ApplySyncStatus[*machinev1applyconfigs.MachineSetStatusApplyConfiguration]( + ctx, + k8sClient, + syncControllerName, + machinev1applyconfigs.MachineSet, + machineSet, + corev1.ConditionTrue, + controllers.ReasonResourceSynchronized, + successMessage, + &machineSet.Generation, + AuthoritativeAPIToSynchronizedAPI(mapiv1beta1.MachineAuthorityMachineAPI), + )).To(Succeed()) + + Expect(k8sClient.Get(ctx, machineSetKey, machineSet)).To(Succeed()) + Expect(machineSet.Status).To(SatisfyAll( + HaveField("AuthoritativeAPI", Equal(mapiv1beta1.MachineAuthorityMachineAPI)), + HaveField("SynchronizedAPI", Equal(mapiv1beta1.MachineAPISynchronized)), + )) + + staleMachineSet = machineSet.DeepCopy() + + By("Switching status.AuthoritativeAPI to Migrating through the migration helper") + + Expect(ApplyMigrationStatus[*machinev1applyconfigs.MachineSetStatusApplyConfiguration]( + ctx, + k8sClient, + migrationControllerName, + machinev1applyconfigs.MachineSet, + machineSet, + mapiv1beta1.MachineAuthorityMigrating, + )).To(Succeed()) + }) + + AfterEach(func() { + testutils.CleanupResources(Default, ctx, cfg, k8sClient, namespace.Name, + &mapiv1beta1.MachineSet{}, + ) + }) + + Context("when reapplying sync status from a fresh post-migration object", func() { + It("should preserve synchronizedAPI", func() { + By("Fetching the MachineSet again after migration was acknowledged") + + freshMachineSet := &mapiv1beta1.MachineSet{} + Expect(k8sClient.Get(ctx, machineSetKey, freshMachineSet)).To(Succeed()) + Expect(freshMachineSet.Status).To(SatisfyAll( + HaveField("AuthoritativeAPI", Equal(mapiv1beta1.MachineAuthorityMigrating)), + HaveField("SynchronizedAPI", Equal(mapiv1beta1.MachineAPISynchronized)), + )) + + By("Reapplying sync status with the sync controller field owner") + + Expect(ApplySyncStatus[*machinev1applyconfigs.MachineSetStatusApplyConfiguration]( + ctx, + k8sClient, + syncControllerName, + machinev1applyconfigs.MachineSet, + freshMachineSet, + corev1.ConditionTrue, + controllers.ReasonResourceSynchronized, + successMessage, + &freshMachineSet.Generation, + AuthoritativeAPIToSynchronizedAPI(freshMachineSet.Status.AuthoritativeAPI), + )).To(Succeed()) + + updatedMachineSet := &mapiv1beta1.MachineSet{} + Expect(k8sClient.Get(ctx, machineSetKey, updatedMachineSet)).To(Succeed()) + Expect(updatedMachineSet.Status).To(SatisfyAll( + HaveField("AuthoritativeAPI", Equal(mapiv1beta1.MachineAuthorityMigrating)), + HaveField("SynchronizedAPI", Equal(mapiv1beta1.MachineAPISynchronized)), + )) + }) + }) + + Context("when reapplying sync status from a stale pre-migration object", func() { + It("should fail with a conflict and preserve synchronizedAPI", func() { + By("Verifying the stale object still reflects the pre-migration state") + + Expect(staleMachineSet.Status).To(SatisfyAll( + HaveField("AuthoritativeAPI", Equal(mapiv1beta1.MachineAuthorityMachineAPI)), + HaveField("SynchronizedAPI", Equal(mapiv1beta1.MachineAPISynchronized)), + )) + + By("Reapplying sync status with the stale resourceVersion") + + err := ApplySyncStatus[*machinev1applyconfigs.MachineSetStatusApplyConfiguration]( + ctx, + k8sClient, + syncControllerName, + machinev1applyconfigs.MachineSet, + staleMachineSet, + corev1.ConditionTrue, + controllers.ReasonResourceSynchronized, + successMessage, + &staleMachineSet.Generation, + AuthoritativeAPIToSynchronizedAPI(staleMachineSet.Status.AuthoritativeAPI), + ) + Expect(err).To(SatisfyAll( + HaveOccurred(), + WithTransform(apierrors.IsConflict, BeTrue()), + )) + + updatedMachineSet := &mapiv1beta1.MachineSet{} + Expect(k8sClient.Get(ctx, machineSetKey, updatedMachineSet)).To(Succeed()) + Expect(updatedMachineSet.Status).To(SatisfyAll( + HaveField("AuthoritativeAPI", Equal(mapiv1beta1.MachineAuthorityMigrating)), + HaveField("SynchronizedAPI", Equal(mapiv1beta1.MachineAPISynchronized)), + )) + }) + }) +}) + +var _ = Describe("ApplySyncStatus for Machine", func() { + const ( + machineMigrationControllerName = "MachineMigrationController" + machineSyncControllerName = "MachineSyncController" + machineSuccessMessage = "Successfully synchronized MAPI Machine to CAPI" + ) + + var ( + namespace *corev1.Namespace + machine *mapiv1beta1.Machine + machineKey client.ObjectKey + ) + + BeforeEach(func() { + By("Creating a namespace and Machine API machine") + + namespace = corev1resourcebuilder.Namespace(). + WithGenerateName("synccommon-machine-syncstatus-"). + Build() + Expect(k8sClient.Create(ctx, namespace)).To(Succeed()) + + machine = machinev1resourcebuilder.Machine(). + WithNamespace(namespace.Name). + WithName("machine"). + WithProviderSpecBuilder(machinev1resourcebuilder.AWSProviderSpec().WithLoadBalancers(nil)). + Build() + Expect(k8sClient.Create(ctx, machine)).To(Succeed()) + + machineKey = client.ObjectKeyFromObject(machine) + + By("Setting status.AuthoritativeAPI to MachineAPI through the migration helper") + + Expect(ApplyMigrationStatus[*machinev1applyconfigs.MachineStatusApplyConfiguration]( + ctx, + k8sClient, + machineMigrationControllerName, + machinev1applyconfigs.Machine, + machine, + mapiv1beta1.MachineAuthorityMachineAPI, + )).To(Succeed()) + + Expect(k8sClient.Get(ctx, machineKey, machine)).To(Succeed()) + + By("Recording synchronized status through the sync helper") + + Expect(ApplySyncStatus[*machinev1applyconfigs.MachineStatusApplyConfiguration]( + ctx, + k8sClient, + machineSyncControllerName, + machinev1applyconfigs.Machine, + machine, + corev1.ConditionTrue, + controllers.ReasonResourceSynchronized, + machineSuccessMessage, + &machine.Generation, + AuthoritativeAPIToSynchronizedAPI(mapiv1beta1.MachineAuthorityMachineAPI), + )).To(Succeed()) + + Expect(k8sClient.Get(ctx, machineKey, machine)).To(Succeed()) + Expect(machine.Status).To(SatisfyAll( + HaveField("AuthoritativeAPI", Equal(mapiv1beta1.MachineAuthorityMachineAPI)), + HaveField("SynchronizedAPI", Equal(mapiv1beta1.MachineAPISynchronized)), + )) + + By("Switching status.AuthoritativeAPI to Migrating through the migration helper") + + Expect(ApplyMigrationStatus[*machinev1applyconfigs.MachineStatusApplyConfiguration]( + ctx, + k8sClient, + machineMigrationControllerName, + machinev1applyconfigs.Machine, + machine, + mapiv1beta1.MachineAuthorityMigrating, + )).To(Succeed()) + }) + + AfterEach(func() { + testutils.CleanupResources(Default, ctx, cfg, k8sClient, namespace.Name, + &mapiv1beta1.Machine{}, + ) + }) + + Context("when reapplying sync status from a fresh post-migration object", func() { + It("should preserve synchronizedAPI", func() { + By("Fetching the Machine again after migration was acknowledged") + + freshMachine := &mapiv1beta1.Machine{} + Expect(k8sClient.Get(ctx, machineKey, freshMachine)).To(Succeed()) + Expect(freshMachine.Status).To(SatisfyAll( + HaveField("AuthoritativeAPI", Equal(mapiv1beta1.MachineAuthorityMigrating)), + HaveField("SynchronizedAPI", Equal(mapiv1beta1.MachineAPISynchronized)), + )) + + By("Reapplying sync status with the sync controller field owner") + + Expect(ApplySyncStatus[*machinev1applyconfigs.MachineStatusApplyConfiguration]( + ctx, + k8sClient, + machineSyncControllerName, + machinev1applyconfigs.Machine, + freshMachine, + corev1.ConditionTrue, + controllers.ReasonResourceSynchronized, + machineSuccessMessage, + &freshMachine.Generation, + AuthoritativeAPIToSynchronizedAPI(freshMachine.Status.AuthoritativeAPI), + )).To(Succeed()) + + updatedMachine := &mapiv1beta1.Machine{} + Expect(k8sClient.Get(ctx, machineKey, updatedMachine)).To(Succeed()) + Expect(updatedMachine.Status).To(SatisfyAll( + HaveField("AuthoritativeAPI", Equal(mapiv1beta1.MachineAuthorityMigrating)), + HaveField("SynchronizedAPI", Equal(mapiv1beta1.MachineAPISynchronized)), + )) + }) + }) +}) diff --git a/pkg/controllers/synccommon/syncstatus_test.go b/pkg/controllers/synccommon/syncstatus_test.go new file mode 100644 index 000000000..32ef56cb9 --- /dev/null +++ b/pkg/controllers/synccommon/syncstatus_test.go @@ -0,0 +1,256 @@ +/* +Copyright 2025 Red Hat, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package synccommon + +import ( + "errors" + "fmt" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + mapiv1beta1 "github.com/openshift/api/machine/v1beta1" + machinev1applyconfigs "github.com/openshift/client-go/machine/applyconfigurations/machine/v1beta1" + "github.com/openshift/cluster-capi-operator/pkg/controllers" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +var _ = Describe("Sync status helpers", func() { + DescribeTable("MigrationDirection should determine the current and desired authorities", func(statusAuthority mapiv1beta1.MachineAuthority, synchronizedAPI mapiv1beta1.SynchronizedAPI, specAuthority mapiv1beta1.MachineAuthority, expectedCurrent mapiv1beta1.MachineAuthority, expectedDesired mapiv1beta1.MachineAuthority, expectedMigrating bool, expectedErrMessage string, expectedErr error) { + currentAuthority, desiredAuthority, isMigrating, err := MigrationDirection(statusAuthority, synchronizedAPI, specAuthority) + + Expect(currentAuthority).To(Equal(expectedCurrent)) + Expect(desiredAuthority).To(Equal(expectedDesired)) + Expect(isMigrating).To(Equal(expectedMigrating)) + + if expectedErr == nil { + Expect(err).NotTo(HaveOccurred()) + return + } + + Expect(err).To(MatchError(expectedErrMessage)) + Expect(errors.Is(err, expectedErr)).To(BeTrue()) + }, + Entry("stable Machine API authority", mapiv1beta1.MachineAuthorityMachineAPI, mapiv1beta1.SynchronizedAPI(""), mapiv1beta1.MachineAuthorityClusterAPI, mapiv1beta1.MachineAuthorityMachineAPI, mapiv1beta1.MachineAuthorityClusterAPI, false, "", nil), + Entry("stable empty authority", mapiv1beta1.MachineAuthority(""), mapiv1beta1.SynchronizedAPI(""), mapiv1beta1.MachineAuthorityMachineAPI, mapiv1beta1.MachineAuthority(""), mapiv1beta1.MachineAuthorityMachineAPI, false, "", nil), + Entry("migrating from Machine API", mapiv1beta1.MachineAuthorityMigrating, mapiv1beta1.MachineAPISynchronized, mapiv1beta1.MachineAuthorityClusterAPI, mapiv1beta1.MachineAuthorityMachineAPI, mapiv1beta1.MachineAuthorityClusterAPI, true, "", nil), + Entry("migrating from Cluster API", mapiv1beta1.MachineAuthorityMigrating, mapiv1beta1.ClusterAPISynchronized, mapiv1beta1.MachineAuthorityMachineAPI, mapiv1beta1.MachineAuthorityClusterAPI, mapiv1beta1.MachineAuthorityMachineAPI, true, "", nil), + Entry("rollback while migrating", mapiv1beta1.MachineAuthorityMigrating, mapiv1beta1.MachineAPISynchronized, mapiv1beta1.MachineAuthorityMachineAPI, mapiv1beta1.MachineAuthorityMachineAPI, mapiv1beta1.MachineAuthorityMachineAPI, true, "", nil), + Entry("migrating without a synchronized API yet", mapiv1beta1.MachineAuthorityMigrating, mapiv1beta1.SynchronizedAPI(""), mapiv1beta1.MachineAuthorityClusterAPI, mapiv1beta1.MachineAuthority(""), mapiv1beta1.MachineAuthorityClusterAPI, true, "missing synchronizedAPI value while authoritativeAPI is Migrating", ErrMissingSynchronizedAPI), + Entry("migrating with an invalid synchronized API", mapiv1beta1.MachineAuthorityMigrating, mapiv1beta1.SynchronizedAPI("BogusAPI"), mapiv1beta1.MachineAuthorityClusterAPI, mapiv1beta1.MachineAuthority(""), mapiv1beta1.MachineAuthorityClusterAPI, true, "invalid synchronizedAPI value: BogusAPI", ErrInvalidSynchronizedAPI), + ) + + Describe("newSyncStatusApplyConfiguration", func() { + type testCase struct { + object client.Object + status corev1.ConditionStatus + generation *int64 + synchronizedAPI *mapiv1beta1.SynchronizedAPI + expectedGeneration int64 + expectedSeverity mapiv1beta1.ConditionSeverity + expectedResourceVer string + } + + DescribeTable("should build apply configurations for supported Machine API objects", func(tc testCase) { + const ( + reason = "SyncComplete" + message = "Machine API object is synchronized" + ) + + var ( + resourceVersion *string + synchronizedGeneration *int64 + condition machinev1applyconfigs.ConditionApplyConfiguration + synchronizedAPI *mapiv1beta1.SynchronizedAPI + ) + + switch o := tc.object.(type) { + case *mapiv1beta1.Machine: + objAC, statusAC, err := newSyncStatusApplyConfiguration[*machinev1applyconfigs.MachineStatusApplyConfiguration](machinev1applyconfigs.Machine, o, tc.status, reason, message, tc.generation, tc.synchronizedAPI) + Expect(err).ToNot(HaveOccurred()) + Expect(objAC.ObjectMetaApplyConfiguration).ToNot(BeNil()) + resourceVersion = objAC.ResourceVersion + synchronizedGeneration = statusAC.SynchronizedGeneration + Expect(statusAC.Conditions).To(HaveLen(1)) + condition = statusAC.Conditions[0] + synchronizedAPI = statusAC.SynchronizedAPI + case *mapiv1beta1.MachineSet: + objAC, statusAC, err := newSyncStatusApplyConfiguration[*machinev1applyconfigs.MachineSetStatusApplyConfiguration](machinev1applyconfigs.MachineSet, o, tc.status, reason, message, tc.generation, tc.synchronizedAPI) + Expect(err).ToNot(HaveOccurred()) + Expect(objAC.ObjectMetaApplyConfiguration).ToNot(BeNil()) + resourceVersion = objAC.ResourceVersion + synchronizedGeneration = statusAC.SynchronizedGeneration + Expect(statusAC.Conditions).To(HaveLen(1)) + condition = statusAC.Conditions[0] + synchronizedAPI = statusAC.SynchronizedAPI + default: + Fail(fmt.Sprintf("unsupported object type %T", tc.object)) + } + + Expect(resourceVersion).ToNot(BeNil()) + Expect(*resourceVersion).To(Equal(tc.expectedResourceVer)) + + Expect(synchronizedGeneration).ToNot(BeNil()) + Expect(*synchronizedGeneration).To(Equal(tc.expectedGeneration)) + + Expect(condition.Type).ToNot(BeNil()) + Expect(*condition.Type).To(Equal(controllers.SynchronizedCondition)) + Expect(condition.Status).ToNot(BeNil()) + Expect(*condition.Status).To(Equal(tc.status)) + Expect(condition.Reason).ToNot(BeNil()) + Expect(*condition.Reason).To(Equal(reason)) + Expect(condition.Message).ToNot(BeNil()) + Expect(*condition.Message).To(Equal(message)) + Expect(condition.Severity).ToNot(BeNil()) + Expect(*condition.Severity).To(Equal(tc.expectedSeverity)) + Expect(condition.LastTransitionTime).ToNot(BeNil()) + + if tc.synchronizedAPI == nil { + Expect(synchronizedAPI).To(BeNil()) + } else { + Expect(synchronizedAPI).ToNot(BeNil()) + Expect(*synchronizedAPI).To(Equal(*tc.synchronizedAPI)) + } + }, + Entry("a Machine while preserving the previous synchronized generation", testCase{ + object: &mapiv1beta1.Machine{ + ObjectMeta: metav1.ObjectMeta{ + Name: "machine-1", + Namespace: "openshift-machine-api", + ResourceVersion: "7", + }, + Status: mapiv1beta1.MachineStatus{ + SynchronizedGeneration: 5, + }, + }, + status: corev1.ConditionTrue, + synchronizedAPI: ptr.To(mapiv1beta1.MachineAPISynchronized), + expectedGeneration: 5, + expectedSeverity: mapiv1beta1.ConditionSeverityNone, + expectedResourceVer: "7", + }), + Entry("a MachineSet while overriding the synchronized generation", testCase{ + object: &mapiv1beta1.MachineSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "machineset-1", + Namespace: "openshift-machine-api", + ResourceVersion: "11", + }, + Status: mapiv1beta1.MachineSetStatus{ + SynchronizedGeneration: 2, + }, + }, + status: corev1.ConditionFalse, + generation: ptr.To(int64(19)), + synchronizedAPI: ptr.To(mapiv1beta1.ClusterAPISynchronized), + expectedGeneration: 19, + expectedSeverity: mapiv1beta1.ConditionSeverityError, + expectedResourceVer: "11", + }), + Entry("a Machine with unknown sync status and no synchronized API", testCase{ + object: &mapiv1beta1.Machine{ + ObjectMeta: metav1.ObjectMeta{ + Name: "machine-2", + Namespace: "openshift-machine-api", + ResourceVersion: "13", + }, + Status: mapiv1beta1.MachineStatus{ + SynchronizedGeneration: 3, + }, + }, + status: corev1.ConditionUnknown, + expectedGeneration: 3, + expectedSeverity: mapiv1beta1.ConditionSeverityInfo, + expectedResourceVer: "13", + }), + ) + + It("should preserve the previous last transition time when the synchronized condition state is unchanged", func() { + lastTransitionTime := metav1.NewTime(time.Unix(1710000000, 0)) + mapiMachine := &mapiv1beta1.Machine{ + ObjectMeta: metav1.ObjectMeta{ + Name: "machine-3", + Namespace: "openshift-machine-api", + ResourceVersion: "17", + }, + Status: mapiv1beta1.MachineStatus{ + Conditions: []mapiv1beta1.Condition{{ + Type: controllers.SynchronizedCondition, + Status: corev1.ConditionTrue, + Reason: "SyncComplete", + Message: "Machine API object is synchronized", + Severity: mapiv1beta1.ConditionSeverityNone, + LastTransitionTime: lastTransitionTime, + }}, + }, + } + + _, statusAC, err := newSyncStatusApplyConfiguration[*machinev1applyconfigs.MachineStatusApplyConfiguration]( + machinev1applyconfigs.Machine, + mapiMachine, + corev1.ConditionTrue, + "SyncComplete", + "Machine API object is synchronized", + nil, + ptr.To(mapiv1beta1.MachineAPISynchronized), + ) + Expect(err).ToNot(HaveOccurred()) + Expect(statusAC.Conditions).To(HaveLen(1)) + Expect(statusAC.Conditions[0].LastTransitionTime).ToNot(BeNil()) + Expect(statusAC.Conditions[0].LastTransitionTime.Time).To(Equal(lastTransitionTime.Time)) + }) + + It("should reject unrecognized condition statuses", func() { + mapiMachine := &mapiv1beta1.Machine{ + ObjectMeta: metav1.ObjectMeta{ + Name: "machine-4", + Namespace: "openshift-machine-api", + }, + } + + _, _, err := newSyncStatusApplyConfiguration[*machinev1applyconfigs.MachineStatusApplyConfiguration]( + machinev1applyconfigs.Machine, + mapiMachine, + corev1.ConditionStatus("NotAConditionStatus"), + "SyncFailed", + "status was invalid", + nil, + nil, + ) + Expect(err).To(MatchError("error unrecognized condition status: NotAConditionStatus")) + Expect(errors.Is(err, errUnrecognizedConditionStatus)).To(BeTrue()) + }) + + It("should reject unsupported Machine API object types", func() { + _, _, err := newSyncStatusApplyConfiguration[*machinev1applyconfigs.MachineStatusApplyConfiguration]( + machinev1applyconfigs.Machine, + &corev1.ConfigMap{}, + corev1.ConditionTrue, + "SyncComplete", + "status was synchronized", + nil, + nil, + ) + Expect(err).To(HaveOccurred()) + Expect(errors.Is(err, errUnsupportedSyncStatusType)).To(BeTrue()) + Expect(err).To(MatchError(ContainSubstring("type does not support setting sync status"))) + }) + }) +}) diff --git a/pkg/conversion/test/fuzz/fuzz.go b/pkg/conversion/test/fuzz/fuzz.go index ad9dd335f..29504ccdf 100644 --- a/pkg/conversion/test/fuzz/fuzz.go +++ b/pkg/conversion/test/fuzz/fuzz.go @@ -763,11 +763,11 @@ func MAPIMachineFuzzerFuncs(providerSpec runtime.Object, providerStatus interfac m.NodeRef = nil } - m.LastOperation = nil // Ignore, this field as it is not present in CAPI. - m.AuthoritativeAPI = "" // Ignore, this field as it is not present in CAPI. - m.SynchronizedGeneration = 0 // Ignore, this field as it is not present in CAPI. - m.SynchronizedAPI = "" // Ignore, this field as it is not present in CAPI. - m.Conditions = nil // Ignore, this field as it is not a 1:1 mapping between CAPI and MAPI but rather a recomputation of the conditions based on other fields. + m.LastOperation = nil // Ignore; this field is not present in CAPI. + m.AuthoritativeAPI = "" // Ignore; this field is not present in CAPI. + m.SynchronizedAPI = "" // Ignore; this field is not present in CAPI. + m.SynchronizedGeneration = 0 // Ignore; this field is not present in CAPI. + m.Conditions = nil // Ignore; this field is not a 1:1 mapping between CAPI and MAPI, but rather a recomputation of the conditions based on other fields. }, } } @@ -810,11 +810,11 @@ func MAPIMachineSetFuzzerFuncs() fuzzer.FuzzerFuncs { func(m *mapiv1beta1.MachineSetStatus, c randfill.Continue) { c.FillNoCustom(m) - m.ObservedGeneration = 0 // Ignore, this field as it shouldn't match between CAPI and MAPI. - m.AuthoritativeAPI = "" // Ignore, this field as it is not present in CAPI. - m.SynchronizedGeneration = 0 // Ignore, this field as it is not present in CAPI. - m.SynchronizedAPI = "" // Ignore, this field as it is not present in CAPI. - m.Conditions = nil // Ignore, this field as it is not a 1:1 mapping between CAPI and MAPI but rather a recomputation of the conditions based on other fields. + m.ObservedGeneration = 0 // Ignore; this field should not match between CAPI and MAPI. + m.AuthoritativeAPI = "" // Ignore; this field is not present in CAPI. + m.SynchronizedAPI = "" // Ignore; this field is not present in CAPI. + m.SynchronizedGeneration = 0 // Ignore; this field is not present in CAPI. + m.Conditions = nil // Ignore; this field is not a 1:1 mapping between CAPI and MAPI, but rather a recomputation of the conditions based on other fields. }, } }