CNTRLPLANE-3383: HO: Add HostedClusterDeleting condition to track deletion progress

api/hypershift/v1beta1/hostedcluster_conditions.go

@@ -266,6 +266,12 @@ const (
 	// cluster's shared ingress. Status reflects observed state: True means
 	// public endpoints are reachable, False means they are not.
 	PublicEndpointExposed ConditionType = "PublicEndpointExposed"
+
+	// HostedClusterDeleting indicates whether the HostedCluster is being deleted and
+	// provides first-class visibility into which phase of deletion the cluster is in.
+	// **False / AsExpected** means the cluster is not being deleted.
+	// **True** means deletion is in progress; the Reason and Message indicate the current phase.
+	HostedClusterDeleting ConditionType = "HostedClusterDeleting"
 )
 
 // Reasons for PublicEndpointExposed condition.
@@ -370,6 +376,17 @@ const (
 	ReEncryptionCompletedReason     = "ReEncryptionCompleted"
 	ReEncryptionFailedReason        = "ReEncryptionFailed"
 	ReEncryptionWaitingForKASReason = "ReEncryptionWaitingForKASConvergence"
+
+	// HostedClusterDeleting reasons track progress through each phase of deletion.
+	DeletionWaitingForNodePoolDeletionReason        = "WaitingForNodePoolDeletion"
+	DeletionWaitingForCAPIClusterDeletionReason     = "WaitingForCAPIClusterDeletion"
+	DeletionWaitingForEndpointServiceDeletionReason = "WaitingForEndpointServiceDeletion"
+	DeletionWaitingForPrivateConnectDeletionReason  = "WaitingForPrivateConnectDeletion"
+	DeletionWaitingForControlPlaneDeletionReason    = "WaitingForControlPlaneDeletion"
+	DeletionWaitingForNamespaceDeletionReason       = "WaitingForNamespaceDeletion"
+	// DeletionCompletedReason indicates all hosted cluster resources have been torn down.
+	// The HostedCluster object itself may still exist briefly until the finalizer is removed.
+	DeletionCompletedReason = "DeletionCompleted"
 )
 
 // Messages.

hypershift-operator/controllers/hostedcluster/hostedcluster_controller.go

@@ -24,6 +24,7 @@ import (
 	"net/netip"
 	"os"
 	"reflect"
+	"sort"
 	"strconv"
 	"strings"
 	"time"
@@ -497,6 +498,14 @@ func (r *HostedClusterReconciler) reconcile(ctx context.Context, req ctrl.Reques
 		}
 	}
 
+	// Keep the steady-state non-deleting condition current.
+	if hcluster.DeletionTimestamp.IsZero() {
+		if err := r.reconcileDeletingConditionSteadyState(ctx, hcluster); err != nil {
+			return ctrl.Result{}, err
+		}
+	}
+
+	// If deleted, clean up and return early.
 	if !hcluster.DeletionTimestamp.IsZero() {
 		// This new condition is necessary for OCM personnel to report any cloud dangling objects to the user.
 		// The grace period is customizable using an annotation called HCDestroyGracePeriodAnnotation. It's a time.Duration annotation.
@@ -3736,11 +3745,39 @@ func deleteControlPlaneOperatorRBAC(ctx context.Context, c client.Client, rbacNa
 	return nil
 }
 
+func (r *HostedClusterReconciler) reconcileDeletingConditionSteadyState(ctx context.Context, hc *hyperv1.HostedCluster) error {
+	if updated := meta.SetStatusCondition(&hc.Status.Conditions, metav1.Condition{
+		Type:               string(hyperv1.HostedClusterDeleting),
+		Status:             metav1.ConditionFalse,
+		Reason:             hyperv1.AsExpectedReason,
+		Message:            "HostedCluster is not being deleted",
+		ObservedGeneration: hc.Generation,
+	}); updated {
+		if err := r.Client.Status().Update(ctx, hc); err != nil {
+			return fmt.Errorf("failed to reconcile HostedClusterDeleting condition: %w", err)
+		}
+	}
+	return nil
+}
+
 //nolint:gocyclo
 func (r *HostedClusterReconciler) delete(ctx context.Context, hc *hyperv1.HostedCluster) (bool, error) {
 	controlPlaneNamespace := manifests.HostedControlPlaneNamespace(hc.Namespace, hc.Name)
 	log := ctrl.LoggerFrom(ctx)
 
+	setDeletionProgress := func(reason, message string) error {
+		if updated := meta.SetStatusCondition(&hc.Status.Conditions, metav1.Condition{
+			Type:               string(hyperv1.HostedClusterDeleting),
+			Status:             metav1.ConditionTrue,
+			Reason:             reason,
+			Message:            message,
+			ObservedGeneration: hc.Generation,
+		}); !updated {
+			return nil
+		}
+		return r.Client.Status().Update(ctx, hc)
+	}
+
 	// Unpause CAPI cluster to allow deletion to proceed
 	if err := pauseCAPICluster(ctx, r.Client, hc, false); err != nil {
 		return false, err
@@ -3774,6 +3811,24 @@ func (r *HostedClusterReconciler) delete(ctx context.Context, hc *hyperv1.Hosted
 		return false, err
 	}
 
+	remainingNodePools, err := listNodePools(ctx, r.Client, hc.Namespace, hc.Name)
+	if err != nil {
+		return false, fmt.Errorf("failed to list NodePools: %w", err)
+	}
+	if len(remainingNodePools) > 0 {
+		npNames := make([]string, len(remainingNodePools))
+		for i := range remainingNodePools {
+			npNames[i] = remainingNodePools[i].Name
+		}
+		sort.Strings(npNames)
+		log.Info("Waiting for NodePool deletion", "remaining", len(remainingNodePools), "nodepools", npNames)
+		if err := setDeletionProgress(hyperv1.DeletionWaitingForNodePoolDeletionReason,
+			fmt.Sprintf("Waiting for %d NodePool(s) to be deleted: %s", len(remainingNodePools), strings.Join(npNames, ", "))); err != nil {
+			return false, fmt.Errorf("failed to update deletion progress: %w", err)
+		}
+		return false, nil
+	}
+
 	p, err := platform.GetPlatform(ctx, hc, nil, "", nil)
 	if err != nil {
 		return false, err
@@ -3797,6 +3852,10 @@ func (r *HostedClusterReconciler) delete(ctx context.Context, hc *hyperv1.Hosted
 
 		if exists {
 			log.Info("Waiting for cluster deletion", "clusterName", hc.Spec.InfraID, "controlPlaneNamespace", controlPlaneNamespace)
+			if err := setDeletionProgress(hyperv1.DeletionWaitingForCAPIClusterDeletionReason,
+				fmt.Sprintf("Waiting for CAPI cluster %s/%s to be deleted", controlPlaneNamespace, hc.Spec.InfraID)); err != nil {
+				return false, fmt.Errorf("failed to update deletion progress: %w", err)
+			}
 			return false, nil
 		} else {
 			// once infra is deleted remove finalizers.
@@ -3836,6 +3895,10 @@ func (r *HostedClusterReconciler) delete(ctx context.Context, hc *hyperv1.Hosted
 		}
 		if exists {
 			log.Info("Waiting for awsendpointservice deletion", "controlPlaneNamespace", controlPlaneNamespace)
+			if err := setDeletionProgress(hyperv1.DeletionWaitingForEndpointServiceDeletionReason,
+				fmt.Sprintf("Waiting for AWS endpoint services in %s to be deleted", controlPlaneNamespace)); err != nil {
+				return false, fmt.Errorf("failed to update deletion progress: %w", err)
+			}
 			return false, nil
 		}
 	}
@@ -3847,6 +3910,10 @@ func (r *HostedClusterReconciler) delete(ctx context.Context, hc *hyperv1.Hosted
 		}
 		if exists {
 			log.Info("Waiting for gcpprivateserviceconnect deletion", "controlPlaneNamespace", controlPlaneNamespace)
+			if err := setDeletionProgress(hyperv1.DeletionWaitingForPrivateConnectDeletionReason,
+				fmt.Sprintf("Waiting for GCP Private Service Connect resources in %s to be deleted", controlPlaneNamespace)); err != nil {
+				return false, fmt.Errorf("failed to update deletion progress: %w", err)
+			}
 			return false, nil
 		}
 	}
@@ -3902,6 +3969,10 @@ func (r *HostedClusterReconciler) delete(ctx context.Context, hc *hyperv1.Hosted
 	}
 	if exists {
 		log.Info("Waiting for hostedcontrolplane deletion", "controlPlaneNamespace", controlPlaneNamespace)
+		if err := setDeletionProgress(hyperv1.DeletionWaitingForControlPlaneDeletionReason,
+			fmt.Sprintf("Waiting for HostedControlPlane %s/%s to be deleted", controlPlaneNamespace, hc.Name)); err != nil {
+			return false, fmt.Errorf("failed to update deletion progress: %w", err)
+		}
 		return false, nil
 	}
 
@@ -3916,6 +3987,9 @@ func (r *HostedClusterReconciler) delete(ctx context.Context, hc *hyperv1.Hosted
 	r.KubevirtInfraClients.Delete(hc.Spec.InfraID)
 
 	if skipNSDeletion := hc.Annotations[hyperv1.SkipControlPlaneNamespaceDeletionAnnotation]; skipNSDeletion == "true" {
+		if err := setDeletionProgress(hyperv1.DeletionCompletedReason, "Deletion completed (namespace deletion skipped)"); err != nil {
+			return false, fmt.Errorf("failed to update deletion progress: %w", err)
+		}
 		return true, nil
 	}
 
@@ -3928,10 +4002,49 @@ func (r *HostedClusterReconciler) delete(ctx context.Context, hc *hyperv1.Hosted
 		return false, err
 	}
 	if exists {
-		log.Info("Waiting for namespace deletion", "controlPlaneNamespace", controlPlaneNamespace)
+		message := fmt.Sprintf("Waiting for namespace %s to be deleted", controlPlaneNamespace)
+
+		// Fetch the namespace to inspect its phase and conditions
+		ns := &corev1.Namespace{}
+		if getErr := r.Client.Get(ctx, types.NamespacedName{Name: controlPlaneNamespace}, ns); getErr == nil {
+			message = fmt.Sprintf("Waiting for namespace %s to be deleted (phase: %s)", controlPlaneNamespace, ns.Status.Phase)
+			var details []string
+			for _, cond := range ns.Status.Conditions {
+				switch cond.Type {
+				case corev1.NamespaceContentRemaining,
+					corev1.NamespaceFinalizersRemaining,
+					corev1.NamespaceDeletionContentFailure:
+					if cond.Status == corev1.ConditionTrue {
+						details = append(details, fmt.Sprintf("%s: %s", cond.Type, cond.Message))
+						log.Info("Namespace deletion blocked",
+							"controlPlaneNamespace", controlPlaneNamespace,
+							"conditionType", cond.Type,
+							"reason", cond.Reason,
+							"message", cond.Message,
+						)
+					}
+				}
+			}
+			if len(details) > 0 {
+				message = fmt.Sprintf("Waiting for namespace %s to be deleted (phase: %s): %s",
+					controlPlaneNamespace, ns.Status.Phase, strings.Join(details, "; "))
+				const maxMessageLen = 1024
+				if len(message) > maxMessageLen {
+					message = message[:maxMessageLen-3] + "..."
+				}
+			}
+		}
+
+		log.Info(message, "controlPlaneNamespace", controlPlaneNamespace)
+		if err := setDeletionProgress(hyperv1.DeletionWaitingForNamespaceDeletionReason, message); err != nil {
+			return false, fmt.Errorf("failed to update deletion progress: %w", err)
+		}
 		return false, nil
 	}
 
+	if err := setDeletionProgress(hyperv1.DeletionCompletedReason, "Deletion completed"); err != nil {
+		return false, fmt.Errorf("failed to update deletion progress: %w", err)
+	}
 	return true, nil
 }