OCPBUGS-80958: Kube apiserver node replace

go.mod

@@ -2,6 +2,10 @@ module github.com/openshift/library-go
 
 go 1.25.0
 
+replace github.com/openshift/api => github.com/bhperry/openshift-api v0.0.0-20260709150816-69d6f1b49426
+
+replace github.com/openshift/client-go => github.com/bhperry/openshift-client-go v0.0.0-20260709151625-a8b452b3b0e4
+
 require (
 	github.com/RangelReale/osincli v0.0.0-20160924135400-fababb0555f2
 	github.com/blang/semver/v4 v4.0.0
@@ -18,7 +22,7 @@ require (
 	github.com/onsi/gomega v1.38.2
 	github.com/opencontainers/go-digest v1.0.0
 	github.com/opencontainers/selinux v1.13.0
-	github.com/openshift/api v0.0.0-20260619095050-5346161d1bf2
+	github.com/openshift/api v0.0.0-20260702202555-ef71f942ef6c
 	github.com/openshift/build-machinery-go v0.0.0-20250530140348-dc5b2804eeee
 	github.com/openshift/client-go v0.0.0-20260618131434-17fd91ed6167
 	github.com/pkg/errors v0.9.1

go.sum

@@ -20,6 +20,10 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/bhperry/openshift-api v0.0.0-20260709150816-69d6f1b49426 h1:zahkrQj9Jei8+t9yXabDHc75Ha+Holy3kQsdcKXv+oY=
+github.com/bhperry/openshift-api v0.0.0-20260709150816-69d6f1b49426/go.mod h1:7WJ3IPaK6nmWT8bDcaNooHqd0H5WepjVqV/10VlkMEM=
+github.com/bhperry/openshift-client-go v0.0.0-20260709151625-a8b452b3b0e4 h1:TE5qVIcGuDyZpBhFACCxpaiRbGCfhl1zhiLzJxj0Y/Q=
+github.com/bhperry/openshift-client-go v0.0.0-20260709151625-a8b452b3b0e4/go.mod h1:3f9XJ35WKsA7bY5Nl5LbFGzjVgqa2MRqHm4+7Sluo2g=
 github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM=
 github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ=
 github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8=
@@ -251,12 +255,8 @@ github.com/opencontainers/image-spec v1.0.2 h1:9yCKha/T5XdGtO0q9Q9a6T5NUCsTn/DrB
 github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
 github.com/opencontainers/selinux v1.13.0 h1:Zza88GWezyT7RLql12URvoxsbLfjFx988+LGaWfbL84=
 github.com/opencontainers/selinux v1.13.0/go.mod h1:XxWTed+A/s5NNq4GmYScVy+9jzXhGBVEOAyucdRUY8s=
-github.com/openshift/api v0.0.0-20260619095050-5346161d1bf2 h1:oZkYkIvQHkQV8pN0oe8AfPfc7y9+svwmsQHwbOWyJBw=
-github.com/openshift/api v0.0.0-20260619095050-5346161d1bf2/go.mod h1:Jm45pE7O6/G0tYYhiLzNyZykTjmf9BfhsKYuGfLLwTE=
 github.com/openshift/build-machinery-go v0.0.0-20250530140348-dc5b2804eeee h1:+Sp5GGnjHDhT/a/nQ1xdp43UscBMr7G5wxsYotyhzJ4=
 github.com/openshift/build-machinery-go v0.0.0-20250530140348-dc5b2804eeee/go.mod h1:8jcm8UPtg2mCAsxfqKil1xrmRMI3a+XU2TZ9fF8A7TE=
-github.com/openshift/client-go v0.0.0-20260618131434-17fd91ed6167 h1:TfqgEkvjjdzmVW3wkXWjh65rF+cjahnrBMbRdraT4Wo=
-github.com/openshift/client-go v0.0.0-20260618131434-17fd91ed6167/go.mod h1:SYV1Wn6Pd27K9olD4WbUtjn4RWU83b9UjG1rbaQsTRE=
 github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=

pkg/operator/staticpod/controller/installer/installer_controller.go

@@ -12,6 +12,7 @@ import (
 	"time"
 
 	"github.com/davecgh/go-spew/spew"
+	"github.com/pkg/errors"
 
 	operatorv1 "github.com/openshift/api/operator/v1"
 	applyoperatorv1 "github.com/openshift/client-go/operator/applyconfigurations/operator/v1"
@@ -30,10 +31,12 @@ import (
 	"k8s.io/apimachinery/pkg/api/equality"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
 	utilerrors "k8s.io/apimachinery/pkg/util/errors"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/client-go/informers"
 	corev1client "k8s.io/client-go/kubernetes/typed/core/v1"
+	corev1listers "k8s.io/client-go/listers/core/v1"
 	"k8s.io/klog/v2"
 	"k8s.io/utils/clock"
 	"k8s.io/utils/ptr"
@@ -94,6 +97,7 @@ type InstallerController struct {
 	configMapsGetter corev1client.ConfigMapsGetter
 	secretsGetter    corev1client.SecretsGetter
 	podsGetter       corev1client.PodsGetter
+	nodeLister       corev1listers.NodeLister
 	eventRecorder    events.Recorder
 	now              func() time.Time // for test plumbing
 
@@ -179,6 +183,7 @@ func NewInstallerController(
 	secrets []revision.RevisionResource,
 	command []string,
 	kubeInformersForTargetNamespace informers.SharedInformerFactory,
+	kubeInformersClusterScoped informers.SharedInformerFactory,
 	operatorClient v1helpers.StaticPodOperatorClient,
 	configMapsGetter corev1client.ConfigMapsGetter,
 	secretsGetter corev1client.SecretsGetter,
@@ -197,6 +202,7 @@ func NewInstallerController(
 		configMapsGetter: configMapsGetter,
 		secretsGetter:    secretsGetter,
 		podsGetter:       podsGetter,
+		nodeLister:       kubeInformersClusterScoped.Core().V1().Nodes().Lister(),
 		eventRecorder:    eventRecorder.WithComponentSuffix("installer-controller"),
 		now:              time.Now,
 		startupMonitorEnabled: func() (bool, error) {
@@ -218,7 +224,12 @@ func NewInstallerController(
 			// informers are needed here because their Getter are cached lister based
 			kubeInformersForTargetNamespace.Core().V1().Pods().Informer(),
 			kubeInformersForTargetNamespace.Core().V1().ConfigMaps().Informer(),
-			kubeInformersForTargetNamespace.Core().V1().Secrets().Informer())
+			kubeInformersForTargetNamespace.Core().V1().Secrets().Informer(),
+		).
+		WithBareInformers(
+			// Don't need to sync on node changes, just wait for informer cache sync
+			kubeInformersClusterScoped.Core().V1().Nodes().Informer(),
+		)
 
 	return c
 }
@@ -599,6 +610,7 @@ func prepareNodeStatusApplyConfigurationFor(nodeStatuses []operatorv1.NodeStatus
 func nodeStatusToNodeStatusApplyConfigurations(nodeStatus operatorv1.NodeStatus) *applyoperatorv1.NodeStatusApplyConfiguration {
 	nodeStatusApplyConfiguration := applyoperatorv1.NodeStatus().
 		WithNodeName(nodeStatus.NodeName).
+		WithNodeUID(nodeStatus.NodeUID).
 		WithCurrentRevision(nodeStatus.CurrentRevision).
 		WithTargetRevision(nodeStatus.TargetRevision).
 		WithLastFailedRevision(nodeStatus.LastFailedRevision).
@@ -615,6 +627,7 @@ func nodeStatusToNodeStatusApplyConfigurations(nodeStatus operatorv1.NodeStatus)
 func nodeStatusApplyConfigToOperatorNodeStatus(nodeStatusApplyConfiguration *applyoperatorv1.NodeStatusApplyConfiguration) *operatorv1.NodeStatus {
 	return &operatorv1.NodeStatus{
 		NodeName:                 ptr.Deref(nodeStatusApplyConfiguration.NodeName, ""),
+		NodeUID:                  ptr.Deref(nodeStatusApplyConfiguration.NodeUID, ""),
 		CurrentRevision:          ptr.Deref(nodeStatusApplyConfiguration.CurrentRevision, 0),
 		TargetRevision:           ptr.Deref(nodeStatusApplyConfiguration.TargetRevision, 0),
 		LastFailedRevision:       ptr.Deref(nodeStatusApplyConfiguration.LastFailedRevision, 0),
@@ -1123,6 +1136,25 @@ func (c InstallerController) ensureRequiredResourcesExist(ctx context.Context, r
 	return fmt.Errorf("missing required resources: %v", aggregatedErr)
 }
 
+func (c *InstallerController) hasNodeUIDChanged(nodeName string, nodeUid types.UID) (bool, error) {
+	if nodeUid == "" {
+		return false, nil
+	}
+	node, err := c.nodeLister.Get(nodeName)
+	if apierrors.IsNotFound(err) {
+		klog.Warningf("Skipping node status update for %s: node not found", nodeName)
+		return true, nil
+	}
+	if err != nil {
+		return false, err
+	}
+	if node.UID != nodeUid {
+		klog.Warningf("Skipping node status update for %s: node UID changed (stored=%s, actual=%s)", nodeName, nodeUid, node.UID)
+		return true, nil
+	}
+	return false, nil
+}
+
 func (c *InstallerController) Sync(ctx context.Context, syncCtx factory.SyncContext) error {
 	operatorSpec, originalOperatorStatus, operatorResourceVersion, err := c.operatorClient.GetStaticPodOperatorState()
 	if err != nil {
@@ -1164,6 +1196,17 @@ func (c *InstallerController) Sync(ctx context.Context, syncCtx factory.SyncCont
 		return nil
 	}
 
+	// Check for stale node status entries with mismatched UID
+	for _, nodeStatus := range originalOperatorStatus.NodeStatuses {
+		if uidChanged, err := c.hasNodeUIDChanged(nodeStatus.NodeName, nodeStatus.NodeUID); err != nil {
+			return errors.Wrapf(err, "error validating uid for node %s", nodeStatus.NodeName)
+		} else if uidChanged {
+			// Avoid applying node statuses when a stale entry is found
+			// Sync will retrigger when the old node status is removed by the node controller
+			return nil
+		}
+	}
+
 	err = c.ensureRequiredResourcesExistFn(ctx, originalOperatorStatus.LatestAvailableRevision)
 
 	// Only manage installation pods when all required certs are present.
@@ -1204,6 +1247,7 @@ func deepCopyNodeStatusWithoutOldFailedState(ns *operatorv1.NodeStatus) *operato
 	if ns.TargetRevision == 0 || ns.TargetRevision != ns.LastFailedRevision {
 		return &operatorv1.NodeStatus{
 			NodeName:        ns.NodeName,
+			NodeUID:         ns.NodeUID,
 			CurrentRevision: ns.CurrentRevision,
 			TargetRevision:  ns.TargetRevision,
 		}

pkg/operator/staticpod/controller/installer/installer_controller_test.go

@@ -555,6 +555,7 @@ func TestNewNodeStateForInstallInProgress(t *testing.T) {
 				[]revision.RevisionResource{{Name: "test-secret"}},
 				[]string{"/bin/true", "--foo=test", "--bar"},
 				kubeInformers,
+				kubeInformers,
 				fakeStaticPodOperatorClient,
 				kubeClient.CoreV1(),
 				kubeClient.CoreV1(),
@@ -690,6 +691,7 @@ func testSync(t *testing.T, firstInstallerBehaviour testSyncInstallerBehaviour,
 		[]revision.RevisionResource{{Name: "test-secret"}},
 		podCommand,
 		kubeInformers,
+		kubeInformers,
 		fakeStaticPodOperatorClient,
 		kubeClient.CoreV1(),
 		kubeClient.CoreV1(),
@@ -1099,6 +1101,7 @@ func TestCreateInstallerPod(t *testing.T) {
 		[]revision.RevisionResource{{Name: "test-secret"}},
 		[]string{"/bin/true"},
 		kubeInformers,
+		kubeInformers,
 		fakeStaticPodOperatorClient,
 		kubeClient.CoreV1(),
 		kubeClient.CoreV1(),
@@ -1268,6 +1271,7 @@ func TestEnsureInstallerPod(t *testing.T) {
 				tt.secrets,
 				[]string{"/bin/true"},
 				kubeInformers,
+				kubeInformers,
 				fakeStaticPodOperatorClient,
 				kubeClient.CoreV1(),
 				kubeClient.CoreV1(),
@@ -2011,6 +2015,7 @@ func TestCreateInstallerPodMultiNode(t *testing.T) {
 				[]revision.RevisionResource{{Name: "test-secret"}},
 				[]string{"/bin/true"},
 				kubeInformers,
+				kubeInformers,
 				fakeStaticPodOperatorClient,
 				kubeClient.CoreV1(),
 				kubeClient.CoreV1(),

pkg/operator/staticpod/controller/node/node_controller.go

@@ -7,6 +7,7 @@ import (
 
 	coreapiv1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/informers"
 	corelisterv1 "k8s.io/client-go/listers/core/v1"
 
@@ -84,22 +85,24 @@ func (c *NodeController) sync(ctx context.Context, syncCtx factory.SyncContext)
 		nodes = append(nodes, extraNodes...)
 	}
 
+	nodeUIDs := map[string]types.UID{}
+	for _, node := range nodes {
+		nodeUIDs[node.Name] = node.UID
+	}
+
 	jsonPatch := jsonpatch.New()
 	var removedNodeStatusesCounter int
 	newTargetNodeStates := []*applyoperatorv1.NodeStatusApplyConfiguration{}
 	// remove entries for missing nodes
 	for i, nodeState := range originalOperatorStatus.NodeStatuses {
-		found := false
-		for _, node := range nodes {
-			if nodeState.NodeName == node.Name {
-				found = true
-			}
-		}
-		if found {
-			newTargetNodeState := applyoperatorv1.NodeStatus().WithNodeName(originalOperatorStatus.NodeStatuses[i].NodeName)
+		currentUID, found := nodeUIDs[nodeState.NodeName]
+		if found && (nodeState.NodeUID == currentUID || nodeState.NodeUID == "") {
+			newTargetNodeState := applyoperatorv1.NodeStatus().
+				WithNodeName(nodeState.NodeName).
+				WithNodeUID(currentUID)
 			newTargetNodeStates = append(newTargetNodeStates, newTargetNodeState)
 		} else {
-			syncCtx.Recorder().Warningf("MasterNodeRemoved", "Observed removal of master node %s", nodeState.NodeName)
+			syncCtx.Recorder().Warningf("MasterNodeRemoved", "Observed removal of master node %s (UID: %s)", nodeState.NodeName, nodeState.NodeUID)
 			// each delete operation is applied to the object,
 			// which modifies the array. Thus, we need to
 			// adjust the indices to find the correct node to remove.
@@ -116,16 +119,19 @@ func (c *NodeController) sync(ctx context.Context, syncCtx factory.SyncContext)
 	for _, node := range nodes {
 		found := false
 		for _, nodeState := range originalOperatorStatus.NodeStatuses {
+			// NodeUID is intentionally ignored here. Nodes that are replaced with the same name will get added in the next sync
+			// to ensure previous nodeStatus from a stale cache isn't merged in with the fresh status.
 			if nodeState.NodeName == node.Name {
 				found = true
+				break
 			}
 		}
 		if found {
 			continue
 		}
 
-		syncCtx.Recorder().Eventf("MasterNodeObserved", "Observed new master node %s", node.Name)
-		newTargetNodeState := applyoperatorv1.NodeStatus().WithNodeName(node.Name)
+		syncCtx.Recorder().Eventf("MasterNodeObserved", "Observed new master node %s (UID: %s)", node.Name, node.UID)
+		newTargetNodeState := applyoperatorv1.NodeStatus().WithNodeName(node.Name).WithNodeUID(node.UID)
 		newTargetNodeStates = append(newTargetNodeStates, newTargetNodeState)
 	}