From 8fe795e27e13c6163f655a927964347d95ac4220 Mon Sep 17 00:00:00 2001 From: Thomas Jungblut Date: Thu, 2 Jul 2026 09:35:36 +0200 Subject: [PATCH] DO-NOT-MERGE: static pod deployer This is just an e2e, not destined for review, we pull individual refactorings out. Signed-off-by: Thomas Jungblut --- .../assets/kms-preflight-installer-pod.yaml | 92 ++++++ .../preflight/assets/kms-preflight-pod.yaml | 23 +- .../encryption/kms/preflight/deployer.go | 1 - .../encryption/kms/preflight/pod_template.go | 87 +++++- .../kms/preflight/pod_template_test.go | 196 ++++++++++++- .../kms/preflight/static_pod_deployer.go | 207 ++++++++++++++ .../kms/preflight/static_pod_deployer_test.go | 263 ++++++++++++++++++ 7 files changed, 853 insertions(+), 16 deletions(-) create mode 100644 pkg/operator/encryption/kms/preflight/assets/kms-preflight-installer-pod.yaml create mode 100644 pkg/operator/encryption/kms/preflight/static_pod_deployer.go create mode 100644 pkg/operator/encryption/kms/preflight/static_pod_deployer_test.go diff --git a/pkg/operator/encryption/kms/preflight/assets/kms-preflight-installer-pod.yaml b/pkg/operator/encryption/kms/preflight/assets/kms-preflight-installer-pod.yaml new file mode 100644 index 0000000000..853d2c2aa3 --- /dev/null +++ b/pkg/operator/encryption/kms/preflight/assets/kms-preflight-installer-pod.yaml @@ -0,0 +1,92 @@ +apiVersion: v1 +kind: Pod +metadata: + name: kms-preflight-installer + namespace: {{.Namespace}} + labels: + app: openshift-kms-preflight-installer +spec: + automountServiceAccountToken: false + nodeSelector: + node-role.kubernetes.io/master: "" + tolerations: + - key: "node-role.kubernetes.io/master" + operator: "Exists" + effect: "NoSchedule" + - key: "node-role.kubernetes.io/master" + operator: "Exists" + effect: "NoExecute" + initContainers: + - name: cleanup + image: {{.InstallerImage}} + imagePullPolicy: IfNotPresent + terminationMessagePolicy: FallbackToLogsOnError + command: + - /bin/sh + - -c + - | + #!/bin/sh + set -euo pipefail + + rm -f /etc/kubernetes/manifests/kms-preflight-pod.yaml + rm -rf /etc/kubernetes/manifests/secrets/kms-preflight-encryption-config + securityContext: + privileged: true + runAsUser: 0 + resources: + requests: + memory: 10Mi + cpu: 10m + volumeMounts: + - mountPath: /etc/kubernetes/manifests + name: manifests-dir + containers: + - name: installer + image: {{.InstallerImage}} + imagePullPolicy: IfNotPresent + terminationMessagePolicy: FallbackToLogsOnError + command: + - /bin/sh + - -c + - | + #!/bin/sh + set -euo pipefail + + target_secret_dir="/etc/kubernetes/manifests/secrets/kms-preflight-encryption-config" + mkdir -p "${target_secret_dir}" + cp -f /install/secret/* "${target_secret_dir}/" + cp -f /install/pod/lb-int.kubeconfig "${target_secret_dir}/lb-int.kubeconfig" + + manifest_path="/etc/kubernetes/manifests/kms-preflight-pod.yaml" + mkdir -p /etc/kubernetes/manifests + cp -f /install/pod/pod.yaml "${manifest_path}" + chmod 0600 "${manifest_path}" + securityContext: + privileged: true + runAsUser: 0 + resources: + requests: + memory: 10Mi + cpu: 10m + volumeMounts: + - mountPath: /etc/kubernetes/manifests + name: manifests-dir + - mountPath: /install/pod + name: pod-manifest + - mountPath: /install/secret + name: encryption-config + readOnly: true + restartPolicy: Never + priorityClassName: system-node-critical + securityContext: + runAsUser: 0 + volumes: + - hostPath: + path: /etc/kubernetes/manifests + name: manifests-dir + - name: pod-manifest + configMap: + name: kms-preflight-pod + - name: encryption-config + secret: + secretName: kms-preflight-encryption-config diff --git a/pkg/operator/encryption/kms/preflight/assets/kms-preflight-pod.yaml b/pkg/operator/encryption/kms/preflight/assets/kms-preflight-pod.yaml index a9990d1846..ccec6965f6 100644 --- a/pkg/operator/encryption/kms/preflight/assets/kms-preflight-pod.yaml +++ b/pkg/operator/encryption/kms/preflight/assets/kms-preflight-pod.yaml @@ -12,7 +12,9 @@ spec: # This is a one-shot preflight check, not a long-running pod. # The operator creates it, waits for completion, and inspects the result. restartPolicy: Never +{{- if not .StaticPod}} serviceAccountName: kms-preflight +{{- end}} priorityClassName: system-cluster-critical nodeSelector: node-role.kubernetes.io/master: "" @@ -24,6 +26,13 @@ spec: - key: "node-role.kubernetes.io/master" operator: "Exists" effect: "NoExecute" +{{- if .StaticPod}} + hostNetwork: true + volumes: + - name: resource-dir + hostPath: + path: /etc/kubernetes/manifests +{{- end}} containers: - name: kms-preflight-check image: {{.OperatorImage}} @@ -32,8 +41,10 @@ spec: - --kms-call-timeout={{.KMSCallTimeout}} - --config-hash=$(CONFIG_HASH) - --pod-name=$(POD_NAME) - - --pod-namespace=$(POD_NAMESPACE){{if .Kubeconfig}} - - --kubeconfig={{.Kubeconfig}}{{end}} + - --pod-namespace=$(POD_NAMESPACE) +{{- if .StaticPod}} + - --kubeconfig=/etc/kubernetes/static-pod-resources/secrets/kms-preflight-encryption-config/lb-int.kubeconfig +{{- end}} env: - name: POD_NAME valueFrom: { fieldRef: { fieldPath: metadata.name } } @@ -46,3 +57,11 @@ spec: requests: memory: 50Mi cpu: 5m +{{- if .StaticPod}} + securityContext: + runAsUser: 0 + volumeMounts: + - name: resource-dir + mountPath: /etc/kubernetes/static-pod-resources + readOnly: true +{{- end}} diff --git a/pkg/operator/encryption/kms/preflight/deployer.go b/pkg/operator/encryption/kms/preflight/deployer.go index 2b6582520a..c6c2f349f0 100644 --- a/pkg/operator/encryption/kms/preflight/deployer.go +++ b/pkg/operator/encryption/kms/preflight/deployer.go @@ -75,7 +75,6 @@ func (d *PodPreflightDeployer) Deploy(ctx context.Context, configHash string, en d.operatorImage, d.operatorCommand, d.kmsCallTimeout, - "", ) if err != nil { return fmt.Errorf("failed to generate preflight pod template: %w", err) diff --git a/pkg/operator/encryption/kms/preflight/pod_template.go b/pkg/operator/encryption/kms/preflight/pod_template.go index 3c079ff097..73433e2959 100644 --- a/pkg/operator/encryption/kms/preflight/pod_template.go +++ b/pkg/operator/encryption/kms/preflight/pod_template.go @@ -12,6 +12,11 @@ import ( "github.com/openshift/library-go/pkg/operator/resource/resourceread" ) +const ( + preflightPodConfigMapPrefix = "kms-preflight-pod" + preflightInstallerPodName = "kms-preflight-installer" +) + type kmsPreflightTemplate struct { PodName string Namespace string @@ -19,38 +24,96 @@ type kmsPreflightTemplate struct { OperatorImage string Command string KMSCallTimeout string - Kubeconfig string + StaticPod bool } -// generatePodTemplate renders the KMS preflight pod YAML template. -// The pod runs the preflight checker as a one-shot command and shares an emptyDir -// volume with the KMS plugin sidecar at /var/run/kmsplugin. -func generatePodTemplate( +func newKmsPreflightTemplate( podName string, namespace string, configHash string, operatorImage string, operatorCommand []string, kmsCallTimeout time.Duration, - kubeconfig string, -) (*corev1.Pod, error) { - rawManifest := mustAsset("assets/kms-preflight-pod.yaml") - + staticPod bool, +) kmsPreflightTemplate { operatorCommandQuoted := make([]string, len(operatorCommand)) for i, cmd := range operatorCommand { operatorCommandQuoted[i] = fmt.Sprintf("%q", cmd) } - tmplVal := kmsPreflightTemplate{ + return kmsPreflightTemplate{ PodName: podName, Namespace: namespace, ConfigHash: configHash, OperatorImage: operatorImage, Command: strings.Join(operatorCommandQuoted, ","), KMSCallTimeout: kmsCallTimeout.String(), - Kubeconfig: kubeconfig, + StaticPod: staticPod, + } +} + +// generatePodTemplate renders the KMS preflight pod YAML template. +// The pod runs the preflight checker as a one-shot command and shares an emptyDir +// volume with the KMS plugin sidecar at /var/run/kmsplugin. +func generatePodTemplate( + podName string, + namespace string, + configHash string, + operatorImage string, + operatorCommand []string, + kmsCallTimeout time.Duration, +) (*corev1.Pod, error) { + return renderPreflightPodTemplate(podName, namespace, configHash, operatorImage, operatorCommand, kmsCallTimeout, false) +} + +// generateStaticPodTemplate renders the KMS preflight static pod YAML template. +func generateStaticPodTemplate( + podName string, + namespace string, + configHash string, + operatorImage string, + operatorCommand []string, + kmsCallTimeout time.Duration, +) (*corev1.Pod, error) { + return renderPreflightPodTemplate(podName, namespace, configHash, operatorImage, operatorCommand, kmsCallTimeout, true) +} + +func renderPreflightPodTemplate( + podName string, + namespace string, + configHash string, + operatorImage string, + operatorCommand []string, + kmsCallTimeout time.Duration, + staticPod bool, +) (*corev1.Pod, error) { + rawManifest := mustAsset("assets/kms-preflight-pod.yaml") + tmplVal := newKmsPreflightTemplate( + podName, namespace, configHash, operatorImage, operatorCommand, kmsCallTimeout, staticPod, + ) + return renderPodTemplate("kms-preflight", string(rawManifest), tmplVal) +} + +type kmsPreflightInstallerTemplate struct { + Namespace string + InstallerImage string +} + +// generateInstallerPodTemplate renders the pod that installs the KMS preflight static pod +// manifest and encryption-config secret onto a control plane node. +func generateInstallerPodTemplate(namespace, operatorImage string) (*corev1.Pod, error) { + rawManifest := mustAsset("assets/kms-preflight-installer-pod.yaml") + + tmplVal := kmsPreflightInstallerTemplate{ + Namespace: namespace, + InstallerImage: operatorImage, } - tmpl, err := template.New("kms-preflight").Parse(string(rawManifest)) + + return renderPodTemplate("kms-preflight-installer", string(rawManifest), tmplVal) +} + +func renderPodTemplate(name, rawManifest string, tmplVal any) (*corev1.Pod, error) { + tmpl, err := template.New(name).Parse(rawManifest) if err != nil { return nil, err } diff --git a/pkg/operator/encryption/kms/preflight/pod_template_test.go b/pkg/operator/encryption/kms/preflight/pod_template_test.go index 063c2c6e76..c28180a26e 100644 --- a/pkg/operator/encryption/kms/preflight/pod_template_test.go +++ b/pkg/operator/encryption/kms/preflight/pod_template_test.go @@ -65,7 +65,6 @@ func TestGeneratePodTemplate(t *testing.T) { "quay.io/openshift/operator:latest", []string{"operator", "kms-preflight"}, 10*time.Second, - "", ) if err != nil { t.Fatalf("unexpected error: %v", err) @@ -79,3 +78,198 @@ func TestGeneratePodTemplate(t *testing.T) { t.Fatalf("pod does not match expected:\ngot: %+v\nwant: %+v", pod, expectedPod) } } + +var expectedStaticPodYAML = ` +apiVersion: v1 +kind: Pod +metadata: + name: kms-preflight + namespace: test-ns + labels: + app: openshift-kms-preflight + annotations: + encryption.apiserver.operator.openshift.io/kms-preflight-config-hash: abc123 +spec: + restartPolicy: Never + priorityClassName: system-cluster-critical + nodeSelector: + node-role.kubernetes.io/master: "" + tolerations: + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoExecute + hostNetwork: true + containers: + - name: kms-preflight-check + image: quay.io/openshift/operator:latest + command: ["operator","kms-preflight"] + args: + - --kms-call-timeout=10s + - --config-hash=$(CONFIG_HASH) + - --pod-name=$(POD_NAME) + - --pod-namespace=$(POD_NAMESPACE) + - --kubeconfig=/etc/kubernetes/static-pod-resources/secrets/kms-preflight-encryption-config/lb-int.kubeconfig + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: CONFIG_HASH + value: abc123 + resources: + requests: + memory: 50Mi + cpu: 5m + securityContext: + runAsUser: 0 + volumeMounts: + - name: resource-dir + mountPath: /etc/kubernetes/static-pod-resources + readOnly: true + volumes: + - name: resource-dir + hostPath: + path: /etc/kubernetes/manifests +` + +func TestGenerateStaticPodTemplate(t *testing.T) { + pod, err := generateStaticPodTemplate( + preflightPodName, + "test-ns", + "abc123", + "quay.io/openshift/operator:latest", + []string{"operator", "kms-preflight"}, + 10*time.Second, + ) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + expectedPod, err := resourceread.ReadPodV1([]byte(expectedStaticPodYAML)) + if err != nil { + t.Fatalf("failed to parse expected pod YAML: %v", err) + } + if !equality.Semantic.DeepEqual(pod, expectedPod) { + t.Fatalf("pod does not match expected:\ngot: %+v\nwant: %+v", pod, expectedPod) + } +} + +var expectedInstallerPodYAML = ` +apiVersion: v1 +kind: Pod +metadata: + name: kms-preflight-installer + namespace: test-ns + labels: + app: openshift-kms-preflight-installer +spec: + automountServiceAccountToken: false + nodeSelector: + node-role.kubernetes.io/master: "" + tolerations: + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoExecute + initContainers: + - name: cleanup + image: quay.io/openshift/operator:latest + imagePullPolicy: IfNotPresent + terminationMessagePolicy: FallbackToLogsOnError + command: + - /bin/sh + - -c + - | + #!/bin/sh + set -euo pipefail + + rm -f /etc/kubernetes/manifests/kms-preflight-pod.yaml + rm -rf /etc/kubernetes/manifests/secrets/kms-preflight-encryption-config + securityContext: + privileged: true + runAsUser: 0 + resources: + requests: + memory: 10Mi + cpu: 10m + volumeMounts: + - mountPath: /etc/kubernetes/manifests + name: manifests-dir + containers: + - name: installer + image: quay.io/openshift/operator:latest + imagePullPolicy: IfNotPresent + terminationMessagePolicy: FallbackToLogsOnError + command: + - /bin/sh + - -c + - | + #!/bin/sh + set -euo pipefail + + target_secret_dir="/etc/kubernetes/manifests/secrets/kms-preflight-encryption-config" + mkdir -p "${target_secret_dir}" + cp -f /install/secret/* "${target_secret_dir}/" + cp -f /install/pod/lb-int.kubeconfig "${target_secret_dir}/lb-int.kubeconfig" + + manifest_path="/etc/kubernetes/manifests/kms-preflight-pod.yaml" + mkdir -p /etc/kubernetes/manifests + cp -f /install/pod/pod.yaml "${manifest_path}" + chmod 0600 "${manifest_path}" + securityContext: + privileged: true + runAsUser: 0 + resources: + requests: + memory: 10Mi + cpu: 10m + volumeMounts: + - mountPath: /etc/kubernetes/manifests + name: manifests-dir + - mountPath: /install/pod + name: pod-manifest + - mountPath: /install/secret + name: encryption-config + readOnly: true + restartPolicy: Never + priorityClassName: system-node-critical + securityContext: + runAsUser: 0 + volumes: + - hostPath: + path: /etc/kubernetes/manifests + name: manifests-dir + - name: pod-manifest + configMap: + name: kms-preflight-pod + - name: encryption-config + secret: + secretName: kms-preflight-encryption-config +` + +func TestGenerateInstallerPodTemplate(t *testing.T) { + pod, err := generateInstallerPodTemplate( + "test-ns", + "quay.io/openshift/operator:latest", + ) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + expectedPod, err := resourceread.ReadPodV1([]byte(expectedInstallerPodYAML)) + if err != nil { + t.Fatalf("failed to parse expected pod YAML: %v", err) + } + if !equality.Semantic.DeepEqual(pod, expectedPod) { + t.Fatalf("pod does not match expected:\ngot: %+v\nwant: %+v", pod, expectedPod) + } +} diff --git a/pkg/operator/encryption/kms/preflight/static_pod_deployer.go b/pkg/operator/encryption/kms/preflight/static_pod_deployer.go new file mode 100644 index 0000000000..73bb261334 --- /dev/null +++ b/pkg/operator/encryption/kms/preflight/static_pod_deployer.go @@ -0,0 +1,207 @@ +package preflight + +import ( + "context" + "errors" + "fmt" + "strings" + "time" + + "github.com/openshift/library-go/pkg/operator/encryption/kms/pluginlifecycle" + "github.com/openshift/library-go/pkg/operator/resource/resourceread" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k8slabels "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/util/uuid" + corev1client "k8s.io/client-go/kubernetes/typed/core/v1" +) + +const ( + nodeKubeconfigsSecretName = "node-kubeconfigs" + nodeKubeconfigSecretKey = "lb-int.kubeconfig" +) + +// StaticPodPreflightDeployer deploys the KMS preflight check as a static pod on a +// control plane node. An installer pod writes the rendered static pod manifest to the +// host manifests directory alongside the static pod manifest. +type StaticPodPreflightDeployer struct { + namespace string + + coreClient corev1client.CoreV1Interface + + operatorImage string + operatorCommand []string + kmsCallTimeout time.Duration +} + +func (d *StaticPodPreflightDeployer) Deploy(ctx context.Context, configHash string, encryptionConfigSecret *corev1.Secret) error { + if configHash == "" { + return fmt.Errorf("configHash is empty") + } + if encryptionConfigSecret == nil { + return fmt.Errorf("encryptionConfigSecret is nil") + } + + if err := d.Cleanup(ctx); err != nil { + return fmt.Errorf("failed to clean up existing preflight resources: %w", err) + } + + encryptionConfigSecret = encryptionConfigSecret.DeepCopy() + encryptionConfigSecret.ObjectMeta = metav1.ObjectMeta{ + Namespace: d.namespace, + Name: preflightEncryptionConfigSecretName, + Labels: labels, + } + if _, err := d.coreClient.Secrets(d.namespace).Create(ctx, encryptionConfigSecret, metav1.CreateOptions{}); err != nil { + return fmt.Errorf("failed to create preflight encryption config secret: %w", err) + } + + pod, err := generateStaticPodTemplate( + preflightPodName, + d.namespace, + configHash, + d.operatorImage, + d.operatorCommand, + d.kmsCallTimeout, + ) + if err != nil { + return fmt.Errorf("failed to generate preflight pod template: %w", err) + } + + err = pluginlifecycle.NewKMSPluginBuilder(). + WithSecretRequired(). + FromEncryptionConfigSecret(d.namespace, preflightEncryptionConfigSecretName, d.coreClient). + AsStaticPod(). + Apply(ctx, &pod.Spec, preflightCheckContainerName) + if err != nil { + return fmt.Errorf("failed to apply preflight plugin: %w", err) + } + + // The kubelet uses the static pod UID to detect manifest changes and terminate prior instances. + // Generate a fresh UID on every deploy so kubelet recreates the pod even when name and filename are unchanged. + pod.UID = uuid.NewUUID() + + podManifest := resourceread.WritePodV1OrDie(pod) + kubeconfig, err := d.nodeKubeconfigData(ctx) + if err != nil { + return fmt.Errorf("failed to load node kubeconfig: %w", err) + } + if _, err = d.coreClient.ConfigMaps(d.namespace).Create(ctx, &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: d.namespace, + Name: preflightPodConfigMapPrefix, + Labels: labels, + }, + Data: map[string]string{ + "pod.yaml": podManifest, + nodeKubeconfigSecretKey: string(kubeconfig), + }, + }, metav1.CreateOptions{}); err != nil { + return fmt.Errorf("failed to create preflight pod configmap: %w", err) + } + + installerPod, err := generateInstallerPodTemplate(d.namespace, d.operatorImage) + if err != nil { + return fmt.Errorf("failed to generate preflight installer pod: %w", err) + } + + if _, err = d.coreClient.Pods(d.namespace).Create(ctx, installerPod, metav1.CreateOptions{}); err != nil { + return fmt.Errorf("failed to create preflight installer pod: %w", err) + } + + return nil +} + +func (d *StaticPodPreflightDeployer) Status(ctx context.Context) (corev1.PodStatus, error) { + pod, err := d.findMirrorPod(ctx) + if err != nil { + return corev1.PodStatus{}, fmt.Errorf("failed to get mirror pod for preflight %s/%s: %w", d.namespace, preflightPodName, err) + } + + return pod.Status, nil +} + +func (d *StaticPodPreflightDeployer) Cleanup(ctx context.Context) error { + var errs []error + + installerPodName := preflightInstallerPodName + err := d.coreClient.Pods(d.namespace).Delete(ctx, installerPodName, metav1.DeleteOptions{}) + if err != nil && !apierrors.IsNotFound(err) { + errs = append(errs, fmt.Errorf("failed to delete installer pod %s/%s: %w", d.namespace, installerPodName, err)) + } + + err = d.coreClient.Secrets(d.namespace).Delete(ctx, preflightEncryptionConfigSecretName, metav1.DeleteOptions{}) + if err != nil && !apierrors.IsNotFound(err) { + errs = append(errs, fmt.Errorf("failed to delete secret %s/%s: %w", d.namespace, preflightEncryptionConfigSecretName, err)) + } + + err = d.coreClient.ConfigMaps(d.namespace).Delete(ctx, preflightPodConfigMapPrefix, metav1.DeleteOptions{}) + if err != nil && !apierrors.IsNotFound(err) { + errs = append(errs, fmt.Errorf("failed to delete configmap %s/%s: %w", d.namespace, preflightPodConfigMapPrefix, err)) + } + + return errors.Join(errs...) +} + +func NewStaticPodPreflightDeployer( + namespace string, + coreClient corev1client.CoreV1Interface, + operatorImage string, + operatorCommand []string, + kmsCallTimeout time.Duration, +) *StaticPodPreflightDeployer { + return &StaticPodPreflightDeployer{ + namespace: namespace, + coreClient: coreClient, + operatorImage: operatorImage, + operatorCommand: operatorCommand, + kmsCallTimeout: kmsCallTimeout, + } +} + +func (d *StaticPodPreflightDeployer) findMirrorPod(ctx context.Context) (*corev1.Pod, error) { + pods, err := d.coreClient.Pods(d.namespace).List(ctx, metav1.ListOptions{ + LabelSelector: k8slabels.Set(labels).String(), + }) + if err != nil { + return nil, err + } + + mirrorPrefix := preflightPodName + "-" + var mirrorPods []corev1.Pod + for _, pod := range pods.Items { + if strings.HasPrefix(pod.Name, mirrorPrefix) { + mirrorPods = append(mirrorPods, pod) + } + } + + switch len(mirrorPods) { + case 0: + return nil, apierrors.NewNotFound(corev1.Resource("pods"), mirrorPrefix+"*") + case 1: + return &mirrorPods[0], nil + default: + return nil, fmt.Errorf("expected one preflight mirror pod, found %d", len(mirrorPods)) + } +} + +func mirrorPodName(nodeName string) string { + return fmt.Sprintf("%s-%s", preflightPodName, nodeName) +} + +func (d *StaticPodPreflightDeployer) nodeKubeconfigData(ctx context.Context) ([]byte, error) { + secret, err := d.coreClient.Secrets(d.namespace).Get(ctx, nodeKubeconfigsSecretName, metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + return mustAsset("assets/lb-int.kubeconfig"), nil + } + return nil, fmt.Errorf("failed to get %s/%s secret: %w", d.namespace, nodeKubeconfigsSecretName, err) + } + + kubeconfig, ok := secret.Data[nodeKubeconfigSecretKey] + if !ok || len(kubeconfig) == 0 { + return nil, fmt.Errorf("secret %s/%s is missing %q data", d.namespace, nodeKubeconfigsSecretName, nodeKubeconfigSecretKey) + } + return kubeconfig, nil +} diff --git a/pkg/operator/encryption/kms/preflight/static_pod_deployer_test.go b/pkg/operator/encryption/kms/preflight/static_pod_deployer_test.go new file mode 100644 index 0000000000..508e289864 --- /dev/null +++ b/pkg/operator/encryption/kms/preflight/static_pod_deployer_test.go @@ -0,0 +1,263 @@ +package preflight + +import ( + "context" + "strings" + "testing" + + "github.com/openshift/library-go/pkg/operator/resource/resourceread" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes/fake" + clienttesting "k8s.io/client-go/testing" +) + +const ( + testNodeName = "master-1" + testMirrorPodName = preflightPodName + "-" + testNodeName +) + +func newTestStaticPodDeployer(t *testing.T, objects ...runtime.Object) (*StaticPodPreflightDeployer, *fake.Clientset) { + t.Helper() + + allObjects := append(testReferenceDataObjects(t), objects...) + kubeClient := fake.NewSimpleClientset(allObjects...) + deployer := NewStaticPodPreflightDeployer( + testNamespace, + kubeClient.CoreV1(), + testOperatorImage, + testOperatorCommand, + testDeployerTimeout, + ) + return deployer, kubeClient +} + +func TestStaticPodPreflightDeployer_Deploy(t *testing.T) { + ctx := context.Background() + deployer, kubeClient := newTestStaticPodDeployer(t) + encryptionConfigSecret := testPreflightEncryptionConfigSecret(t) + + if err := deployer.Deploy(ctx, testConfigHash, encryptionConfigSecret); err != nil { + t.Fatalf("Deploy() error = %v", err) + } + + actions := kubeClient.Actions() + if len(actions) != 8 { + t.Fatalf("expected 8 actions, got %d: %#v", len(actions), actions) + } + + if !actions[0].Matches("delete", "pods") { + t.Fatalf("unexpected action: %#v", actions[0]) + } + if !actions[1].Matches("delete", "secrets") { + t.Fatalf("unexpected action: %#v", actions[1]) + } + if !actions[2].Matches("delete", "configmaps") { + t.Fatalf("unexpected action: %#v", actions[2]) + } + if !actions[3].Matches("create", "secrets") { + t.Fatalf("unexpected action: %#v", actions[3]) + } + secretCreateAction, ok := actions[3].(clienttesting.CreateAction) + if !ok { + t.Fatalf("expected CreateAction, got %T", actions[3]) + } + createdSecret, ok := secretCreateAction.GetObject().(*corev1.Secret) + if !ok { + t.Fatalf("expected *corev1.Secret, got %T", secretCreateAction.GetObject()) + } + if createdSecret.Name != preflightEncryptionConfigSecretName { + t.Fatalf("expected secret name %q, got %q", preflightEncryptionConfigSecretName, createdSecret.Name) + } + + if !actions[4].Matches("get", "secrets") { + t.Fatalf("unexpected action: %#v", actions[4]) + } + if !actions[5].Matches("get", "secrets") { + t.Fatalf("unexpected action: %#v", actions[5]) + } + if !actions[6].Matches("create", "configmaps") { + t.Fatalf("unexpected action: %#v", actions[6]) + } + configMapCreateAction, ok := actions[6].(clienttesting.CreateAction) + if !ok { + t.Fatalf("expected CreateAction, got %T", actions[6]) + } + createdConfigMap, ok := configMapCreateAction.GetObject().(*corev1.ConfigMap) + if !ok { + t.Fatalf("expected *corev1.ConfigMap, got %T", configMapCreateAction.GetObject()) + } + if createdConfigMap.Name != preflightPodConfigMapPrefix { + t.Fatalf("expected configmap name %q, got %q", preflightPodConfigMapPrefix, createdConfigMap.Name) + } + if _, ok := createdConfigMap.Data["pod.yaml"]; !ok { + t.Fatalf("expected pod.yaml key in configmap data") + } + if len(createdConfigMap.Data[nodeKubeconfigSecretKey]) == 0 { + t.Fatalf("expected %q key in configmap data", nodeKubeconfigSecretKey) + } + writtenPod, err := resourceread.ReadPodV1([]byte(createdConfigMap.Data["pod.yaml"])) + if err != nil { + t.Fatalf("failed to parse pod.yaml from configmap: %v", err) + } + if writtenPod.UID == "" { + t.Fatal("expected static pod manifest to include a UID") + } + + if !actions[7].Matches("create", "pods") { + t.Fatalf("unexpected action: %#v", actions[7]) + } + createAction, ok := actions[7].(clienttesting.CreateAction) + if !ok { + t.Fatalf("expected CreateAction, got %T", actions[7]) + } + if createAction.GetNamespace() != testNamespace { + t.Fatalf("unexpected namespace %q", createAction.GetNamespace()) + } +} + +func TestStaticPodPreflightDeployer_Status(t *testing.T) { + scenarios := []struct { + name string + objects []runtime.Object + expectPhase corev1.PodPhase + expectErr string + }{ + { + name: "missing mirror pod returns error", + expectErr: "failed to get mirror pod", + }, + { + name: "mirror pod returns pod status", + objects: []runtime.Object{ + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: testMirrorPodName, + Namespace: testNamespace, + Labels: labels, + }, + Status: corev1.PodStatus{Phase: corev1.PodRunning}, + }, + }, + expectPhase: corev1.PodRunning, + }, + } + + for _, scenario := range scenarios { + t.Run(scenario.name, func(t *testing.T) { + deployer, _ := newTestStaticPodDeployer(t, scenario.objects...) + + status, err := deployer.Status(context.Background()) + if scenario.expectErr != "" { + if err == nil || !strings.Contains(err.Error(), scenario.expectErr) { + t.Fatalf("expected error containing %q, got %v", scenario.expectErr, err) + } + return + } + if err != nil { + t.Fatalf("Status() error = %v", err) + } + if status.Phase != scenario.expectPhase { + t.Fatalf("expected phase %q, got %q", scenario.expectPhase, status.Phase) + } + }) + } +} + +func TestStaticPodPreflightDeployer_Cleanup(t *testing.T) { + existingInstallerPod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: preflightInstallerPodName, + Namespace: testNamespace, + }, + } + existingSecret := testPreflightEncryptionConfigSecret(t) + existingSecret.Labels = labels + existingConfigMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: preflightPodConfigMapPrefix, + Namespace: testNamespace, + Labels: labels, + }, + } + + deployer, kubeClient := newTestStaticPodDeployer(t, existingInstallerPod, existingSecret, existingConfigMap) + if err := deployer.Cleanup(context.Background()); err != nil { + t.Fatalf("Cleanup() error = %v", err) + } + + actions := kubeClient.Actions() + if len(actions) != 3 { + t.Fatalf("expected 3 actions, got %d: %#v", len(actions), actions) + } + if !actions[0].Matches("delete", "pods") { + t.Fatalf("unexpected action: %#v", actions[0]) + } + if !actions[1].Matches("delete", "secrets") { + t.Fatalf("unexpected action: %#v", actions[1]) + } + if !actions[2].Matches("delete", "configmaps") { + t.Fatalf("unexpected action: %#v", actions[2]) + } +} + +func TestStaticPodPreflightDeployer_Deploy_installerCreateFailure(t *testing.T) { + deployer, kubeClient := newTestStaticPodDeployer(t) + kubeClient.PrependReactor("create", "pods", func(action clienttesting.Action) (bool, runtime.Object, error) { + return true, nil, apierrors.NewForbidden(corev1.Resource("pods"), preflightInstallerPodName, nil) + }) + + err := deployer.Deploy(context.Background(), testConfigHash, testPreflightEncryptionConfigSecret(t)) + if err == nil || !strings.Contains(err.Error(), "failed to create preflight installer pod") { + t.Fatalf("expected installer pod create error, got %v", err) + } +} + +func TestMirrorPodName(t *testing.T) { + if got := mirrorPodName("node-a"); got != "kms-preflight-node-a" { + t.Fatalf("unexpected mirror pod name %q", got) + } +} + +func TestStaticPodPreflightDeployer_nodeKubeconfigData_fromCluster(t *testing.T) { + const kubeconfig = "apiVersion: v1\nkind: Config\n" + kubeClient := fake.NewSimpleClientset(&corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: nodeKubeconfigsSecretName, + Namespace: testNamespace, + }, + Type: corev1.SecretTypeOpaque, + Data: map[string][]byte{ + nodeKubeconfigSecretKey: []byte(kubeconfig), + }, + }) + deployer := NewStaticPodPreflightDeployer( + testNamespace, + kubeClient.CoreV1(), + testOperatorImage, + testOperatorCommand, + testDeployerTimeout, + ) + + got, err := deployer.nodeKubeconfigData(context.Background()) + if err != nil { + t.Fatalf("nodeKubeconfigData() error = %v", err) + } + if string(got) != kubeconfig { + t.Fatalf("expected cluster kubeconfig, got %q", got) + } +} + +func TestStaticPodPreflightDeployer_nodeKubeconfigData_embeddedFallback(t *testing.T) { + deployer, _ := newTestStaticPodDeployer(t) + + got, err := deployer.nodeKubeconfigData(context.Background()) + if err != nil { + t.Fatalf("nodeKubeconfigData() error = %v", err) + } + if len(got) == 0 { + t.Fatal("expected embedded kubeconfig fallback") + } +}