Adding KAS-O helpers and assertions

test/library/encryption/assertion_kas.go

@@ -0,0 +1,66 @@
+// Copied from KAS-O test/library/encryption/assertion.go:
+// https://github.com/openshift/cluster-kube-apiserver-operator/blob/main/test/library/encryption/assertion.go
+package encryption
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+
+	configv1 "github.com/openshift/api/config/v1"
+)
+
+var DefaultTargetGRs = []schema.GroupResource{
+	{Group: "", Resource: "secrets"},
+	{Group: "", Resource: "configmaps"},
+}
+
+func AssertSecretOfLifeEncrypted(t testing.TB, clientSet ClientSet, resource runtime.Object) {
+	t.Helper()
+	secret, ok := resource.(*corev1.Secret)
+	if !ok {
+		t.Fatalf("expected *corev1.Secret, got %T", resource)
+	}
+	rawValue := GetRawSecretOfLife(t, clientSet, secret.Namespace)
+	if strings.Contains(rawValue, string(secret.Data["quote"])) {
+		t.Errorf("secret not encrypted, etcd value contains quote in plain text")
+	}
+}
+
+func AssertSecretOfLifeNotEncrypted(t testing.TB, clientSet ClientSet, resource runtime.Object) {
+	t.Helper()
+	secret, ok := resource.(*corev1.Secret)
+	if !ok {
+		t.Fatalf("expected *corev1.Secret, got %T", resource)
+	}
+	rawValue := GetRawSecretOfLife(t, clientSet, secret.Namespace)
+	if !strings.Contains(rawValue, string(secret.Data["quote"])) {
+		t.Errorf("secret not decrypted, etcd value does not contain quote in plain text")
+	}
+}
+
+func AssertSecretsAndConfigMaps(t testing.TB, clientSet ClientSet, expectedMode configv1.EncryptionType, namespace, labelSelector string) {
+	t.Helper()
+	assertSecrets(t, clientSet.Etcd, string(expectedMode))
+	assertConfigMaps(t, clientSet.Etcd, string(expectedMode))
+	AssertLastMigratedKey(t, clientSet.Kube, DefaultTargetGRs, namespace, labelSelector)
+}
+
+func assertSecrets(t testing.TB, etcdClient EtcdClient, expectedMode string) {
+	t.Logf("Checking if all Secrets where encrypted/decrypted for %q mode", expectedMode)
+	totalSecrets, err := VerifyResources(t, etcdClient, "/kubernetes.io/secrets/", expectedMode, false)
+	t.Logf("Verified %d Secrets", totalSecrets)
+	require.NoError(t, err)
+}
+
+func assertConfigMaps(t testing.TB, etcdClient EtcdClient, expectedMode string) {
+	t.Logf("Checking if all ConfigMaps where encrypted/decrypted for %q mode", expectedMode)
+	totalConfigMaps, err := VerifyResources(t, etcdClient, "/kubernetes.io/configmaps/", expectedMode, false)
+	t.Logf("Verified %d ConfigMaps", totalConfigMaps)
+	require.NoError(t, err)
+}

test/library/encryption/helpers.go

@@ -7,6 +7,7 @@ import (
 	"sort"
 	"strconv"
 	"strings"
+	"sync"
 	"testing"
 	"time"
 
@@ -85,30 +86,47 @@ func WaitForNextEncryptionKeyRotation() WaitForRotationCompleteFunc {
 	}
 }
 
-func SetAndWaitForEncryptionType(ctx context.Context, t testing.TB, provider EncryptionProvider, defaultTargetGRs []schema.GroupResource, namespace, labelSelector string) ClientSet {
+// ApplyEncryptionProviderIfNeeded runs provider.Setup and updates APIServer/cluster when
+// the desired encryption config differs from the current cluster config.
+func ApplyEncryptionProviderIfNeeded(ctx context.Context, t testing.TB, provider EncryptionProvider) (configv1.APIServerEncryption, bool) {
 	t.Helper()
 
-	t.Logf("Starting encryption e2e test for %q mode", provider.Type)
-
 	clientSet := GetClients(t)
-	lastMigratedKeyMeta, err := GetLastKeyMeta(t, clientSet.Kube, namespace, labelSelector)
-	require.NoError(t, err)
-
-	apiServer, err := clientSet.ApiServerConfig.Get(ctx, "cluster", metav1.GetOptions{})
-	require.NoError(t, err)
-	previousEncryption := apiServer.Spec.Encryption
-	needsUpdate := !equality.Semantic.DeepEqual(previousEncryption, provider.APIServerEncryption)
-	if needsUpdate {
+	var previousEncryption configv1.APIServerEncryption
+	var needsUpdate bool
+	err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
+		apiServer, err := clientSet.ApiServerConfig.Get(ctx, "cluster", metav1.GetOptions{})
+		if err != nil {
+			return err
+		}
+		previousEncryption = apiServer.Spec.Encryption
+		needsUpdate = !equality.Semantic.DeepEqual(previousEncryption, provider.APIServerEncryption)
+		if !needsUpdate {
+			t.Logf("APIServer is already configured to use %q mode", provider.Type)
+			return nil
+		}
 		if provider.Setup != nil {
 			provider.Setup(ctx, t)
 		}
 		t.Logf("Updating encryption configuration for APIServer from %#v to %#v", previousEncryption, provider.APIServerEncryption)
 		apiServer.Spec.Encryption = provider.APIServerEncryption
 		_, err = clientSet.ApiServerConfig.Update(ctx, apiServer, metav1.UpdateOptions{})
-		require.NoError(t, err)
-	} else {
-		t.Logf("APIServer is already configured to use %q mode", provider.Type)
-	}
+		return err
+	})
+	require.NoError(t, err)
+	return previousEncryption, needsUpdate
+}
+
+func SetAndWaitForEncryptionType(ctx context.Context, t testing.TB, provider EncryptionProvider, defaultTargetGRs []schema.GroupResource, namespace, labelSelector string) ClientSet {
+	t.Helper()
+
+	t.Logf("Starting encryption e2e test for %q mode", provider.Type)
+
+	clientSet := GetClients(t)
+	lastMigratedKeyMeta, err := GetLastKeyMeta(t, clientSet.Kube, namespace, labelSelector)
+	require.NoError(t, err)
+
+	previousEncryption, needsUpdate := ApplyEncryptionProviderIfNeeded(ctx, t, provider)
 
 	// KMS-to-KMS migration: when both old and new are KMS but the config differs,
 	// the key controller creates a new key. We must wait for the next migrated key
@@ -475,3 +493,35 @@ func WaitForCurrentKeyMigrated(t testing.TB, kubeClient kubernetes.Interface, pr
 		require.NoError(t, newErr)
 	}
 }
+
+// inParallel returns a single testStep that runs the given steps
+// concurrently and waits for all to finish before returning.
+// Panics are caught and reported via t.Errorf. Failures from
+// t.FailNow/require (runtime.Goexit) are handled naturally since
+// the testing framework already records the error on t.
+func inParallel(steps ...testStep) testStep {
+	if len(steps) == 1 {
+		return steps[0]
+	}
+	names := make([]string, len(steps))
+	for i, s := range steps {
+		names[i] = s.name
+	}
+	return testStep{
+		name: strings.Join(names, " | "),
+		testFunc: func(t testing.TB) {
+			var wg sync.WaitGroup
+			for _, s := range steps {
+				wg.Go(func() {
+					defer func() {
+						if r := recover(); r != nil {
+							t.Errorf("step %q panicked: %v", s.name, r)
+						}
+					}()
+					s.testFunc(t)
+				})
+			}
+			wg.Wait()
+		},
+	}
+}

test/library/encryption/helpers_kas.go

@@ -0,0 +1,64 @@
+// Copied from KAS-O test/library/encryption/helpers.go:
+// https://github.com/openshift/cluster-kube-apiserver-operator/blob/main/test/library/encryption/helpers.go
+package encryption
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+)
+
+const secretOfLifeName = "secret-of-life"
+
+func CreateAndStoreSecretOfLife(t testing.TB, clientSet ClientSet, namespace string) runtime.Object {
+	t.Helper()
+	ctx := context.TODO()
+
+	oldSecret, err := clientSet.Kube.CoreV1().Secrets(namespace).Get(ctx, secretOfLifeName, metav1.GetOptions{})
+	if err != nil && !errors.IsNotFound(err) {
+		t.Fatalf("Failed to check if the secret already exists: %v", err)
+	}
+	if oldSecret != nil && len(oldSecret.Name) > 0 {
+		t.Log("The secret already exists, removing it first")
+		require.NoError(t, clientSet.Kube.CoreV1().Secrets(namespace).Delete(ctx, oldSecret.Name, metav1.DeleteOptions{}))
+	}
+
+	t.Logf("Creating %q in %s namespace", secretOfLifeName, namespace)
+	rawSecret := SecretOfLife(t, namespace)
+	secret, err := clientSet.Kube.CoreV1().Secrets(namespace).Create(ctx, rawSecret.(*corev1.Secret), metav1.CreateOptions{})
+	require.NoError(t, err)
+	return secret
+}
+
+func SecretOfLife(_ testing.TB, namespace string) runtime.Object {
+	return &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      secretOfLifeName,
+			Namespace: namespace,
+		},
+		Data: map[string][]byte{
+			"quote": []byte("I have no special talents. I am only passionately curious"),
+		},
+	}
+}
+
+func GetRawSecretOfLife(t testing.TB, clientSet ClientSet, namespace string) string {
+	t.Helper()
+	timeout, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
+	defer cancel()
+
+	secretOfLifeKey := fmt.Sprintf("/kubernetes.io/secrets/%s/%s", namespace, secretOfLifeName)
+	resp, err := clientSet.Etcd.Get(timeout, secretOfLifeKey)
+	require.NoError(t, err)
+	require.Len(t, resp.Kvs, 1, "expected exactly one key from etcd for secret-of-life")
+
+	return string(resp.Kvs[0].Value)
+}