diff --git a/test/library/encryption/assertion_kas.go b/test/library/encryption/assertion_kas.go new file mode 100644 index 0000000000..a7a73150c4 --- /dev/null +++ b/test/library/encryption/assertion_kas.go @@ -0,0 +1,66 @@ +// Copied from KAS-O test/library/encryption/assertion.go: +// https://github.com/openshift/cluster-kube-apiserver-operator/blob/main/test/library/encryption/assertion.go +package encryption + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/require" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + + configv1 "github.com/openshift/api/config/v1" +) + +var DefaultTargetGRs = []schema.GroupResource{ + {Group: "", Resource: "secrets"}, + {Group: "", Resource: "configmaps"}, +} + +func AssertSecretOfLifeEncrypted(t testing.TB, clientSet ClientSet, resource runtime.Object) { + t.Helper() + secret, ok := resource.(*corev1.Secret) + if !ok { + t.Fatalf("expected *corev1.Secret, got %T", resource) + } + rawValue := GetRawSecretOfLife(t, clientSet, secret.Namespace) + if strings.Contains(rawValue, string(secret.Data["quote"])) { + t.Errorf("secret not encrypted, etcd value contains quote in plain text") + } +} + +func AssertSecretOfLifeNotEncrypted(t testing.TB, clientSet ClientSet, resource runtime.Object) { + t.Helper() + secret, ok := resource.(*corev1.Secret) + if !ok { + t.Fatalf("expected *corev1.Secret, got %T", resource) + } + rawValue := GetRawSecretOfLife(t, clientSet, secret.Namespace) + if !strings.Contains(rawValue, string(secret.Data["quote"])) { + t.Errorf("secret not decrypted, etcd value does not contain quote in plain text") + } +} + +func AssertSecretsAndConfigMaps(t testing.TB, clientSet ClientSet, expectedMode configv1.EncryptionType, namespace, labelSelector string) { + t.Helper() + assertSecrets(t, clientSet.Etcd, string(expectedMode)) + assertConfigMaps(t, clientSet.Etcd, string(expectedMode)) + AssertLastMigratedKey(t, clientSet.Kube, DefaultTargetGRs, namespace, labelSelector) +} + +func assertSecrets(t testing.TB, etcdClient EtcdClient, expectedMode string) { + t.Logf("Checking if all Secrets where encrypted/decrypted for %q mode", expectedMode) + totalSecrets, err := VerifyResources(t, etcdClient, "/kubernetes.io/secrets/", expectedMode, false) + t.Logf("Verified %d Secrets", totalSecrets) + require.NoError(t, err) +} + +func assertConfigMaps(t testing.TB, etcdClient EtcdClient, expectedMode string) { + t.Logf("Checking if all ConfigMaps where encrypted/decrypted for %q mode", expectedMode) + totalConfigMaps, err := VerifyResources(t, etcdClient, "/kubernetes.io/configmaps/", expectedMode, false) + t.Logf("Verified %d ConfigMaps", totalConfigMaps) + require.NoError(t, err) +} diff --git a/test/library/encryption/helpers.go b/test/library/encryption/helpers.go index e0f836bee2..ab1b79db75 100644 --- a/test/library/encryption/helpers.go +++ b/test/library/encryption/helpers.go @@ -7,6 +7,7 @@ import ( "sort" "strconv" "strings" + "sync" "testing" "time" @@ -85,30 +86,47 @@ func WaitForNextEncryptionKeyRotation() WaitForRotationCompleteFunc { } } -func SetAndWaitForEncryptionType(ctx context.Context, t testing.TB, provider EncryptionProvider, defaultTargetGRs []schema.GroupResource, namespace, labelSelector string) ClientSet { +// ApplyEncryptionProviderIfNeeded runs provider.Setup and updates APIServer/cluster when +// the desired encryption config differs from the current cluster config. +func ApplyEncryptionProviderIfNeeded(ctx context.Context, t testing.TB, provider EncryptionProvider) (configv1.APIServerEncryption, bool) { t.Helper() - t.Logf("Starting encryption e2e test for %q mode", provider.Type) - clientSet := GetClients(t) - lastMigratedKeyMeta, err := GetLastKeyMeta(t, clientSet.Kube, namespace, labelSelector) - require.NoError(t, err) - - apiServer, err := clientSet.ApiServerConfig.Get(ctx, "cluster", metav1.GetOptions{}) - require.NoError(t, err) - previousEncryption := apiServer.Spec.Encryption - needsUpdate := !equality.Semantic.DeepEqual(previousEncryption, provider.APIServerEncryption) - if needsUpdate { + var previousEncryption configv1.APIServerEncryption + var needsUpdate bool + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + apiServer, err := clientSet.ApiServerConfig.Get(ctx, "cluster", metav1.GetOptions{}) + if err != nil { + return err + } + previousEncryption = apiServer.Spec.Encryption + needsUpdate = !equality.Semantic.DeepEqual(previousEncryption, provider.APIServerEncryption) + if !needsUpdate { + t.Logf("APIServer is already configured to use %q mode", provider.Type) + return nil + } if provider.Setup != nil { provider.Setup(ctx, t) } t.Logf("Updating encryption configuration for APIServer from %#v to %#v", previousEncryption, provider.APIServerEncryption) apiServer.Spec.Encryption = provider.APIServerEncryption _, err = clientSet.ApiServerConfig.Update(ctx, apiServer, metav1.UpdateOptions{}) - require.NoError(t, err) - } else { - t.Logf("APIServer is already configured to use %q mode", provider.Type) - } + return err + }) + require.NoError(t, err) + return previousEncryption, needsUpdate +} + +func SetAndWaitForEncryptionType(ctx context.Context, t testing.TB, provider EncryptionProvider, defaultTargetGRs []schema.GroupResource, namespace, labelSelector string) ClientSet { + t.Helper() + + t.Logf("Starting encryption e2e test for %q mode", provider.Type) + + clientSet := GetClients(t) + lastMigratedKeyMeta, err := GetLastKeyMeta(t, clientSet.Kube, namespace, labelSelector) + require.NoError(t, err) + + previousEncryption, needsUpdate := ApplyEncryptionProviderIfNeeded(ctx, t, provider) // KMS-to-KMS migration: when both old and new are KMS but the config differs, // the key controller creates a new key. We must wait for the next migrated key @@ -475,3 +493,35 @@ func WaitForCurrentKeyMigrated(t testing.TB, kubeClient kubernetes.Interface, pr require.NoError(t, newErr) } } + +// inParallel returns a single testStep that runs the given steps +// concurrently and waits for all to finish before returning. +// Panics are caught and reported via t.Errorf. Failures from +// t.FailNow/require (runtime.Goexit) are handled naturally since +// the testing framework already records the error on t. +func inParallel(steps ...testStep) testStep { + if len(steps) == 1 { + return steps[0] + } + names := make([]string, len(steps)) + for i, s := range steps { + names[i] = s.name + } + return testStep{ + name: strings.Join(names, " | "), + testFunc: func(t testing.TB) { + var wg sync.WaitGroup + for _, s := range steps { + wg.Go(func() { + defer func() { + if r := recover(); r != nil { + t.Errorf("step %q panicked: %v", s.name, r) + } + }() + s.testFunc(t) + }) + } + wg.Wait() + }, + } +} diff --git a/test/library/encryption/helpers_kas.go b/test/library/encryption/helpers_kas.go new file mode 100644 index 0000000000..28f9f97c74 --- /dev/null +++ b/test/library/encryption/helpers_kas.go @@ -0,0 +1,64 @@ +// Copied from KAS-O test/library/encryption/helpers.go: +// https://github.com/openshift/cluster-kube-apiserver-operator/blob/main/test/library/encryption/helpers.go +package encryption + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/require" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" +) + +const secretOfLifeName = "secret-of-life" + +func CreateAndStoreSecretOfLife(t testing.TB, clientSet ClientSet, namespace string) runtime.Object { + t.Helper() + ctx := context.TODO() + + oldSecret, err := clientSet.Kube.CoreV1().Secrets(namespace).Get(ctx, secretOfLifeName, metav1.GetOptions{}) + if err != nil && !errors.IsNotFound(err) { + t.Fatalf("Failed to check if the secret already exists: %v", err) + } + if oldSecret != nil && len(oldSecret.Name) > 0 { + t.Log("The secret already exists, removing it first") + require.NoError(t, clientSet.Kube.CoreV1().Secrets(namespace).Delete(ctx, oldSecret.Name, metav1.DeleteOptions{})) + } + + t.Logf("Creating %q in %s namespace", secretOfLifeName, namespace) + rawSecret := SecretOfLife(t, namespace) + secret, err := clientSet.Kube.CoreV1().Secrets(namespace).Create(ctx, rawSecret.(*corev1.Secret), metav1.CreateOptions{}) + require.NoError(t, err) + return secret +} + +func SecretOfLife(_ testing.TB, namespace string) runtime.Object { + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: secretOfLifeName, + Namespace: namespace, + }, + Data: map[string][]byte{ + "quote": []byte("I have no special talents. I am only passionately curious"), + }, + } +} + +func GetRawSecretOfLife(t testing.TB, clientSet ClientSet, namespace string) string { + t.Helper() + timeout, cancel := context.WithTimeout(context.Background(), 10*time.Minute) + defer cancel() + + secretOfLifeKey := fmt.Sprintf("/kubernetes.io/secrets/%s/%s", namespace, secretOfLifeName) + resp, err := clientSet.Etcd.Get(timeout, secretOfLifeKey) + require.NoError(t, err) + require.Len(t, resp.Kvs, 1, "expected exactly one key from etcd for secret-of-life") + + return string(resp.Kvs[0].Value) +} diff --git a/test/library/encryption/scenarios.go b/test/library/encryption/scenarios.go index e49c56f882..01a2d72c0a 100644 --- a/test/library/encryption/scenarios.go +++ b/test/library/encryption/scenarios.go @@ -181,48 +181,96 @@ func ShuffleEncryptionProviders(providers []EncryptionProvider) []EncryptionProv // It creates a resource, migrates through each provider, // verifies the resource is encrypted after each migration, and finally // switches to identity (off). -func TestEncryptionProvidersMigration(ctx context.Context, t testing.TB, scenario ProvidersMigrationScenario) { - if len(scenario.EncryptionProviders) < 2 { - t.Fatalf("ProvidersMigrationScenario requires at least 2 encryption providers, got %d", len(scenario.EncryptionProviders)) +// Pass one scenario for a single operator, or multiple scenarios to run +// KAS/Auth/OAS migration together (shared EncryptionProviders required). +func TestEncryptionProvidersMigration(ctx context.Context, t testing.TB, migrationScenarios ...ProvidersMigrationScenario) { + if len(migrationScenarios) == 0 { + t.Fatalf("TestEncryptionProvidersMigration requires at least one scenario") } - for _, provider := range scenario.EncryptionProviders { + providers := migrationScenarios[0].EncryptionProviders + if len(providers) < 2 { + t.Fatalf("ProvidersMigrationScenario requires at least 2 encryption providers, got %d", len(providers)) + } + for _, provider := range providers { if provider.Type == configv1.EncryptionTypeIdentity || provider.Type == "" { t.Fatalf("Unsupported encryption provider %q passed", provider.Type) } } - // step 1: create the resource - scenarios := []testStep{ - {name: fmt.Sprintf("CreateAndStore%s", scenario.ResourceName), testFunc: func(t testing.TB) { + multi := len(migrationScenarios) > 1 + + // step 1: create the resource(s) + scenarios := []testStep{} + if multi { + createSteps := make([]testStep, len(migrationScenarios)) + for i, scenario := range migrationScenarios { + createSteps[i] = testStep{name: fmt.Sprintf("CreateAndStore%s", scenario.ResourceName), testFunc: func(t testing.TB) { + e := NewE(t) + scenario.CreateResourceFunc(e, GetClients(e), scenario.Namespace) + }} + } + scenarios = append(scenarios, inParallel(createSteps...)) + } else { + scenario := migrationScenarios[0] + scenarios = append(scenarios, testStep{name: fmt.Sprintf("CreateAndStore%s", scenario.ResourceName), testFunc: func(t testing.TB) { e := NewE(t) scenario.CreateResourceFunc(e, GetClients(e), scenario.Namespace) - }}, + }}) } // step 2: migrate through each provider in sequence - for i, provider := range scenario.EncryptionProviders { + for i, provider := range providers { prefix := "EncryptWith" if i > 0 { prefix = "MigrateTo" } - scenarios = append(scenarios, - testStep{name: fmt.Sprintf("%s%s", prefix, strings.ToUpper(string(provider.Type))), testFunc: func(t testing.TB) { + stepName := fmt.Sprintf("%s%s", prefix, strings.ToUpper(string(provider.Type))) + if multi { + scenarios = append(scenarios, testStep{name: stepName, testFunc: func(t testing.TB) { + runMigrationStepMultiOperator(ctx, t, migrationScenarios, provider) + }}) + assertSteps := make([]testStep, len(migrationScenarios)) + for i, scenario := range migrationScenarios { + assertSteps[i] = testStep{name: fmt.Sprintf("Assert%sEncrypted", scenario.ResourceName), testFunc: func(t testing.TB) { + e := NewE(t) + scenario.AssertResourceEncryptedFunc(e, GetClients(e), scenario.ResourceFunc(e, scenario.Namespace)) + }} + } + scenarios = append(scenarios, inParallel(assertSteps...)) + } else { + scenario := migrationScenarios[0] + scenarios = append(scenarios, testStep{name: stepName, testFunc: func(t testing.TB) { TestEncryptionType(ctx, t, scenario.BasicScenario, provider) - }}, - testStep{name: fmt.Sprintf("Assert%sEncrypted", scenario.ResourceName), testFunc: func(t testing.TB) { + }}) + scenarios = append(scenarios, testStep{name: fmt.Sprintf("Assert%sEncrypted", scenario.ResourceName), testFunc: func(t testing.TB) { e := NewE(t) scenario.AssertResourceEncryptedFunc(e, GetClients(e), scenario.ResourceFunc(e, scenario.Namespace)) - }}, - ) + }}) + } } // step 3: switch to identity (off) to verify the resource is re-written unencrypted - scenarios = append(scenarios, testStep{name: fmt.Sprintf("OffIdentityAndAssert%sNotEncrypted", scenario.ResourceName), testFunc: func(t testing.TB) { - TestEncryptionTypeIdentity(ctx, t, scenario.BasicScenario) - e := NewE(t) - scenario.AssertResourceNotEncryptedFunc(e, GetClients(e), scenario.ResourceFunc(e, scenario.Namespace)) - }}) + if multi { + scenarios = append(scenarios, testStep{name: "OffIdentity", testFunc: func(t testing.TB) { + runMigrationStepMultiOperator(ctx, t, migrationScenarios, EncryptionProvider{APIServerEncryption: configv1.APIServerEncryption{Type: configv1.EncryptionTypeIdentity}}) + }}) + assertSteps := make([]testStep, len(migrationScenarios)) + for i, scenario := range migrationScenarios { + assertSteps[i] = testStep{name: fmt.Sprintf("Assert%sNotEncrypted", scenario.ResourceName), testFunc: func(t testing.TB) { + e := NewE(t) + scenario.AssertResourceNotEncryptedFunc(e, GetClients(e), scenario.ResourceFunc(e, scenario.Namespace)) + }} + } + scenarios = append(scenarios, inParallel(assertSteps...)) + } else { + scenario := migrationScenarios[0] + scenarios = append(scenarios, testStep{name: fmt.Sprintf("OffIdentityAndAssert%sNotEncrypted", scenario.ResourceName), testFunc: func(t testing.TB) { + TestEncryptionTypeIdentity(ctx, t, scenario.BasicScenario) + e := NewE(t) + scenario.AssertResourceNotEncryptedFunc(e, GetClients(e), scenario.ResourceFunc(e, scenario.Namespace)) + }}) + } // run scenarios for _, testScenario := range scenarios { @@ -282,6 +330,50 @@ func TestEncryptionRotation(ctx context.Context, t testing.TB, scenario Rotation // TODO: assert conditions - operator and encryption migration controller must report status as active not progressing, and not failing for all scenarios } +func runMigrationStepMultiOperator(ctx context.Context, t testing.TB, migrationScenarios []ProvidersMigrationScenario, provider EncryptionProvider) { + t.Helper() + t.Logf("Starting multi-operator encryption e2e test for %q mode", provider.Type) + + clientSet := GetClients(t) + prevKeyMeta := make([]EncryptionKeyMeta, len(migrationScenarios)) + for i, scenario := range migrationScenarios { + keyMeta, err := GetLastKeyMeta(t, clientSet.Kube, scenario.Namespace, scenario.LabelSelector) + require.NoError(t, err) + prevKeyMeta[i] = keyMeta + } + + // APIServer config is cluster-wide: update once (with conflict retry), then wait per operator in parallel. + _, clusterConfigUpdated := ApplyEncryptionProviderIfNeeded(ctx, t, provider) + + waitSteps := make([]testStep, len(migrationScenarios)) + for i, scenario := range migrationScenarios { + waitSteps[i] = testStep{ + name: scenario.ResourceName, + testFunc: func(st testing.TB) { + waitAndAssertOperatorEncryption(st, clientSet, scenario, provider, prevKeyMeta[i], clusterConfigUpdated) + }, + } + } + inParallel(waitSteps...).testFunc(t) +} + +func waitAndAssertOperatorEncryption(t testing.TB, clientSet ClientSet, scenario ProvidersMigrationScenario, provider EncryptionProvider, prevKeyMeta EncryptionKeyMeta, clusterConfigUpdated bool) { + t.Helper() + bs := scenario.BasicScenario + t.Logf("Waiting for encryption key migration for %q", scenario.ResourceName) + if clusterConfigUpdated { + WaitForNextMigratedKey(t, clientSet.Kube, prevKeyMeta, bs.TargetGRs, bs.Namespace, bs.LabelSelector) + } else { + WaitForEncryptionKeyBasedOn(t, clientSet.Kube, prevKeyMeta, provider.Type, bs.TargetGRs, bs.Namespace, bs.LabelSelector) + } + e := NewE(t, PrintEventsOnFailure(bs.OperatorNamespace)) + bs.AssertFunc(e, clientSet, provider.Type, bs.Namespace, bs.LabelSelector) + switch provider.Type { + case configv1.EncryptionTypeAESCBC, configv1.EncryptionTypeAESGCM, configv1.EncryptionTypeKMS: + AssertEncryptionConfig(e, clientSet, bs.EncryptionConfigSecretName, bs.EncryptionConfigSecretNamespace, bs.TargetGRs) + } +} + // ApplyEncryption applies the given encryption config to apiserver/cluster // without waiting for completion. func ApplyEncryption(ctx context.Context, t testing.TB, encryption configv1.APIServerEncryption) {