[WIP] Replace oc commands in OperatorSource automated test with client-go

Signed-off-by: Pratik Jagrut <pjagrut@redhat.com>

Author: Pratik Jagrut

make/test.mk

@@ -107,7 +107,7 @@ test-operator-source: push-operator-app-registry
 	$(Q)sed -e "s,REPLACE_NAMESPACE,$(DEVCONSOLE_APPR_NAMESPACE)," ./$(OPSRC_DIR)/operatorsource.yaml | sed -e "s,REPLACE_OPERATOR_SOURCE_NAME,$(OPSRC_NAME)," | oc apply -f -
 	$(Q)sed -e "s,REPLACE_APPR_REPOSITORY,$(DEVCONSOLE_APPR_REPOSITORY)," ./$(OPSRC_DIR)/catalogsourceconfig.yaml | oc apply -f -
 	$(Q)sed -e "s,REPLACE_APPR_REPOSITORY,$(DEVCONSOLE_APPR_REPOSITORY)," ./$(OPSRC_DIR)/subscription.yaml | oc apply -f -
-	$(Q)sleep 30
+	$(Q)sleep 15
 	$(Q)OPSRC_NAME=$(OPSRC_NAME) \
 	DEVCONSOLE_OPERATOR_VERSION=$(DEVCONSOLE_OPERATOR_VERSION) \
 	go test -vet off ${V_FLAG} $(shell go list ./... | grep $(OPSRC_DIR)) -failfast

test/operatorsource/basic_test.go

@@ -1,146 +1,108 @@
 package operatorsource
 
 import (
-	"bytes"
 	"fmt"
+	"github.com/stretchr/testify/require"
+	corev1 "k8s.io/api/core/v1"
 	"os"
-	"os/exec"
-	"strings"
 	"testing"
-
-	"github.com/stretchr/testify/require"
+	"time"
 )
 
-const ShellToUse = "bash"
-
-func Shellout(command string) (string, string, error) {
-	var stdout bytes.Buffer
-	var stderr bytes.Buffer
-	cmd := exec.Command(ShellToUse, "-c", command)
-	cmd.Stdout = &stdout
-	cmd.Stderr = &stderr
-	err := cmd.Run()
-	return stdout.String(), stderr.String(), err
-}
-
-func Test_OperatorSource_oc_commands(t *testing.T) {
+var (
+	Client               = NewTestClient()
+	namespace            = "openshift-operators"
+	subName              = "devconsole"
+	label                = "name=devconsole-operator"
+	subscription, suberr = Client.GetSubscription(subName, namespace)
+)
 
-	defer CleanUp(t)
+func Test_OperatorSource(t *testing.T) {
 
-	t.Run("login", func(t *testing.T) { Login(t) })
-	t.Run("subscription", func(t *testing.T) { Subscription(t) })
-	t.Run("install plan", func(t *testing.T) { InstallPlan(t) })
-	t.Run("operator pod", func(t *testing.T) { OperatorPod(t) })
-}
+	pod, err := Client.GetPodByLabel(label, namespace)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer CleanUp(t, pod)
+	retryInterval := time.Second * 10
+	timeout := time.Second * 120
 
-func Login(t *testing.T) {
-	// Start - Login to oc
-	out, _, err := Shellout("oc login -u " + os.Getenv("OC_LOGIN_USERNAME") + " -p " + os.Getenv("OC_LOGIN_PASSWORD"))
+	err = Client.WaitForOperatorDeployment(t, pod.Name, namespace, retryInterval, timeout)
 	if err != nil {
-		t.Fatalf("error: %v\n", err)
+		t.Fatal(err)
 	} else {
-		require.True(t, strings.Contains(out, "Login successful."), "Expecting successful login")
+		t.Run("subscription", func(t *testing.T) { Subscription(t) })
+		t.Run("install plan", func(t *testing.T) { InstallPlan(t) })
+		t.Run("operator pod", func(t *testing.T) { OperatorPod(t) })
 	}
 }
 
 func Subscription(t *testing.T) {
 	// 1) Verify that the subscription was created
-	out, errout, err := Shellout("oc get sub devconsole -n openshift-operators")
-	if err != nil {
-		t.Logf("stdout: %s\n", out)
-		t.Logf("stderr: %s\n", errout)
-		t.Fatalf("error: %v\n", err)
-	} else {
-		require.True(t, strings.Contains(out, "devconsole"), "Expecting the subscription name to be found")
-		require.True(t, strings.Contains(out, "installed-custom-openshift-operators"), "Expecting the subscription namespace to be found")
+	if suberr != nil {
+		t.Fatal(suberr)
 	}
+	fmt.Printf("Subscription Name: %s\nCatalog Source: %s\n", subscription.Name, subscription.Spec.CatalogSource)
+	require.Equal(t, subName, subscription.Name)
+	require.Equal(t, "installed-custom-openshift-operators", subscription.Spec.CatalogSource)
 }
 
 func InstallPlan(t *testing.T) {
 	// 2) Find the name of the install plan
-	out, errout, err := Shellout("oc get sub devconsole -n openshift-operators -o jsonpath='{.status.installplan.name}'")
-	var installPlan string
+	installPlanName := subscription.Status.Install.Name
+	fmt.Printf("Install Plan Name: %s\n", installPlanName)
+	installPlan, err := Client.GetInstallPlan(installPlanName, namespace)
 	if err != nil {
-		t.Logf("stdout: %s\n", out)
-		t.Logf("stderr: %s\n", errout)
-		t.Fatalf("error: %v\n", err)
-	} else {
-		installPlan = out
+		t.Fatal(err)
 	}
-
-	// 3) Verify the install plan
-	out, errout, err = Shellout(fmt.Sprintf("oc get installplan %s -n openshift-operators", installPlan))
-	if err != nil {
-		t.Logf("stdout: %s\n", out)
-		t.Logf("stderr: %s\n", errout)
-		t.Fatalf("error: %v\n", err)
-	} else {
-		require.True(t, strings.Contains(out, installPlan), "Expecting the Install Plan name to be found")
-		require.True(t, strings.Contains(out, "devconsole-operator.v0.1.0"), "Expecting the Operator release to be found")
-		require.True(t, strings.Contains(out, "Automatic"), "Expecting the approval method to be found")
-		require.True(t, strings.Contains(out, "true"), "Expecting the approved state to be found")
+	fmt.Printf("CSV: %v\n", installPlan.Spec.ClusterServiceVersionNames[0])
+	fmt.Printf("Install Plan Approval: %v\n", installPlan.Spec.Approval)
+	fmt.Printf("Install Plan Approved: %v\n", installPlan.Spec.Approved)
+
+	require.Equal(t, "devconsole-operator.v0.1.0", installPlan.Spec.ClusterServiceVersionNames[0])
+	require.Equal(t, "Automatic", string(installPlan.Spec.Approval))
+	if !installPlan.Spec.Approved {
+		require.FailNow(t, "Install plan approved is false")
 	}
 }
 
 func OperatorPod(t *testing.T) {
-	// Verify that the operator's pod is running
-	out, errout, err := Shellout("oc get pods  -l name=devconsole-operator -n openshift-operators -o jsonpath='{.items[*].status.phase}'")
+	// 3) Check operator pod status, fail status != Running
+	pod, err := Client.GetPodByLabel(label, namespace)
 	if err != nil {
-		t.Logf("stdout: %s\n", out)
-		t.Logf("stderr: %s\n", errout)
-		t.Fatalf("error: %v\n", err)
-	} else {
-		require.True(t, strings.Contains(out, "Running"), "Expecting the state of the Operator pod to be running")
+		t.Fatal(err)
 	}
+	fmt.Printf("Pod Name: %v\nPod status: %v\n", pod.Name, pod.Status.Phase)
+	require.Equal(t, pod.Status.Phase, corev1.PodRunning)
 }
 
-func CleanUp(t *testing.T) {
+func CleanUp(t *testing.T, pod *corev1.Pod) {
 	// Clean up resources
-	operatorSourceName := os.Getenv("OPSRC_NAME")
 	operatorVersion := os.Getenv("DEVCONSOLE_OPERATOR_VERSION")
 
-	out, errout, err := Shellout(fmt.Sprintf("oc delete opsrc %s -n openshift-marketplace", operatorSourceName))
+	err := Client.Delete("installplan", subscription.Status.Install.Name, namespace)
 	if err != nil {
-		t.Logf("stdout: %s\n", out)
-		t.Logf("stderr: %s\n", errout)
-		t.Logf("error: %v\n", err)
-	} else {
-		t.Logf(out)
+		t.Logf("Error: %v\n", err)
 	}
 
-	out, errout, err = Shellout("oc delete sub devconsole -n openshift-operators")
+	err = Client.Delete("catsrc", subscription.Spec.CatalogSource, namespace)
 	if err != nil {
-		t.Logf("stdout: %s\n", out)
-		t.Logf("stderr: %s\n", errout)
-		t.Logf("error: %v\n", err)
-	} else {
-		t.Logf(out)
+		t.Logf("Error: %v\n", err)
 	}
 
-	out, errout, err = Shellout("oc delete catsrc installed-custom-openshift-operators -n openshift-operators")
+	err = Client.Delete("sub", subName, namespace)
 	if err != nil {
-		t.Logf("stdout: %s\n", out)
-		t.Logf("stderr: %s\n", errout)
-		t.Logf("error: %v\n", err)
-	} else {
-		t.Logf(out)
+		t.Logf("Error: %v\n", err)
 	}
 
-	out, errout, err = Shellout("oc delete csc installed-custom-openshift-operators -n openshift-marketplace")
+	csv := fmt.Sprintf("devconsole-operator.v%s", operatorVersion)
+	err = Client.Delete("csv", csv, namespace)
 	if err != nil {
-		t.Logf("stdout: %s\n", out)
-		t.Logf("stderr: %s\n", errout)
-		t.Logf("error: %v\n", err)
-	} else {
-		t.Logf(out)
+		t.Logf("Error: %v\n", err)
 	}
 
-	out, errout, err = Shellout(fmt.Sprintf("oc delete csv devconsole-operator.v%s -n openshift-operators", operatorVersion))
+	err = Client.Delete("pod", pod.Name, namespace)
 	if err != nil {
-		t.Logf("stdout: %s\n", out)
-		t.Logf("stderr: %s\n", errout)
-		t.Logf("error: %v\n", err)
-	} else {
-		t.Logf(out)
+		t.Logf("Error: %v\n", err)
 	}
 }

test/operatorsource/opsrc_helpers.go

@@ -0,0 +1,151 @@
+package operatorsource
+
+import (
+	"errors"
+	"fmt"
+	apis_v1alpha1 "github.com/operator-framework/operator-lifecycle-manager/pkg/api/apis/operators/v1alpha1"
+	client "github.com/operator-framework/operator-lifecycle-manager/pkg/api/client"
+	v1alpha1 "github.com/operator-framework/operator-lifecycle-manager/pkg/api/client/clientset/versioned/typed/operators/v1alpha1"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/clientcmd"
+	"log"
+	"os"
+	"testing"
+	"time"
+)
+
+//ClientSetK8sCoreAPI returns new Clientset for the given config, use to interact with K8s resources like pods
+func ClientSetK8sCoreAPI(kubeconfig string) *kubernetes.Clientset {
+	config, err := clientcmd.BuildConfigFromFlags("", kubeconfig)
+	if err != nil {
+		log.Fatal(err)
+	}
+	clientset, err := kubernetes.NewForConfig(config)
+	if err != nil {
+		log.Fatal(err)
+	}
+	return clientset
+}
+
+//ClientSet Creates clientset for given config, use to interact with custom resources
+func ClientSet(kubeconfig string) v1alpha1.OperatorsV1alpha1Interface {
+	client, err := client.NewClient(kubeconfig)
+	if err != nil {
+		log.Fatal(err)
+	}
+	return client.OperatorsV1alpha1()
+}
+
+// TestClient wraps all the clientsets required while testing
+type TestClient struct {
+	K8sClient      *kubernetes.Clientset
+	OperatorClient v1alpha1.OperatorsV1alpha1Interface
+}
+
+//NewTestClient initialises the TestClient
+func NewTestClient() *TestClient {
+	kubeconfig := os.Getenv("KUBECONFIG")
+
+	return &TestClient{
+		K8sClient:      ClientSetK8sCoreAPI(kubeconfig),
+		OperatorClient: ClientSet(kubeconfig),
+	}
+}
+
+// GetPodByLabel is a function that takes label and namespace and returns the pod and error
+func (tc *TestClient) GetPodByLabel(label string, namespace string) (*corev1.Pod, error) {
+
+	pods, err := tc.K8sClient.CoreV1().Pods(namespace).List(metav1.ListOptions{LabelSelector: label})
+	if err != nil {
+		return nil, err
+	}
+	if len(pods.Items) == 0 {
+		return nil, nil
+	}
+	return &pods.Items[0], nil
+}
+
+//GetSubscription returns subscription struct
+func (tc *TestClient) GetSubscription(subName, namespace string) (*apis_v1alpha1.Subscription, error) {
+	subscription, err := tc.OperatorClient.Subscriptions(namespace).Get(subName, metav1.GetOptions{})
+	if err != nil {
+		return nil, err
+	}
+
+	return subscription, nil
+}
+
+//GetInstallPlan returns install plan struct
+func (tc *TestClient) GetInstallPlan(installPlanName, namespace string) (*apis_v1alpha1.InstallPlan, error) {
+
+	installPlan, err := tc.OperatorClient.InstallPlans(namespace).Get(installPlanName, metav1.GetOptions{})
+	if err != nil {
+		return nil, err
+	}
+	return installPlan, nil
+}
+
+//Delete takes kind, name and namespace of the resource and deletes it. Returns an error if one occurs.
+func (tc *TestClient) Delete(resource, name, namespace string) error {
+	switch resource {
+	case "subscription", "sub":
+		err := tc.OperatorClient.Subscriptions(namespace).Delete(name, &metav1.DeleteOptions{})
+		if err != nil {
+			return err
+		}
+		return nil
+	case "installplan":
+		err := tc.OperatorClient.InstallPlans(namespace).Delete(name, &metav1.DeleteOptions{})
+		if err != nil {
+			return err
+		}
+		return nil
+	case "catalogsource", "catsrc":
+		err := tc.OperatorClient.CatalogSources(namespace).Delete(name, &metav1.DeleteOptions{})
+		if err != nil {
+			return err
+		}
+		return nil
+	case "clusterserviceversion", "csv":
+		err := tc.OperatorClient.ClusterServiceVersions(namespace).Delete(name, &metav1.DeleteOptions{})
+		if err != nil {
+			return err
+		}
+		return nil
+	case "pod":
+		err := tc.K8sClient.CoreV1().Pods(namespace).Delete(name, &metav1.DeleteOptions{})
+		if err != nil {
+			return err
+		}
+		return nil
+	default:
+		option := fmt.Sprintf("Invalid resource: %s", resource)
+		return errors.New(option)
+
+	}
+}
+
+//WaitForOperatorDeployment takes pod struct and wait till pods gets in runnig state
+func (tc *TestClient) WaitForOperatorDeployment(t *testing.T, name, namespace string, retryInterval, timeout time.Duration) error {
+	err := wait.Poll(retryInterval, timeout, func() (bool, error) {
+		pod, err := tc.K8sClient.CoreV1().Pods(namespace).Get(name, metav1.GetOptions{})
+		if err != nil {
+			return false, err
+		}
+		if pod == nil {
+			return false, errors.New("pod returned empty")
+		}
+		if pod.Status.Phase == corev1.PodRunning {
+			return true, nil
+		}
+		t.Logf("Pod %s Status: %s", pod.Name, pod.Status.Phase)
+		return false, nil
+	})
+	if err != nil {
+		return err
+	}
+	return nil
+}