diff --git a/openshift/cmd/ovn-kubernetes-tests-ext/main.go b/openshift/cmd/ovn-kubernetes-tests-ext/main.go index 9f90ff0a20..8163b5128b 100644 --- a/openshift/cmd/ovn-kubernetes-tests-ext/main.go +++ b/openshift/cmd/ovn-kubernetes-tests-ext/main.go @@ -4,6 +4,7 @@ import ( "os" "strings" + exutil "github.com/openshift/origin/test/extended/util" "github.com/ovn-kubernetes/ovn-kubernetes/openshift/test" ocpdeploymentconfig "github.com/ovn-kubernetes/ovn-kubernetes/openshift/test/deploymentconfig" "github.com/ovn-kubernetes/ovn-kubernetes/openshift/test/generated" @@ -11,6 +12,8 @@ import ( // import ovn-kubernetes tests _ "github.com/ovn-kubernetes/ovn-kubernetes/test/e2e" + // import OTP migrated tests + _ "github.com/ovn-kubernetes/ovn-kubernetes/openshift/test/otp" "github.com/ovn-kubernetes/ovn-kubernetes/test/e2e/deploymentconfig" "github.com/ovn-kubernetes/ovn-kubernetes/test/e2e/infraprovider" @@ -130,14 +133,19 @@ func main() { if err := initializeTestFramework(os.Getenv("TEST_PROVIDER"), cfg); err != nil { panic(err) } + exutil.WithCleanup(func() {}) }) informingTests := sets.New(test.InformingTests...) blockingTests := sets.New(test.BlockingTests...) specs.Walk(func(spec *extensiontests.ExtensionTestSpec) { - for _, label := range getTestExtensionLabels() { - spec.Labels.Insert(label) + isOTP := strings.Contains(spec.Name, "[OTP]") + + if !isOTP { + for _, label := range getTestExtensionLabels() { + spec.Labels.Insert(label) + } } // Exclude Network Segmentation tests on SingleReplica topology (e.g., MicroShift, SNO) @@ -150,14 +158,22 @@ func main() { spec.Name += " " + annotations } - // prepend other labels by matching on existing spec labels - for _, label := range getPrependLabels(spec.Labels) { - spec.Labels.Insert(label) - } + if isOTP { + if spec.Labels.Has("Level0") { + spec.Name = "[Level0] " + spec.Name + } + } else { + // prepend other labels by matching on existing spec labels + for _, label := range getPrependLabels(spec.Labels) { + spec.Labels.Insert(label) + } - spec.Name = generatePrependedLabelsStr(spec.Labels) + " " + spec.Name // prepend ginkgo labels to test name + spec.Name = generatePrependedLabelsStr(spec.Labels) + " " + spec.Name + } switch { + case isOTP: + spec.Lifecycle = extensiontests.LifecycleInforming case informingTests.Has(spec.Name): spec.Lifecycle = extensiontests.LifecycleInforming case blockingTests.Has(spec.Name): diff --git a/openshift/go.mod b/openshift/go.mod index 0d5f1aa884..df7a84db09 100644 --- a/openshift/go.mod +++ b/openshift/go.mod @@ -10,34 +10,39 @@ require ( github.com/openshift-eng/openshift-tests-extension v0.0.0-20260127124016-0fed2b824818 github.com/openshift/api v0.0.0-20260311143357-f6ee4c095675 github.com/openshift/client-go v0.0.0-20260306160707-3935d929fc7d + github.com/openshift/origin v1.5.0-alpha.3.0.20260310231025-5d3fd0545b5d github.com/ovn-kubernetes/ovn-kubernetes/go-controller v1.0.0 github.com/ovn-kubernetes/ovn-kubernetes/test/e2e v0.0.0-00010101000000-000000000000 - github.com/spf13/cobra v1.10.0 + github.com/spf13/cobra v1.10.1 k8s.io/api v0.35.1 k8s.io/apimachinery v0.35.1 k8s.io/client-go v0.35.1 k8s.io/component-base v0.35.1 k8s.io/kubernetes v1.35.1 + k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2 ) require ( cel.dev/expr v0.25.1 // indirect cyphar.com/go-pathrs v0.2.1 // indirect + github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect github.com/JeffAshton/win_pdh v0.0.0-20161109143554-76bb4ee9f0ab // indirect + github.com/MakeNowJust/heredoc v1.0.0 // indirect github.com/Masterminds/semver/v3 v3.4.0 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect github.com/Microsoft/hnslib v0.1.2 // indirect github.com/NYTimes/gziphandler v1.1.1 // indirect - github.com/antlr4-go/antlr/v4 v4.13.0 // indirect + github.com/antlr4-go/antlr/v4 v4.13.1 // indirect github.com/armon/circbuf v0.0.0-20190214190532-5111143e8da2 // indirect - github.com/aws/aws-sdk-go v1.44.204 // indirect + github.com/aws/aws-sdk-go v1.50.25 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/blang/semver/v4 v4.0.0 // indirect - github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cenkalti/backoff/v5 v5.0.3 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/chai2010/gettext-go v1.0.2 // indirect github.com/clarketm/json v1.17.1 // indirect github.com/container-storage-interface/spec v1.9.0 // indirect + github.com/containerd/containerd v1.7.32 // indirect github.com/containerd/containerd/api v1.9.0 // indirect github.com/containerd/errdefs v1.0.0 // indirect github.com/containerd/errdefs/pkg v0.3.0 // indirect @@ -60,16 +65,21 @@ require ( github.com/docker/docker v28.2.2+incompatible // indirect github.com/docker/go-connections v0.5.0 // indirect github.com/docker/go-units v0.5.0 // indirect - github.com/emicklei/go-restful/v3 v3.12.2 // indirect + github.com/emicklei/go-restful/v3 v3.13.0 // indirect github.com/euank/go-kmsg-parser v2.0.0+incompatible // indirect github.com/evanphx/json-patch/v5 v5.9.11 // indirect + github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f // indirect + github.com/fatih/camelcase v1.0.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.9.0 // indirect + github.com/fsouza/go-dockerclient v1.12.0 // indirect github.com/fxamacker/cbor/v2 v2.9.0 // indirect github.com/gaissmai/cidrtree v0.1.4 // indirect + github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32 // indirect + github.com/go-errors/errors v1.4.2 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect - github.com/go-openapi/jsonpointer v0.21.1 // indirect + github.com/go-openapi/jsonpointer v0.21.2 // indirect github.com/go-openapi/jsonreference v0.21.0 // indirect github.com/go-openapi/swag v0.23.1 // indirect github.com/go-task/slim-sprig/v3 v3.0.0 // indirect @@ -78,7 +88,7 @@ require ( github.com/golang/protobuf v1.5.4 // indirect github.com/google/btree v1.1.3 // indirect github.com/google/cadvisor v0.53.0 // indirect - github.com/google/cel-go v0.26.0 // indirect + github.com/google/cel-go v0.26.1 // indirect github.com/google/gnostic-models v0.7.0 // indirect github.com/google/go-cmp v0.7.0 // indirect github.com/google/goexpect v0.0.0-20210430020637-ab937bf7fd6f // indirect @@ -86,9 +96,11 @@ require ( github.com/google/pprof v0.0.0-20250820193118-f64d9cf942d6 // indirect github.com/google/uuid v1.6.0 // indirect github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // indirect + github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 // indirect github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/josharian/native v1.1.0 // indirect github.com/json-iterator/go v1.1.12 // indirect @@ -98,8 +110,10 @@ require ( github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.7.7 // indirect github.com/k8snetworkplumbingwg/sriovnet v1.2.1-0.20250818105516-24ab680f94f3 // indirect github.com/karrick/godirwalk v1.17.0 // indirect + github.com/klauspost/compress v1.18.0 // indirect github.com/kylelemons/godebug v1.1.0 // indirect github.com/libopenstorage/openstorage v1.0.0 // indirect + github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect github.com/mailru/easyjson v0.9.0 // indirect github.com/mdlayher/arp v0.0.0-20220512170110-6706a2966875 // indirect github.com/mdlayher/ethernet v0.0.0-20220221185849-529eae5b6118 // indirect @@ -107,15 +121,22 @@ require ( github.com/mdlayher/packet v1.1.2 // indirect github.com/mdlayher/socket v0.5.1 // indirect github.com/metallb/frr-k8s v0.0.21 // indirect - github.com/miekg/dns v1.1.43 // indirect + github.com/miekg/dns v1.1.68 // indirect github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible // indirect + github.com/mitchellh/go-wordwrap v1.0.1 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect + github.com/moby/patternmatcher v0.6.0 // indirect github.com/moby/spdystream v0.5.1 // indirect github.com/moby/sys/mountinfo v0.7.2 // indirect + github.com/moby/sys/sequential v0.6.0 // indirect + github.com/moby/sys/user v0.4.0 // indirect github.com/moby/sys/userns v0.1.0 // indirect + github.com/moby/term v0.5.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect + github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect + github.com/morikuni/aec v1.0.0 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect github.com/onsi/ginkgo v1.16.5 // indirect @@ -123,42 +144,48 @@ require ( github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.1 // indirect github.com/opencontainers/runtime-spec v1.2.1 // indirect - github.com/opencontainers/selinux v1.13.0 // indirect + github.com/opencontainers/selinux v1.13.1 // indirect github.com/openshift-kni/k8sreporter v1.0.6 // indirect github.com/openshift/custom-resource-status v1.1.2 // indirect github.com/openshift/library-go v0.0.0-20260303171201-5d9eb6295ff6 // indirect github.com/ovn-kubernetes/dpu-simulator/lib/dpusim v0.0.0-20260507161134-3aec48e5cb25 // indirect + github.com/pborman/uuid v1.2.0 // indirect + github.com/peterbourgon/diskv v2.0.1+incompatible // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_golang v1.23.2 // indirect github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.66.1 // indirect - github.com/prometheus/procfs v0.16.1 // indirect + github.com/prometheus/procfs v0.17.0 // indirect github.com/robfig/cron/v3 v3.0.1 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/safchain/ethtool v0.6.2 // indirect github.com/sirupsen/logrus v1.9.3 // indirect github.com/spf13/afero v1.14.0 // indirect github.com/spf13/pflag v1.0.9 // indirect - github.com/stoewer/go-strcase v1.3.0 // indirect + github.com/stoewer/go-strcase v1.3.1 // indirect github.com/stretchr/objx v0.5.2 // indirect github.com/stretchr/testify v1.11.1 // indirect + github.com/tidwall/gjson v1.18.0 // indirect + github.com/tidwall/match v1.1.1 // indirect + github.com/tidwall/pretty v1.2.1 // indirect github.com/urfave/cli/v2 v2.27.2 // indirect github.com/vincent-petithory/dataurl v1.0.0 // indirect github.com/vishvananda/netlink v1.3.2-0.20260320193013-72a8cd7e0a73 // indirect github.com/vishvananda/netns v0.0.5 // indirect github.com/x448/float16 v0.8.4 // indirect + github.com/xlab/treeprint v1.2.0 // indirect github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 // indirect go.etcd.io/etcd/api/v3 v3.6.5 // indirect go.etcd.io/etcd/client/pkg/v3 v3.6.5 // indirect go.etcd.io/etcd/client/v3 v3.6.5 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/contrib/instrumentation/github.com/emicklei/go-restful/otelrestful v0.44.0 // indirect - go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 // indirect go.opentelemetry.io/otel v1.43.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.34.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.36.0 // indirect go.opentelemetry.io/otel/metric v1.43.0 // indirect go.opentelemetry.io/otel/sdk v1.43.0 // indirect go.opentelemetry.io/otel/trace v1.43.0 // indirect @@ -170,7 +197,7 @@ require ( go.yaml.in/yaml/v2 v2.4.3 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/crypto v0.49.0 // indirect - golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect + golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b // indirect golang.org/x/mod v0.33.0 // indirect golang.org/x/net v0.52.0 // indirect golang.org/x/oauth2 v0.35.0 // indirect @@ -178,7 +205,7 @@ require ( golang.org/x/sys v0.42.0 // indirect golang.org/x/term v0.41.0 // indirect golang.org/x/text v0.35.0 // indirect - golang.org/x/time v0.11.0 // indirect + golang.org/x/time v0.12.0 // indirect golang.org/x/tools v0.42.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9 // indirect @@ -186,6 +213,7 @@ require ( google.golang.org/protobuf v1.36.11 // indirect gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect gopkg.in/gcfg.v1 v1.2.3 // indirect + gopkg.in/go-jose/go-jose.v2 v2.6.3 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect @@ -193,7 +221,9 @@ require ( gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/apiextensions-apiserver v0.35.1 // indirect k8s.io/apiserver v0.35.1 // indirect + k8s.io/cli-runtime v0.33.4 // indirect k8s.io/cloud-provider v0.35.1 // indirect + k8s.io/cluster-bootstrap v0.0.0 // indirect k8s.io/component-helpers v0.35.1 // indirect k8s.io/controller-manager v0.35.1 // indirect k8s.io/cri-api v0.34.1 // indirect @@ -202,20 +232,25 @@ require ( k8s.io/dynamic-resource-allocation v0.35.1 // indirect k8s.io/klog/v2 v2.130.1 // indirect k8s.io/kms v0.35.1 // indirect + k8s.io/kube-aggregator v0.35.1 // indirect k8s.io/kube-openapi v0.0.0-20260304202019-5b3e3fdb0acf // indirect k8s.io/kube-scheduler v0.34.1 // indirect k8s.io/kubectl v0.35.1 // indirect k8s.io/kubelet v0.35.1 // indirect k8s.io/mount-utils v0.34.1 // indirect k8s.io/pod-security-admission v0.35.1 // indirect - k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2 // indirect + k8s.io/sample-apiserver v0.0.0 // indirect kubevirt.io/api v1.4.0 // indirect kubevirt.io/containerized-data-importer-api v1.57.0-alpha1 // indirect kubevirt.io/controller-lifecycle-operator-sdk/api v0.0.0-20220329064328-f3cc58c6ed90 // indirect sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.33.0 // indirect sigs.k8s.io/controller-runtime v0.23.1 // indirect + sigs.k8s.io/gateway-api v1.4.0 // indirect sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect sigs.k8s.io/knftables v0.0.20 // indirect + sigs.k8s.io/kube-storage-version-migrator v0.0.6-0.20230721195810-5c8923c5ff96 // indirect + sigs.k8s.io/kustomize/api v0.20.1 // indirect + sigs.k8s.io/kustomize/kyaml v0.20.1 // indirect sigs.k8s.io/network-policy-api v0.1.5 // indirect sigs.k8s.io/randfill v1.0.0 // indirect sigs.k8s.io/structured-merge-diff/v6 v6.3.2-0.20260122202528-d9cc6641c482 // indirect @@ -238,6 +273,7 @@ replace ( k8s.io/apiserver => github.com/openshift/kubernetes/staging/src/k8s.io/apiserver v0.0.0-20260313023409-2034d92b4a3a k8s.io/client-go => github.com/openshift/kubernetes/staging/src/k8s.io/client-go v0.0.0-20260313023409-2034d92b4a3a k8s.io/cloud-provider => github.com/openshift/kubernetes/staging/src/k8s.io/cloud-provider v0.0.0-20260313023409-2034d92b4a3a + k8s.io/cluster-bootstrap => github.com/openshift/kubernetes/staging/src/k8s.io/cluster-bootstrap v0.0.0-20260313023409-2034d92b4a3a k8s.io/component-base => github.com/openshift/kubernetes/staging/src/k8s.io/component-base v0.0.0-20260313023409-2034d92b4a3a k8s.io/component-helpers => github.com/openshift/kubernetes/staging/src/k8s.io/component-helpers v0.0.0-20260313023409-2034d92b4a3a k8s.io/controller-manager => github.com/openshift/kubernetes/staging/src/k8s.io/controller-manager v0.0.0-20260313023409-2034d92b4a3a @@ -252,4 +288,5 @@ replace ( k8s.io/kubernetes => github.com/openshift/kubernetes v1.30.1-0.20260313023409-2034d92b4a3a k8s.io/mount-utils => github.com/openshift/kubernetes/staging/src/k8s.io/mount-utils v0.0.0-20260313023409-2034d92b4a3a k8s.io/pod-security-admission => github.com/openshift/kubernetes/staging/src/k8s.io/pod-security-admission v0.0.0-20260313023409-2034d92b4a3a + k8s.io/sample-apiserver => github.com/openshift/kubernetes/staging/src/k8s.io/sample-apiserver v0.0.0-20260313023409-2034d92b4a3a ) diff --git a/openshift/go.sum b/openshift/go.sum index c5229e924a..c9006b4b97 100644 --- a/openshift/go.sum +++ b/openshift/go.sum @@ -4,15 +4,21 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= cyphar.com/go-pathrs v0.2.1 h1:9nx1vOgwVvX1mNBWDu93+vaceedpbsDqo+XuBGL40b8= cyphar.com/go-pathrs v0.2.1/go.mod h1:y8f1EMG7r+hCuFf/rXsKqMJrJAUoADZGNh5/vZPKcGc= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 h1:bvDV9vkmnHYOMsOr4WLk+Vo07yKIzd94sVoIqshQ4bU= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0= github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/JeffAshton/win_pdh v0.0.0-20161109143554-76bb4ee9f0ab h1:UKkYhof1njT1/xq4SEg5z+VpTgjmNeHwPGRQl7takDI= github.com/JeffAshton/win_pdh v0.0.0-20161109143554-76bb4ee9f0ab/go.mod h1:3VYc5hodBMJ5+l/7J4xAyMeuM2PNuepvHlGs8yilUCA= +github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ= +github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE= github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/Microsoft/hcsshim v0.13.0 h1:/BcXOiS6Qi7N9XqUcv27vkIuVOkBEcWstd2pMlWSeaA= +github.com/Microsoft/hcsshim v0.13.0/go.mod h1:9KWJ/8DgU+QzYGupX4tzMhRQE8h6w90lH6HAaclpEok= github.com/Microsoft/hnslib v0.1.2 h1:CshjwTQsNx1o7BIA1XO8HtgDsiCqn+b6kGjL/tIxXQQ= github.com/Microsoft/hnslib v0.1.2/go.mod h1:5vTyBey4N/VI2ZTNh2gdWhkPMefSbCFYjpvVwye+qtI= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= @@ -20,15 +26,15 @@ github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cq github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= -github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI= -github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g= +github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ= +github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw= github.com/armon/circbuf v0.0.0-20190214190532-5111143e8da2 h1:7Ip0wMmLHLRJdrloDxZfhMm0xrLXZS8+COSu2bXmEQs= github.com/armon/circbuf v0.0.0-20190214190532-5111143e8da2/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/aws/aws-sdk-go v1.44.204 h1:7/tPUXfNOHB390A63t6fJIwmlwVQAkAwcbzKsU2/6OQ= -github.com/aws/aws-sdk-go v1.44.204/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/aws/aws-sdk-go v1.50.25 h1:vhiHtLYybv1Nhx3Kv18BBC6L0aPJHaG9aeEsr92W99c= +github.com/aws/aws-sdk-go v1.50.25/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= github.com/aymanbagabas/go-udiff v0.2.0/go.mod h1:RE4Ex0qsGkTAJoQdQQCA0uG+nAzJO/pI/QwceO5fgrA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -36,13 +42,13 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r github.com/bits-and-blooms/bitset v1.22.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= -github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= -github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chai2010/gettext-go v1.0.2 h1:1Lwwip6Q2QGsAdl/ZKPCwTe9fe0CjlUbqj5bFNSjIRk= +github.com/chai2010/gettext-go v1.0.2/go.mod h1:y+wnP2cHYaVj19NZhYKAwEMH2CI1gNHeQQ+5AjwawxA= github.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc/go.mod h1:X4/0JoqgTIPSFcRA/P6INZzIuyqdFY5rm8tb41s9okk= github.com/charmbracelet/colorprofile v0.3.1/go.mod h1:/GkGusxNs8VB/RSOh3fu0TJmQ4ICMMPApIIVn0KszZ0= github.com/charmbracelet/lipgloss v1.1.0/go.mod h1:/6Q8FR2o+kj8rz4Dq0zQc3vYf7X+B0binUUBwA0aL30= @@ -64,6 +70,8 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/container-storage-interface/spec v1.9.0 h1:zKtX4STsq31Knz3gciCYCi1SXtO2HJDecIjDVboYavY= github.com/container-storage-interface/spec v1.9.0/go.mod h1:ZfDu+3ZRyeVqxZM0Ds19MVLkN2d1XJ5MAfi1L3VjlT0= +github.com/containerd/containerd v1.7.32 h1:S54xuVcPxeLaYgaRABtpJ2VyVUVsy0IGf7qHBs+sbY8= +github.com/containerd/containerd v1.7.32/go.mod h1:jdwD6s/BhV4XVJGrvtziNPVA+83n66TwptVaPKprq4E= github.com/containerd/containerd/api v1.9.0 h1:HZ/licowTRazus+wt9fM6r/9BQO7S0vD5lMcWspGIg0= github.com/containerd/containerd/api v1.9.0/go.mod h1:GhghKFmTR3hNtyznBoQ0EMWr9ju5AqHjcZPsSpTKutI= github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= @@ -95,6 +103,8 @@ github.com/coreos/vcontext v0.0.0-20230201181013-d72178a18687/go.mod h1:Salmysdw github.com/cpuguy83/go-md2man/v2 v2.0.6 h1:XJtiaUW6dEEqVuZiMTn1ldk455QWwEIsMIJlo5vtkx0= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= +github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= github.com/cyphar/filepath-securejoin v0.6.0 h1:BtGB77njd6SVO6VztOHfPxKitJvd/VPT+OFBFMOi1Is= github.com/cyphar/filepath-securejoin v0.6.0/go.mod h1:A8hd4EnAeyujCJRrICiOWqjS1AX0a9kM5XL+NwKoYSc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -117,8 +127,9 @@ github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT github.com/emicklei/go-restful v2.15.0+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.16.0+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= -github.com/emicklei/go-restful/v3 v3.12.2 h1:DhwDP0vY3k8ZzE0RunuJy8GhNpPL6zqLkDf9B/a0/xU= github.com/emicklei/go-restful/v3 v3.12.2/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/emicklei/go-restful/v3 v3.13.0 h1:C4Bl2xDndpU6nJ4bc1jXd+uTmYPVUwkD6bFY/oTyCes= +github.com/emicklei/go-restful/v3 v3.13.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -127,24 +138,34 @@ github.com/euank/go-kmsg-parser v2.0.0+incompatible h1:cHD53+PLQuuQyLZeriD1V/esu github.com/euank/go-kmsg-parser v2.0.0+incompatible/go.mod h1:MhmAMZ8V4CYH4ybgdRwPr2TU5ThnS43puaKEMpja1uw= github.com/evanphx/json-patch/v5 v5.9.11 h1:/8HVnzMq13/3x9TPvjG08wUGqBTmZBsCWzjTM0wiaDU= github.com/evanphx/json-patch/v5 v5.9.11/go.mod h1:3j+LviiESTElxA4p3EMKAB9HXj3/XEtnUf6OZxqIQTM= +github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f h1:Wl78ApPPB2Wvf/TIe2xdyJxTlb6obmF18d8QdkxNDu4= +github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f/go.mod h1:OSYXu++VVOHnXeitef/D8n/6y4QV8uLHSFXX4NeXMGc= +github.com/fatih/camelcase v1.0.0 h1:hxNvNX/xYBp0ovncs8WyWZrOrpBNub/JfaMvbURyft8= +github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/fsouza/go-dockerclient v1.12.0 h1:S2f2crEUbBNCFiF06kR/GvioEB8EMsb3Td/bpawD+aU= +github.com/fsouza/go-dockerclient v1.12.0/go.mod h1:YWUtjg8japrqD/80L98nTtCoxQFp5B5wrSsnyeB5lFo= github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= github.com/gaissmai/cidrtree v0.1.4 h1:/aYnv1LIwjtSDHNr1eNN99WJeh6vLrB+Sgr1tRMhHDc= github.com/gaissmai/cidrtree v0.1.4/go.mod h1:nrjEeeMZmvoJpLcSvZ3qIVFxw/+9GHKi7wDHHmHKGRI= github.com/getkin/kin-openapi v0.76.0/go.mod h1:660oXbgy5JFMKreazJaQTw7o+X00qeSyhcnluiMv+Xg= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32 h1:Mn26/9ZMNWSw9C9ERFA1PUxfmGpolnw2v0bKOREu5ew= +github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32/go.mod h1:GIjDIg/heH5DOkXY3YJ/wNhfHsQHoXGjl8G8amsYQ1I= github.com/gkampitakis/ciinfo v0.3.2 h1:JcuOPk8ZU7nZQjdUhctuhQofk7BGHuIy0c9Ez8BNhXs= github.com/gkampitakis/ciinfo v0.3.2/go.mod h1:1NIwaOcFChN4fa/B0hEBdAb6npDlFL8Bwx4dfRLRqAo= github.com/gkampitakis/go-diff v1.3.2 h1:Qyn0J9XJSDTgnsgHRdz9Zp24RaJeKMUHg2+PDZZdC4M= github.com/gkampitakis/go-diff v1.3.2/go.mod h1:LLgOrpqleQe26cte8s36HTWcTmMEur6OPYerdAAS9tk= github.com/gkampitakis/go-snaps v0.5.15 h1:amyJrvM1D33cPHwVrjo9jQxX8g/7E2wYdZ+01KS3zGE= github.com/gkampitakis/go-snaps v0.5.15/go.mod h1:HNpx/9GoKisdhw9AFOBT1N7DBs9DiHo/hGheFGBZ+mc= +github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= +github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= @@ -162,8 +183,8 @@ github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34 github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= -github.com/go-openapi/jsonpointer v0.21.1 h1:whnzv/pNXtK2FbX/W9yJfRmE2gsmkfahjMKB0fZvcic= -github.com/go-openapi/jsonpointer v0.21.1/go.mod h1:50I1STOfbY1ycR8jGz8DaMeLCdXiI6aDteEdRNNzpdk= +github.com/go-openapi/jsonpointer v0.21.2 h1:AqQaNADVwq/VnkCmQg6ogE+M3FOsKTytwges0JdwVuA= +github.com/go-openapi/jsonpointer v0.21.2/go.mod h1:50I1STOfbY1ycR8jGz8DaMeLCdXiI6aDteEdRNNzpdk= github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg= github.com/go-openapi/jsonreference v0.19.6/go.mod h1:diGHMEHg2IqXZGKxqyvWdfWU/aim5Dprw5bqpKkTvns= @@ -214,8 +235,8 @@ github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= github.com/google/cadvisor v0.53.0 h1:pmveUw2VBlr/T2SBE9Fsp8gdLhKWyOBkECGbaas9mcI= github.com/google/cadvisor v0.53.0/go.mod h1:Tz3zf/exzFfdWd1T/U/9eNst0ZR2C6CIV62LJATj5tg= -github.com/google/cel-go v0.26.0 h1:DPGjXackMpJWH680oGY4lZhYjIameYmR+/6RBdDGmaI= -github.com/google/cel-go v0.26.0/go.mod h1:A9O8OU9rdvrK5MQyrqfIxo1a0u4g3sF8KB6PUIaryMM= +github.com/google/cel-go v0.26.1 h1:iPbVVEdkhTX++hpe3lzSk7D3G3QSYqLGoHOcEio+UXQ= +github.com/google/cel-go v0.26.1/go.mod h1:A9O8OU9rdvrK5MQyrqfIxo1a0u4g3sF8KB6PUIaryMM= github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo= github.com/google/gnostic-models v0.7.0/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -241,6 +262,7 @@ github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8/go.mod h1:K1liHPHnj73 github.com/google/pprof v0.0.0-20250403155104-27863c87afa6/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= github.com/google/pprof v0.0.0-20250820193118-f64d9cf942d6 h1:EEHtgt9IwisQ2AZ4pIsMjahcegHh6rmhqxzIRQIyepY= github.com/google/pprof v0.0.0-20250820193118-f64d9cf942d6/go.mod h1:I6V7YzU0XDpsHqbsyrghnFZLO1gwK6NPTNvmetQIk9U= +github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= @@ -250,7 +272,9 @@ github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97Dwqy github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 h1:JeSE6pjso5THxAzdVpqr6/geYxZytqFMBCOtn/ujyeo= github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674/go.mod h1:r4w70xmWCQKmi1ONH4KIaBptdivuRPyosB9RmPlGEwA= +github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 h1:+ngKgrYPPJrOjhax5N+uePQ0Fh1Z7PheYoUI/0nzkPA= github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw= github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1 h1:qnpSQwGEnkcRpTqNOIR6bJbR0gAorgP9CSALpRcKoAA= github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1/go.mod h1:lXGCsh6c22WGtjr+qGHj1otzZpV/1kwTMAqkwZsnWRU= github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.0 h1:FbSCl+KggFl+Ocym490i/EyXF4lPgLoUtcSWquBM0Rs= @@ -263,7 +287,9 @@ github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpO github.com/ianlancetaylor/demangle v0.0.0-20240312041847-bd984b5ce465/go.mod h1:gx7rwoVhcfuVKG5uya9Hs3Sxj7EIvldVofAWIUtGouw= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jonboulle/clockwork v0.5.0 h1:Hyh9A8u51kptdkR+cqRpT1EebBwTn1oK9YfGYbdFz6I= github.com/jonboulle/clockwork v0.5.0/go.mod h1:3mZlmanh0g2NDKO5TWZVJAfofYk64M7XN3SzBPjZF60= @@ -274,6 +300,8 @@ github.com/josharian/native v1.1.0 h1:uuaP0hAbW7Y4l0ZRQ6C9zfb7Mg1mbFKry/xzDAfmtL github.com/josharian/native v1.1.0/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w= github.com/joshdk/go-junit v1.0.0 h1:S86cUKIdwBHWwA6xCmFlf3RTLfVXYQfvanM5Uh+K6GE= github.com/joshdk/go-junit v1.0.0/go.mod h1:TiiV0PqkaNfFXjEiyjWM3XXrhVyCa1K4Zfga6W52ung= +github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= @@ -308,6 +336,8 @@ github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+ github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs= github.com/libopenstorage/openstorage v1.0.0 h1:GLPam7/0mpdP8ZZtKjbfcXJBTIA/T1O6CBErVEFEyIM= github.com/libopenstorage/openstorage v1.0.0/go.mod h1:Sp1sIObHjat1BeXhfMqLZ14wnOzEhNx2YQedreMcUyc= +github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de h1:9TO3cAIGXtEhnIaL+V+BEER86oLrvS+kWobKpbJuye0= +github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE= github.com/lithammer/dedent v1.1.0 h1:VNzHMVCBNG1j0fh3OrsFRkVUwStdDArbgBWoPAffktY= github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc= github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= @@ -339,18 +369,26 @@ github.com/metallb/metallb v0.14.9 h1:rjhftr7b0vv56c8pXm7UDo7ad61EwKT6lbGGocrN/V github.com/metallb/metallb v0.14.9/go.mod h1:qUh1zVwYAfp3JLxhZrDH20j55QvYwCkI37QU4gUG3ns= github.com/mfridman/tparse v0.18.0 h1:wh6dzOKaIwkUGyKgOntDW4liXSo37qg5AXbIhkMV3vE= github.com/mfridman/tparse v0.18.0/go.mod h1:gEvqZTuCgEhPbYk/2lS3Kcxg1GmTxxU7kTC8DvP0i/A= -github.com/miekg/dns v1.1.43 h1:JKfpVSCB84vrAmHzyrsxB5NAr5kLoMXZArPSw7Qlgyg= -github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4= +github.com/miekg/dns v1.1.68 h1:jsSRkNozw7G/mnmXULynzMNIsgY2dHC8LO6U6Ij2JEA= +github.com/miekg/dns v1.1.68/go.mod h1:fujopn7TB3Pu3JM69XaawiU0wqjpL9/8xGop5UrTPps= github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible h1:aKW/4cBs+yK6gpqU3K/oIwk9Q/XICqd3zOX/UFuvqmk= github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4= +github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0= +github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= +github.com/moby/patternmatcher v0.6.0 h1:GmP9lR19aU5GqSSFko+5pRqHi+Ohk1O69aFiKkVGiPk= +github.com/moby/patternmatcher v0.6.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= github.com/moby/spdystream v0.5.0/go.mod h1:xBAYlnt/ay+11ShkdFKNAG7LsyK/tmNBVvVOwrfMgdI= github.com/moby/spdystream v0.5.1 h1:9sNYeYZUcci9R6/w7KDaFWEWeV4LStVG78Mpyq/Zm/Y= github.com/moby/spdystream v0.5.1/go.mod h1:xBAYlnt/ay+11ShkdFKNAG7LsyK/tmNBVvVOwrfMgdI= github.com/moby/sys/mountinfo v0.7.2 h1:1shs6aH5s4o5H2zQLn796ADW1wMrIwHsyJ2v9KouLrg= github.com/moby/sys/mountinfo v0.7.2/go.mod h1:1YOa8w8Ih7uW0wALDUgT1dTTSBrZ+HiBLGws92L2RU4= +github.com/moby/sys/sequential v0.6.0 h1:qrx7XFUd/5DxtqcoH1h438hF5TmOvzC/lspjy7zgvCU= +github.com/moby/sys/sequential v0.6.0/go.mod h1:uyv8EUTrca5PnDsdMGXhZe6CCe8U/UiTWd+lL+7b/Ko= +github.com/moby/sys/user v0.4.0 h1:jhcMKit7SA80hivmFJcbB1vqmw//wU61Zdui2eQXuMs= +github.com/moby/sys/user v0.4.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85uoHjs= github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g= github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28= github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= @@ -365,12 +403,16 @@ github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFd github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw= github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= +github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 h1:n6/2gBQ3RWajuToeY6ZtZTIKv2v7ThUy5KKusIT0yc0= +github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4= github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= @@ -398,8 +440,8 @@ github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJw github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= github.com/opencontainers/runtime-spec v1.2.1 h1:S4k4ryNgEpxW1dzyqffOmhI1BHYcjzU8lpJfSlR0xww= github.com/opencontainers/runtime-spec v1.2.1/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= -github.com/opencontainers/selinux v1.13.0 h1:Zza88GWezyT7RLql12URvoxsbLfjFx988+LGaWfbL84= -github.com/opencontainers/selinux v1.13.0/go.mod h1:XxWTed+A/s5NNq4GmYScVy+9jzXhGBVEOAyucdRUY8s= +github.com/opencontainers/selinux v1.13.1 h1:A8nNeceYngH9Ow++M+VVEwJVpdFmrlxsN22F+ISDCJE= +github.com/opencontainers/selinux v1.13.1/go.mod h1:S10WXZ/osk2kWOYKy1x2f/eXF5ZHJoUs8UU/2caNRbg= github.com/openshift-eng/openshift-tests-extension v0.0.0-20260127124016-0fed2b824818 h1:jJLE/aCAqDf8U4wc3bE1IEKgIxbb0ICjCNVFA49x/8s= github.com/openshift-eng/openshift-tests-extension v0.0.0-20260127124016-0fed2b824818/go.mod h1:6gkP5f2HL0meusT0Aim8icAspcD1cG055xxBZ9yC68M= github.com/openshift-kni/k8sreporter v1.0.6 h1:aaxDzZx3s9bo1I3nopR63RGVZxcJgR94j5X87aDihYo= @@ -424,6 +466,8 @@ github.com/openshift/kubernetes/staging/src/k8s.io/client-go v0.0.0-202603130234 github.com/openshift/kubernetes/staging/src/k8s.io/client-go v0.0.0-20260313023409-2034d92b4a3a/go.mod h1:sxLPgzwe7FZzbSa2jq8GNA8cy11BWiWWruFwYo29fis= github.com/openshift/kubernetes/staging/src/k8s.io/cloud-provider v0.0.0-20260313023409-2034d92b4a3a h1:sKeTe1RQlC6MTQpl7vTTc5JEzdR023h7wdTpTuiXBIg= github.com/openshift/kubernetes/staging/src/k8s.io/cloud-provider v0.0.0-20260313023409-2034d92b4a3a/go.mod h1:0RXKUJuaCwy8B4rj3V35zJ52G26pF5n2P4vQVwpgKFk= +github.com/openshift/kubernetes/staging/src/k8s.io/cluster-bootstrap v0.0.0-20260313023409-2034d92b4a3a h1:2l4sLOk39OywkVzTGUclSNyHiiKUkMUaK5cnL7AxmSM= +github.com/openshift/kubernetes/staging/src/k8s.io/cluster-bootstrap v0.0.0-20260313023409-2034d92b4a3a/go.mod h1:gMUCCRWiGtUnB6NpqpXAoNTnvyjCF1Jn2/Af6Oa4Uj4= github.com/openshift/kubernetes/staging/src/k8s.io/component-base v0.0.0-20260313023409-2034d92b4a3a h1:LDfkMgpDBm6LjZ9dtOnb7EAJ5mKoKe3wvB47oY6zI/E= github.com/openshift/kubernetes/staging/src/k8s.io/component-base v0.0.0-20260313023409-2034d92b4a3a/go.mod h1:UNoqyTPeVLhnE4PLihJHeu1Wn7s0/PVFxbbx901ww14= github.com/openshift/kubernetes/staging/src/k8s.io/component-helpers v0.0.0-20260313023409-2034d92b4a3a h1:JbOUoV7CXD3cNW3OklzcD7J1GColOXrpSoVj3wsAvlg= @@ -450,13 +494,20 @@ github.com/openshift/kubernetes/staging/src/k8s.io/mount-utils v0.0.0-2026031302 github.com/openshift/kubernetes/staging/src/k8s.io/mount-utils v0.0.0-20260313023409-2034d92b4a3a/go.mod h1:sL4snITWGw8a7Yqz2+zD+X44bEq+HtRTihlZLArDNqI= github.com/openshift/kubernetes/staging/src/k8s.io/pod-security-admission v0.0.0-20260313023409-2034d92b4a3a h1:9LfgYMod0ZiYvk2NZBzoHE/Ntpgt7qcK2n6mdChoTMA= github.com/openshift/kubernetes/staging/src/k8s.io/pod-security-admission v0.0.0-20260313023409-2034d92b4a3a/go.mod h1:6wqqkK0+5hV+CLJ3uz9A1lkjxXRvkbq+5RnZdUZx/H8= +github.com/openshift/kubernetes/staging/src/k8s.io/sample-apiserver v0.0.0-20260313023409-2034d92b4a3a h1:HeGw4Eg8Ev0LqzOcdG7Ms+39TynUGFbH55bfdW6El+0= +github.com/openshift/kubernetes/staging/src/k8s.io/sample-apiserver v0.0.0-20260313023409-2034d92b4a3a/go.mod h1:CnFDBq5NGnfOSMeOP8l4SNYJrxK6Z1kUaKdu3Qq9Uik= github.com/openshift/library-go v0.0.0-20260303171201-5d9eb6295ff6 h1:xjqy0OolrFdJ+ofI/aD0+2k9+MSk5anP5dXifFt539Q= github.com/openshift/library-go v0.0.0-20260303171201-5d9eb6295ff6/go.mod h1:D797O/ssKTNglbrGchjIguFq+DbyRYdeds5w4/VTrKM= github.com/openshift/onsi-ginkgo/v2 v2.6.1-0.20260303184444-1cc650aa0565 h1:3/q8qM4HbFa+Een8wgzpwO8W6mO7Po+MwY6uxiXi/ac= github.com/openshift/onsi-ginkgo/v2 v2.6.1-0.20260303184444-1cc650aa0565/go.mod h1:ArE1D/XhNXBXCBkKOLkbsb2c81dQHCRcF5zwn/ykDRo= +github.com/openshift/origin v1.5.0-alpha.3.0.20260310231025-5d3fd0545b5d h1:P5WxSxg/87liefXJZmQvWvvwHY/D+Q0oE3/OOO2PPbU= +github.com/openshift/origin v1.5.0-alpha.3.0.20260310231025-5d3fd0545b5d/go.mod h1:e/erYniFBXOFFzOwXxVxkPI7zXFbv1WD1tgOZnw89xc= github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0= github.com/ovn-kubernetes/dpu-simulator/lib/dpusim v0.0.0-20260507161134-3aec48e5cb25 h1:uujE+MtjuaWOz1+nt5uwj193CQEXOKR8qzhsRRV7t9o= github.com/ovn-kubernetes/dpu-simulator/lib/dpusim v0.0.0-20260507161134-3aec48e5cb25/go.mod h1:fkgl6NeQ4GO+DImnBM1MQKA5fT3BeALC+pOBaNvlVLs= +github.com/pborman/uuid v1.2.0 h1:J7Q5mO4ysT1dv8hyrUGHb9+ooztCXu1D8MY8DZYsu3g= +github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= +github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -472,8 +523,8 @@ github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNw github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs= github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA= -github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= -github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= +github.com/prometheus/procfs v0.17.0 h1:FuLQ+05u4ZI+SS/w9+BWEM2TXiHKsUQ9TADiRH7DuK0= +github.com/prometheus/procfs v0.17.0/go.mod h1:oPQLaDAMRbA+u8H5Pbfq+dl3VDAvHxMUOVhe0wYB2zw= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= @@ -487,6 +538,8 @@ github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/safchain/ethtool v0.6.2 h1:O3ZPFAKEUEfbtE6J/feEe2Ft7dIJ2Sy8t4SdMRiIMHY= github.com/safchain/ethtool v0.6.2/go.mod h1:VS7cn+bP3Px3rIq55xImBiZGHVLNyBh5dqG6dDQy8+I= +github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= +github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js= @@ -494,16 +547,15 @@ github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/afero v1.14.0 h1:9tH6MapGnn/j0eb0yIXiLjERO8RB6xIVZRDCX7PtqWA= github.com/spf13/afero v1.14.0/go.mod h1:acJQ8t0ohCGuMN3O+Pv0V0hgMxNYDlvdk+VTfyZmbYo= -github.com/spf13/cobra v1.10.0 h1:a5/WeUlSDCvV5a45ljW2ZFtV0bTDpkfSAj3uqB6Sc+0= -github.com/spf13/cobra v1.10.0/go.mod h1:9dhySC7dnTtEiqzmqfkLj47BslqLCUPMXjG2lj/NgoE= +github.com/spf13/cobra v1.10.1 h1:lJeBwCfmrnXthfAupyUTzJ/J4Nc1RsHC/mSRU2dll/s= +github.com/spf13/cobra v1.10.1/go.mod h1:7SmJGaTHFVBY0jW4NXGluQoLvhqFQM+6XSKD+P4XaB0= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/pflag v1.0.8/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY= github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= -github.com/stoewer/go-strcase v1.3.0 h1:g0eASXYtp+yvN9fK8sH94oCIk0fau9uV1/ZdJ0AVEzs= -github.com/stoewer/go-strcase v1.3.0/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8wodgtPmh1xo= +github.com/stoewer/go-strcase v1.3.1 h1:iS0MdW+kVTxgMoE1LAZyMiYJFKlOzLooE4MxjirtkAs= +github.com/stoewer/go-strcase v1.3.1/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8wodgtPmh1xo= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -547,6 +599,8 @@ github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/xiang90/probing v0.0.0-20221125231312-a49e3df8f510 h1:S2dVYn90KE98chqDkyE9Z4N61UnQd+KOfgp5Iu53llk= github.com/xiang90/probing v0.0.0-20221125231312-a49e3df8f510/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xlab/treeprint v1.2.0 h1:HzHnuAF1plUN2zGlAFHbSQP2qJ0ZAD3XF5XD7OesXRQ= +github.com/xlab/treeprint v1.2.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0= github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM= github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 h1:+qGGcbkzsfDQNPPe9UDgpxAWQrhbbBXOYJFQDq/dtJw= github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913/go.mod h1:4aEEwZQutDLsQv2Deui4iYQ6DWTxR14g6m8Wv88+Xqk= @@ -575,8 +629,8 @@ go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib/instrumentation/github.com/emicklei/go-restful/otelrestful v0.44.0 h1:KemlMZlVwBSEGaO91WKgp41BBFsnWqqj9sKRwmOqC40= go.opentelemetry.io/contrib/instrumentation/github.com/emicklei/go-restful/otelrestful v0.44.0/go.mod h1:uq8DrRaen3suIWTpdR/JNHCGpurSvMv9D5Nr5CU5TXc= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0 h1:x7wzEgXfnzJcHDwStJT+mxOz4etr2EcexjqhBvmoakw= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0/go.mod h1:rg+RlpR5dKwaS95IyyZqj5Wd4E13lk/msnTS0Xl9lJM= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0 h1:q4XOmH/0opmeuJtPsbFNivyl7bCt7yRBbeEm2sC/XtQ= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0/go.mod h1:snMWehoOh2wsEwnvvwtDyFCxVeDAODenXHtn5vzrKjo= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 h1:F7Jx+6hwnZ41NSFTO5q4LYDtJRXBf2PD0rNBkeB/lus= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0/go.mod h1:UHB22Z8QsdRDrnAtX4PntOl36ajSxcdUMt1sF7Y6E7Q= go.opentelemetry.io/contrib/propagators/b3 v1.19.0 h1:ulz44cpm6V5oAeg5Aw9HyqGFMS6XM7untlMEhD7YzzA= @@ -585,8 +639,8 @@ go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I= go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0 h1:88Y4s2C8oTui1LGM6bTWkw0ICGcOLCAI5l6zsD1j20k= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0/go.mod h1:Vl1/iaggsuRlrHf/hfPJPvVag77kKyvrLeD10kpMl+A= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.34.0 h1:tgJ0uaNS4c98WRNUEx5U3aDlrDOI5Rs+1Vifcw4DJ8U= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.34.0/go.mod h1:U7HYyW0zt/a9x5J1Kjs+r1f/d4ZHnYFclhYY2+YbeoE= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.36.0 h1:JgtbA0xkWHnTmYk7YusopJFX6uleBmAuZ8n05NEh8nQ= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.36.0/go.mod h1:179AK5aar5R3eS9FucPy6rggvU0g52cvKId8pv4+v0c= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0 h1:3iZJKlCZufyRzPzlQhUIWVmfltrXuGyfjREgGP3UUjc= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.43.0/go.mod h1:/G+nUPfhq2e+qiXMGxMwumDrP5jtzU+mWN7/sjT2rak= go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM= @@ -637,8 +691,8 @@ golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4= golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20220909182711-5c715a9e8561/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE= -golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= -golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= +golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b h1:M2rDM6z3Fhozi9O7NWsxAkg/yqS/lQJ6PmkyIV3YP+o= +golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b/go.mod h1:3//PLf8L/X+8b4vuAfHzxeRUl04Adcb341+IGKfnqS8= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= @@ -686,7 +740,6 @@ golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= @@ -742,7 +795,6 @@ golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -759,7 +811,6 @@ golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -788,7 +839,6 @@ golang.org/x/telemetry v0.0.0-20250908211612-aef8a434d053/go.mod h1:+nZKN+XVh4LC golang.org/x/telemetry v0.0.0-20251008203120-078029d740a8/go.mod h1:Pi4ztBfryZoJEkyFTI5/Ocsu2jXyDr6iSdgJiYE/uwE= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= @@ -812,7 +862,6 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= @@ -830,8 +879,8 @@ golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8= golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA= golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= -golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0= -golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= +golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE= +golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= @@ -917,6 +966,8 @@ gopkg.in/evanphx/json-patch.v4 v4.13.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWM gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/gcfg.v1 v1.2.3 h1:m8OOJ4ccYHnx2f4gQwpno8nAX5OGOh7RLaaz0pj3Ogs= gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= +gopkg.in/go-jose/go-jose.v2 v2.6.3 h1:nt80fvSDlhKWQgSWyHyy5CfmlQr+asih51R8PTWNKKs= +gopkg.in/go-jose/go-jose.v2 v2.6.3/go.mod h1:zzZDPkNNw/c9IE7Z9jr11mBZQhKQTMzoEEIoEdZlFBI= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= @@ -941,6 +992,8 @@ gotest.tools/v3 v3.5.0 h1:Ljk6PdHdOhAb5aDMWXjDLMMhph+BpztA4v1QdqEW2eY= gotest.tools/v3 v3.5.0/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +k8s.io/cli-runtime v0.33.4 h1:V8NSxGfh24XzZVhXmIGzsApdBpGq0RQS2u/Fz1GvJwk= +k8s.io/cli-runtime v0.33.4/go.mod h1:V+ilyokfqjT5OI+XE+O515K7jihtr0/uncwoyVqXaIU= k8s.io/code-generator v0.22.7/go.mod h1:iOZwYADSgFPNGWfqHFfg1V0TNJnl1t0WyZluQp4baqU= k8s.io/code-generator v0.23.3/go.mod h1:S0Q1JVA+kSzTI1oUvbKAxZY/DYbA/ZUb4Uknog12ETk= k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= @@ -958,6 +1011,8 @@ k8s.io/klog/v2 v2.40.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/klog/v2 v2.80.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/kube-aggregator v0.35.1 h1:LN+btMJ3yp7biqVgT/0LF6SKIKLyfPU0R+JJ1mycs2I= +k8s.io/kube-aggregator v0.35.1/go.mod h1:HQSjPQfOFRzcv7biQ7jV3cEfKHG+bczpLCfh4QfvxZU= k8s.io/kube-openapi v0.0.0-20211109043538-20434351676c/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw= k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk= k8s.io/kube-openapi v0.0.0-20220124234850-424119656bbf/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk= @@ -979,11 +1034,19 @@ sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.33.0 h1:qPrZsv1cwQiFe sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.33.0/go.mod h1:Ve9uj1L+deCXFrPOk1LpFXqTg7LCFzFso6PA48q/XZw= sigs.k8s.io/controller-runtime v0.23.1 h1:TjJSM80Nf43Mg21+RCy3J70aj/W6KyvDtOlpKf+PupE= sigs.k8s.io/controller-runtime v0.23.1/go.mod h1:B6COOxKptp+YaUT5q4l6LqUJTRpizbgf9KSRNdQGns0= +sigs.k8s.io/gateway-api v1.4.0 h1:ZwlNM6zOHq0h3WUX2gfByPs2yAEsy/EenYJB78jpQfQ= +sigs.k8s.io/gateway-api v1.4.0/go.mod h1:AR5RSqciWP98OPckEjOjh2XJhAe2Na4LHyXD2FUY7Qk= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg= sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= sigs.k8s.io/knftables v0.0.20 h1:eU2NWpgcJ/wgb4Fy0cX3klK6nDjERvZRdYgkORLU0Tc= sigs.k8s.io/knftables v0.0.20/go.mod h1:f/5ZLKYEUPUhVjUCg6l80ACdL7CIIyeL0DxfgojGRTk= +sigs.k8s.io/kube-storage-version-migrator v0.0.6-0.20230721195810-5c8923c5ff96 h1:PFWFSkpArPNJxFX4ZKWAk9NSeRoZaXschn+ULa4xVek= +sigs.k8s.io/kube-storage-version-migrator v0.0.6-0.20230721195810-5c8923c5ff96/go.mod h1:EOBQyBowOUsd7U4CJnMHNE0ri+zCXyouGdLwC/jZU+I= +sigs.k8s.io/kustomize/api v0.20.1 h1:iWP1Ydh3/lmldBnH/S5RXgT98vWYMaTUL1ADcr+Sv7I= +sigs.k8s.io/kustomize/api v0.20.1/go.mod h1:t6hUFxO+Ph0VxIk1sKp1WS0dOjbPCtLJ4p8aADLwqjM= +sigs.k8s.io/kustomize/kyaml v0.20.1 h1:PCMnA2mrVbRP3NIB6v9kYCAc38uvFLVs8j/CD567A78= +sigs.k8s.io/kustomize/kyaml v0.20.1/go.mod h1:0EmkQHRUsJxY8Ug9Niig1pUMSCGHxQ5RklbpV/Ri6po= sigs.k8s.io/network-policy-api v0.1.5 h1:xyS7VAaM9EfyB428oFk7WjWaCK6B129i+ILUF4C8l6E= sigs.k8s.io/network-policy-api v0.1.5/go.mod h1:D7Nkr43VLNd7iYryemnj8qf0N/WjBzTZDxYA+g4u1/Y= sigs.k8s.io/randfill v0.0.0-20250304075658-069ef1bbf016/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= diff --git a/openshift/pkg/otp/testdata/bindata.go b/openshift/pkg/otp/testdata/bindata.go new file mode 100644 index 0000000000..aeed529c0c --- /dev/null +++ b/openshift/pkg/otp/testdata/bindata.go @@ -0,0 +1,44 @@ +package testdata + +import ( + "embed" + "io/fs" + "os" + "path/filepath" + "sync" +) + +//go:embed all:networking +var embeddedFS embed.FS + +var ( + extractOnce sync.Once + extractDir string +) + +func FixturePath(elem ...string) string { + extractOnce.Do(func() { + dir, err := os.MkdirTemp("", "otp-testdata-") + if err != nil { + panic(err) + } + if err := fs.WalkDir(embeddedFS, ".", func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + target := filepath.Join(dir, path) + if d.IsDir() { + return os.MkdirAll(target, 0755) + } + data, err := embeddedFS.ReadFile(path) + if err != nil { + return err + } + return os.WriteFile(target, data, 0644) + }); err != nil { + panic(err) + } + extractDir = dir + }) + return filepath.Join(append([]string{extractDir}, elem...)...) +} diff --git a/openshift/pkg/otp/testdata/networking/OWNERS b/openshift/pkg/otp/testdata/networking/OWNERS new file mode 100644 index 0000000000..18ff1b3f4f --- /dev/null +++ b/openshift/pkg/otp/testdata/networking/OWNERS @@ -0,0 +1,4 @@ +approvers: + - zhaozhanqi + - asood-rh + - jechen0648 diff --git a/openshift/pkg/otp/testdata/networking/hello-pod-daemonset.yaml b/openshift/pkg/otp/testdata/networking/hello-pod-daemonset.yaml new file mode 100644 index 0000000000..fffd147e26 --- /dev/null +++ b/openshift/pkg/otp/testdata/networking/hello-pod-daemonset.yaml @@ -0,0 +1,29 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: hello-daemonset +spec: + selector: + matchLabels: + name: hello-pod + template: + metadata: + labels: + name: hello-pod + spec: + nodeSelector: + kubernetes.io/os: linux + securityContext: + seccompProfile: + type: RuntimeDefault + containers: + - image: quay.io/openshifttest/hello-sdn@sha256:c89445416459e7adea9a5a416b3365ed3d74f2491beb904d61dc8d1eb89a72a4 + name: hello-pod + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + terminationGracePeriodSeconds: 10 + tolerations: + - operator: Exists + diff --git a/openshift/pkg/otp/testdata/networking/hellosdn.yaml b/openshift/pkg/otp/testdata/networking/hellosdn.yaml new file mode 100644 index 0000000000..324566242e --- /dev/null +++ b/openshift/pkg/otp/testdata/networking/hellosdn.yaml @@ -0,0 +1,58 @@ +{ + "apiVersion": "v1", + "kind": "List", + "items": [ + { + "apiVersion": "v1", + "kind": "ReplicationController", + "metadata": { + "labels": { + "name": "hello-sdn" + }, + "name": "hello-sdn" + }, + "spec": { + "replicas": 2, + "template": { + "metadata": { + "labels": { + "name": "hellosdn" + } + }, + "spec": { + "containers": [ + { + "image": "quay.io/openshifttest/hello-sdn@sha256:c89445416459e7adea9a5a416b3365ed3d74f2491beb904d61dc8d1eb89a72a4", + "name": "hellosdn", + "imagePullPolicy": "IfNotPresent" + } + ] + } + } + } + }, + { + "apiVersion": "v1", + "kind": "Service", + "metadata": { + "labels": { + "name": "hello-service" + }, + "name": "hello-service" + }, + "spec": { + "ports": [ + { + "name": "http", + "port": 27017, + "protocol": "TCP", + "targetPort": 8080 + } + ], + "selector": { + "name": "hellosdn" + } + } + } + ] +} diff --git a/openshift/pkg/otp/testdata/networking/network_segmentation/multicast/multicast-rc.json b/openshift/pkg/otp/testdata/networking/network_segmentation/multicast/multicast-rc.json new file mode 100644 index 0000000000..14e87ec099 --- /dev/null +++ b/openshift/pkg/otp/testdata/networking/network_segmentation/multicast/multicast-rc.json @@ -0,0 +1,47 @@ +{ + "apiVersion": "template.openshift.io/v1", + "kind": "Template", + "metadata": { + "name": "mcast-pod-template" + }, + "objects": [ + { + "apiVersion": "v1", + "kind": "ReplicationController", + "metadata": { + "labels": { + "name": "${{RCNAME}}" + }, + "name": "${{RCNAME}}" + }, + "spec": { + "replicas": "${{REPLICAS}}", + "template": { + "metadata": { + "labels": { + "name": "${{RCNAME}}" + } + }, + "spec": { + "containers": [ + { + "image": "quay.io/openshifttest/mcast-pod@sha256:aa2a8fdcfeddb09097e6a7201cfdb062db8d92d523cbcf758ef7d3545a1f5776", + "name": "mcast-rc" + } + ] + } + } + } + } + ], + "parameters": [ + { + "name": "RCNAME" + }, + { + "name": "REPLICAS", + "displayName": "Replicas", + "value": "3" + } + ] +} diff --git a/openshift/pkg/otp/testdata/networking/network_segmentation/networkpolicy/allow-to-all-namespaces.yaml b/openshift/pkg/otp/testdata/networking/network_segmentation/networkpolicy/allow-to-all-namespaces.yaml new file mode 100644 index 0000000000..1118615cc4 --- /dev/null +++ b/openshift/pkg/otp/testdata/networking/network_segmentation/networkpolicy/allow-to-all-namespaces.yaml @@ -0,0 +1,14 @@ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-to-all-namespaces +spec: + podSelector: {} + egress: + - ports: + - protocol: TCP + port: 8080 + to: + - namespaceSelector: {} + policyTypes: + - Egress diff --git a/openshift/pkg/otp/testdata/networking/network_segmentation/rc-ping-for-pod-template.yaml b/openshift/pkg/otp/testdata/networking/network_segmentation/rc-ping-for-pod-template.yaml new file mode 100644 index 0000000000..a7fb676cde --- /dev/null +++ b/openshift/pkg/otp/testdata/networking/network_segmentation/rc-ping-for-pod-template.yaml @@ -0,0 +1,34 @@ +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + name: rc-ping-for-pod-template +objects: +- kind: ReplicationController + apiVersion: v1 + metadata: + name: "${PODNAME}" + namespace: "${NAMESPACE}" + labels: + name: "${PODNAME}" + spec: + replicas: ${{REPLICAS}} + template: + metadata: + labels: + name: "${PODNAME}" + spec: + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + containers: + - image: "quay.io/openshifttest/hello-sdn@sha256:c89445416459e7adea9a5a416b3365ed3d74f2491beb904d61dc8d1eb89a72a4" + name: "test-pod" + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] +parameters: +- name: PODNAME +- name: NAMESPACE +- name: REPLICAS diff --git a/openshift/pkg/otp/testdata/networking/network_segmentation/sctp/load-sctp-module.yaml b/openshift/pkg/otp/testdata/networking/network_segmentation/sctp/load-sctp-module.yaml new file mode 100644 index 0000000000..e13a6dfebb --- /dev/null +++ b/openshift/pkg/otp/testdata/networking/network_segmentation/sctp/load-sctp-module.yaml @@ -0,0 +1,23 @@ +apiVersion: machineconfiguration.openshift.io/v1 +kind: MachineConfig +metadata: + labels: + machineconfiguration.openshift.io/role: worker + name: load-sctp-module +spec: + config: + ignition: + version: 2.2.0 + storage: + files: + - contents: + source: data:, + verification: {} + filesystem: root + mode: 420 + path: /etc/modprobe.d/sctp-blacklist.conf + - contents: + source: data:text/plain;charset=utf-8,sctp + filesystem: root + mode: 420 + path: /etc/modules-load.d/sctp-load.conf diff --git a/openshift/pkg/otp/testdata/networking/network_segmentation/udn/cudn_crd_dualstack_template.yaml b/openshift/pkg/otp/testdata/networking/network_segmentation/udn/cudn_crd_dualstack_template.yaml new file mode 100644 index 0000000000..5cdf7e1ac4 --- /dev/null +++ b/openshift/pkg/otp/testdata/networking/network_segmentation/udn/cudn_crd_dualstack_template.yaml @@ -0,0 +1,32 @@ +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + name: cudn-crd-dualstack-template +objects: + - apiVersion: k8s.ovn.org/v1 + kind: ClusterUserDefinedNetwork + metadata: + name: "${CRDNAME}" + spec: + namespaceSelector: + matchLabels: + "${LABELKEY}": "${LABELVALUE}" + network: + topology: Layer3 + layer3: + role: "${ROLE}" + subnets: + - cidr: "${IPv4CIDR}" + hostSubnet: ${{IPv4PREFIX}} + - cidr: "${IPv6CIDR}" + hostSubnet: ${{IPv6PREFIX}} +parameters: +- name: CRDNAME +- name: LABELVALUE +- name: LABELKEY +- name: ROLE +- name: IPv4CIDR +- name: IPv4PREFIX +- name: IPv6CIDR +- name: IPv6PREFIX + diff --git a/openshift/pkg/otp/testdata/networking/network_segmentation/udn/cudn_crd_layer2_dualstack_template.yaml b/openshift/pkg/otp/testdata/networking/network_segmentation/udn/cudn_crd_layer2_dualstack_template.yaml new file mode 100644 index 0000000000..7a95dd3b27 --- /dev/null +++ b/openshift/pkg/otp/testdata/networking/network_segmentation/udn/cudn_crd_layer2_dualstack_template.yaml @@ -0,0 +1,26 @@ +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + name: cudn-crd-layer2-dualstack-template +objects: + - apiVersion: k8s.ovn.org/v1 + kind: ClusterUserDefinedNetwork + metadata: + name: "${CRDNAME}" + spec: + namespaceSelector: + matchLabels: + "${LABELKEY}": "${LABELVALUE}" + network: + topology: Layer2 + layer2: + role: "${ROLE}" + subnets: ["${IPv4CIDR}", "${IPv6CIDR}"] +parameters: +- name: CRDNAME +- name: LABELVALUE +- name: LABELKEY +- name: ROLE +- name: IPv4CIDR +- name: IPv6CIDR + diff --git a/openshift/pkg/otp/testdata/networking/network_segmentation/udn/cudn_crd_layer2_singlestack_template.yaml b/openshift/pkg/otp/testdata/networking/network_segmentation/udn/cudn_crd_layer2_singlestack_template.yaml new file mode 100644 index 0000000000..4ab4cf7a55 --- /dev/null +++ b/openshift/pkg/otp/testdata/networking/network_segmentation/udn/cudn_crd_layer2_singlestack_template.yaml @@ -0,0 +1,25 @@ +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + name: cudn-crd-layer2-singlestack-template +objects: + - apiVersion: k8s.ovn.org/v1 + kind: ClusterUserDefinedNetwork + metadata: + name: "${CRDNAME}" + spec: + namespaceSelector: + matchLabels: + "${LABELKEY}": "${LABELVALUE}" + network: + topology: Layer2 + layer2: + role: "${ROLE}" + subnets: ["${CIDR}"] +parameters: +- name: CRDNAME +- name: LABELVALUE +- name: LABELKEY +- name: CIDR +- name: ROLE + diff --git a/openshift/pkg/otp/testdata/networking/network_segmentation/udn/cudn_crd_localnet_singlestack_template.yaml b/openshift/pkg/otp/testdata/networking/network_segmentation/udn/cudn_crd_localnet_singlestack_template.yaml new file mode 100644 index 0000000000..dcbfc2c9ef --- /dev/null +++ b/openshift/pkg/otp/testdata/networking/network_segmentation/udn/cudn_crd_localnet_singlestack_template.yaml @@ -0,0 +1,30 @@ +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + name: cudn-crd-localnet-singlestack-template +objects: + - apiVersion: k8s.ovn.org/v1 + kind: ClusterUserDefinedNetwork + metadata: + name: "${CRDNAME}" + spec: + namespaceSelector: + matchLabels: + "${LABELKEY}": "${LABELVALUE}" + network: + topology: Localnet + localnet: + role: "${ROLE}" + physicalNetworkName: "${PHYSICALNETWORK}" + subnets: + - ${{SUBNET}} + excludeSubnets: + - ${{EXCLUDESUBNET}} +parameters: +- name: CRDNAME +- name: LABELVALUE +- name: LABELKEY +- name: ROLE +- name: PHYSICALNETWORK +- name: SUBNET +- name: EXCLUDESUBNET diff --git a/openshift/pkg/otp/testdata/networking/network_segmentation/udn/cudn_crd_localnet_singlestack_with_vlan_template.yaml b/openshift/pkg/otp/testdata/networking/network_segmentation/udn/cudn_crd_localnet_singlestack_with_vlan_template.yaml new file mode 100644 index 0000000000..17cbe7369e --- /dev/null +++ b/openshift/pkg/otp/testdata/networking/network_segmentation/udn/cudn_crd_localnet_singlestack_with_vlan_template.yaml @@ -0,0 +1,34 @@ +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + name: cudn-crd-localnet-singlestack-with-vlan-template +objects: + - apiVersion: k8s.ovn.org/v1 + kind: ClusterUserDefinedNetwork + metadata: + name: "${CRDNAME}" + spec: + namespaceSelector: + matchLabels: + "${LABELKEY}": "${LABELVALUE}" + network: + topology: Localnet + localnet: + role: "${ROLE}" + physicalNetworkName: "${PHYSICALNETWORK}" + vlan: + mode: Access + access: + id: 50 + subnets: + - ${{SUBNET}} + excludeSubnets: + - ${{EXCLUDESUBNET}} +parameters: +- name: CRDNAME +- name: LABELVALUE +- name: LABELKEY +- name: ROLE +- name: PHYSICALNETWORK +- name: SUBNET +- name: EXCLUDESUBNET diff --git a/openshift/pkg/otp/testdata/networking/network_segmentation/udn/cudn_crd_matchexp_dualstack_template.yaml b/openshift/pkg/otp/testdata/networking/network_segmentation/udn/cudn_crd_matchexp_dualstack_template.yaml new file mode 100644 index 0000000000..09006425f5 --- /dev/null +++ b/openshift/pkg/otp/testdata/networking/network_segmentation/udn/cudn_crd_matchexp_dualstack_template.yaml @@ -0,0 +1,38 @@ +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + name: cudn-crd-matchexp-dualstack-template +objects: + - apiVersion: k8s.ovn.org/v1 + kind: ClusterUserDefinedNetwork + metadata: + name: "${CRDNAME}" + spec: + namespaceSelector: + matchExpressions: + - key: "${KEY}" + operator: "${OPERATOR}" + values: + - "${VALUE1}" + - "${VALUE2}" + network: + topology: Layer3 + layer3: + role: "${ROLE}" + subnets: + - cidr: "${IPv4CIDR}" + hostSubnet: ${{IPv4PREFIX}} + - cidr: "${IPv6CIDR}" + hostSubnet: ${{IPv6PREFIX}} +parameters: +- name: CRDNAME +- name: VALUE1 +- name: VALUE2 +- name: KEY +- name: OPERATOR +- name: PREFIX +- name: ROLE +- name: IPv4CIDR +- name: IPv4PREFIX +- name: IPv6CIDR +- name: IPv6PREFIX \ No newline at end of file diff --git a/openshift/pkg/otp/testdata/networking/network_segmentation/udn/cudn_crd_matchexp_layer2_dualstack_template.yaml b/openshift/pkg/otp/testdata/networking/network_segmentation/udn/cudn_crd_matchexp_layer2_dualstack_template.yaml new file mode 100644 index 0000000000..faff9f4112 --- /dev/null +++ b/openshift/pkg/otp/testdata/networking/network_segmentation/udn/cudn_crd_matchexp_layer2_dualstack_template.yaml @@ -0,0 +1,33 @@ +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + name: cudn-crd-matchexp-layer2-dualstack-template +objects: + - apiVersion: k8s.ovn.org/v1 + kind: ClusterUserDefinedNetwork + metadata: + name: "${CRDNAME}" + spec: + namespaceSelector: + matchExpressions: + - key: "${KEY}" + operator: "${OPERATOR}" + values: + - "${VALUE1}" + - "${VALUE2}" + network: + topology: Layer2 + layer2: + role: "${ROLE}" + subnets: ["${IPv4CIDR}", "${IPv6CIDR}"] +parameters: +- name: CRDNAME +- name: VALUE1 +- name: VALUE2 +- name: KEY +- name: OPERATOR +- name: PREFIX +- name: ROLE +- name: IPv4CIDR +- name: IPv6CIDR + diff --git a/openshift/pkg/otp/testdata/networking/network_segmentation/udn/cudn_crd_matchexp_layer2_singlestack_template.yaml b/openshift/pkg/otp/testdata/networking/network_segmentation/udn/cudn_crd_matchexp_layer2_singlestack_template.yaml new file mode 100644 index 0000000000..b84c19f4dd --- /dev/null +++ b/openshift/pkg/otp/testdata/networking/network_segmentation/udn/cudn_crd_matchexp_layer2_singlestack_template.yaml @@ -0,0 +1,30 @@ +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + name: cudn-crd-matchexp-layer2-singlestack-template +objects: + - apiVersion: k8s.ovn.org/v1 + kind: ClusterUserDefinedNetwork + metadata: + name: "${CRDNAME}" + spec: + namespaceSelector: + matchExpressions: + - key: "${KEY}" + operator: "${OPERATOR}" + values: + - "${VALUE1}" + - "${VALUE2}" + network: + topology: Layer2 + layer2: + role: "${ROLE}" + subnets: ["${CIDR}"] +parameters: +- name: CRDNAME +- name: VALUE1 +- name: VALUE2 +- name: KEY +- name: OPERATOR +- name: CIDR +- name: ROLE diff --git a/openshift/pkg/otp/testdata/networking/network_segmentation/udn/cudn_crd_matchexp_singlestack_template.yaml b/openshift/pkg/otp/testdata/networking/network_segmentation/udn/cudn_crd_matchexp_singlestack_template.yaml new file mode 100644 index 0000000000..f75e604e88 --- /dev/null +++ b/openshift/pkg/otp/testdata/networking/network_segmentation/udn/cudn_crd_matchexp_singlestack_template.yaml @@ -0,0 +1,36 @@ +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + name: cudn-crd-matchexp-singlestack-template +objects: + - apiVersion: k8s.ovn.org/v1 + kind: ClusterUserDefinedNetwork + metadata: + name: "${CRDNAME}" + spec: + namespaceSelector: + matchExpressions: + - key: "${KEY}" + operator: "${OPERATOR}" + values: + - "${VALUE1}" + - "${VALUE2}" + network: + topology: Layer3 + layer3: + role: "${ROLE}" + subnets: + - cidr: "${CIDR}" + hostSubnet: ${{PREFIX}} +parameters: +- name: CRDNAME +- name: VALUE1 +- name: VALUE2 +- name: KEY +- name: OPERATOR +- name: CIDR +- name: PREFIX +- name: ROLE + + + diff --git a/openshift/pkg/otp/testdata/networking/network_segmentation/udn/cudn_crd_singlestack_template.yaml b/openshift/pkg/otp/testdata/networking/network_segmentation/udn/cudn_crd_singlestack_template.yaml new file mode 100644 index 0000000000..d24e452319 --- /dev/null +++ b/openshift/pkg/otp/testdata/networking/network_segmentation/udn/cudn_crd_singlestack_template.yaml @@ -0,0 +1,28 @@ +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + name: cudn-crd-singlestack-template +objects: + - apiVersion: k8s.ovn.org/v1 + kind: ClusterUserDefinedNetwork + metadata: + name: "${CRDNAME}" + spec: + namespaceSelector: + matchLabels: + "${LABELKEY}": "${LABELVALUE}" + network: + topology: Layer3 + layer3: + role: "${ROLE}" + subnets: + - cidr: "${CIDR}" + hostSubnet: ${{PREFIX}} +parameters: +- name: CRDNAME +- name: LABELVALUE +- name: LABELKEY +- name: CIDR +- name: PREFIX +- name: ROLE + diff --git a/openshift/pkg/otp/testdata/networking/network_segmentation/udn/udn_crd_dualstack2_template.yaml b/openshift/pkg/otp/testdata/networking/network_segmentation/udn/udn_crd_dualstack2_template.yaml new file mode 100644 index 0000000000..6ad3c891fc --- /dev/null +++ b/openshift/pkg/otp/testdata/networking/network_segmentation/udn/udn_crd_dualstack2_template.yaml @@ -0,0 +1,27 @@ +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + name: udn-crd-dualstack-template +objects: +- apiVersion: k8s.ovn.org/v1 + kind: UserDefinedNetwork + metadata: + name: "${CRDNAME}" + namespace: "${NAMESPACE}" + spec: + topology: Layer3 + layer3: + role: "${ROLE}" + subnets: + - cidr: "${IPv4CIDR}" + hostSubnet: ${{IPv4PREFIX}} + - cidr: "${IPv6CIDR}" + hostSubnet: ${{IPv6PREFIX}} +parameters: +- name: CRDNAME +- name: NAMESPACE +- name: ROLE +- name: IPv4CIDR +- name: IPv4PREFIX +- name: IPv6CIDR +- name: IPv6PREFIX diff --git a/openshift/pkg/otp/testdata/networking/network_segmentation/udn/udn_crd_layer2_dualstack_template.yaml b/openshift/pkg/otp/testdata/networking/network_segmentation/udn/udn_crd_layer2_dualstack_template.yaml new file mode 100644 index 0000000000..509d665154 --- /dev/null +++ b/openshift/pkg/otp/testdata/networking/network_segmentation/udn/udn_crd_layer2_dualstack_template.yaml @@ -0,0 +1,23 @@ +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + name: udn-crd-layer2-dualstack-template +objects: +- apiVersion: k8s.ovn.org/v1 + kind: UserDefinedNetwork + metadata: + name: "${CRDNAME}" + namespace: "${NAMESPACE}" + spec: + topology: Layer2 + layer2: + role: "${ROLE}" + subnets: ["${IPv4CIDR}","${IPv6CIDR}"] +parameters: +- name: CRDNAME +- name: NAMESPACE +- name: ROLE +- name: IPv4CIDR +- name: IPv6CIDR + + diff --git a/openshift/pkg/otp/testdata/networking/network_segmentation/udn/udn_crd_layer2_singlestack_template.yaml b/openshift/pkg/otp/testdata/networking/network_segmentation/udn/udn_crd_layer2_singlestack_template.yaml new file mode 100644 index 0000000000..011bb03243 --- /dev/null +++ b/openshift/pkg/otp/testdata/networking/network_segmentation/udn/udn_crd_layer2_singlestack_template.yaml @@ -0,0 +1,20 @@ +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + name: udn-crd-layer2-singlestack-template +objects: +- apiVersion: k8s.ovn.org/v1 + kind: UserDefinedNetwork + metadata: + name: "${CRDNAME}" + namespace: "${NAMESPACE}" + spec: + topology: Layer2 + layer2: + role: "${ROLE}" + subnets: ["${CIDR}"] +parameters: +- name: CRDNAME +- name: NAMESPACE +- name: CIDR +- name: ROLE diff --git a/openshift/pkg/otp/testdata/networking/network_segmentation/udn/udn_crd_singlestack_template.yaml b/openshift/pkg/otp/testdata/networking/network_segmentation/udn/udn_crd_singlestack_template.yaml new file mode 100644 index 0000000000..dc2b1aa11b --- /dev/null +++ b/openshift/pkg/otp/testdata/networking/network_segmentation/udn/udn_crd_singlestack_template.yaml @@ -0,0 +1,23 @@ +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + name: udn-crd-singlestack-template +objects: +- apiVersion: k8s.ovn.org/v1 + kind: UserDefinedNetwork + metadata: + name: "${CRDNAME}" + namespace: "${NAMESPACE}" + spec: + topology: Layer3 + layer3: + role: "${ROLE}" + subnets: + - cidr: "${CIDR}" + hostSubnet: ${{PREFIX}} +parameters: +- name: CRDNAME +- name: NAMESPACE +- name: CIDR +- name: PREFIX +- name: ROLE diff --git a/openshift/pkg/otp/testdata/networking/network_segmentation/udn/udn_nad_template.yaml b/openshift/pkg/otp/testdata/networking/network_segmentation/udn/udn_nad_template.yaml new file mode 100644 index 0000000000..4e1420b1b8 --- /dev/null +++ b/openshift/pkg/otp/testdata/networking/network_segmentation/udn/udn_nad_template.yaml @@ -0,0 +1,28 @@ +apiVersion: template.openshift.io/v1 +kind: Template +objects: +- kind: NetworkAttachmentDefinition + apiVersion: k8s.cni.cncf.io/v1 + metadata: + name: "${NADNAME}" + namespace: "${NAMESPACE}" + spec: + config: |2 + { + "cniVersion": "0.3.1", + "name": "${NAD_NETWORK_NAME}", + "type": "ovn-k8s-cni-overlay", + "topology":"${TOPOLOGY}", + "subnets": "${SUBNET}", + "netAttachDefName": "${NET_ATTACH_DEF_NAME}", + "role": "${ROLE}" + } + +parameters: +- name: NADNAME +- name: NAMESPACE +- name: NAD_NETWORK_NAME +- name: TOPOLOGY +- name: SUBNET +- name: NET_ATTACH_DEF_NAME +- name: ROLE diff --git a/openshift/pkg/otp/testdata/networking/network_segmentation/udn/udn_statefulset_template.yaml b/openshift/pkg/otp/testdata/networking/network_segmentation/udn/udn_statefulset_template.yaml new file mode 100644 index 0000000000..0bf30fdf4d --- /dev/null +++ b/openshift/pkg/otp/testdata/networking/network_segmentation/udn/udn_statefulset_template.yaml @@ -0,0 +1,46 @@ +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + name: udn-statefulset-template +objects: +- apiVersion: apps/v1 + kind: StatefulSet + metadata: + labels: + app: ${LABEL} + name: ${LABEL} + name: ${NAME} + namespace: ${NAMESPACE} + spec: + replicas: 5 + selector: + matchLabels: + app: ${LABEL} + serviceName: hello + template: + metadata: + labels: + app: ${LABEL} + annotations: + k8s.v1.cni.cncf.io/networks: '[{"name": "${NETWORK_NAME}","interface": "${INTERFACE_NAME}"}]' + spec: + containers: + - image: quay.io/openshifttest/hello-sdn@sha256:c89445416459e7adea9a5a416b3365ed3d74f2491beb904d61dc8d1eb89a72a4 + name: hello + ports: + - containerPort: 8080 + name: web + protocol: TCP + resources: + limits: + memory: 340Mi + restartPolicy: Always +parameters: +- name: NAME + value: "hello" +- name: NAMESPACE +- name: LABEL + value: "hello" +- name: NETWORK_NAME +- name: INTERFACE_NAME + value: "ovn-udn1" diff --git a/openshift/pkg/otp/testdata/networking/network_segmentation/udn/udn_test_pod_annotation_template.yaml b/openshift/pkg/otp/testdata/networking/network_segmentation/udn/udn_test_pod_annotation_template.yaml new file mode 100644 index 0000000000..d79924ecf4 --- /dev/null +++ b/openshift/pkg/otp/testdata/networking/network_segmentation/udn/udn_test_pod_annotation_template.yaml @@ -0,0 +1,31 @@ +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + name: udn-pod-annotation-template +objects: +- kind: Pod + apiVersion: v1 + metadata: + name: "${NAME}" + namespace: "${NAMESPACE}" + labels: + name: "${LABEL}" + annotations: + k8s.v1.cni.cncf.io/networks: "${ANNOTATION}" + spec: + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + containers: + - image: "quay.io/openshifttest/hello-sdn@sha256:c89445416459e7adea9a5a416b3365ed3d74f2491beb904d61dc8d1eb89a72a4" + name: hello-pod + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] +parameters: +- name: NAME +- name: NAMESPACE +- name: LABEL +- name: ANNOTATION diff --git a/openshift/pkg/otp/testdata/networking/network_segmentation/udn/udn_test_pod_annotation_template_node.yaml b/openshift/pkg/otp/testdata/networking/network_segmentation/udn/udn_test_pod_annotation_template_node.yaml new file mode 100644 index 0000000000..7550f7e26f --- /dev/null +++ b/openshift/pkg/otp/testdata/networking/network_segmentation/udn/udn_test_pod_annotation_template_node.yaml @@ -0,0 +1,33 @@ +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + name: udn-pod-annotation-template-node +objects: + - kind: Pod + apiVersion: v1 + metadata: + name: "${NAME}" + namespace: "${NAMESPACE}" + labels: + name: "${LABEL}" + annotations: + k8s.v1.cni.cncf.io/networks: "${NADNAME}" + spec: + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + containers: + - image: "quay.io/openshifttest/hello-sdn@sha256:c89445416459e7adea9a5a416b3365ed3d74f2491beb904d61dc8d1eb89a72a4" + name: hello-pod + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + nodeName: "${NODENAME}" +parameters: + - name: NAME + - name: NAMESPACE + - name: LABEL + - name: NADNAME + - name: NODENAME diff --git a/openshift/pkg/otp/testdata/networking/network_segmentation/udn/udn_test_pod_liveness_template.yaml b/openshift/pkg/otp/testdata/networking/network_segmentation/udn/udn_test_pod_liveness_template.yaml new file mode 100644 index 0000000000..6d569f6961 --- /dev/null +++ b/openshift/pkg/otp/testdata/networking/network_segmentation/udn/udn_test_pod_liveness_template.yaml @@ -0,0 +1,33 @@ +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + name: udn-pod-template +objects: +- kind: Pod + apiVersion: v1 + metadata: + name: "${NAME}" + namespace: "${NAMESPACE}" + labels: + name: "${LABEL}" + spec: + containers: + - name: hello-pod + image: "quay.io/openshifttest/hello-sdn@sha256:c89445416459e7adea9a5a416b3365ed3d74f2491beb904d61dc8d1eb89a72a4" + securityContext: + capabilities: + add: ["NET_ADMIN"] + privileged: true + livenessProbe: + httpGet: + path: / + port: ${{PORT}} + failureThreshold: ${{FAILURETHRESHOLD}} + periodSeconds: ${{PERIODSECONDS}} +parameters: +- name: NAME +- name: NAMESPACE +- name: LABEL +- name: PORT +- name: FAILURETHRESHOLD +- name: PERIODSECONDS diff --git a/openshift/pkg/otp/testdata/networking/network_segmentation/udn/udn_test_pod_readiness_template.yaml b/openshift/pkg/otp/testdata/networking/network_segmentation/udn/udn_test_pod_readiness_template.yaml new file mode 100644 index 0000000000..7aa1843ac5 --- /dev/null +++ b/openshift/pkg/otp/testdata/networking/network_segmentation/udn/udn_test_pod_readiness_template.yaml @@ -0,0 +1,33 @@ +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + name: udn-pod-template +objects: +- kind: Pod + apiVersion: v1 + metadata: + name: "${NAME}" + namespace: "${NAMESPACE}" + labels: + name: "${LABEL}" + spec: + containers: + - name: hello-pod + image: "quay.io/openshifttest/hello-sdn@sha256:c89445416459e7adea9a5a416b3365ed3d74f2491beb904d61dc8d1eb89a72a4" + securityContext: + capabilities: + add: ["NET_ADMIN"] + privileged: true + readinessProbe: + httpGet: + path: / + port: ${{PORT}} + failureThreshold: ${{FAILURETHRESHOLD}} + periodSeconds: ${{PERIODSECONDS}} +parameters: +- name: NAME +- name: NAMESPACE +- name: LABEL +- name: PORT +- name: FAILURETHRESHOLD +- name: PERIODSECONDS diff --git a/openshift/pkg/otp/testdata/networking/network_segmentation/udn/udn_test_pod_startup_template.yaml b/openshift/pkg/otp/testdata/networking/network_segmentation/udn/udn_test_pod_startup_template.yaml new file mode 100644 index 0000000000..d692cf7af5 --- /dev/null +++ b/openshift/pkg/otp/testdata/networking/network_segmentation/udn/udn_test_pod_startup_template.yaml @@ -0,0 +1,33 @@ +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + name: udn-pod-template +objects: +- kind: Pod + apiVersion: v1 + metadata: + name: "${NAME}" + namespace: "${NAMESPACE}" + labels: + name: "${LABEL}" + spec: + containers: + - name: hello-pod + image: "quay.io/openshifttest/hello-sdn@sha256:c89445416459e7adea9a5a416b3365ed3d74f2491beb904d61dc8d1eb89a72a4" + securityContext: + capabilities: + add: ["NET_ADMIN"] + privileged: true + startupProbe: + httpGet: + path: / + port: ${{PORT}} + failureThreshold: ${{FAILURETHRESHOLD}} + periodSeconds: ${{PERIODSECONDS}} +parameters: +- name: NAME +- name: NAMESPACE +- name: LABEL +- name: PORT +- name: FAILURETHRESHOLD +- name: PERIODSECONDS diff --git a/openshift/pkg/otp/testdata/networking/network_segmentation/udn/udn_test_pod_template.yaml b/openshift/pkg/otp/testdata/networking/network_segmentation/udn/udn_test_pod_template.yaml new file mode 100644 index 0000000000..51516d95e0 --- /dev/null +++ b/openshift/pkg/otp/testdata/networking/network_segmentation/udn/udn_test_pod_template.yaml @@ -0,0 +1,28 @@ +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + name: udn-pod-template +objects: +- kind: Pod + apiVersion: v1 + metadata: + name: "${NAME}" + namespace: "${NAMESPACE}" + labels: + name: "${LABEL}" + spec: + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + containers: + - image: "quay.io/openshifttest/hello-sdn@sha256:c89445416459e7adea9a5a416b3365ed3d74f2491beb904d61dc8d1eb89a72a4" + name: hello-pod + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] +parameters: +- name: NAME +- name: NAMESPACE +- name: LABEL diff --git a/openshift/pkg/otp/testdata/networking/network_segmentation/udn/udn_test_pod_template_node.yaml b/openshift/pkg/otp/testdata/networking/network_segmentation/udn/udn_test_pod_template_node.yaml new file mode 100644 index 0000000000..84d9918747 --- /dev/null +++ b/openshift/pkg/otp/testdata/networking/network_segmentation/udn/udn_test_pod_template_node.yaml @@ -0,0 +1,30 @@ +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + name: udn-pod-template-node +objects: +- kind: Pod + apiVersion: v1 + metadata: + name: "${NAME}" + namespace: "${NAMESPACE}" + labels: + name: "${LABEL}" + spec: + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + containers: + - image: "quay.io/openshifttest/hello-sdn@sha256:c89445416459e7adea9a5a416b3365ed3d74f2491beb904d61dc8d1eb89a72a4" + name: hello-pod + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + nodeName: "${NODENAME}" +parameters: +- name: NAME +- name: NAMESPACE +- name: LABEL +- name: NODENAME diff --git a/openshift/pkg/otp/testdata/networking/network_segmentation/udn/udn_with_multiplenetworkpolicy.yaml b/openshift/pkg/otp/testdata/networking/network_segmentation/udn/udn_with_multiplenetworkpolicy.yaml new file mode 100644 index 0000000000..985158c26a --- /dev/null +++ b/openshift/pkg/otp/testdata/networking/network_segmentation/udn/udn_with_multiplenetworkpolicy.yaml @@ -0,0 +1,20 @@ +apiVersion: k8s.cni.cncf.io/v1beta1 +kind: MultiNetworkPolicy +metadata: + name: ingress-allow-same-podselector-with-same-namespaceselector + annotations: + k8s.v1.cni.cncf.io/policy-for: project75624/dualstack +spec: + podSelector: + matchLabels: + name: blue-pod + policyTypes: + - Ingress + ingress: + - from: + - namespaceSelector: + matchLabels: + user: project75624 + podSelector: + matchLabels: + name: blue-pod diff --git a/openshift/pkg/otp/testdata/networking/networkpolicy/allow-egress-red.yaml b/openshift/pkg/otp/testdata/networking/networkpolicy/allow-egress-red.yaml new file mode 100644 index 0000000000..62b45b6938 --- /dev/null +++ b/openshift/pkg/otp/testdata/networking/networkpolicy/allow-egress-red.yaml @@ -0,0 +1,16 @@ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-egress-to-red +spec: + podSelector: {} + egress: + - to: + - namespaceSelector: + matchLabels: + team: qe + podSelector: + matchLabels: + type: red + policyTypes: + - Egress diff --git a/openshift/pkg/otp/testdata/networking/networkpolicy/allow-from-all-namespaces.yaml b/openshift/pkg/otp/testdata/networking/networkpolicy/allow-from-all-namespaces.yaml new file mode 100644 index 0000000000..c2b91ae2ae --- /dev/null +++ b/openshift/pkg/otp/testdata/networking/networkpolicy/allow-from-all-namespaces.yaml @@ -0,0 +1,14 @@ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-from-all-namespaces +spec: + podSelector: {} + ingress: + - ports: + - protocol: TCP + port: 8080 + from: + - namespaceSelector: {} + policyTypes: + - Ingress diff --git a/openshift/pkg/otp/testdata/networking/networkpolicy/allow-from-same-namespace.yaml b/openshift/pkg/otp/testdata/networking/networkpolicy/allow-from-same-namespace.yaml new file mode 100644 index 0000000000..f652524c25 --- /dev/null +++ b/openshift/pkg/otp/testdata/networking/networkpolicy/allow-from-same-namespace.yaml @@ -0,0 +1,11 @@ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-from-same-namespace +spec: + ingress: + - from: + - podSelector: {} + podSelector: {} + policyTypes: + - Ingress diff --git a/openshift/pkg/otp/testdata/networking/networkpolicy/allow-ingress-red.yaml b/openshift/pkg/otp/testdata/networking/networkpolicy/allow-ingress-red.yaml new file mode 100644 index 0000000000..395a40c781 --- /dev/null +++ b/openshift/pkg/otp/testdata/networking/networkpolicy/allow-ingress-red.yaml @@ -0,0 +1,10 @@ +kind: NetworkPolicy +apiVersion: networking.k8s.io/v1 +metadata: + name: allow-ingress-to-red +spec: + podSelector: + matchLabels: + type: red + ingress: + - {} diff --git a/openshift/pkg/otp/testdata/networking/networkpolicy/allow-same-namespace.yaml b/openshift/pkg/otp/testdata/networking/networkpolicy/allow-same-namespace.yaml new file mode 100644 index 0000000000..6f799dd3eb --- /dev/null +++ b/openshift/pkg/otp/testdata/networking/networkpolicy/allow-same-namespace.yaml @@ -0,0 +1,15 @@ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-same-namespace +spec: + egress: + - to: + - podSelector: {} + ingress: + - from: + - podSelector: {} + podSelector: {} + policyTypes: + - Ingress + - Egress diff --git a/openshift/pkg/otp/testdata/networking/networkpolicy/allow-to-openshift-dns.yaml b/openshift/pkg/otp/testdata/networking/networkpolicy/allow-to-openshift-dns.yaml new file mode 100644 index 0000000000..3e256a2341 --- /dev/null +++ b/openshift/pkg/otp/testdata/networking/networkpolicy/allow-to-openshift-dns.yaml @@ -0,0 +1,19 @@ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-to-openshift-dns +spec: + egress: + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: openshift-dns + ports: + - port: 5353 + protocol: TCP + - port: 5353 + protocol: UDP + podSelector: {} + policyTypes: + - Egress + diff --git a/openshift/pkg/otp/testdata/networking/networkpolicy/allow-to-same-namespace.yaml b/openshift/pkg/otp/testdata/networking/networkpolicy/allow-to-same-namespace.yaml new file mode 100644 index 0000000000..d7e1a8ede3 --- /dev/null +++ b/openshift/pkg/otp/testdata/networking/networkpolicy/allow-to-same-namespace.yaml @@ -0,0 +1,11 @@ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-to-same-namespace +spec: + egress: + - to: + - podSelector: {} + podSelector: {} + policyTypes: + - Egress diff --git a/openshift/pkg/otp/testdata/networking/networkpolicy/default-allow-egress.yaml b/openshift/pkg/otp/testdata/networking/networkpolicy/default-allow-egress.yaml new file mode 100644 index 0000000000..3d01b94faf --- /dev/null +++ b/openshift/pkg/otp/testdata/networking/networkpolicy/default-allow-egress.yaml @@ -0,0 +1,11 @@ +kind: NetworkPolicy +apiVersion: networking.k8s.io/v1 +metadata: + name: default-allow-egress +spec: + podSelector: {} + egress: + - {} + policyTypes: + - Egress + diff --git a/openshift/pkg/otp/testdata/networking/networkpolicy/default-deny-egress.yaml b/openshift/pkg/otp/testdata/networking/networkpolicy/default-deny-egress.yaml new file mode 100644 index 0000000000..b94abe2220 --- /dev/null +++ b/openshift/pkg/otp/testdata/networking/networkpolicy/default-deny-egress.yaml @@ -0,0 +1,8 @@ +kind: NetworkPolicy +apiVersion: networking.k8s.io/v1 +metadata: + name: default-deny-egress +spec: + podSelector: {} + policyTypes: + - Egress diff --git a/openshift/pkg/otp/testdata/networking/networkpolicy/default-deny-ingress.yaml b/openshift/pkg/otp/testdata/networking/networkpolicy/default-deny-ingress.yaml new file mode 100644 index 0000000000..9a6234572c --- /dev/null +++ b/openshift/pkg/otp/testdata/networking/networkpolicy/default-deny-ingress.yaml @@ -0,0 +1,8 @@ +kind: NetworkPolicy +apiVersion: networking.k8s.io/v1 +metadata: + name: default-deny-ingress +spec: + podSelector: {} + policyTypes: + - Ingress diff --git a/openshift/pkg/otp/testdata/networking/networkpolicy/egress-allow-all.yaml b/openshift/pkg/otp/testdata/networking/networkpolicy/egress-allow-all.yaml new file mode 100644 index 0000000000..b0b82ca477 --- /dev/null +++ b/openshift/pkg/otp/testdata/networking/networkpolicy/egress-allow-all.yaml @@ -0,0 +1,12 @@ +kind: NetworkPolicy +apiVersion: networking.k8s.io/v1 +metadata: + name: allow-all-egress +spec: + podSelector: + matchLabels: + name: test-pods + egress: + - {} + policyTypes: + - Egress diff --git a/openshift/pkg/otp/testdata/networking/networkpolicy/egress-ingress-62524.yaml b/openshift/pkg/otp/testdata/networking/networkpolicy/egress-ingress-62524.yaml new file mode 100644 index 0000000000..39c65c43fe --- /dev/null +++ b/openshift/pkg/otp/testdata/networking/networkpolicy/egress-ingress-62524.yaml @@ -0,0 +1,27 @@ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: egress-ingress-62524.test +spec: + egress: + - to: + - namespaceSelector: + matchLabels: + team: openshift-networking + podSelector: + matchLabels: + name: hello-pod + ingress: + - from: + - namespaceSelector: + matchLabels: + team: openshift-networking + podSelector: + matchLabels: + name: test-pods + podSelector: + matchLabels: + name: test-pods + policyTypes: + - Ingress + - Egress diff --git a/openshift/pkg/otp/testdata/networking/networkpolicy/egress_49696.yaml b/openshift/pkg/otp/testdata/networking/networkpolicy/egress_49696.yaml new file mode 100644 index 0000000000..72d4ecb5cd --- /dev/null +++ b/openshift/pkg/otp/testdata/networking/networkpolicy/egress_49696.yaml @@ -0,0 +1,15 @@ +kind: NetworkPolicy +apiVersion: networking.k8s.io/v1 +metadata: + name: egress-all-otherpod +spec: + podSelector: + matchLabels: + name: hellosdn + egress: + - to: + - namespaceSelector: + matchLabels: + team: openshift + policyTypes: + - Egress diff --git a/openshift/pkg/otp/testdata/networking/networkpolicy/generic-networkpolicy-template.yaml b/openshift/pkg/otp/testdata/networking/networkpolicy/generic-networkpolicy-template.yaml new file mode 100644 index 0000000000..15fc0953a0 --- /dev/null +++ b/openshift/pkg/otp/testdata/networking/networkpolicy/generic-networkpolicy-template.yaml @@ -0,0 +1,38 @@ +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + name: generic_networkpolicy_template +objects: +- kind: NetworkPolicy + apiVersion: networking.k8s.io/v1 + metadata: + name: "${NAME}" + namespace: "${NAMESPACE}" + spec: + podSelector: {} + "${POLICY}": + - "${DIRECTION1}": + - podSelector : {} + namespaceSelector: + "${NAMESPACESEL1}": + "${NAMESPACESELKEY1}": "${NAMESPACESELVAL1}" + - "${DIRECTION2}": + - podSelector : {} + namespaceSelector: + "${NAMESPACESEL2}": + "${NAMESPACESELKEY2}": "${NAMESPACESELVAL2}" + policyTypes: + - "${POLICYTYPE}" +parameters: +- name: NAME +- name: NAMESPACE +- name: POLICY +- name: DIRECTION1 +- name: NAMESPACESEL1 +- name: NAMESPACESELKEY1 +- name: NAMESPACESELVAL1 +- name: DIRECTION2 +- name: NAMESPACESEL2 +- name: NAMESPACESELKEY2 +- name: NAMESPACESELVAL2 +- name: POLICYTYPE diff --git a/openshift/pkg/otp/testdata/networking/networkpolicy/ingress-allow-all.yaml b/openshift/pkg/otp/testdata/networking/networkpolicy/ingress-allow-all.yaml new file mode 100644 index 0000000000..65d71cb5fa --- /dev/null +++ b/openshift/pkg/otp/testdata/networking/networkpolicy/ingress-allow-all.yaml @@ -0,0 +1,12 @@ +kind: NetworkPolicy +apiVersion: networking.k8s.io/v1 +metadata: + name: allow-all-ingress +spec: + podSelector: + matchLabels: + name: hellosdn + ingress: + - {} + policyTypes: + - Ingress diff --git a/openshift/pkg/otp/testdata/networking/networkpolicy/ingress_49696.yaml b/openshift/pkg/otp/testdata/networking/networkpolicy/ingress_49696.yaml new file mode 100644 index 0000000000..2dad3fe86f --- /dev/null +++ b/openshift/pkg/otp/testdata/networking/networkpolicy/ingress_49696.yaml @@ -0,0 +1,18 @@ +kind: NetworkPolicy +apiVersion: networking.k8s.io/v1 +metadata: + name: allow-all-ingress +spec: + podSelector: + matchLabels: + name: hellosdn + ingress: + - from: + - namespaceSelector: + matchLabels: + team: operations + podSelector: + matchLabels: + name: test-pods + policyTypes: + - Ingress diff --git a/openshift/pkg/otp/testdata/networking/networkpolicy/ipblock/ipBlock-egress-dual-CIDRs-template.yaml b/openshift/pkg/otp/testdata/networking/networkpolicy/ipblock/ipBlock-egress-dual-CIDRs-template.yaml new file mode 100644 index 0000000000..15743974fc --- /dev/null +++ b/openshift/pkg/otp/testdata/networking/networkpolicy/ipblock/ipBlock-egress-dual-CIDRs-template.yaml @@ -0,0 +1,25 @@ +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + name: ipBlock_egress_dual_cidrs_template +objects: +- kind: NetworkPolicy + apiVersion: networking.k8s.io/v1 + metadata: + name: "${NAME}" + namespace: "${NAMESPACE}" + spec: + podSelector: {} + egress: + - to: + - ipBlock: + cidr: "${cidrIpv4}" + - ipBlock: + cidr: "${cidrIpv6}" + policyTypes: + - Egress +parameters: +- name: NAME +- name: cidrIpv4 +- name: cidrIpv6 +- name: NAMESPACE diff --git a/openshift/pkg/otp/testdata/networking/networkpolicy/ipblock/ipBlock-egress-dual-multiple-CIDRs-template.yaml b/openshift/pkg/otp/testdata/networking/networkpolicy/ipblock/ipBlock-egress-dual-multiple-CIDRs-template.yaml new file mode 100644 index 0000000000..5c630f7daf --- /dev/null +++ b/openshift/pkg/otp/testdata/networking/networkpolicy/ipblock/ipBlock-egress-dual-multiple-CIDRs-template.yaml @@ -0,0 +1,37 @@ +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + name: ipBlockEgressDual_multiple_cidrs_template +objects: +- kind: NetworkPolicy + apiVersion: networking.k8s.io/v1 + metadata: + name: "${NAME}" + namespace: "${NAMESPACE}" + spec: + podSelector: {} + egress: + - to: + - ipBlock: + cidr: "${cidrIpv4}" + - ipBlock: + cidr: "${cidrIpv6}" + - ipBlock: + cidr: "${cidr2Ipv4}" + - ipBlock: + cidr: "${cidr2Ipv6}" + - ipBlock: + cidr: "${cidr3Ipv4}" + - ipBlock: + cidr: "${cidr3Ipv6}" + policyTypes: + - Egress +parameters: +- name: NAME +- name: cidrIpv4 +- name: cidrIpv6 +- name: cidr2Ipv4 +- name: cidr2Ipv6 +- name: cidr3Ipv4 +- name: cidr3Ipv6 +- name: NAMESPACE diff --git a/openshift/pkg/otp/testdata/networking/networkpolicy/ipblock/ipBlock-egress-except-dual-CIDRs-template.yaml b/openshift/pkg/otp/testdata/networking/networkpolicy/ipblock/ipBlock-egress-except-dual-CIDRs-template.yaml new file mode 100644 index 0000000000..2a1ee30f3d --- /dev/null +++ b/openshift/pkg/otp/testdata/networking/networkpolicy/ipblock/ipBlock-egress-except-dual-CIDRs-template.yaml @@ -0,0 +1,31 @@ +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + name: ipBlock_egress_dual_cidrs_template +objects: +- kind: NetworkPolicy + apiVersion: networking.k8s.io/v1 + metadata: + name: "${NAME}" + namespace: "${NAMESPACE}" + spec: + podSelector: {} + egress: + - to: + - ipBlock: + cidr: "${CIDR_IPv4}" + except: + - "${EXCEPT_IPv4}" + - ipBlock: + cidr: "${CIDR_IPv6}" + except: + - "${EXCEPT_IPv6}" + policyTypes: + - Egress +parameters: +- name: NAME +- name: CIDR_IPv4 +- name: CIDR_IPv6 +- name: NAMESPACE +- name: EXCEPT_IPv4 +- name: EXCEPT_IPv6 diff --git a/openshift/pkg/otp/testdata/networking/networkpolicy/ipblock/ipBlock-egress-except-single-CIDR-template.yaml b/openshift/pkg/otp/testdata/networking/networkpolicy/ipblock/ipBlock-egress-except-single-CIDR-template.yaml new file mode 100644 index 0000000000..1b712fad9c --- /dev/null +++ b/openshift/pkg/otp/testdata/networking/networkpolicy/ipblock/ipBlock-egress-except-single-CIDR-template.yaml @@ -0,0 +1,25 @@ +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + name: ipBlockEgressSingle_cidr_template +objects: +- kind: NetworkPolicy + apiVersion: networking.k8s.io/v1 + metadata: + name: "${NAME}" + namespace: "${NAMESPACE}" + spec: + podSelector: {} + egress: + - to: + - ipBlock: + cidr: "${CIDR}" + except: + - "${EXCEPT}" + policyTypes: + - Egress +parameters: +- name: NAME +- name: CIDR +- name: NAMESPACE +- name: EXCEPT diff --git a/openshift/pkg/otp/testdata/networking/networkpolicy/ipblock/ipBlock-egress-single-CIDR-template.yaml b/openshift/pkg/otp/testdata/networking/networkpolicy/ipblock/ipBlock-egress-single-CIDR-template.yaml new file mode 100644 index 0000000000..89d47524d1 --- /dev/null +++ b/openshift/pkg/otp/testdata/networking/networkpolicy/ipblock/ipBlock-egress-single-CIDR-template.yaml @@ -0,0 +1,22 @@ +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + name: ipBlockEgressSingle_cidr_template +objects: +- kind: NetworkPolicy + apiVersion: networking.k8s.io/v1 + metadata: + name: "${NAME}" + namespace: "${NAMESPACE}" + spec: + podSelector: {} + egress: + - to: + - ipBlock: + cidr: "${CIDR}" + policyTypes: + - Egress +parameters: +- name: NAME +- name: CIDR +- name: NAMESPACE diff --git a/openshift/pkg/otp/testdata/networking/networkpolicy/ipblock/ipBlock-egress-single-multiple-CIDRs-template.yaml b/openshift/pkg/otp/testdata/networking/networkpolicy/ipblock/ipBlock-egress-single-multiple-CIDRs-template.yaml new file mode 100644 index 0000000000..d6bbaedb8e --- /dev/null +++ b/openshift/pkg/otp/testdata/networking/networkpolicy/ipblock/ipBlock-egress-single-multiple-CIDRs-template.yaml @@ -0,0 +1,28 @@ +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + name: ipBlockEgressSingle_multiple_cidrs_template +objects: +- kind: NetworkPolicy + apiVersion: networking.k8s.io/v1 + metadata: + name: "${NAME}" + namespace: "${NAMESPACE}" + spec: + podSelector: {} + egress: + - to: + - ipBlock: + cidr: "${CIDR}" + - ipBlock: + cidr: "${CIDR2}" + - ipBlock: + cidr: "${CIDR3}" + policyTypes: + - Egress +parameters: +- name: NAME +- name: CIDR +- name: CIDR2 +- name: CIDR3 +- name: NAMESPACE diff --git a/openshift/pkg/otp/testdata/networking/networkpolicy/ipblock/ipBlock-ingress-dual-CIDRs-template.yaml b/openshift/pkg/otp/testdata/networking/networkpolicy/ipblock/ipBlock-ingress-dual-CIDRs-template.yaml new file mode 100644 index 0000000000..d1567bba28 --- /dev/null +++ b/openshift/pkg/otp/testdata/networking/networkpolicy/ipblock/ipBlock-ingress-dual-CIDRs-template.yaml @@ -0,0 +1,23 @@ +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + name: ipBlock_ingress_dual_cidrs_template +objects: +- kind: NetworkPolicy + apiVersion: networking.k8s.io/v1 + metadata: + name: "${NAME}" + namespace: "${NAMESPACE}" + spec: + podSelector: {} + ingress: + - from: + - ipBlock: + cidr: "${cidrIpv4}" + - ipBlock: + cidr: "${cidrIpv6}" +parameters: +- name: NAME +- name: cidrIpv4 +- name: cidrIpv6 +- name: NAMESPACE diff --git a/openshift/pkg/otp/testdata/networking/networkpolicy/ipblock/ipBlock-ingress-dual-multiple-CIDRs-template.yaml b/openshift/pkg/otp/testdata/networking/networkpolicy/ipblock/ipBlock-ingress-dual-multiple-CIDRs-template.yaml new file mode 100644 index 0000000000..12d49b89bd --- /dev/null +++ b/openshift/pkg/otp/testdata/networking/networkpolicy/ipblock/ipBlock-ingress-dual-multiple-CIDRs-template.yaml @@ -0,0 +1,35 @@ +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + name: ipBlock_ingress_dual_multiple_cidrs_template +objects: +- kind: NetworkPolicy + apiVersion: networking.k8s.io/v1 + metadata: + name: "${NAME}" + namespace: "${NAMESPACE}" + spec: + podSelector: {} + ingress: + - from: + - ipBlock: + cidr: "${cidrIpv4}" + - ipBlock: + cidr: "${cidrIpv6}" + - ipBlock: + cidr: "${cidr2Ipv4}" + - ipBlock: + cidr: "${cidr2Ipv6}" + - ipBlock: + cidr: "${cidr3Ipv4}" + - ipBlock: + cidr: "${cidr3Ipv6}" +parameters: +- name: NAME +- name: cidrIpv4 +- name: cidrIpv6 +- name: cidr2Ipv4 +- name: cidr2Ipv6 +- name: cidr3Ipv4 +- name: cidr3Ipv6 +- name: NAMESPACE diff --git a/openshift/pkg/otp/testdata/networking/networkpolicy/ipblock/ipBlock-ingress-single-CIDR-template.yaml b/openshift/pkg/otp/testdata/networking/networkpolicy/ipblock/ipBlock-ingress-single-CIDR-template.yaml new file mode 100644 index 0000000000..9dfe14642f --- /dev/null +++ b/openshift/pkg/otp/testdata/networking/networkpolicy/ipblock/ipBlock-ingress-single-CIDR-template.yaml @@ -0,0 +1,20 @@ +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + name: ipBlockIngressSingle_cidr_template +objects: +- kind: NetworkPolicy + apiVersion: networking.k8s.io/v1 + metadata: + name: "${NAME}" + namespace: "${NAMESPACE}" + spec: + podSelector: {} + ingress: + - from: + - ipBlock: + cidr: "${CIDR}" +parameters: +- name: NAME +- name: CIDR +- name: NAMESPACE diff --git a/openshift/pkg/otp/testdata/networking/networkpolicy/ipblock/ipBlock-ingress-single-multiple-CIDRs-template.yaml b/openshift/pkg/otp/testdata/networking/networkpolicy/ipblock/ipBlock-ingress-single-multiple-CIDRs-template.yaml new file mode 100644 index 0000000000..8318cc166c --- /dev/null +++ b/openshift/pkg/otp/testdata/networking/networkpolicy/ipblock/ipBlock-ingress-single-multiple-CIDRs-template.yaml @@ -0,0 +1,26 @@ +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + name: ipBlockIngressSingle_multiple_cidrs_template +objects: +- kind: NetworkPolicy + apiVersion: networking.k8s.io/v1 + metadata: + name: "${NAME}" + namespace: "${NAMESPACE}" + spec: + podSelector: {} + ingress: + - from: + - ipBlock: + cidr: "${CIDR}" + - ipBlock: + cidr: "${CIDR2}" + - ipBlock: + cidr: "${CIDR3}" +parameters: +- name: NAME +- name: CIDR +- name: CIDR2 +- name: CIDR3 +- name: NAMESPACE diff --git a/openshift/pkg/otp/testdata/networking/networkpolicy/netpol-30920-75540.yaml b/openshift/pkg/otp/testdata/networking/networkpolicy/netpol-30920-75540.yaml new file mode 100644 index 0000000000..c3bcbb4d7b --- /dev/null +++ b/openshift/pkg/otp/testdata/networking/networkpolicy/netpol-30920-75540.yaml @@ -0,0 +1,16 @@ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: OCPBUGS-30920 +spec: + ingress: + - from: + - namespaceSelector: + matchExpressions: + - key: name + operator: In + values: + - invalid value + podSelector: {} + policyTypes: + - Ingress diff --git a/openshift/pkg/otp/testdata/networking/networkpolicy/vap-npprotection-blockdeletion.yaml b/openshift/pkg/otp/testdata/networking/networkpolicy/vap-npprotection-blockdeletion.yaml new file mode 100644 index 0000000000..223b5ac5a8 --- /dev/null +++ b/openshift/pkg/otp/testdata/networking/networkpolicy/vap-npprotection-blockdeletion.yaml @@ -0,0 +1,27 @@ +kind: ValidatingAdmissionPolicy +apiVersion: admissionregistration.k8s.io/v1 +metadata: + name: fine-grained-network-policy-protection +spec: + matchConstraints: + resourceRules: + - operations: + - DELETE + apiGroups: + - networking.k8s.io + apiVersions: + - v1 + resources: + - networkpolicies + validations: + - expression: '!(has(oldObject.metadata.labels) && oldObject.metadata.labels[''npprotection/blockdeletion''] == "true")' + message: "Cannot delete NetworkPolicy with 'npprotection/blockdeletion: true' label. This policy is not allowed for deletion." +--- +kind: ValidatingAdmissionPolicyBinding +apiVersion: admissionregistration.k8s.io/v1 +metadata: + name: fine-grained-network-policy-protection-binding +spec: + policyName: fine-grained-network-policy-protection + validationActions: + - Deny diff --git a/openshift/pkg/otp/testdata/networking/ping-for-pod-specific-node-template.yaml b/openshift/pkg/otp/testdata/networking/ping-for-pod-specific-node-template.yaml new file mode 100644 index 0000000000..492a645178 --- /dev/null +++ b/openshift/pkg/otp/testdata/networking/ping-for-pod-specific-node-template.yaml @@ -0,0 +1,29 @@ +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + name: hello-pod-specific-node-template +objects: +- kind: Pod + apiVersion: v1 + metadata: + name: "${NAME}" + namespace: "${NAMESPACE}" + labels: + name: hello-pod + spec: + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + containers: + - image: "quay.io/openshifttest/hello-sdn@sha256:c89445416459e7adea9a5a416b3365ed3d74f2491beb904d61dc8d1eb89a72a4" + name: hello-pod + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + nodeName: "${NODENAME}" +parameters: +- name: NAME +- name: NAMESPACE +- name: NODENAME diff --git a/openshift/pkg/otp/testdata/networking/ping-for-pod-template.yaml b/openshift/pkg/otp/testdata/networking/ping-for-pod-template.yaml new file mode 100644 index 0000000000..5d00c5657e --- /dev/null +++ b/openshift/pkg/otp/testdata/networking/ping-for-pod-template.yaml @@ -0,0 +1,27 @@ +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + name: hello-pod-template +objects: +- kind: Pod + apiVersion: v1 + metadata: + name: "${NAME}" + namespace: "${NAMESPACE}" + labels: + name: hello-pod + spec: + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + containers: + - image: "quay.io/openshifttest/hello-sdn@sha256:c89445416459e7adea9a5a416b3365ed3d74f2491beb904d61dc8d1eb89a72a4" + name: hello-pod + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] +parameters: +- name: NAME +- name: NAMESPACE diff --git a/openshift/pkg/otp/testdata/networking/service-generic-template.yaml b/openshift/pkg/otp/testdata/networking/service-generic-template.yaml new file mode 100644 index 0000000000..251c69176f --- /dev/null +++ b/openshift/pkg/otp/testdata/networking/service-generic-template.yaml @@ -0,0 +1,34 @@ +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + name: test-service-template +objects: +- kind: Service + apiVersion: v1 + metadata: + name: "${SERVICENAME}" + namespace: "${NAMESPACE}" + labels: + name: test-service + spec: + internalTrafficPolicy: "${internalTrafficPolicy}" + externalTrafficPolicy: "${externalTrafficPolicy}" + ipFamilyPolicy: "${ipFamilyPolicy}" + ports: + - name: http + port: 27017 + protocol: "${PROTOCOL}" + targetPort: 8080 + selector: + name: "${SELECTOR}" + type: "${serviceType}" +parameters: +- name: SERVICENAME +- name: NAMESPACE +- name: internalTrafficPolicy +- name: externalTrafficPolicy +- name: ipFamilyPolicy +- name: PROTOCOL +- name: SELECTOR +- name: serviceType +- name: PORT diff --git a/openshift/pkg/otp/testdata/networking/statefulset-hello.yaml b/openshift/pkg/otp/testdata/networking/statefulset-hello.yaml new file mode 100644 index 0000000000..b5ecec1a9a --- /dev/null +++ b/openshift/pkg/otp/testdata/networking/statefulset-hello.yaml @@ -0,0 +1,29 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + labels: + app: hello + name: hello + name: hello +spec: + replicas: 1 + selector: + matchLabels: + app: hello + serviceName: hello + template: + metadata: + labels: + app: hello + spec: + containers: + - image: quay.io/openshifttest/hello-sdn@sha256:c89445416459e7adea9a5a416b3365ed3d74f2491beb904d61dc8d1eb89a72a4 + name: hello + ports: + - containerPort: 8080 + name: web + protocol: TCP + resources: + limits: + memory: 340Mi + restartPolicy: Always diff --git a/openshift/pkg/otp/testdata/networking/testpod.yaml b/openshift/pkg/otp/testdata/networking/testpod.yaml new file mode 100644 index 0000000000..2f2b9e1f1c --- /dev/null +++ b/openshift/pkg/otp/testdata/networking/testpod.yaml @@ -0,0 +1,70 @@ +{ + "apiVersion": "v1", + "kind": "List", + "items": [ + { + "apiVersion": "v1", + "kind": "ReplicationController", + "metadata": { + "labels": { + "name": "test-rc" + }, + "name": "test-rc" + }, + "spec": { + "replicas": 2, + "template": { + "metadata": { + "labels": { + "name": "test-pods" + } + }, + "spec": { + "containers": [ + { + "image": "quay.io/openshifttest/hello-sdn@sha256:c89445416459e7adea9a5a416b3365ed3d74f2491beb904d61dc8d1eb89a72a4", + "name": "test-pod", + "imagePullPolicy": "IfNotPresent" + } + ], + "tolerations": [ + { + "effect": "NoSchedule", + "key": "node-role.kubernetes.io/outposts", + "operator": "Exists" + }, + { + "effect": "NoSchedule", + "key": "node-role.kubernetes.io/edge", + "operator": "Exists" + } + ] + } + } + } + }, + { + "apiVersion": "v1", + "kind": "Service", + "metadata": { + "labels": { + "name": "test-service" + }, + "name": "test-service" + }, + "spec": { + "ports": [ + { + "name": "http", + "port": 27017, + "protocol": "TCP", + "targetPort": 8080 + } + ], + "selector": { + "name": "test-pods" + } + } + } + ] +} diff --git a/openshift/pkg/otp/utils/network_segmentation_utils.go b/openshift/pkg/otp/utils/network_segmentation_utils.go new file mode 100644 index 0000000000..c97406f016 --- /dev/null +++ b/openshift/pkg/otp/utils/network_segmentation_utils.go @@ -0,0 +1,1112 @@ +package otputils + +import ( + "context" + "fmt" + "net" + "path/filepath" + "regexp" + "strconv" + "strings" + "time" + + o "github.com/onsi/gomega" + exutil "github.com/openshift/origin/test/extended/util" + "github.com/ovn-kubernetes/ovn-kubernetes/openshift/pkg/otp/testdata" + "k8s.io/apimachinery/pkg/util/wait" + e2e "k8s.io/kubernetes/test/e2e/framework" + e2eoutput "k8s.io/kubernetes/test/e2e/framework/pod/output" + netutils "k8s.io/utils/net" +) + +// UDN pod resource types + +type UdnPodResource struct { + Name string + Namespace string + Label string + Template string +} + +type UdnPodResourceNode struct { + Name string + Namespace string + Label string + Nodename string + Template string +} + +type UdnPodSecNADResource struct { + Name string + Namespace string + Label string + Annotation string + Template string +} + +type UdnPodSecNADResourceNode struct { + Name string + Namespace string + Label string + Nadname string + Nodename string + Template string +} + +type UdnNetDefResource struct { + Nadname string + Namespace string + NadNetworkName string + Topology string + Subnet string + NetAttachDefName string + Role string + Template string +} + +type UdnCRDResource struct { + Crdname string + Namespace string + IPv4cidr string + IPv4prefix int32 + IPv6cidr string + IPv6prefix int32 + Cidr string + Prefix int32 + Role string + Template string +} + +type CudnCRDResource struct { + Crdname string + Labelvalue string + Labelkey string + Key string + Operator string + Values []string + IPv4cidr string + IPv4prefix int32 + IPv6cidr string + IPv6prefix int32 + Cidr string + Prefix int32 + Role string + Physicalnetworkname string + Subnet string + Excludesubnet string + Template string +} + +type UdnPodWithProbeResource struct { + Name string + Namespace string + Label string + Port int + Failurethreshold int + Periodseconds int + Template string +} + +type ReplicationControllerPingPodResource struct { + Name string + Replicas int + Namespace string + Template string +} + +// RunOcWithRetry runs an oc command with retry on i/o timeout errors +func RunOcWithRetry(oc *exutil.CLI, cmd string, args ...string) (string, error) { + var err error + var output string + maxRetries := 5 + + for numRetries := 0; numRetries < maxRetries; numRetries++ { + if numRetries > 0 { + e2e.Logf("Retrying oc command (retry count=%v/%v)", numRetries+1, maxRetries) + } + + output, err = oc.Run(cmd).Args(args...).Output() + if err != nil { + if strings.Contains(strings.ToLower(err.Error()), "i/o timeout") { + e2e.Logf("Warning: oc command encountered i/o timeout.\nerr=%v\n)", err) + continue + } + return output, err + } + break + } + return output, err +} + +func (pod *UdnPodResource) CreateUdnPod(oc *exutil.CLI) { + err := wait.Poll(5*time.Second, 20*time.Second, func() (bool, error) { + err1 := ApplyResourceFromTemplateByAdmin(oc, "--ignore-unknown-parameters=true", "-f", pod.Template, "-p", "NAME="+pod.Name, "NAMESPACE="+pod.Namespace, "LABEL="+pod.Label) + if err1 != nil { + e2e.Logf("the err:%v, and try next round", err1) + return false, nil + } + return true, nil + }) + o.Expect(err).NotTo(o.HaveOccurred(), fmt.Sprintf("fail to create pod %v", pod.Name)) +} + +func (pod *UdnPodResourceNode) CreateUdnPodNode(oc *exutil.CLI) { + err := wait.Poll(5*time.Second, 20*time.Second, func() (bool, error) { + err1 := ApplyResourceFromTemplateByAdmin(oc, "--ignore-unknown-parameters=true", "-f", pod.Template, "-p", "NAME="+pod.Name, "NAMESPACE="+pod.Namespace, "LABEL="+pod.Label, "NODENAME="+pod.Nodename) + if err1 != nil { + e2e.Logf("the err:%v, and try next round", err1) + return false, nil + } + return true, nil + }) + o.Expect(err).NotTo(o.HaveOccurred(), fmt.Sprintf("fail to create pod %v", pod.Name)) +} + +func (pod *UdnPodWithProbeResource) CreateUdnPodWithProbe(oc *exutil.CLI) { + err := wait.Poll(5*time.Second, 20*time.Second, func() (bool, error) { + err1 := ApplyResourceFromTemplateByAdmin(oc, "--ignore-unknown-parameters=true", "-f", pod.Template, "-p", "NAME="+pod.Name, "NAMESPACE="+pod.Namespace, "LABEL="+pod.Label, "PORT="+strconv.Itoa(pod.Port), "FAILURETHRESHOLD="+strconv.Itoa(pod.Failurethreshold), "PERIODSECONDS="+strconv.Itoa(pod.Periodseconds)) + if err1 != nil { + e2e.Logf("the err:%v, and try next round", err1) + return false, nil + } + return true, nil + }) + o.Expect(err).NotTo(o.HaveOccurred(), fmt.Sprintf("fail to create pod %v", pod.Name)) +} + +func (pod *UdnPodSecNADResource) CreateUdnPodWithSecNAD(oc *exutil.CLI) { + err := wait.Poll(5*time.Second, 20*time.Second, func() (bool, error) { + err1 := ApplyResourceFromTemplateByAdmin(oc, "--ignore-unknown-parameters=true", "-f", pod.Template, "-p", "NAME="+pod.Name, "NAMESPACE="+pod.Namespace, "LABEL="+pod.Label, "ANNOTATION="+pod.Annotation) + if err1 != nil { + e2e.Logf("the err:%v, and try next round", err1) + return false, nil + } + return true, nil + }) + o.Expect(err).NotTo(o.HaveOccurred(), fmt.Sprintf("fail to create pod %v", pod.Name)) +} + +func (pod *UdnPodSecNADResourceNode) CreateUdnPodWithSecNADNode(oc *exutil.CLI) { + err := wait.Poll(5*time.Second, 20*time.Second, func() (bool, error) { + err1 := ApplyResourceFromTemplateByAdmin(oc, "--ignore-unknown-parameters=true", "-f", pod.Template, "-p", "NAME="+pod.Name, "NAMESPACE="+pod.Namespace, "LABEL="+pod.Label, "NADNAME="+pod.Nadname, "NODENAME="+pod.Nodename) + if err1 != nil { + e2e.Logf("the err:%v, and try next round", err1) + return false, nil + } + return true, nil + }) + o.Expect(err).NotTo(o.HaveOccurred(), fmt.Sprintf("fail to create pod %v", pod.Name)) +} + +func (nad *UdnNetDefResource) CreateUdnNad(oc *exutil.CLI) { + err := wait.Poll(5*time.Second, 20*time.Second, func() (bool, error) { + err1 := ApplyResourceFromTemplateByAdmin(oc, "--ignore-unknown-parameters=true", "-f", nad.Template, "-p", "NADNAME="+nad.Nadname, "NAMESPACE="+nad.Namespace, "NAD_NETWORK_NAME="+nad.NadNetworkName, "TOPOLOGY="+nad.Topology, "SUBNET="+nad.Subnet, "NET_ATTACH_DEF_NAME="+nad.NetAttachDefName, "ROLE="+nad.Role) + if err1 != nil { + e2e.Logf("the err:%v, and try next round", err1) + return false, nil + } + return true, nil + }) + o.Expect(err).NotTo(o.HaveOccurred(), fmt.Sprintf("fail to create nad %v", nad.Nadname)) +} + +func (nad *UdnNetDefResource) DeleteUdnNetDef(oc *exutil.CLI) { + RemoveResource(oc, false, true, "net-attach-def", nad.Nadname, "-n", nad.Namespace) +} + +// GetPodIPUDN returns IPv6 and IPv4 in vars in order on dual stack respectively and main IP in case of single stack (v4 or v6) in 1st var, and nil in 2nd var +func GetPodIPUDN(oc *exutil.CLI, namespace string, podName string, netName string) (string, string) { + ipStack := CheckIPStackType(oc) + cmdIPv4 := "ip a sho " + netName + " | awk 'NR==3{print $2}' |grep -Eo '((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])'" + cmdIPv6 := "ip -o -6 addr show dev " + netName + " | awk '$3 == \"inet6\" && $6 == \"global\" {print $4}' | cut -d'/' -f1" + switch ipStack { + case "ipv4single": + podIPv4, err := ExecCommandInSpecificPod(oc, namespace, podName, cmdIPv4) + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("The UDN pod %s IPv4 in namespace %s is %q", podName, namespace, podIPv4) + return podIPv4, "" + case "ipv6single": + podIPv6, err := ExecCommandInSpecificPod(oc, namespace, podName, cmdIPv6) + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("The UDN pod %s IPv6 in namespace %s is %q", podName, namespace, podIPv6) + return podIPv6, "" + default: + podIPv4, err := ExecCommandInSpecificPod(oc, namespace, podName, cmdIPv4) + o.Expect(err).NotTo(o.HaveOccurred()) + podIPv6, err := ExecCommandInSpecificPod(oc, namespace, podName, cmdIPv6) + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("The UDN pod's %s IPv6 and IPv4 IP in namespace %s is %q %q", podName, namespace, podIPv6, podIPv4) + return podIPv6, podIPv4 + } +} + +// CurlPod2PodPassUDN checks connectivity across udn pods regardless of network addressing type on cluster +func CurlPod2PodPassUDN(oc *exutil.CLI, namespaceSrc string, podNameSrc string, namespaceDst string, podNameDst string) { + podIP1, podIP2 := GetPodIPUDN(oc, namespaceDst, podNameDst, "ovn-udn1") + if podIP2 != "" { + _, err := e2eoutput.RunHostCmd(namespaceSrc, podNameSrc, "curl --connect-timeout 2 -s "+net.JoinHostPort(podIP1, "8080")) + o.Expect(err).NotTo(o.HaveOccurred()) + _, err = e2eoutput.RunHostCmd(namespaceSrc, podNameSrc, "curl --connect-timeout 2 -s "+net.JoinHostPort(podIP2, "8080")) + o.Expect(err).NotTo(o.HaveOccurred()) + } else { + _, err := e2eoutput.RunHostCmd(namespaceSrc, podNameSrc, "curl --connect-timeout 2 -s "+net.JoinHostPort(podIP1, "8080")) + o.Expect(err).NotTo(o.HaveOccurred()) + } +} + +// CurlPod2PodFailUDN ensures no connectivity from a udn pod to pod regardless of network addressing type on cluster +func CurlPod2PodFailUDN(oc *exutil.CLI, namespaceSrc string, podNameSrc string, namespaceDst string, podNameDst string) { + podIP1, podIP2 := GetPodIPUDN(oc, namespaceDst, podNameDst, "ovn-udn1") + if podIP2 != "" { + _, err := e2eoutput.RunHostCmd(namespaceSrc, podNameSrc, "curl --connect-timeout 2 -s "+net.JoinHostPort(podIP1, "8080")) + o.Expect(err).To(o.HaveOccurred()) + _, err = e2eoutput.RunHostCmd(namespaceSrc, podNameSrc, "curl --connect-timeout 2 -s "+net.JoinHostPort(podIP2, "8080")) + o.Expect(err).To(o.HaveOccurred()) + } else { + _, err := e2eoutput.RunHostCmd(namespaceSrc, podNameSrc, "curl --connect-timeout 2 -s "+net.JoinHostPort(podIP1, "8080")) + o.Expect(err).To(o.HaveOccurred()) + } +} + +func CurlNode2PodFailUDN(oc *exutil.CLI, nodeName string, namespaceDst string, podNameDst string) { + podIP1, podIP2 := GetPodIPUDN(oc, namespaceDst, podNameDst, "ovn-udn1") + if podIP2 != "" { + podv4URL := net.JoinHostPort(podIP2, "8080") + _, err := DebugNode(oc, nodeName, "curl", podv4URL, "-s", "--connect-timeout", "2") + o.Expect(err).To(o.HaveOccurred()) + } + podURL := net.JoinHostPort(podIP1, "8080") + _, err := DebugNode(oc, nodeName, "curl", podURL, "-s", "--connect-timeout", "2") + o.Expect(err).To(o.HaveOccurred()) +} + +func CurlUDNPod2PodPassMultiNetwork(oc *exutil.CLI, namespaceSrc string, namespaceDst string, podNameSrc string, netNameInterface string, podNameDst string, netNameDst string) { + podIP1, podIP2 := GetPodIPUDN(oc, namespaceDst, podNameDst, netNameDst) + if podIP2 != "" { + _, err := e2eoutput.RunHostCmd(namespaceSrc, podNameSrc, "curl --interface "+netNameInterface+" --connect-timeout 2 -s "+net.JoinHostPort(podIP1, "8080")) + o.Expect(err).NotTo(o.HaveOccurred()) + _, err = e2eoutput.RunHostCmd(namespaceSrc, podNameSrc, "curl --interface "+netNameInterface+" --connect-timeout 2 -s "+net.JoinHostPort(podIP2, "8080")) + o.Expect(err).NotTo(o.HaveOccurred()) + } else { + _, err := e2eoutput.RunHostCmd(namespaceSrc, podNameSrc, "curl --interface "+netNameInterface+" --connect-timeout 2 -s "+net.JoinHostPort(podIP1, "8080")) + o.Expect(err).NotTo(o.HaveOccurred()) + } +} + +func CurlUDNPod2PodFailMultiNetwork(oc *exutil.CLI, namespaceSrc string, namespaceDst string, podNameSrc string, netNameInterface string, podNameDst string, netNameDst string) { + podIP1, podIP2 := GetPodIPUDN(oc, namespaceDst, podNameDst, netNameDst) + if podIP2 != "" { + _, err := e2eoutput.RunHostCmd(namespaceSrc, podNameSrc, "curl --interface "+netNameInterface+" --connect-timeout 2 -s "+net.JoinHostPort(podIP1, "8080")) + o.Expect(err).To(o.HaveOccurred()) + _, err = e2eoutput.RunHostCmd(namespaceSrc, podNameSrc, "curl --interface "+netNameInterface+" --connect-timeout 2 -s "+net.JoinHostPort(podIP2, "8080")) + o.Expect(err).To(o.HaveOccurred()) + } else { + _, err := e2eoutput.RunHostCmd(namespaceSrc, podNameSrc, "curl --interface "+netNameInterface+" --connect-timeout 2 -s "+net.JoinHostPort(podIP1, "8080")) + o.Expect(err).To(o.HaveOccurred()) + } +} + +func (udncrd *UdnCRDResource) CreateUdnCRDSingleStack(oc *exutil.CLI) { + err := wait.PollUntilContextTimeout(context.TODO(), 2*time.Second, 10*time.Second, false, func(ctx context.Context) (bool, error) { + err1 := ApplyResourceFromTemplateByAdmin(oc, "--ignore-unknown-parameters=true", "-f", udncrd.Template, "-p", "CRDNAME="+udncrd.Crdname, "NAMESPACE="+udncrd.Namespace, "CIDR="+udncrd.Cidr, "PREFIX="+strconv.Itoa(int(udncrd.Prefix)), "ROLE="+udncrd.Role) + if err1 != nil { + e2e.Logf("the err:%v, and try next round", err1) + return false, nil + } + return true, nil + }) + o.Expect(err).NotTo(o.HaveOccurred(), fmt.Sprintf("fail to create udn CRD %s", udncrd.Crdname)) +} + +func (udncrd *UdnCRDResource) CreateUdnCRDDualStack(oc *exutil.CLI) { + err := wait.PollUntilContextTimeout(context.TODO(), 2*time.Second, 10*time.Second, false, func(ctx context.Context) (bool, error) { + err1 := ApplyResourceFromTemplateByAdmin(oc, "--ignore-unknown-parameters=true", "-f", udncrd.Template, "-p", "CRDNAME="+udncrd.Crdname, "NAMESPACE="+udncrd.Namespace, "IPv4CIDR="+udncrd.IPv4cidr, "IPv4PREFIX="+strconv.Itoa(int(udncrd.IPv4prefix)), "IPv6CIDR="+udncrd.IPv6cidr, "IPv6PREFIX="+strconv.Itoa(int(udncrd.IPv6prefix)), "ROLE="+udncrd.Role) + if err1 != nil { + e2e.Logf("the err:%v, and try next round", err1) + return false, nil + } + return true, nil + }) + o.Expect(err).NotTo(o.HaveOccurred(), fmt.Sprintf("fail to create udn CRD %s", udncrd.Crdname)) +} + +func (cudncrd *CudnCRDResource) CreateCUDNCRDSingleStack(oc *exutil.CLI) { + err := wait.PollUntilContextTimeout(context.TODO(), 2*time.Second, 10*time.Second, false, func(ctx context.Context) (bool, error) { + err1 := ApplyResourceFromTemplateByAdmin(oc, "--ignore-unknown-parameters=true", "-f", cudncrd.Template, "-p", "CRDNAME="+cudncrd.Crdname, "LABELKEY="+cudncrd.Labelkey, "LABELVALUE="+cudncrd.Labelvalue, + "CIDR="+cudncrd.Cidr, "PREFIX="+strconv.Itoa(int(cudncrd.Prefix)), "ROLE="+cudncrd.Role) + if err1 != nil { + e2e.Logf("the err:%v, and try next round", err1) + return false, nil + } + return true, nil + }) + o.Expect(err).NotTo(o.HaveOccurred(), fmt.Sprintf("fail to create cudn CRD %s", cudncrd.Crdname)) +} + +func (cudncrd *CudnCRDResource) CreateCUDNCRDDualStack(oc *exutil.CLI) { + err := wait.PollUntilContextTimeout(context.TODO(), 2*time.Second, 10*time.Second, false, func(ctx context.Context) (bool, error) { + err1 := ApplyResourceFromTemplateByAdmin(oc, "--ignore-unknown-parameters=true", "-f", cudncrd.Template, "-p", "CRDNAME="+cudncrd.Crdname, "LABELKEY="+cudncrd.Labelkey, "LABELVALUE="+cudncrd.Labelvalue, + "IPv4CIDR="+cudncrd.IPv4cidr, "IPv4PREFIX="+strconv.Itoa(int(cudncrd.IPv4prefix)), "IPv6CIDR="+cudncrd.IPv6cidr, "IPv6PREFIX="+strconv.Itoa(int(cudncrd.IPv6prefix)), "ROLE="+cudncrd.Role) + if err1 != nil { + e2e.Logf("the err:%v, and try next round", err1) + return false, nil + } + return true, nil + }) + o.Expect(err).NotTo(o.HaveOccurred(), fmt.Sprintf("fail to create cudn CRD %s", cudncrd.Crdname)) +} + +func (cudncrd *CudnCRDResource) CreateCUDNCRDMatchExpSingleStack(oc *exutil.CLI) { + o.Expect(len(cudncrd.Values)).To(o.BeNumerically(">=", 2), fmt.Sprintf("CUDN %q requires at least 2 match-expression values", cudncrd.Crdname)) + err := wait.PollUntilContextTimeout(context.TODO(), 2*time.Second, 20*time.Second, false, func(ctx context.Context) (bool, error) { + err1 := ApplyResourceFromTemplateByAdmin(oc, "--ignore-unknown-parameters=true", "-f", cudncrd.Template, "-p", "CRDNAME="+cudncrd.Crdname, "KEY="+cudncrd.Key, "OPERATOR="+cudncrd.Operator, "VALUE1="+cudncrd.Values[0], "VALUE2="+cudncrd.Values[1], + "CIDR="+cudncrd.Cidr, "PREFIX="+strconv.Itoa(int(cudncrd.Prefix)), "ROLE="+cudncrd.Role) + if err1 != nil { + e2e.Logf("the err:%v, and try next round", err1) + return false, nil + } + return true, nil + }) + o.Expect(err).NotTo(o.HaveOccurred(), fmt.Sprintf("fail to create cudn CRD %s", cudncrd.Crdname)) +} + +func (cudncrd *CudnCRDResource) CreateCUDNCRDMatchExpDualStack(oc *exutil.CLI) { + o.Expect(len(cudncrd.Values)).To(o.BeNumerically(">=", 2), fmt.Sprintf("CUDN %q requires at least 2 match-expression values", cudncrd.Crdname)) + err := wait.PollUntilContextTimeout(context.TODO(), 2*time.Second, 20*time.Second, false, func(ctx context.Context) (bool, error) { + err1 := ApplyResourceFromTemplateByAdmin(oc, "--ignore-unknown-parameters=true", "-f", cudncrd.Template, "-p", "CRDNAME="+cudncrd.Crdname, "KEY="+cudncrd.Key, "OPERATOR="+cudncrd.Operator, "VALUE1="+cudncrd.Values[0], "VALUE2="+cudncrd.Values[1], + "IPv4CIDR="+cudncrd.IPv4cidr, "IPv4PREFIX="+strconv.Itoa(int(cudncrd.IPv4prefix)), "IPv6CIDR="+cudncrd.IPv6cidr, "IPv6PREFIX="+strconv.Itoa(int(cudncrd.IPv6prefix)), "ROLE="+cudncrd.Role) + if err1 != nil { + e2e.Logf("the err:%v, and try next round", err1) + return false, nil + } + return true, nil + }) + o.Expect(err).NotTo(o.HaveOccurred(), fmt.Sprintf("fail to create cudn CRD %s", cudncrd.Crdname)) +} + +func (udncrd *UdnCRDResource) DeleteUdnCRDDef(oc *exutil.CLI) { + RemoveResource(oc, true, true, "UserDefinedNetwork", udncrd.Crdname, "-n", udncrd.Namespace) +} + +func WaitUDNCRDApplied(oc *exutil.CLI, ns, crdName string) error { + checkErr := wait.PollUntilContextTimeout(context.TODO(), 3*time.Second, 60*time.Second, false, func(ctx context.Context) (bool, error) { + output, efErr := oc.AsAdmin().WithoutNamespace().Run("wait").Args("UserDefinedNetwork/"+crdName, "-n", ns, "--for", "condition=NetworkAllocationSucceeded=True").Output() + if efErr != nil { + e2e.Logf("Failed to get UDN %v, error: %s. Trying again", crdName, efErr) + return false, nil + } + if !strings.Contains(output, fmt.Sprintf("userdefinednetwork.k8s.ovn.org/%s condition met", crdName)) { + e2e.Logf("UDN CRD was not applied yet, trying again. \n %s", output) + return false, nil + } + return true, nil + }) + return checkErr +} + +func WaitCUDNCRDApplied(oc *exutil.CLI, crdName string) error { + checkErr := wait.PollUntilContextTimeout(context.TODO(), 3*time.Second, 30*time.Second, false, func(ctx context.Context) (bool, error) { + output, efErr := oc.AsAdmin().WithoutNamespace().Run("wait").Args("ClusterUserDefinedNetwork/"+crdName, "--for", "condition=NetworkCreated=True").Output() + if efErr != nil { + e2e.Logf("Failed to get CUDN %v, error: %s. Trying again", crdName, efErr) + return false, nil + } + if !strings.Contains(output, fmt.Sprintf("clusteruserdefinednetwork.k8s.ovn.org/%s condition met", crdName)) { + e2e.Logf("CUDN CRD was not applied yet, trying again. \n %s", output) + return false, nil + } + return true, nil + }) + return checkErr +} + +func (udncrd *UdnCRDResource) CreateLayer2DualStackUDNCRD(oc *exutil.CLI) { + err := wait.PollUntilContextTimeout(context.TODO(), 2*time.Second, 20*time.Second, false, func(ctx context.Context) (bool, error) { + err1 := ApplyResourceFromTemplateByAdmin(oc, "--ignore-unknown-parameters=true", "-f", udncrd.Template, "-p", "CRDNAME="+udncrd.Crdname, "NAMESPACE="+udncrd.Namespace, "IPv4CIDR="+udncrd.IPv4cidr, "IPv6CIDR="+udncrd.IPv6cidr, "ROLE="+udncrd.Role) + if err1 != nil { + e2e.Logf("the err:%v, and try next round", err1) + return false, nil + } + return true, nil + }) + o.Expect(err).NotTo(o.HaveOccurred(), fmt.Sprintf("fail to create udn CRD %s", udncrd.Crdname)) +} + +func (udncrd *UdnCRDResource) CreateLayer2SingleStackUDNCRD(oc *exutil.CLI) { + err := wait.PollUntilContextTimeout(context.TODO(), 2*time.Second, 20*time.Second, false, func(ctx context.Context) (bool, error) { + err1 := ApplyResourceFromTemplateByAdmin(oc, "--ignore-unknown-parameters=true", "-f", udncrd.Template, "-p", "CRDNAME="+udncrd.Crdname, "NAMESPACE="+udncrd.Namespace, "CIDR="+udncrd.Cidr, "ROLE="+udncrd.Role) + if err1 != nil { + e2e.Logf("the err:%v, and try next round", err1) + return false, nil + } + return true, nil + }) + o.Expect(err).NotTo(o.HaveOccurred(), fmt.Sprintf("fail to create udn CRD %s", udncrd.Crdname)) +} + +func (cudncrd *CudnCRDResource) CreateLayer2SingleStackCUDNCRD(oc *exutil.CLI) { + err := wait.PollUntilContextTimeout(context.TODO(), 2*time.Second, 20*time.Second, false, func(ctx context.Context) (bool, error) { + err1 := ApplyResourceFromTemplateByAdmin(oc, "--ignore-unknown-parameters=true", "-f", cudncrd.Template, "-p", "CRDNAME="+cudncrd.Crdname, "LABELKEY="+cudncrd.Labelkey, "LABELVALUE="+cudncrd.Labelvalue, + "CIDR="+cudncrd.Cidr, "ROLE="+cudncrd.Role) + if err1 != nil { + e2e.Logf("the err:%v, and try next round", err1) + return false, nil + } + return true, nil + }) + o.Expect(err).NotTo(o.HaveOccurred(), fmt.Sprintf("fail to create cudn CRD %s", cudncrd.Crdname)) +} + +func (cudncrd *CudnCRDResource) CreateLayer2DualStackCUDNCRD(oc *exutil.CLI) { + err := wait.PollUntilContextTimeout(context.TODO(), 2*time.Second, 20*time.Second, false, func(ctx context.Context) (bool, error) { + err1 := ApplyResourceFromTemplateByAdmin(oc, "--ignore-unknown-parameters=true", "-f", cudncrd.Template, "-p", "CRDNAME="+cudncrd.Crdname, "LABELKEY="+cudncrd.Labelkey, "LABELVALUE="+cudncrd.Labelvalue, + "IPv4CIDR="+cudncrd.IPv4cidr, "IPv6CIDR="+cudncrd.IPv6cidr, "ROLE="+cudncrd.Role) + if err1 != nil { + e2e.Logf("the err:%v, and try next round", err1) + return false, nil + } + return true, nil + }) + o.Expect(err).NotTo(o.HaveOccurred(), fmt.Sprintf("fail to create cudn CRD %s", cudncrd.Crdname)) +} + +func (cudncrd *CudnCRDResource) CreateLayer2CUDNCRDMatchExpSingleStack(oc *exutil.CLI) { + o.Expect(len(cudncrd.Values)).To(o.BeNumerically(">=", 2), fmt.Sprintf("CUDN %q requires at least 2 match-expression values", cudncrd.Crdname)) + err := wait.PollUntilContextTimeout(context.TODO(), 2*time.Second, 20*time.Second, false, func(ctx context.Context) (bool, error) { + err1 := ApplyResourceFromTemplateByAdmin(oc, "--ignore-unknown-parameters=true", "-f", cudncrd.Template, "-p", "CRDNAME="+cudncrd.Crdname, "KEY="+cudncrd.Key, "OPERATOR="+cudncrd.Operator, "VALUE1="+cudncrd.Values[0], "VALUE2="+cudncrd.Values[1], + "CIDR="+cudncrd.Cidr, "ROLE="+cudncrd.Role) + if err1 != nil { + e2e.Logf("the err:%v, and try next round", err1) + return false, nil + } + return true, nil + }) + o.Expect(err).NotTo(o.HaveOccurred(), fmt.Sprintf("fail to create cudn CRD %s", cudncrd.Crdname)) +} + +func (cudncrd *CudnCRDResource) CreateLayer2CUDNCRDMatchExpDualStack(oc *exutil.CLI) { + o.Expect(len(cudncrd.Values)).To(o.BeNumerically(">=", 2), fmt.Sprintf("CUDN %q requires at least 2 match-expression values", cudncrd.Crdname)) + err := wait.PollUntilContextTimeout(context.TODO(), 2*time.Second, 20*time.Second, false, func(ctx context.Context) (bool, error) { + err1 := ApplyResourceFromTemplateByAdmin(oc, "--ignore-unknown-parameters=true", "-f", cudncrd.Template, "-p", "CRDNAME="+cudncrd.Crdname, "KEY="+cudncrd.Key, "OPERATOR="+cudncrd.Operator, "VALUE1="+cudncrd.Values[0], "VALUE2="+cudncrd.Values[1], + "IPv4CIDR="+cudncrd.IPv4cidr, "IPv6CIDR="+cudncrd.IPv6cidr, "ROLE="+cudncrd.Role) + if err1 != nil { + e2e.Logf("the err:%v, and try next round", err1) + return false, nil + } + return true, nil + }) + o.Expect(err).NotTo(o.HaveOccurred(), fmt.Sprintf("fail to create cudn CRD %s", cudncrd.Crdname)) +} + +func (cudncrd *CudnCRDResource) CreateLayer3LocalnetCUDNCRD(oc *exutil.CLI) { + err := wait.PollUntilContextTimeout(context.TODO(), 2*time.Second, 20*time.Second, false, func(ctx context.Context) (bool, error) { + err1 := ApplyResourceFromTemplateByAdmin(oc, "--ignore-unknown-parameters=true", "-f", cudncrd.Template, "-p", "CRDNAME="+cudncrd.Crdname, "LABELKEY="+cudncrd.Labelkey, "LABELVALUE="+cudncrd.Labelvalue, "PHYSICALNETWORK="+cudncrd.Physicalnetworkname, "SUBNET="+cudncrd.Subnet, "EXCLUDESUBNET="+cudncrd.Excludesubnet, "ROLE="+cudncrd.Role) + if err1 != nil { + e2e.Logf("the err:%v, and try next round", err1) + return false, nil + } + return true, nil + }) + o.Expect(err).NotTo(o.HaveOccurred(), fmt.Sprintf("fail to create cudn CRD %s", cudncrd.Crdname)) +} + +func CheckPodCIDRsOverlap(oc *exutil.CLI, namespace string, ipStack string, Pods []string, netName string) bool { + var subnetsIPv4 []*net.IPNet + var subnetsIPv6 []*net.IPNet + var subnets []*net.IPNet + cmdIPv4 := "ip a sho " + netName + " | awk 'NR==3{print $2}'" + cmdIPv6 := "ip -o -6 addr show dev " + netName + " | awk '$3 == \"inet6\" && $6 == \"global\" {print $4}'" + for _, pod := range Pods { + if ipStack == "dualstack" { + podIPv4, ipv4Err := ExecCommandInSpecificPod(oc, namespace, pod, cmdIPv4) + o.Expect(ipv4Err).NotTo(o.HaveOccurred()) + podIPv6, ipv6Err := ExecCommandInSpecificPod(oc, namespace, pod, cmdIPv6) + o.Expect(ipv6Err).NotTo(o.HaveOccurred()) + _, subnetIPv4, err := net.ParseCIDR(strings.TrimSpace(podIPv4)) + o.Expect(err).NotTo(o.HaveOccurred()) + subnetsIPv4 = append(subnetsIPv4, subnetIPv4) + _, subnetIPv6, err := net.ParseCIDR(strings.TrimSpace(podIPv6)) + o.Expect(err).NotTo(o.HaveOccurred()) + subnetsIPv6 = append(subnetsIPv6, subnetIPv6) + } else { + if ipStack == "ipv6single" { + podIPv6, ipv6Err := ExecCommandInSpecificPod(oc, namespace, pod, cmdIPv6) + o.Expect(ipv6Err).NotTo(o.HaveOccurred()) + _, subnet, err := net.ParseCIDR(strings.TrimSpace(podIPv6)) + o.Expect(err).NotTo(o.HaveOccurred()) + subnets = append(subnets, subnet) + } else { + podIPv4, ipv4Err := ExecCommandInSpecificPod(oc, namespace, pod, cmdIPv4) + o.Expect(ipv4Err).NotTo(o.HaveOccurred()) + _, subnet, err := net.ParseCIDR(strings.TrimSpace(podIPv4)) + o.Expect(err).NotTo(o.HaveOccurred()) + subnets = append(subnets, subnet) + } + } + } + if ipStack == "dualstack" { + return subnetsIPv4[0].Contains(subnetsIPv4[1].IP) || subnetsIPv4[1].Contains(subnetsIPv4[0].IP) || + subnetsIPv6[0].Contains(subnetsIPv6[1].IP) || subnetsIPv6[1].Contains(subnetsIPv6[0].IP) + } + return subnets[0].Contains(subnets[1].IP) || subnets[1].Contains(subnets[0].IP) +} + +func ApplyL3UDNtoNamespace(oc *exutil.CLI, namespace string, udnSelector int) error { + udnCRDSingleStack := testdata.FixturePath("networking", "network_segmentation", "udn", "udn_crd_singlestack_template.yaml") + udnCRDdualStack := testdata.FixturePath("networking", "network_segmentation", "udn", "udn_crd_dualstack2_template.yaml") + + SkipIfNoFeatureGate(oc, "NetworkSegmentation") + + ipStackType := CheckIPStackType(oc) + var cidr, ipv4cidr, ipv6cidr []string + var prefix, ipv4prefix, ipv6prefix int32 + if ipStackType == "ipv4single" { + cidr = []string{"10.150.0.0/16", "10.151.0.0/16", "10.151.0.0/16"} + prefix = 24 + } else { + if ipStackType == "ipv6single" { + cidr = []string{"2010:100:200::0/60", "2011:100:200::0/60", "2011:100:200::0/60"} + prefix = 64 + } else { + ipv4cidr = []string{"10.150.0.0/16", "10.151.0.0/16", "10.151.0.0/16"} + ipv4prefix = 24 + ipv6cidr = []string{"2010:100:200::0/60", "2011:100:200::0/60", "2011:100:200::0/60"} + ipv6prefix = 64 + } + } + + var udncrd UdnCRDResource + if ipStackType == "dualstack" { + udncrd = UdnCRDResource{ + Crdname: "l3-network-" + namespace, + Namespace: namespace, + Role: "Primary", + IPv4cidr: ipv4cidr[udnSelector], + IPv4prefix: ipv4prefix, + IPv6cidr: ipv6cidr[udnSelector], + IPv6prefix: ipv6prefix, + Template: udnCRDdualStack, + } + udncrd.CreateUdnCRDDualStack(oc) + } else { + udncrd = UdnCRDResource{ + Crdname: "l3-network-" + namespace, + Namespace: namespace, + Role: "Primary", + Cidr: cidr[udnSelector], + Prefix: prefix, + Template: udnCRDSingleStack, + } + udncrd.CreateUdnCRDSingleStack(oc) + } + err := WaitUDNCRDApplied(oc, namespace, udncrd.Crdname) + return err +} + +func ApplyCUDNtoMatchLabelNS(oc *exutil.CLI, matchLabelKey, matchValue, crdName, ipv4cidr, ipv6cidr, cidr, topology string) (CudnCRDResource, error) { + var ( + testDataDirUDN = testdata.FixturePath("networking/network_segmentation/udn") + cudnCRDSingleStack = filepath.Join(testDataDirUDN, "cudn_crd_singlestack_template.yaml") + cudnCRDdualStack = filepath.Join(testDataDirUDN, "cudn_crd_dualstack_template.yaml") + cudnCRDL2dualStack = filepath.Join(testDataDirUDN, "cudn_crd_layer2_dualstack_template.yaml") + cudnCRDL2SingleStack = filepath.Join(testDataDirUDN, "cudn_crd_layer2_singlestack_template.yaml") + ) + + ipStackType := CheckIPStackType(oc) + cudncrd := CudnCRDResource{ + Crdname: crdName, + Labelkey: matchLabelKey, + Labelvalue: matchValue, + Role: "Primary", + Template: cudnCRDSingleStack, + } + + switch topology { + case "layer3": + switch ipStackType { + case "dualstack": + cudncrd.IPv4cidr = ipv4cidr + cudncrd.IPv4prefix = 24 + cudncrd.IPv6cidr = ipv6cidr + cudncrd.IPv6prefix = 64 + cudncrd.Template = cudnCRDdualStack + cudncrd.CreateCUDNCRDDualStack(oc) + case "ipv6single": + cudncrd.Prefix = 64 + cudncrd.Cidr = cidr + cudncrd.Template = cudnCRDSingleStack + cudncrd.CreateCUDNCRDSingleStack(oc) + case "ipv4single": + cudncrd.Prefix = 24 + cudncrd.Cidr = cidr + cudncrd.Template = cudnCRDSingleStack + cudncrd.CreateCUDNCRDSingleStack(oc) + } + case "layer2": + switch ipStackType { + case "dualstack": + cudncrd.IPv4cidr = ipv4cidr + cudncrd.IPv6cidr = ipv6cidr + cudncrd.Template = cudnCRDL2dualStack + cudncrd.CreateLayer2DualStackCUDNCRD(oc) + default: + cudncrd.Cidr = cidr + cudncrd.Template = cudnCRDL2SingleStack + cudncrd.CreateLayer2SingleStackCUDNCRD(oc) + } + } + err := WaitCUDNCRDApplied(oc, cudncrd.Crdname) + if err != nil { + return cudncrd, err + } + return cudncrd, nil +} + +func ApplyLocalnetCUDNtoMatchLabelNS(oc *exutil.CLI, matchLabelKey, matchValue, crdName, physicalNetworkName, subnet, excludeSubnet string, vlan bool) (CudnCRDResource, error) { + var ( + testDataDirUDN = testdata.FixturePath("networking/network_segmentation/udn") + cudnCRDLocalnetSingleStack = filepath.Join(testDataDirUDN, "cudn_crd_localnet_singlestack_template.yaml") + cudnCRDLocalnetSingleStackWithVlan = filepath.Join(testDataDirUDN, "cudn_crd_localnet_singlestack_with_vlan_template.yaml") + ) + + cudncrd := CudnCRDResource{ + Crdname: crdName, + Labelkey: matchLabelKey, + Labelvalue: matchValue, + Physicalnetworkname: physicalNetworkName, + Subnet: subnet, + Excludesubnet: excludeSubnet, + Role: "Secondary", + } + + if vlan { + cudncrd.Template = cudnCRDLocalnetSingleStackWithVlan + } else { + cudncrd.Template = cudnCRDLocalnetSingleStack + } + + cudncrd.CreateLayer3LocalnetCUDNCRD(oc) + err := WaitCUDNCRDApplied(oc, cudncrd.Crdname) + if err != nil { + return cudncrd, err + } + return cudncrd, nil +} + +func PingPod2PodPass(oc *exutil.CLI, namespaceSrc string, podNameSrc string, namespaceDst string, podNameDst string) { + podIP1, podIP2 := GetPodIP(oc, namespaceDst, podNameDst) + if podIP2 != "" { + _, err := e2eoutput.RunHostCmd(namespaceSrc, podNameSrc, "ping6 -c4 "+podIP1) + o.Expect(err).NotTo(o.HaveOccurred()) + _, err = e2eoutput.RunHostCmd(namespaceSrc, podNameSrc, "ping -c4 "+podIP2) + o.Expect(err).NotTo(o.HaveOccurred()) + } else { + if netutils.IsIPv6String(podIP1) { + _, err := e2eoutput.RunHostCmd(namespaceSrc, podNameSrc, "ping6 -c4 "+podIP1) + o.Expect(err).NotTo(o.HaveOccurred()) + } else { + _, err := e2eoutput.RunHostCmd(namespaceSrc, podNameSrc, "ping -c4 "+podIP1) + o.Expect(err).NotTo(o.HaveOccurred()) + } + } +} + +func PingPod2PodFail(oc *exutil.CLI, namespaceSrc string, podNameSrc string, namespaceDst string, podNameDst string) { + podIP1, podIP2 := GetPodIP(oc, namespaceDst, podNameDst) + if podIP2 != "" { + _, err := e2eoutput.RunHostCmd(namespaceSrc, podNameSrc, "ping6 -c4 "+podIP1) + o.Expect(err).To(o.HaveOccurred()) + _, err = e2eoutput.RunHostCmd(namespaceSrc, podNameSrc, "ping -c4 "+podIP2) + o.Expect(err).To(o.HaveOccurred()) + } else { + if netutils.IsIPv6String(podIP1) { + _, err := e2eoutput.RunHostCmd(namespaceSrc, podNameSrc, "ping6 -c4 "+podIP1) + o.Expect(err).To(o.HaveOccurred()) + } else { + _, err := e2eoutput.RunHostCmd(namespaceSrc, podNameSrc, "ping -c4 "+podIP1) + o.Expect(err).To(o.HaveOccurred()) + } + } +} + +func VerifyConnPod2Pod(oc *exutil.CLI, namespaceSrc string, podNameSrc string, namespaceDst string, podNameDst string, protocol string, port int, pass bool) { + e2e.Logf("==== Check %s traffic ====", protocol) + // kill socat process before sending/listen traffic + for _, nsPod := range [][]string{{namespaceSrc, podNameSrc}, {namespaceDst, podNameDst}} { + e2eoutput.RunHostCmd(nsPod[0], nsPod[1], "killall socat") + } + var clientOpt, serverOpt string + switch protocol { + case "UDP": + clientOpt = "udp-connect" + serverOpt = "udp6-listen" + case "SCTP": + clientOpt = "sctp-connect" + serverOpt = "sctp6-listen" + default: + e2e.Failf("protocol is not specified") + } + + e2e.Logf("Listening on port %s on dst pod %s", strconv.Itoa(port), podNameDst) + serverCmd, serverCmdOutput, _, serverCmdErr := oc.AsAdmin().WithoutNamespace().Run("exec").Args("-n", namespaceDst, podNameDst, "--", "socat", "-", serverOpt+":"+strconv.Itoa(port)+",fork").Background() + o.Expect(serverCmdErr).NotTo(o.HaveOccurred()) + defer func() { + if serverCmd != nil && serverCmd.Process != nil { + _ = serverCmd.Process.Kill() + } + }() + + e2e.Logf("Check %s process enabled in the dst pod %s", protocol, podNameDst) + o.Eventually(func() string { + msg, err := e2eoutput.RunHostCmd(namespaceDst, podNameDst, "ps aux | grep socat") + o.Expect(err).NotTo(o.HaveOccurred()) + return msg + }, "30s", "5s").Should(o.ContainSubstring(serverOpt), "No expected process running on dst pod") + + e2e.Logf("Sending %s packets from src pod %s to dst pod %s", protocol, podNameSrc, podNameDst) + podIP1, podIP2 := GetPodIP(oc, namespaceDst, podNameDst) + if pass { + if podIP2 != "" { + clientCmd := fmt.Sprintf("echo hello | socat - %s:%s", clientOpt, net.JoinHostPort(podIP1, strconv.Itoa(port))) + _, clientCmdErr := e2eoutput.RunHostCmd(namespaceSrc, podNameSrc, clientCmd) + o.Expect(clientCmdErr).NotTo(o.HaveOccurred()) + clientCmd = fmt.Sprintf("echo hello | socat - %s:%s", clientOpt, net.JoinHostPort(podIP2, strconv.Itoa(port))) + _, clientCmdErr = e2eoutput.RunHostCmd(namespaceSrc, podNameSrc, clientCmd) + o.Expect(clientCmdErr).NotTo(o.HaveOccurred()) + e2e.Logf("output on server side: %s", serverCmdOutput.String()) + o.Expect(strings.Count(serverCmdOutput.String(), "hello") == 2).To(o.BeTrue()) + } else { + clientCmd := fmt.Sprintf("timeout 10 sh -c 'echo hello | socat - %s:%s'", clientOpt, net.JoinHostPort(podIP1, strconv.Itoa(port))) + _, clientCmdErr := e2eoutput.RunHostCmd(namespaceSrc, podNameSrc, clientCmd) + o.Expect(clientCmdErr).NotTo(o.HaveOccurred()) + e2e.Logf("output on server side: %s", serverCmdOutput.String()) + o.Expect(strings.Contains(serverCmdOutput.String(), "hello")).To(o.BeTrue()) + } + } else { + if podIP2 != "" { + clientCmd := fmt.Sprintf("timeout 10 sh -c 'echo hello | socat - %s:%s'", clientOpt, net.JoinHostPort(podIP1, strconv.Itoa(port))) + e2eoutput.RunHostCmd(namespaceSrc, podNameSrc, clientCmd) + clientCmd = fmt.Sprintf("timeout 10 sh -c 'echo hello | socat - %s:%s'", clientOpt, net.JoinHostPort(podIP2, strconv.Itoa(port))) + e2eoutput.RunHostCmd(namespaceSrc, podNameSrc, clientCmd) + e2e.Logf("output on server side: %s", serverCmdOutput.String()) + o.Expect(strings.Contains(serverCmdOutput.String(), "hello")).To(o.BeFalse()) + } else { + clientCmd := fmt.Sprintf("timeout 10 sh -c 'echo hello | socat - %s:%s'", clientOpt, net.JoinHostPort(podIP1, strconv.Itoa(port))) + e2eoutput.RunHostCmd(namespaceSrc, podNameSrc, clientCmd) + e2e.Logf("output on server side: %s", serverCmdOutput.String()) + o.Expect(strings.Contains(serverCmdOutput.String(), "hello")).To(o.BeFalse()) + } + } +} + +func CreateGeneralUDNCRD(oc *exutil.CLI, namespace, crdName, ipv4cidr, ipv6cidr, cidr, layer string) { + var ( + testDataDirUDN = testdata.FixturePath("networking/network_segmentation/udn") + udnCRDdualStack = filepath.Join(testDataDirUDN, "udn_crd_dualstack2_template.yaml") + udnCRDSingleStack = filepath.Join(testDataDirUDN, "udn_crd_singlestack_template.yaml") + udnCRDLayer2dualStack = filepath.Join(testDataDirUDN, "udn_crd_layer2_dualstack_template.yaml") + udnCRDLayer2SingleStack = filepath.Join(testDataDirUDN, "udn_crd_layer2_singlestack_template.yaml") + ) + + ipStackType := CheckIPStackType(oc) + var udncrd UdnCRDResource + switch layer { + case "layer3": + switch ipStackType { + case "dualstack": + udncrd = UdnCRDResource{ + Crdname: crdName, + Namespace: namespace, + Role: "Primary", + IPv4cidr: ipv4cidr, + IPv4prefix: 24, + IPv6cidr: ipv6cidr, + IPv6prefix: 64, + Template: udnCRDdualStack, + } + udncrd.CreateUdnCRDDualStack(oc) + case "ipv6single": + udncrd = UdnCRDResource{ + Crdname: crdName, + Namespace: namespace, + Role: "Primary", + Cidr: cidr, + Prefix: 64, + Template: udnCRDSingleStack, + } + udncrd.CreateUdnCRDSingleStack(oc) + default: + udncrd = UdnCRDResource{ + Crdname: crdName, + Namespace: namespace, + Role: "Primary", + Cidr: cidr, + Prefix: 24, + Template: udnCRDSingleStack, + } + udncrd.CreateUdnCRDSingleStack(oc) + } + err := WaitUDNCRDApplied(oc, namespace, udncrd.Crdname) + o.Expect(err).NotTo(o.HaveOccurred()) + + case "layer2": + switch ipStackType { + case "dualstack": + udncrd = UdnCRDResource{ + Crdname: crdName, + Namespace: namespace, + Role: "Primary", + IPv4cidr: ipv4cidr, + IPv6cidr: ipv6cidr, + Template: udnCRDLayer2dualStack, + } + udncrd.CreateLayer2DualStackUDNCRD(oc) + + default: + udncrd = UdnCRDResource{ + Crdname: crdName, + Namespace: namespace, + Role: "Primary", + Cidr: cidr, + Template: udnCRDLayer2SingleStack, + } + udncrd.CreateLayer2SingleStackUDNCRD(oc) + } + err := WaitUDNCRDApplied(oc, namespace, udncrd.Crdname) + o.Expect(err).NotTo(o.HaveOccurred()) + default: + e2e.Logf("Not surpport UDN type for now.") + } +} + +func CreateCUDNCRD(oc *exutil.CLI, key, crdName, ipv4cidr, ipv6cidr, cidr, layer string, values []string) (CudnCRDResource, error) { + var ( + testDataDirUDN = testdata.FixturePath("networking/network_segmentation/udn") + cudnCRDL3dualStack = filepath.Join(testDataDirUDN, "cudn_crd_matchexp_dualstack_template.yaml") + cudnCRDL3SingleStack = filepath.Join(testDataDirUDN, "cudn_crd_matchexp_singlestack_template.yaml") + cudnCRDLayer2dualStack = filepath.Join(testDataDirUDN, "cudn_crd_matchexp_layer2_dualstack_template.yaml") + cudnCRDLayer2SingleStack = filepath.Join(testDataDirUDN, "cudn_crd_matchexp_layer2_singlestack_template.yaml") + ) + + ipStackType := CheckIPStackType(oc) + cudncrd := CudnCRDResource{ + Crdname: crdName, + Key: key, + Operator: "In", + Values: values, + Role: "Primary", + Template: cudnCRDL3dualStack, + } + + switch layer { + case "layer3": + switch ipStackType { + case "dualstack": + cudncrd.IPv4cidr = ipv4cidr + cudncrd.IPv4prefix = 24 + cudncrd.IPv6cidr = ipv6cidr + cudncrd.IPv6prefix = 64 + cudncrd.Template = cudnCRDL3dualStack + cudncrd.CreateCUDNCRDMatchExpDualStack(oc) + case "ipv6single": + cudncrd.Prefix = 64 + cudncrd.Cidr = cidr + cudncrd.Template = cudnCRDL3SingleStack + cudncrd.CreateCUDNCRDMatchExpSingleStack(oc) + default: + cudncrd.Prefix = 24 + cudncrd.Cidr = cidr + cudncrd.Template = cudnCRDL3SingleStack + cudncrd.CreateCUDNCRDMatchExpSingleStack(oc) + } + case "layer2": + switch ipStackType { + case "dualstack": + cudncrd.IPv4cidr = ipv4cidr + cudncrd.IPv6cidr = ipv6cidr + cudncrd.Template = cudnCRDLayer2dualStack + cudncrd.CreateLayer2CUDNCRDMatchExpDualStack(oc) + default: + cudncrd.Cidr = cidr + cudncrd.Template = cudnCRDLayer2SingleStack + cudncrd.CreateLayer2CUDNCRDMatchExpSingleStack(oc) + } + + default: + e2e.Logf("Not supported UDN type for now.") + } + err := WaitCUDNCRDApplied(oc, cudncrd.Crdname) + if err != nil { + return cudncrd, err + } + return cudncrd, nil +} + +// Multicast utility functions + +// DisableMulticast disables multicast on specific namespace +func DisableMulticast(oc *exutil.CLI, ns string) { + _, err := RunOcWithRetry(oc.AsAdmin().WithoutNamespace(), "annotate", "namespace", ns, "k8s.ovn.org/multicast-enabled-") + o.Expect(err).NotTo(o.HaveOccurred()) +} + +// ChkMcastTraffic sends omping traffic on all multicast pods +func ChkMcastTraffic(oc *exutil.CLI, namespace string, podList []string, ipList []string, mcastip string, port string) bool { + pktFile := make([]string, len(podList)) + for i, podName := range podList { + pktFile[i] = "/tmp/" + GetRandomString() + ".txt" + StartMcastTrafficOnPod(oc, namespace, podName, ipList, pktFile[i], mcastip, port) + } + // wait for omping packets send and receive + time.Sleep(30 * time.Second) + for i, podName := range podList { + if !ChkMcatRcvOnPod(oc, namespace, podName, ipList[i], ipList, mcastip, pktFile[i]) { + return false + } + } + return true +} + +// StartMcastTrafficOnPod sends multicast traffic via omping in a goroutine so +// all pods start omping concurrently — omping requires all peers running simultaneously. +func StartMcastTrafficOnPod(oc *exutil.CLI, ns string, pod string, ipList []string, pktfile string, mcastip string, port string) { + ipStr := strings.Join(ipList, " ") + if port == "" { + port = "4321" + } + go func() { + ompingCmd := "omping " + "-q " + "-p " + port + " -c 20 -T 20 -m " + mcastip + " " + ipStr + " > " + fmt.Sprintf("%s", pktfile) + " &" + _, err := e2eoutput.RunHostCmd(ns, pod, ompingCmd) + o.Expect(err).NotTo(o.HaveOccurred()) + }() +} + +// ChkMcatRcvOnPod checks omping send/receive results on a pod +func ChkMcatRcvOnPod(oc *exutil.CLI, ns string, pod string, podip string, iplist []string, mcastip string, pktfile string) bool { + catCmd := "cat " + fmt.Sprintf("%s", pktfile) + outPut, err := e2eoutput.RunHostCmd(ns, pod, catCmd) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(outPut).NotTo(o.BeEmpty()) + for _, neighborip := range iplist { + if neighborip != podip { + reg1 := regexp.MustCompile(regexp.QuoteMeta(neighborip) + `.*joined \(S,G\) = \(\*,\s*` + regexp.QuoteMeta(mcastip) + `\), pinging`) + reg2 := regexp.MustCompile(regexp.QuoteMeta(neighborip) + `.*multicast, xmt/rcv/%loss = \d+/(\d+)/\d+%`) + + match1 := reg1.MatchString(outPut) + match2 := reg2.FindStringSubmatch(outPut) + o.Expect(match2).ShouldNot(o.BeNil()) + pktNum, _ := strconv.Atoi(match2[1]) + e2e.Logf("Received packets on pod %v from ip %v is %v", pod, neighborip, pktNum) + if pktNum == 0 || !match1 { + return false + } + } + } + return true +} + +// GetPodIPv4UDNList gets ipv4 addresses of udn pods +func GetPodIPv4UDNList(oc *exutil.CLI, namespace string, podList []string) []string { + var ipList []string + ipStackType := CheckIPStackType(oc) + o.Expect(ipStackType).ShouldNot(o.Equal("ipv6single")) + for _, podName := range podList { + podIP1, podIP2 := GetPodIPUDN(oc, namespace, podName, "ovn-udn1") + if ipStackType == "dualstack" { + ipList = append(ipList, podIP2) + } else { + ipList = append(ipList, podIP1) + } + } + e2e.Logf("The ipv4list for pods is %v", ipList) + return ipList +} + +// GetPodIPv6UDNList gets ipv6 addresses of udn pods +func GetPodIPv6UDNList(oc *exutil.CLI, namespace string, podList []string) []string { + var ipList []string + ipStackType := CheckIPStackType(oc) + o.Expect(ipStackType).ShouldNot(o.Equal("ipv4single")) + for _, podName := range podList { + podIP1, _ := GetPodIPUDN(oc, namespace, podName, "ovn-udn1") + ipList = append(ipList, podIP1) + } + e2e.Logf("The ipv6list for pods is %v", ipList) + return ipList +} + +// GetPodIPv4List gets ipv4 addresses of default pods +func GetPodIPv4List(oc *exutil.CLI, namespace string, podList []string) []string { + var ipList []string + ipStackType := CheckIPStackType(oc) + o.Expect(ipStackType).ShouldNot(o.Equal("ipv6single")) + for _, podName := range podList { + podIP := GetPodIPv4(oc, namespace, podName) + ipList = append(ipList, podIP) + } + e2e.Logf("The ipv4list for pods is %v", ipList) + return ipList +} + +// GetPodIPv6List gets ipv6 addresses of default pods +func GetPodIPv6List(oc *exutil.CLI, namespace string, podList []string) []string { + var ipList []string + ipStackType := CheckIPStackType(oc) + o.Expect(ipStackType).ShouldNot(o.Equal("ipv4single")) + for _, podName := range podList { + podIP := GetPodIPv6(oc, namespace, podName, ipStackType) + ipList = append(ipList, podIP) + } + e2e.Logf("The ipv6list for pods is %v", ipList) + return ipList +} + +// ChkMcastAddress checks netstat during sending multicast traffic +func ChkMcastAddress(oc *exutil.CLI, ns string, pod string, intf string, mcastip string) { + netstatCmd := "netstat -ng" + outPut, err := e2eoutput.RunHostCmd(ns, pod, netstatCmd) + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("netstat result is %v: /n", outPut) + reg := regexp.MustCompile(regexp.QuoteMeta(intf) + `\s+\d+\s+` + regexp.QuoteMeta(mcastip)) + matchRes := reg.MatchString(outPut) + o.Expect(matchRes).Should(o.BeTrue()) +} + +// GetPodIPUDNv4 returns IPv4 address of specific interface +func GetPodIPUDNv4(oc *exutil.CLI, namespace string, podName string, netName string) string { + ipStack := CheckIPStackType(oc) + ip1, ip2 := GetPodIPUDN(oc, namespace, podName, netName) + if ipStack == "ipv4single" { + return ip1 + } else if ipStack == "dualstack" { + return ip2 + } + return "" +} + +// GetPodIPUDNv6 returns IPv6 address of specific interface +func GetPodIPUDNv6(oc *exutil.CLI, namespace string, podName string, netName string) string { + ipStack := CheckIPStackType(oc) + ip1, _ := GetPodIPUDN(oc, namespace, podName, netName) + if ipStack == "ipv6single" || ipStack == "dualstack" { + return ip1 + } + return "" +} + +// ReplicationControllerPingPodResource methods + +func (rcPingPod *ReplicationControllerPingPodResource) CreateReplicaController(oc *exutil.CLI) { + e2e.Logf("Creating replication controller from template") + replicasString := fmt.Sprintf("REPLICAS=%v", rcPingPod.Replicas) + err := wait.Poll(5*time.Second, 20*time.Second, func() (bool, error) { + err1 := ApplyResourceFromTemplateByAdmin(oc, "--ignore-unknown-parameters=true", "-f", rcPingPod.Template, "-p", "PODNAME="+rcPingPod.Name, + "NAMESPACE="+rcPingPod.Namespace, replicasString) + if err1 != nil { + e2e.Logf("Error creating replication controller:%v, and trying again", err1) + return false, nil + } + return true, nil + }) + o.Expect(err).NotTo(o.HaveOccurred(), fmt.Sprintf("Failed to create replicationcontroller %v", rcPingPod.Name)) +} diff --git a/openshift/pkg/otp/utils/utils.go b/openshift/pkg/otp/utils/utils.go new file mode 100644 index 0000000000..460b6c4404 --- /dev/null +++ b/openshift/pkg/otp/utils/utils.go @@ -0,0 +1,4854 @@ +package otputils + +import ( + "context" + "encoding/json" + "fmt" + "io" + "math/rand" + "net" + "os" + "os/exec" + "path/filepath" + "regexp" + "strconv" + "strings" + "time" + + g "github.com/onsi/ginkgo/v2" + o "github.com/onsi/gomega" + exutil "github.com/openshift/origin/test/extended/util" + "github.com/ovn-kubernetes/ovn-kubernetes/openshift/pkg/otp/testdata" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/wait" + e2e "k8s.io/kubernetes/test/e2e/framework" + e2enode "k8s.io/kubernetes/test/e2e/framework/node" + e2eoutput "k8s.io/kubernetes/test/e2e/framework/pod/output" + netutils "k8s.io/utils/net" +) + +type PingPodResource struct { + Name string + Namespace string + Template string +} + +type PingPodResourceNode struct { + Name string + Namespace string + Nodename string + Template string +} + +type PingPodResourceWinNode struct { + Name string + Namespace string + Image string + Nodename string + Template string +} + +type EgressIPResource1 struct { + Name string + Template string + EgressIP1 string + EgressIP2 string + NsLabelKey string + NsLabelValue string + PodLabelKey string + PodLabelValue string +} + +type EgressFirewall1 struct { + Name string + Namespace string + Template string +} + +type EgressFirewall2 struct { + Name string + Namespace string + Ruletype string + Cidr string + Template string +} + +type IpBlockCIDRsDual struct { + Name string + Namespace string + CidrIpv4 string + CidrIpv6 string + Cidr2Ipv4 string + Cidr2Ipv6 string + Cidr3Ipv4 string + Cidr3Ipv6 string + Template string +} + +type IpBlockCIDRsSingle struct { + Name string + Namespace string + Cidr string + Cidr2 string + Cidr3 string + Template string +} +type IpBlockCIDRsExceptDual struct { + Name string + Namespace string + CidrIpv4 string + CidrIpv4Except string + CidrIpv6 string + CidrIpv6Except string + Cidr2Ipv4 string + Cidr2Ipv4Except string + Cidr2Ipv6 string + Cidr2Ipv6Except string + Cidr3Ipv4 string + Cidr3Ipv4Except string + Cidr3Ipv6 string + Cidr3Ipv6Except string + Template string +} +type IpBlockCIDRsExceptSingle struct { + Name string + Namespace string + Cidr string + Except string + Cidr2 string + Except2 string + Cidr3 string + Except3 string + Template string +} + +type GenericServiceResource struct { + Servicename string + Namespace string + Protocol string + Selector string + ServiceType string + IpFamilyPolicy string + ExternalTrafficPolicy string + InternalTrafficPolicy string + Template string +} + +type WindowGenericServiceResource struct { + Servicename string + Namespace string + Protocol string + Selector string + ServiceType string + IpFamilyPolicy string + ExternalTrafficPolicy string + InternalTrafficPolicy string + Template string +} + +type ExternalIPService struct { + Name string + Namespace string + ExternalIP string + Template string +} + +type ExternalIPPod struct { + Name string + Namespace string + Template string +} + +type NodePortService struct { + Name string + Namespace string + NodeName string + Template string +} + +type EgressPolicy struct { + Name string + Namespace string + CidrSelector string + Template string +} +type AclSettings struct { + DenySetting string `json:"deny"` + AllowSetting string `json:"allow"` +} + +type EgressrouterMultipleDst struct { + Name string + Namespace string + Reservedip string + Gateway string + Destinationip1 string + Destinationip2 string + Destinationip3 string + Template string +} + +type EgressrouterRedSDN struct { + Name string + Namespace string + Reservedip string + Gateway string + Destinationip string + Labelkey string + Labelvalue string + Template string +} + +type EgressFirewall5 struct { + Name string + Namespace string + Ruletype1 string + Rulename1 string + Rulevalue1 string + Protocol1 string + Portnumber1 int + Ruletype2 string + Rulename2 string + Rulevalue2 string + Protocol2 string + Portnumber2 int + Template string +} + +type EgressNetworkpolicy struct { + Name string + Namespace string + Ruletype string + Rulename string + Rulevalue string + Template string +} + +type SvcEndpontDetails struct { + OvnKubeNodePod string + NodeName string + PodIP string +} + +type MigrationDetails struct { + Name string + Template string + Namespace string + Virtualmachinesintance string +} + +type KubeletKillerPod struct { + Name string + Namespace string + Nodename string + Template string +} + +type HttpserverPodResourceNode struct { + Name string + Namespace string + Containerport int32 + Hostport int32 + Nodename string + Template string +} + +// struct for using nncp to create VF on sriov node +type VRFResource struct { + Name string + Intfname string + Nodename string + Tableid int + Template string +} + +// struct to create a pod with named port +type NamedPortPodResource struct { + Name string + Namespace string + PodLabelKey string + PodLabelVal string + Portname string + Containerport int32 + Template string +} + +func (pod *PingPodResource) CreatePingPod(oc *exutil.CLI) { + err := wait.Poll(5*time.Second, 20*time.Second, func() (bool, error) { + err1 := ApplyResourceFromTemplateByAdmin(oc, "--ignore-unknown-parameters=true", "-f", pod.Template, "-p", "NAME="+pod.Name, "NAMESPACE="+pod.Namespace) + if err1 != nil { + e2e.Logf("the err:%v, and try next round", err1) + return false, nil + } + return true, nil + }) + o.Expect(err).NotTo(o.HaveOccurred(), fmt.Sprintf("fail to create pod %v", pod.Name)) +} + +func (pod *PingPodResourceNode) CreatePingPodNode(oc *exutil.CLI) { + err := wait.Poll(3*time.Second, 20*time.Second, func() (bool, error) { + err1 := ApplyResourceFromTemplateByAdmin(oc, "--ignore-unknown-parameters=true", "-f", pod.Template, "-p", "NAME="+pod.Name, "NAMESPACE="+pod.Namespace, "NODENAME="+pod.Nodename) + if err1 != nil { + e2e.Logf("the err:%v, and try next round", err1) + return false, nil + } + return true, nil + }) + o.Expect(err).NotTo(o.HaveOccurred(), fmt.Sprintf("fail to create pod %v", pod.Name)) +} + +func (pod *PingPodResourceWinNode) CreatePingPodWinNode(oc *exutil.CLI) { + err := wait.Poll(5*time.Second, 20*time.Second, func() (bool, error) { + err1 := ApplyResourceFromTemplate(oc, "--ignore-unknown-parameters=true", "-f", pod.Template, "-p", "NAME="+pod.Name, "NAMESPACE="+pod.Namespace, "IMAGE="+pod.Image, "NODENAME="+pod.Nodename) + if err1 != nil { + e2e.Logf("the err:%v, and try next round", err1) + return false, nil + } + return true, nil + }) + o.Expect(err).NotTo(o.HaveOccurred(), fmt.Sprintf("fail to create pod %v", pod.Name)) +} + +func ApplyResourceFromTemplate(oc *exutil.CLI, parameters ...string) error { + var configFile string + err := wait.Poll(3*time.Second, 15*time.Second, func() (bool, error) { + output, err := oc.Run("process").Args(parameters...).OutputToFile(GetRandomString() + "ping-pod.json") + if err != nil { + e2e.Logf("the err:%v, and try next round", err) + return false, nil + } + configFile = output + return true, nil + }) + o.Expect(err).NotTo(o.HaveOccurred(), fmt.Sprintf("fail to process %v", parameters)) + + e2e.Logf("the file of resource is %s", configFile) + applyArgs := []string{"-f", configFile} + for i, p := range parameters { + if p == "-n" && i+1 < len(parameters) { + applyArgs = append(applyArgs, "-n", parameters[i+1]) + break + } + } + return oc.WithoutNamespace().Run("apply").Args(applyArgs...).Execute() +} + +func (egressIP *EgressIPResource1) CreateEgressIPObject1(oc *exutil.CLI) { + err := wait.Poll(5*time.Second, 20*time.Second, func() (bool, error) { + err1 := ApplyResourceFromTemplateByAdmin(oc, "--ignore-unknown-parameters=true", "-f", egressIP.Template, "-p", "NAME="+egressIP.Name, "EGRESSIP1="+egressIP.EgressIP1, "EGRESSIP2="+egressIP.EgressIP2) + if err1 != nil { + e2e.Logf("the err:%v, and try next round", err1) + return false, nil + } + return true, nil + }) + o.Expect(err).NotTo(o.HaveOccurred(), fmt.Sprintf("fail to create EgressIP %v", egressIP.Name)) +} + +func (egressIP *EgressIPResource1) DeleteEgressIPObject1(oc *exutil.CLI) { + RemoveResource(oc, true, true, "egressip", egressIP.Name) +} + +func (egressIP *EgressIPResource1) CreateEgressIPObject2(oc *exutil.CLI) { + err := wait.Poll(5*time.Second, 20*time.Second, func() (bool, error) { + err1 := ApplyResourceFromTemplateByAdmin(oc, "--ignore-unknown-parameters=true", "-f", egressIP.Template, "-p", "NAME="+egressIP.Name, "EGRESSIP1="+egressIP.EgressIP1, "NSLABELKEY="+egressIP.NsLabelKey, "NSLABELVALUE="+egressIP.NsLabelValue, "PODLABELKEY="+egressIP.PodLabelKey, "PODLABELVALUE="+egressIP.PodLabelValue) + if err1 != nil { + e2e.Logf("the err:%v, and try next round", err1) + return false, nil + } + return true, nil + }) + o.Expect(err).NotTo(o.HaveOccurred(), fmt.Sprintf("fail to create EgressIP %v", egressIP.Name)) +} + +func (egressFirewall *EgressFirewall1) CreateEgressFWObject1(oc *exutil.CLI) { + err := wait.Poll(5*time.Second, 20*time.Second, func() (bool, error) { + err1 := ApplyResourceFromTemplateByAdmin(oc, "--ignore-unknown-parameters=true", "-f", egressFirewall.Template, "-p", "NAME="+egressFirewall.Name, "NAMESPACE="+egressFirewall.Namespace) + if err1 != nil { + e2e.Logf("the err:%v, and try next round", err1) + return false, nil + } + return true, nil + }) + o.Expect(err).NotTo(o.HaveOccurred(), fmt.Sprintf("fail to create EgressFW %v", egressFirewall.Name)) +} + +func (egressFirewall *EgressFirewall1) DeleteEgressFWObject1(oc *exutil.CLI) { + RemoveResource(oc, true, true, "egressfirewall", egressFirewall.Name, "-n", egressFirewall.Namespace) +} + +func (egressFirewall *EgressFirewall2) CreateEgressFW2Object(oc *exutil.CLI) { + err := wait.Poll(5*time.Second, 20*time.Second, func() (bool, error) { + err1 := ApplyResourceFromTemplateByAdmin(oc, "--ignore-unknown-parameters=true", "-f", egressFirewall.Template, "-p", "NAME="+egressFirewall.Name, "NAMESPACE="+egressFirewall.Namespace, "RULETYPE="+egressFirewall.Ruletype, "CIDR="+egressFirewall.Cidr) + if err1 != nil { + e2e.Logf("the err:%v, and try next round", err1) + return false, nil + } + return true, nil + }) + o.Expect(err).NotTo(o.HaveOccurred(), fmt.Sprintf("fail to create EgressFW2 %v", egressFirewall.Name)) +} + +func (EFW *EgressFirewall5) CreateEgressFW5Object(oc *exutil.CLI) { + err := wait.Poll(5*time.Second, 20*time.Second, func() (bool, error) { + parameters := []string{"--ignore-unknown-parameters=true", "-f", EFW.Template, "-p", "NAME=" + EFW.Name, "NAMESPACE=" + EFW.Namespace, "RULETYPE1=" + EFW.Ruletype1, "RULENAME1=" + EFW.Rulename1, "RULEVALUE1=" + EFW.Rulevalue1, "PROTOCOL1=" + EFW.Protocol1, "PORTNUMBER1=" + strconv.Itoa(EFW.Portnumber1), "RULETYPE2=" + EFW.Ruletype2, "RULENAME2=" + EFW.Rulename2, "RULEVALUE2=" + EFW.Rulevalue2, "PROTOCOL2=" + EFW.Protocol2, "PORTNUMBER2=" + strconv.Itoa(EFW.Portnumber2)} + err1 := ApplyResourceFromTemplateByAdmin(oc, parameters...) + if err1 != nil { + e2e.Logf("the err:%v, and try next round", err1) + return false, nil + } + return true, nil + }) + o.Expect(err).NotTo(o.HaveOccurred(), fmt.Sprintf("fail to create EgressFW2 %v", EFW.Name)) +} + +func (eNPL *EgressNetworkpolicy) CreateEgressNetworkPolicyObj(oc *exutil.CLI) { + err := wait.Poll(5*time.Second, 20*time.Second, func() (bool, error) { + parameters := []string{"--ignore-unknown-parameters=true", "-f", eNPL.Template, "-p", "NAME=" + eNPL.Name, "NAMESPACE=" + eNPL.Namespace, "RULETYPE=" + eNPL.Ruletype, "RULENAME=" + eNPL.Rulename, "RULEVALUE=" + eNPL.Rulevalue} + err1 := ApplyResourceFromTemplateByAdmin(oc, parameters...) + if err1 != nil { + e2e.Logf("the err:%v, and try next round", err1) + return false, nil + } + return true, nil + }) + o.Expect(err).NotTo(o.HaveOccurred(), fmt.Sprintf("Failed to create EgressNetworkPolicy %v in Namespace %v", eNPL.Name, eNPL.Namespace)) +} + +// Single CIDR on Dual stack +func (ipBlock_policy *IpBlockCIDRsDual) CreateipBlockCIDRObjectDual(oc *exutil.CLI) { + err := wait.Poll(5*time.Second, 20*time.Second, func() (bool, error) { + err1 := ApplyResourceFromTemplateByAdmin(oc, "--ignore-unknown-parameters=true", "-f", ipBlock_policy.Template, "-p", "NAME="+ipBlock_policy.Name, "NAMESPACE="+ipBlock_policy.Namespace, "cidrIpv6="+ipBlock_policy.CidrIpv6, "cidrIpv4="+ipBlock_policy.CidrIpv4) + if err1 != nil { + e2e.Logf("the err:%v, and try next round", err1) + return false, nil + } + return true, nil + }) + o.Expect(err).NotTo(o.HaveOccurred(), fmt.Sprintf("fail to create network policy %v", ipBlock_policy.Name)) +} + +// Single CIDR on single stack +func (ipBlock_policy *IpBlockCIDRsSingle) CreateipBlockCIDRObjectSingle(oc *exutil.CLI) { + err := wait.Poll(5*time.Second, 20*time.Second, func() (bool, error) { + err1 := ApplyResourceFromTemplateByAdmin(oc, "--ignore-unknown-parameters=true", "-f", ipBlock_policy.Template, "-p", "NAME="+ipBlock_policy.Name, "NAMESPACE="+ipBlock_policy.Namespace, "CIDR="+ipBlock_policy.Cidr) + if err1 != nil { + e2e.Logf("the err:%v, and try next round", err1) + return false, nil + } + return true, nil + }) + o.Expect(err).NotTo(o.HaveOccurred(), fmt.Sprintf("fail to create network policy %v", ipBlock_policy.Name)) +} + +// Single IP Block with except clause on Dual stack +func (ipBlock_except_policy *IpBlockCIDRsExceptDual) CreateipBlockExceptObjectDual(oc *exutil.CLI) { + err := wait.Poll(5*time.Second, 20*time.Second, func() (bool, error) { + + policyApplyError := ApplyResourceFromTemplateByAdmin(oc, "--ignore-unknown-parameters=true", "-f", ipBlock_except_policy.Template, "-p", "NAME="+ipBlock_except_policy.Name, "NAMESPACE="+ipBlock_except_policy.Namespace, "CIDR_IPv6="+ipBlock_except_policy.CidrIpv6, "EXCEPT_IPv6="+ipBlock_except_policy.CidrIpv6Except, "CIDR_IPv4="+ipBlock_except_policy.CidrIpv4, "EXCEPT_IPv4="+ipBlock_except_policy.CidrIpv4Except) + if policyApplyError != nil { + e2e.Logf("the err:%v, and try next round", policyApplyError) + return false, nil + } + return true, nil + }) + o.Expect(err).NotTo(o.HaveOccurred(), fmt.Sprintf("fail to create network policy %v", ipBlock_except_policy.Name)) +} + +// Single IP Block with except clause on Single stack +func (ipBlock_except_policy *IpBlockCIDRsExceptSingle) CreateipBlockExceptObjectSingle(oc *exutil.CLI, except bool) { + err := wait.Poll(5*time.Second, 20*time.Second, func() (bool, error) { + + policyApplyError := ApplyResourceFromTemplateByAdmin(oc, "--ignore-unknown-parameters=true", "-f", ipBlock_except_policy.Template, "-p", "NAME="+ipBlock_except_policy.Name, "NAMESPACE="+ipBlock_except_policy.Namespace, "CIDR="+ipBlock_except_policy.Cidr, "EXCEPT="+ipBlock_except_policy.Except) + if policyApplyError != nil { + e2e.Logf("the err:%v, and try next round", policyApplyError) + return false, nil + } + return true, nil + }) + o.Expect(err).NotTo(o.HaveOccurred(), fmt.Sprintf("fail to create network policy %v", ipBlock_except_policy.Name)) +} + +// Function to create ingress or egress policy with multiple CIDRs on Dual Stack Cluster +func (ipBlock_cidrs_policy *IpBlockCIDRsDual) CreateIPBlockMultipleCIDRsObjectDual(oc *exutil.CLI) { + err := wait.Poll(5*time.Second, 20*time.Second, func() (bool, error) { + err1 := ApplyResourceFromTemplateByAdmin(oc, "--ignore-unknown-parameters=true", "-f", ipBlock_cidrs_policy.Template, "-p", "NAME="+ipBlock_cidrs_policy.Name, "NAMESPACE="+ipBlock_cidrs_policy.Namespace, "cidrIpv6="+ipBlock_cidrs_policy.CidrIpv6, "cidrIpv4="+ipBlock_cidrs_policy.CidrIpv4, "cidr2Ipv4="+ipBlock_cidrs_policy.Cidr2Ipv4, "cidr2Ipv6="+ipBlock_cidrs_policy.Cidr2Ipv6, "cidr3Ipv4="+ipBlock_cidrs_policy.Cidr3Ipv4, "cidr3Ipv6="+ipBlock_cidrs_policy.Cidr3Ipv6) + if err1 != nil { + e2e.Logf("the err:%v, and try next round", err1) + return false, nil + } + return true, nil + }) + o.Expect(err).NotTo(o.HaveOccurred(), fmt.Sprintf("fail to create network policy %v", ipBlock_cidrs_policy.Name)) +} + +// Function to create ingress or egress policy with multiple CIDRs on Single Stack Cluster +func (ipBlock_cidrs_policy *IpBlockCIDRsSingle) CreateIPBlockMultipleCIDRsObjectSingle(oc *exutil.CLI) { + err := wait.Poll(5*time.Second, 20*time.Second, func() (bool, error) { + err1 := ApplyResourceFromTemplateByAdmin(oc, "--ignore-unknown-parameters=true", "-f", ipBlock_cidrs_policy.Template, "-p", "NAME="+ipBlock_cidrs_policy.Name, "NAMESPACE="+ipBlock_cidrs_policy.Namespace, "CIDR="+ipBlock_cidrs_policy.Cidr, "CIDR2="+ipBlock_cidrs_policy.Cidr2, "CIDR3="+ipBlock_cidrs_policy.Cidr3) + if err1 != nil { + e2e.Logf("the err:%v, and try next round", err1) + return false, nil + } + return true, nil + }) + o.Expect(err).NotTo(o.HaveOccurred(), fmt.Sprintf("fail to create network policy %v", ipBlock_cidrs_policy.Name)) +} + +func (service *GenericServiceResource) CreateServiceFromParams(oc *exutil.CLI) { + err := wait.Poll(3*time.Second, 20*time.Second, func() (bool, error) { + err1 := ApplyResourceFromTemplateByAdmin(oc, "--ignore-unknown-parameters=true", "-f", service.Template, "-p", "SERVICENAME="+service.Servicename, "NAMESPACE="+service.Namespace, "PROTOCOL="+service.Protocol, "SELECTOR="+service.Selector, "serviceType="+service.ServiceType, "ipFamilyPolicy="+service.IpFamilyPolicy, "internalTrafficPolicy="+service.InternalTrafficPolicy, "externalTrafficPolicy="+service.ExternalTrafficPolicy) + if err1 != nil { + e2e.Logf("the err:%v, and try next round", err1) + return false, nil + } + return true, nil + }) + o.Expect(err).NotTo(o.HaveOccurred(), fmt.Sprintf("fail to create svc %v", service.Servicename)) +} + +func (service *WindowGenericServiceResource) CreateWinServiceFromParams(oc *exutil.CLI) { + err := wait.Poll(5*time.Second, 20*time.Second, func() (bool, error) { + err1 := ApplyResourceFromTemplateByAdmin(oc, "--ignore-unknown-parameters=true", "-f", service.Template, "-p", "SERVICENAME="+service.Servicename, "NAMESPACE="+service.Namespace, "PROTOCOL="+service.Protocol, "SELECTOR="+service.Selector, "serviceType="+service.ServiceType, "ipFamilyPolicy="+service.IpFamilyPolicy, "internalTrafficPolicy="+service.InternalTrafficPolicy, "externalTrafficPolicy="+service.ExternalTrafficPolicy) + if err1 != nil { + e2e.Logf("the err:%v, and try next round", err1) + return false, nil + } + return true, nil + }) + o.Expect(err).NotTo(o.HaveOccurred(), fmt.Sprintf("fail to create svc %v", service.Servicename)) +} + +func (egressrouter *EgressrouterMultipleDst) CreateEgressRouterMultipeDst(oc *exutil.CLI) { + err := wait.Poll(5*time.Second, 20*time.Second, func() (bool, error) { + err1 := ApplyResourceFromTemplateByAdmin(oc, "--ignore-unknown-parameters=true", "-f", egressrouter.Template, "-p", "NAME="+egressrouter.Name, "NAMESPACE="+egressrouter.Namespace, "RESERVEDIP="+egressrouter.Reservedip, "GATEWAY="+egressrouter.Gateway, "DSTIP1="+egressrouter.Destinationip1, "DSTIP2="+egressrouter.Destinationip2, "DSTIP3="+egressrouter.Destinationip3) + if err1 != nil { + e2e.Logf("the err:%v, and try next round", err1) + return false, nil + } + return true, nil + }) + o.Expect(err).NotTo(o.HaveOccurred(), fmt.Sprintf("fail to create egressrouter %v", egressrouter.Name)) +} + +func (egressrouter *EgressrouterRedSDN) CreateEgressRouterRedSDN(oc *exutil.CLI) { + err := wait.Poll(5*time.Second, 20*time.Second, func() (bool, error) { + err1 := ApplyResourceFromTemplateByAdmin(oc, "--ignore-unknown-parameters=true", "-f", egressrouter.Template, "-p", "NAME="+egressrouter.Name, "NAMESPACE="+egressrouter.Namespace, "RESERVEDIP="+egressrouter.Reservedip, "GATEWAY="+egressrouter.Gateway, "DSTIP="+egressrouter.Destinationip, "LABELKEY="+egressrouter.Labelkey, "LABELVALUE="+egressrouter.Labelvalue) + if err1 != nil { + e2e.Logf("the err:%v, and try next round", err1) + return false, nil + } + return true, nil + }) + o.Expect(err).NotTo(o.HaveOccurred(), fmt.Sprintf("fail to create egressrouter %v", egressrouter.Name)) +} + +func (egressFirewall *EgressFirewall2) DeleteEgressFW2Object(oc *exutil.CLI) { + RemoveResource(oc, true, true, "egressfirewall", egressFirewall.Name, "-n", egressFirewall.Namespace) +} + +func (pod *PingPodResource) DeletePingPod(oc *exutil.CLI) { + RemoveResource(oc, false, true, "pod", pod.Name, "-n", pod.Namespace) +} + +func (pod *PingPodResourceNode) DeletePingPodNode(oc *exutil.CLI) { + RemoveResource(oc, false, true, "pod", pod.Name, "-n", pod.Namespace) +} + +func RemoveResource(oc *exutil.CLI, asAdmin bool, withoutNamespace bool, parameters ...string) { + output, err := DoAction(oc, "delete", asAdmin, withoutNamespace, parameters...) + if err != nil && (strings.Contains(output, "NotFound") || strings.Contains(output, "No resources found")) { + e2e.Logf("the resource is deleted already") + return + } + o.Expect(err).NotTo(o.HaveOccurred()) + + err = wait.Poll(3*time.Second, 120*time.Second, func() (bool, error) { + output, err := DoAction(oc, "get", asAdmin, withoutNamespace, parameters...) + if err != nil && (strings.Contains(output, "NotFound") || strings.Contains(output, "No resources found")) { + e2e.Logf("the resource is delete successfully") + return true, nil + } + return false, nil + }) + o.Expect(err).NotTo(o.HaveOccurred(), fmt.Sprintf("fail to delete resource %v", parameters)) +} + +func DoAction(oc *exutil.CLI, action string, asAdmin bool, withoutNamespace bool, parameters ...string) (string, error) { + if asAdmin && withoutNamespace { + return oc.AsAdmin().WithoutNamespace().Run(action).Args(parameters...).Output() + } + if asAdmin && !withoutNamespace { + return oc.AsAdmin().Run(action).Args(parameters...).Output() + } + if !asAdmin && withoutNamespace { + return oc.WithoutNamespace().Run(action).Args(parameters...).Output() + } + if !asAdmin && !withoutNamespace { + return oc.Run(action).Args(parameters...).Output() + } + return "", nil +} + +func ApplyResourceFromTemplateByAdmin(oc *exutil.CLI, parameters ...string) error { + var configFile string + err := wait.Poll(3*time.Second, 15*time.Second, func() (bool, error) { + output, err := oc.AsAdmin().Run("process").Args(parameters...).OutputToFile(GetRandomString() + "resource.json") + if err != nil { + e2e.Logf("the err:%v, and try next round", err) + return false, nil + } + configFile = output + return true, nil + }) + o.Expect(err).NotTo(o.HaveOccurred(), fmt.Sprintf("as admin fail to process %v", parameters)) + + e2e.Logf("the file of resource is %s", configFile) + applyArgs := []string{"-f", configFile} + for i, p := range parameters { + if p == "-n" && i+1 < len(parameters) { + applyArgs = append(applyArgs, "-n", parameters[i+1]) + break + } + } + return oc.WithoutNamespace().AsAdmin().Run("apply").Args(applyArgs...).Execute() +} + +func GetRandomString() string { + chars := "abcdefghijklmnopqrstuvwxyz0123456789" + seed := rand.New(rand.NewSource(time.Now().UnixNano())) + buffer := make([]byte, 8) + for index := range buffer { + buffer[index] = chars[seed.Intn(len(chars))] + } + return string(buffer) +} + +func GetPodStatus(oc *exutil.CLI, namespace string, podName string) (string, error) { + podStatus, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("pod", "-n", namespace, podName, "-o=jsonpath={.status.phase}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("The pod %s status in namespace %s is %q", podName, namespace, podStatus) + return podStatus, err +} + +func CheckPodReady(oc *exutil.CLI, namespace string, podName string) (bool, error) { + podOutPut, err := GetPodStatus(oc, namespace, podName) + status := []string{"Running", "Ready", "Complete", "Succeeded"} + return Contains(status, podOutPut), err +} + +func CheckPodNotReady(oc *exutil.CLI, namespace string, podName string) (bool, error) { + podOutPut, err := GetPodStatus(oc, namespace, podName) + status := []string{"Pending", "ContainerCreating"} + return Contains(status, podOutPut), err +} + +func Contains(s []string, str string) bool { + for _, v := range s { + if v == str { + return true + } + } + + return false +} + +func WaitPodReady(oc *exutil.CLI, namespace string, podName string) { + err := wait.Poll(5*time.Second, 60*time.Second, func() (bool, error) { + status, err1 := CheckPodReady(oc, namespace, podName) + if err1 != nil { + e2e.Logf("the err:%v, wait for pod %v to become ready.", err1, podName) + return status, err1 + } + if !status { + return status, nil + } + return status, nil + }) + + if err != nil { + podDescribe := DescribePod(oc, namespace, podName) + e2e.Logf("oc describe pod %v.", podName) + e2e.Logf("%s", podDescribe) + } + o.Expect(err).NotTo(o.HaveOccurred(), fmt.Sprintf("pod %v is not ready", podName)) +} + +func DescribePod(oc *exutil.CLI, namespace string, podName string) string { + podDescribe, err := oc.AsAdmin().WithoutNamespace().Run("describe").Args("pod", "-n", namespace, podName).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("The pod %s status is %q", podName, podDescribe) + return podDescribe +} + +func ExecCommandInSpecificPod(oc *exutil.CLI, namespace string, podName string, command string) (string, error) { + e2e.Logf("The command is: %v", command) + command1 := []string{"-n", namespace, podName, "--", "bash", "-c", command} + msg, err := oc.AsAdmin().WithoutNamespace().Run("exec").Args(command1...).Output() + if err != nil { + e2e.Logf("Execute command failed with err:%v and output is %v.", err, msg) + return msg, err + } + o.Expect(err).NotTo(o.HaveOccurred()) + return msg, nil +} + +func ExecCommandInNetworkingPod(oc *exutil.CLI, command string) (string, error) { + var cmd []string + podName, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("pods", "-n", "openshift-ovn-kubernetes", "-l", "app=ovnkube-node", "-o=jsonpath={.items[0].metadata.name}").Output() + if err != nil { + e2e.Logf("Cannot get ovn-kubernetes pods, errors: %v", err) + return "", err + } + cmd = []string{"-n", "openshift-ovn-kubernetes", "-c", "ovnkube-controller", podName, "--", "/bin/sh", "-c", command} + + msg, err := oc.WithoutNamespace().AsAdmin().Run("exec").Args(cmd...).Output() + if err != nil { + e2e.Logf("Execute command failed with err:%v .", err) + return "", err + } + o.Expect(err).NotTo(o.HaveOccurred()) + return msg, nil +} + +func GetDefaultInterface(oc *exutil.CLI) (string, error) { + getDefaultInterfaceCmd := "/usr/sbin/ip -4 route show default" + int1, err := ExecCommandInNetworkingPod(oc, getDefaultInterfaceCmd) + if err != nil { + e2e.Logf("Cannot get default interface, errors: %v", err) + return "", err + } + defInterface := strings.Split(int1, " ")[4] + e2e.Logf("Get the default inteface: %s", defInterface) + return defInterface, nil +} + +func GetDefaultSubnet(oc *exutil.CLI) (string, error) { + int1, _ := GetDefaultInterface(oc) + getDefaultSubnetCmd := "/usr/sbin/ip -4 -brief a show " + int1 + subnet1, err := ExecCommandInNetworkingPod(oc, getDefaultSubnetCmd) + defSubnet := strings.Fields(subnet1)[2] + if err != nil { + e2e.Logf("Cannot get default subnet, errors: %v", err) + return "", err + } + e2e.Logf("Get the default subnet: %s", defSubnet) + return defSubnet, nil +} + +// Hosts function return the host network CIDR +func Hosts(cidr string) ([]string, error) { + ip, ipnet, err := net.ParseCIDR(cidr) + e2e.Logf("in Hosts function, ip: %v, ipnet: %v", ip, ipnet) + if err != nil { + return nil, err + } + + var ips []string + for ip := ip.Mask(ipnet.Mask); ipnet.Contains(ip); Inc(ip) { + ips = append(ips, ip.String()) + } + // remove network address and broadcast address + return ips[1 : len(ips)-1], nil +} + +func Inc(ip net.IP) { + for j := len(ip) - 1; j >= 0; j-- { + ip[j]++ + if ip[j] > 0 { + break + } + } +} + +func FindUnUsedIPs(oc *exutil.CLI, cidr string, number int) []string { + ipRange, _ := Hosts(cidr) + var ipUnused = []string{} + //shuffle the ips slice + rand.Seed(time.Now().UnixNano()) + rand.Shuffle(len(ipRange), func(i, j int) { ipRange[i], ipRange[j] = ipRange[j], ipRange[i] }) + for _, ip := range ipRange { + if len(ipUnused) < number { + pingCmd := "ping -c4 -t1 " + ip + _, err := ExecCommandInNetworkingPod(oc, pingCmd) + if err != nil { + e2e.Logf("%s is not used!\n", ip) + ipUnused = append(ipUnused, ip) + } + } else { + break + } + + } + return ipUnused +} + +func IpEchoServer() string { + return "172.31.249.80:9095" +} + +func CheckPlatform(oc *exutil.CLI) string { + output, _ := oc.WithoutNamespace().AsAdmin().Run("get").Args("infrastructure", "cluster", "-o=jsonpath={.status.platformStatus.type}").Output() + return strings.ToLower(output) +} + +func CheckNetworkType(oc *exutil.CLI) string { + output, _ := oc.WithoutNamespace().AsAdmin().Run("get").Args("network.operator", "cluster", "-o=jsonpath={.spec.defaultNetwork.type}").Output() + return strings.ToLower(output) +} + +func GetDefaultIPv6Subnet(oc *exutil.CLI) (string, error) { + int1, _ := GetDefaultInterface(oc) + getDefaultSubnetCmd := "/usr/sbin/ip -6 -brief a show " + int1 + subnet1, err := ExecCommandInNetworkingPod(oc, getDefaultSubnetCmd) + if err != nil { + e2e.Logf("Cannot get default ipv6 subnet, errors: %v", err) + return "", err + } + defSubnet := strings.Fields(subnet1)[2] + e2e.Logf("Get the default ipv6 subnet: %s", defSubnet) + return defSubnet, nil +} + +func FindUnUsedIPv6(oc *exutil.CLI, cidr string, number int) ([]string, error) { + ip, ipnet, err := net.ParseCIDR(cidr) + if err != nil { + return nil, err + } + + number += 2 + var ips []string + var i = 0 + for ip := ip.Mask(ipnet.Mask); ipnet.Contains(ip); Inc(ip) { + //Not use the first two IPv6 addresses , such as 2620:52:0:4e:: , 2620:52:0:4e::1 + if i == 0 || i == 1 { + i++ + continue + } + //Start to detect the IPv6 adress is used or not + if i < number { + pingCmd := "ping -c4 -t1 -6 " + ip.String() + _, err := ExecCommandInNetworkingPod(oc, pingCmd) + if err != nil { + e2e.Logf("%s is not used!\n", ip) + ips = append(ips, ip.String()) + i++ + } + } else { + break + } + + } + + return ips, nil +} + +func Ipv6EchoServer(isIPv6 bool) string { + if isIPv6 { + return "[2620:52:0:4974:def4:1ff:fee7:8144]:8085" + } + return "10.73.116.56:8085" +} + +func CheckIPStackType(oc *exutil.CLI) string { + svcNetwork, err := oc.WithoutNamespace().AsAdmin().Run("get").Args("network.operator", "cluster", "-o=jsonpath={.spec.serviceNetwork}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + if strings.Count(svcNetwork, ":") >= 2 && strings.Count(svcNetwork, ".") >= 2 { + return "dualstack" + } else if strings.Count(svcNetwork, ":") >= 2 { + return "ipv6single" + } else if strings.Count(svcNetwork, ".") >= 2 { + return "ipv4single" + } + return "" +} + +func InstallSctpModule(oc *exutil.CLI, configFile string) { + status, _ := oc.AsAdmin().Run("get").Args("machineconfigs").Output() + if !strings.Contains(status, "load-sctp-module") { + err := oc.WithoutNamespace().AsAdmin().Run("create").Args("-f", configFile).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + } +} + +func CheckSctpModule(oc *exutil.CLI, nodeName, namespace string) { + defer RecoverNamespaceRestricted(oc, namespace) + SetNamespacePrivileged(oc, namespace) + err := wait.Poll(30*time.Second, 15*time.Minute, func() (bool, error) { + // Check nodes status to make sure all nodes are up after rebooting caused by load-sctp-module + nodesStatus, err := oc.AsAdmin().Run("get").Args("node").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("oc_get_nodes: %v", nodesStatus) + status, _ := oc.AsAdmin().Run("debug").Args("node/"+nodeName, "--", "cat", "/sys/module/sctp/initstate").Output() + if strings.Contains(status, "live") { + e2e.Logf("stcp module is installed in the %s", nodeName) + return true, nil + } + return false, nil + }) + o.Expect(err).NotTo(o.HaveOccurred(), "stcp module is installed in the nodes") +} + +func GetPodIPv4(oc *exutil.CLI, namespace string, podName string) string { + podIPv4, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("pod", "-n", namespace, podName, "-o=jsonpath={.status.podIPs[0].ip}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("The pod %s IP in namespace %s is %q", podName, namespace, podIPv4) + return podIPv4 +} + +func GetPodIPv6(oc *exutil.CLI, namespace string, podName string, ipStack string) string { + if ipStack == "ipv6single" { + podIPv6, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("pod", "-n", namespace, podName, "-o=jsonpath={.status.podIPs[0].ip}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("The pod %s IP in namespace %s is %q", podName, namespace, podIPv6) + return podIPv6 + } else if ipStack == "dualstack" { + podIPv6, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("pod", "-n", namespace, podName, "-o=jsonpath={.status.podIPs[1].ip}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("The pod %s IP in namespace %s is %q", podName, namespace, podIPv6) + return podIPv6 + } + return "" +} + +// For normal user to create resources in the specified namespace from the file (not template) +func CreateResourceFromFile(oc *exutil.CLI, ns, file string) { + err := oc.AsAdmin().WithoutNamespace().Run("create").Args("-f", file, "-n", ns).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) +} + +func WaitForPodWithLabelReady(oc *exutil.CLI, ns, label string) error { + return wait.Poll(5*time.Second, 5*time.Minute, func() (bool, error) { + status, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("pod", "-n", ns, "-l", label, "-ojsonpath={.items[*].status.conditions[?(@.type==\"Ready\")].status}").Output() + e2e.Logf("the Ready status of pod is %v", status) + if err != nil || status == "" { + e2e.Logf("failed to get pod status: %v, retrying...", err) + return false, nil + } + if strings.Contains(status, "False") { + e2e.Logf("the pod Ready status not met; wanted True but got %v, retrying...", status) + return false, nil + } + return true, nil + }) +} + +func WaitForPodWithLabelGone(oc *exutil.CLI, ns, label string) error { + errWait := wait.Poll(5*time.Second, 10*time.Minute, func() (bool, error) { + podsOutput, _ := oc.AsAdmin().WithoutNamespace().Run("get").Args("pod", "-n", ns, "-l", label).Output() + if strings.Contains(podsOutput, "NotFound") || strings.Contains(podsOutput, "No resources found") { + e2e.Logf("the resource is deleted already") + return true, nil + } + e2e.Logf("Wait for pods to be deleted, retrying...") + return false, nil + }) + if errWait != nil { + return fmt.Errorf("case: %v\nerror: %s", g.CurrentSpecReport().FullText(), fmt.Sprintf("pod with lable %v in ns %v is not gone", label, ns)) + } + return nil + +} + +func GetSvcIPv4(oc *exutil.CLI, namespace string, svcName string) string { + svcIPv4, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("service", "-n", namespace, svcName, "-o=jsonpath={.spec.clusterIPs[0]}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("The service %s IPv4 in namespace %s is %q", svcName, namespace, svcIPv4) + return svcIPv4 +} + +func GetSvcIPv6(oc *exutil.CLI, namespace string, svcName string) string { + svcIPv6, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("service", "-n", namespace, svcName, "-o=jsonpath={.spec.clusterIPs[1]}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("The service %s IPv6 in namespace %s is %q", svcName, namespace, svcIPv6) + return svcIPv6 +} + +func GetSvcIPv6SingleStack(oc *exutil.CLI, namespace string, svcName string) string { + svcIPv6, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("service", "-n", namespace, svcName, "-o=jsonpath={.spec.clusterIPs[0]}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("The service %s IPv6 in namespace %s is %q", svcName, namespace, svcIPv6) + return svcIPv6 +} + +func GetSvcIPdualstack(oc *exutil.CLI, namespace string, svcName string) (string, string) { + svcIPv4, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("service", "-n", namespace, svcName, "-o=jsonpath={.spec.clusterIPs[0]}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("The service %s IPv4 in namespace %s is %q", svcName, namespace, svcIPv4) + svcIPv6, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("service", "-n", namespace, svcName, "-o=jsonpath={.spec.clusterIPs[1]}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("The service %s IPv6 in namespace %s is %q", svcName, namespace, svcIPv6) + return svcIPv4, svcIPv6 +} + +// check if a configmap is created in specific namespace [usage: CheckConfigMap(oc, namesapce, configmapName)] +func CheckConfigMap(oc *exutil.CLI, ns, configmapName string) error { + return wait.Poll(5*time.Second, 3*time.Minute, func() (bool, error) { + searchOutput, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("cm", "-n", ns).Output() + if err != nil { + e2e.Logf("failed to get configmap: %v", err) + return false, nil + } + if strings.Contains(searchOutput, configmapName) { + e2e.Logf("configmap %v found", configmapName) + return true, nil + } + return false, nil + }) +} + +func SshRunCmd(host string, user string, cmd string) error { + privateKey := os.Getenv("SSH_CLOUD_PRIV_KEY") + if privateKey == "" { + privateKey = "../internal/config/keys/openshift-qe.pem" + } + sshClient := SshClient{User: user, Host: host, Port: 22, PrivateKey: privateKey} + return sshClient.Run(cmd) +} + +// For Admin to patch a resource in the specified namespace +func PatchResourceAsAdmin(oc *exutil.CLI, resource, patch string, nameSpace ...string) { + var cargs []string + if len(nameSpace) > 0 { + cargs = []string{resource, "-p", patch, "-n", nameSpace[0], "--type=merge"} + } else { + cargs = []string{resource, "-p", patch, "--type=merge"} + } + err := oc.AsAdmin().WithoutNamespace().Run("patch").Args(cargs...).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) +} + +// Check network operator status in intervals until timeout +func CheckNetworkOperatorState(oc *exutil.CLI, interval int, timeout int) { + errCheck := wait.Poll(time.Duration(interval)*time.Second, time.Duration(timeout)*time.Second, func() (bool, error) { + output, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("co", "network").Output() + if err != nil { + e2e.Logf("Fail to get clusteroperator network, error:%s. Trying again", err) + return false, nil + } + matched, _ := regexp.MatchString("True.*False.*False", output) + e2e.Logf("Network operator state is:%s", output) + o.Expect(matched).To(o.BeTrue()) + return false, nil + }) + o.Expect(errCheck.Error()).To(o.ContainSubstring("timed out waiting for the condition")) +} + +func GetNodeIPv4(oc *exutil.CLI, namespace, nodeName string) string { + output, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("-n", oc.Namespace(), "node", nodeName, "-o=jsonpath={.status.addresses[?(@.type==\"InternalIP\")].address}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + if err != nil { + e2e.Logf("Cannot get node default interface ipv4 address, errors: %v", err) + } + + // when egressIP is applied to a node, it would be listed as internal IP for the node, thus, there could be more than one IPs shown as internal IP + // use RE to match out to first internal IP + re := regexp.MustCompile(`(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)){3}`) + nodeipv4 := re.FindAllString(output, -1)[0] + e2e.Logf("The IPv4 of node's default interface is %q", nodeipv4) + return nodeipv4 +} + +// Return IPv6 and IPv4 in vars respectively for Dual Stack and IPv4/IPv6 in 2nd var for single stack Clusters, and var1 will be nil in those cases +func GetNodeIP(oc *exutil.CLI, nodeName string) (string, string) { + ipStack := CheckIPStackType(oc) + if (ipStack == "ipv6single") || (ipStack == "ipv4single") { + e2e.Logf("Its a Single Stack Cluster, either IPv4 or IPv6") + InternalIP, err := oc.AsAdmin().Run("get").Args("node", nodeName, "-o=jsonpath={.status.addresses[?(@.type==\"InternalIP\")].address}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("The node's Internal IP is %q", InternalIP) + return "", InternalIP + } + e2e.Logf("Its a Dual Stack Cluster") + InternalIP1, err := oc.AsAdmin().Run("get").Args("node", nodeName, "-o=jsonpath={.status.addresses[0].address}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("The node's 1st Internal IP is %q", InternalIP1) + InternalIP2, err := oc.AsAdmin().Run("get").Args("node", nodeName, "-o=jsonpath={.status.addresses[1].address}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("The node's 2nd Internal IP is %q", InternalIP2) + if netutils.IsIPv6String(InternalIP1) { + return InternalIP1, InternalIP2 + } + return InternalIP2, InternalIP1 +} + +// get CLuster Manager's leader info +func GetLeaderInfo(oc *exutil.CLI, namespace string, cmName string, networkType string) string { + if networkType == "ovnkubernetes" { + linuxNodeList, err := GetAllNodesbyOSType(oc, "linux") + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(linuxNodeList).NotTo(o.BeEmpty()) + podName, getPodNameErr := GetOVNKPodOnNode(oc, namespace, cmName, linuxNodeList[0]) + o.Expect(getPodNameErr).NotTo(o.HaveOccurred()) + o.Expect(podName).NotTo(o.BeEmpty()) + return podName + } + output, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("cm", "openshift-network-controller", "-n", namespace, "-o=jsonpath={.metadata.annotations.control-plane\\.alpha\\.kubernetes\\.io\\/leader}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + var sdnAnnotations map[string]interface{} + json.Unmarshal([]byte(output), &sdnAnnotations) + leaderNodeName := sdnAnnotations["holderIdentity"].(string) + o.Expect(leaderNodeName).NotTo(o.BeEmpty()) + ocGetPods, podErr := oc.AsAdmin().WithoutNamespace().Run("get").Args("-n", "openshift-sdn", "pod", "-l app=sdn", "-o=wide").OutputToFile("ocgetpods.txt") + defer os.RemoveAll(ocGetPods) + o.Expect(podErr).NotTo(o.HaveOccurred()) + rawGrepOutput, rawGrepErr := exec.Command("bash", "-c", "cat "+ocGetPods+" | grep "+leaderNodeName+" | awk '{print $1}'").Output() + o.Expect(rawGrepErr).NotTo(o.HaveOccurred()) + leaderPodName := strings.TrimSpace(string(rawGrepOutput)) + e2e.Logf("The leader Pod's Name: %v", leaderPodName) + return leaderPodName +} + +func CheckSDNMetrics(oc *exutil.CLI, url string, metrics string) { + var metricsOutput []byte + var metricsLog []byte + olmToken, err := GetSAToken(oc) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(olmToken).NotTo(o.BeEmpty()) + metricsErr := wait.Poll(5*time.Second, 10*time.Second, func() (bool, error) { + output, err := oc.AsAdmin().WithoutNamespace().Run("exec").Args("-n", "openshift-monitoring", "-c", "prometheus", "prometheus-k8s-0", "--", "curl", "-k", "-H", fmt.Sprintf("Authorization: Bearer %v", olmToken), fmt.Sprintf("%s", url)).OutputToFile("metrics.txt") + if err != nil { + e2e.Logf("Can't get metrics and try again, the error is:%s", err) + return false, nil + } + metricsLog, _ = exec.Command("bash", "-c", "cat "+output+" ").Output() + metricsString := string(metricsLog) + if strings.Contains(metricsString, "ovnkube_controller_pod") { + metricsOutput, _ = exec.Command("bash", "-c", "cat "+output+" | grep "+metrics+" | awk 'NR==1{print $2}'").Output() + } else { + metricsOutput, _ = exec.Command("bash", "-c", "cat "+output+" | grep "+metrics+" | awk 'NR==3{print $2}'").Output() + } + metricsValue := strings.TrimSpace(string(metricsOutput)) + if metricsValue != "" { + e2e.Logf("The output of the metrics for %s is : %v", metrics, metricsValue) + } else { + e2e.Logf("Can't get metrics for %s:", metrics) + return false, nil + } + return true, nil + }) + o.Expect(metricsErr).NotTo(o.HaveOccurred(), fmt.Sprintf("Fail to get metric and the error is:%s", metricsErr)) +} + +func GetEgressCIDRs(oc *exutil.CLI, node string) string { + output, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("hostsubnet", node, "-o=jsonpath={.egressCIDRs}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("egressCIDR for hostsubnet node %v is: %v", node, output) + return output +} + +// get egressIP from a node +// When they are multiple egressIPs on the node, egressIp list is in format of ["10.0.247.116","10.0.156.51"] +// as an example from the output of command "oc get hostsubnet -o=jsonpath={.egressIPs}" +// convert the iplist into an array of ip addresses +func GetEgressIPByKind(oc *exutil.CLI, kind string, kindName string, expectedNum int) ([]string, error) { + var ip = []string{} + iplist, err := oc.AsAdmin().WithoutNamespace().Run("get").Args(kind, kindName, "-o=jsonpath={.egressIPs}").Output() + isIPListEmpty := (iplist == "" || iplist == "[]") + if expectedNum == 0 { + // Add waiting time for egressIP removed + egressIPEmptyErr := wait.Poll(30*time.Second, 5*time.Minute, func() (bool, error) { + iplist, err = oc.AsAdmin().WithoutNamespace().Run("get").Args(kind, kindName, "-o=jsonpath={.egressIPs}").Output() + if iplist == "" || iplist == "[]" { + e2e.Logf("EgressIP list is empty") + return true, nil + } + e2e.Logf("EgressIP list is %s, not removed, or have err:%v, and try next round", iplist, err) + return false, nil + }) + return ip, egressIPEmptyErr + } + if !isIPListEmpty && iplist != "[]" { + ip = strings.Split(iplist[2:len(iplist)-2], "\",\"") + } + if isIPListEmpty || len(ip) < expectedNum || err != nil { + err = wait.Poll(30*time.Second, 5*time.Minute, func() (bool, error) { + iplist, err = oc.AsAdmin().WithoutNamespace().Run("get").Args(kind, kindName, "-o=jsonpath={.egressIPs}").Output() + if len(iplist) > 0 && iplist != "[]" { + ip = strings.Split(iplist[2:len(iplist)-2], "\",\"") + } + if len(ip) < expectedNum || err != nil { + e2e.Logf("only got %d egressIP, or have err:%v, and try next round", len(ip), err) + return false, nil + } + if len(iplist) > 0 && len(ip) == expectedNum { + e2e.Logf("Found egressIP list for %v %v is: %v", kind, kindName, iplist) + return true, nil + } + return false, nil + }) + e2e.Logf("Only got %d egressIP, or have err:%v", len(ip), err) + return ip, err + } + return ip, nil +} + +func GetPodName(oc *exutil.CLI, namespace string, label string) []string { + var podName []string + podNameAll, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("-n", namespace, "pod", "-l", label, "-ojsonpath={.items[*].metadata.name}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + podName = strings.Split(podNameAll, " ") + o.Expect(len(podName)).NotTo(o.BeEquivalentTo(0)) + e2e.Logf("The pod(s) are %v ", podName) + return podName +} + +// starting from first node, compare its subnet with subnet of subsequent nodes in the list +// until two nodes with same subnet found, otherwise, return false to indicate that no two nodes with same subnet found +func FindTwoNodesWithSameSubnet(oc *exutil.CLI, nodeList *v1.NodeList) (bool, [2]string) { + var nodes [2]string + for i := 0; i < (len(nodeList.Items) - 1); i++ { + for j := i + 1; j < len(nodeList.Items); j++ { + firstSub := GetIfaddrFromNode(nodeList.Items[i].Name, oc) + secondSub := GetIfaddrFromNode(nodeList.Items[j].Name, oc) + if firstSub == secondSub { + e2e.Logf("Found nodes with same subnet.") + nodes[0] = nodeList.Items[i].Name + nodes[1] = nodeList.Items[j].Name + return true, nodes + } + } + } + return false, nodes +} + +func GetSDNMetrics(oc *exutil.CLI, podName string) string { + var metricsLog string + metricsErr := wait.Poll(5*time.Second, 10*time.Second, func() (bool, error) { + output, err := oc.AsAdmin().WithoutNamespace().Run("exec").Args("-n", "openshift-sdn", fmt.Sprintf("%s", podName), "--", "curl", "localhost:29100/metrics").OutputToFile("metrics.txt") + if err != nil { + e2e.Logf("Can't get metrics and try again, the error is:%s", err) + return false, nil + } + metricsLog = output + return true, nil + }) + o.Expect(metricsErr).NotTo(o.HaveOccurred(), fmt.Sprintf("Fail to get metric and the error is:%s", metricsErr)) + return metricsLog +} + +func GetOVNMetrics(oc *exutil.CLI, url string) string { + var metricsLog string + olmToken, err := GetSAToken(oc) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(olmToken).NotTo(o.BeEmpty()) + metricsErr := wait.Poll(5*time.Second, 10*time.Second, func() (bool, error) { + output, err := oc.AsAdmin().WithoutNamespace().Run("exec").Args("-n", "openshift-monitoring", "-c", "prometheus", "prometheus-k8s-0", "--", "env", "-u", "HTTPS_PROXY", "-u", "https_proxy", "curl", "-k", "-H", fmt.Sprintf("Authorization: Bearer %v", olmToken), fmt.Sprintf("%s", url)).OutputToFile("metrics.txt") + if err != nil { + e2e.Logf("Can't get metrics and try again, the error is:%s", err) + return false, nil + } + metricsLog = output + return true, nil + }) + o.Expect(metricsErr).NotTo(o.HaveOccurred(), fmt.Sprintf("Fail to get metric and the error is:%s", metricsErr)) + return metricsLog +} + +func CheckIPsec(oc *exutil.CLI) string { + output, err := oc.WithoutNamespace().AsAdmin().Run("get").Args("network.operator", "cluster", "-o=jsonpath={.spec.defaultNetwork.ovnKubernetesConfig.ipsecConfig.mode}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + if output == "" { + // if have {} in 4.15+, that means it upgraded from previous version and with ipsec enabled. + output, err = oc.WithoutNamespace().AsAdmin().Run("get").Args("network.operator", "cluster", "-o=jsonpath={.spec.defaultNetwork.ovnKubernetesConfig.ipsecConfig}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + } + e2e.Logf("The ipsec state is === %v ===", output) + return output +} + +func GetAssignedEIPInEIPObject(oc *exutil.CLI, egressIPObject string) []map[string]string { + timeout := EstimateTimeoutForEgressIP(oc) + var egressIPs string + egressipErr := wait.Poll(10*time.Second, timeout, func() (bool, error) { + egressIPStatus, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("egressip", egressIPObject, "-ojsonpath={.status.items}").Output() + if err != nil { + e2e.Logf("Wait to get EgressIP object applied,try next round. %v", err) + return false, nil + } + if egressIPStatus == "" { + e2e.Logf("Wait to get EgressIP object applied,try next round. %v", err) + return false, nil + } + egressIPs = egressIPStatus + e2e.Logf("egressIPStatus: %v", egressIPs) + return true, nil + }) + o.Expect(egressipErr).NotTo(o.HaveOccurred(), fmt.Sprintf("Failed to apply egressIPs:%s", egressipErr)) + + var egressIPJsonMap []map[string]string + json.Unmarshal([]byte(egressIPs), &egressIPJsonMap) + e2e.Logf("egressIPJsonMap:%v", egressIPJsonMap) + return egressIPJsonMap +} + +func RebootNode(oc *exutil.CLI, nodeName string) { + e2e.Logf("\nRebooting node %s....", nodeName) + _, err1 := DebugNodeWithChroot(oc, nodeName, "shutdown", "-r", "+1") + o.Expect(err1).NotTo(o.HaveOccurred()) +} + +func CheckNodeStatus(oc *exutil.CLI, nodeName string, expectedStatus string) { + var expectedStatus1 string + if expectedStatus == "Ready" { + expectedStatus1 = "True" + } else if expectedStatus == "NotReady" { + expectedStatus1 = "Unknown" + } else { + err1 := fmt.Errorf("TBD supported node status") + o.Expect(err1).NotTo(o.HaveOccurred()) + } + err := wait.Poll(5*time.Second, 15*time.Minute, func() (bool, error) { + statusOutput, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("nodes", nodeName, "-ojsonpath={.status.conditions[-1].status}").Output() + if err != nil { + e2e.Logf("\nGet node status with error : %v", err) + return false, nil + } + e2e.Logf("Expect Node %s in state %v, kubelet status is %s", nodeName, expectedStatus, statusOutput) + if statusOutput != expectedStatus1 { + return false, nil + } + return true, nil + }) + o.Expect(err).NotTo(o.HaveOccurred(), fmt.Sprintf("Node %s is not in expected status %s", nodeName, expectedStatus)) +} + +func UpdateEgressIPObject(oc *exutil.CLI, egressIPObjectName string, egressIP string) { + PatchResourceAsAdmin(oc, "egressip/"+egressIPObjectName, "{\"spec\":{\"egressIPs\":[\""+egressIP+"\"]}}") + egressipErr := wait.Poll(10*time.Second, 180*time.Second, func() (bool, error) { + output, err := oc.WithoutNamespace().AsAdmin().Run("get").Args("egressip", egressIPObjectName, "-o=jsonpath={.status.items[*]}").Output() + if err != nil { + e2e.Logf("Wait to get EgressIP object applied,try next round. %v", err) + return false, nil + } + if !strings.Contains(output, egressIP) { + e2e.Logf("Wait for new IP %s applied,try next round.", egressIP) + e2e.Logf("%s", output) + return false, nil + } + e2e.Logf("%s", output) + return true, nil + }) + o.Expect(egressipErr).NotTo(o.HaveOccurred(), fmt.Sprintf("Failed to apply new egressIP %s:%v", egressIP, egressipErr)) +} + +func GetTwoNodesSameSubnet(oc *exutil.CLI, nodeList *v1.NodeList) (bool, []string) { + var egressNodes []string + if len(nodeList.Items) < 2 { + e2e.Logf("Not enough nodes available for the test, skip the case!!") + return false, nil + } + platform := CheckPlatform(oc) + if strings.Contains(platform, "aws") { + e2e.Logf("find the two nodes that have same subnet") + check, nodes := FindTwoNodesWithSameSubnet(oc, nodeList) + if check { + egressNodes = nodes[:2] + } else { + e2e.Logf("No more than 2 worker nodes in same subnet, skip the test!!!") + return false, nil + } + } else { + e2e.Logf("since worker nodes all have same subnet, just pick first two nodes as egress nodes") + egressNodes = append(egressNodes, nodeList.Items[0].Name) + egressNodes = append(egressNodes, nodeList.Items[1].Name) + } + return true, egressNodes +} + +/* +GetSvcIP returns IPv6 and IPv4 in vars in order on dual stack respectively and main Svc IP in case of single stack (v4 or v6) in 1st var, and nil in 2nd var. +LoadBalancer svc will return Ingress VIP in var1, v4 or v6 and NodePort svc will return Ingress SvcIP in var1 and NodePort in var2 +*/ +func GetSvcIP(oc *exutil.CLI, namespace string, svcName string) (string, string) { + ipStack := CheckIPStackType(oc) + svctype, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("service", "-n", namespace, svcName, "-o=jsonpath={.spec.type}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + ipFamilyType, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("service", "-n", namespace, svcName, "-o=jsonpath={.spec.IpFamilyPolicy}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + if (svctype == "ClusterIP") || (svctype == "NodePort") { + if (ipStack == "ipv6single") || (ipStack == "ipv4single") { + svcIP, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("service", "-n", namespace, svcName, "-o=jsonpath={.spec.clusterIPs[0]}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + if svctype == "ClusterIP" { + e2e.Logf("The service %s IP in namespace %s is %q", svcName, namespace, svcIP) + return svcIP, "" + } + nodePort, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("service", "-n", namespace, svcName, "-o=jsonpath={.spec.ports[*].nodePort}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("The NodePort service %s IP and NodePort in namespace %s is %s %s", svcName, namespace, svcIP, nodePort) + return svcIP, nodePort + + } else if (ipStack == "dualstack" && ipFamilyType == "PreferDualStack") || (ipStack == "dualstack" && ipFamilyType == "RequireDualStack") { + ipFamilyPrecedence, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("service", "-n", namespace, svcName, "-o=jsonpath={.spec.ipFamilies[0]}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + //if IPv4 is listed first in ipFamilies then clustrIPs allocation will take order as Ipv4 first and then Ipv6 else reverse + svcIPv4, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("service", "-n", namespace, svcName, "-o=jsonpath={.spec.clusterIPs[0]}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("The service %s IP in namespace %s is %q", svcName, namespace, svcIPv4) + svcIPv6, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("service", "-n", namespace, svcName, "-o=jsonpath={.spec.clusterIPs[1]}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("The service %s IP in namespace %s is %q", svcName, namespace, svcIPv6) + /*As stated Nodeport type svc will return node port value in 2nd var. We don't care about what svc address is coming in 1st var as we evetually going to get + node IPs later and use that in curl operation to node_ip:nodeport*/ + if ipFamilyPrecedence == "IPv4" { + e2e.Logf("The ipFamilyPrecedence is Ipv4, Ipv6") + switch svctype { + case "NodePort": + nodePort, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("service", "-n", namespace, svcName, "-o=jsonpath={.spec.ports[*].nodePort}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("The Dual Stack NodePort service %s IP and NodePort in namespace %s is %s %s", svcName, namespace, svcIPv4, nodePort) + return svcIPv4, nodePort + default: + return svcIPv6, svcIPv4 + } + } else { + e2e.Logf("The ipFamilyPrecedence is Ipv6, Ipv4") + switch svctype { + case "NodePort": + nodePort, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("service", "-n", namespace, svcName, "-o=jsonpath={.spec.ports[*].nodePort}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("The Dual Stack NodePort service %s IP and NodePort in namespace %s is %s %s", svcName, namespace, svcIPv6, nodePort) + return svcIPv6, nodePort + default: + svcIPv4, svcIPv6 = svcIPv6, svcIPv4 + return svcIPv6, svcIPv4 + } + } + } else { + //Its a Dual Stack Cluster with SingleStack ipFamilyPolicy + svcIP, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("service", "-n", namespace, svcName, "-o=jsonpath={.spec.clusterIPs[0]}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("The service %s IP in namespace %s is %q", svcName, namespace, svcIP) + return svcIP, "" + } + } else { + //Loadbalancer will be supported for single stack Ipv4 here for mostly GCP,Azure. We can take further enhancements wrt Metal platforms in Metallb utils later + e2e.Logf("The serviceType is LoadBalancer") + platform := CheckPlatform(oc) + var jsonString string + if platform == "aws" { + jsonString = "-o=jsonpath={.status.loadBalancer.ingress[0].hostname}" + } else { + jsonString = "-o=jsonpath={.status.loadBalancer.ingress[0].ip}" + } + + err := wait.Poll(30*time.Second, 300*time.Second, func() (bool, error) { + svcIP, er := oc.AsAdmin().WithoutNamespace().Run("get").Args("service", "-n", namespace, svcName, jsonString).Output() + o.Expect(er).NotTo(o.HaveOccurred()) + if svcIP == "" { + e2e.Logf("Waiting for lb service IP assignment. Trying again...") + return false, nil + } + return true, nil + }) + o.Expect(err).NotTo(o.HaveOccurred(), fmt.Sprintf("fail to assign lb svc IP to %v", svcName)) + lbSvcIP, _ := oc.AsAdmin().WithoutNamespace().Run("get").Args("service", "-n", namespace, svcName, jsonString).Output() + e2e.Logf("The %s lb service Ingress VIP in namespace %s is %q", svcName, namespace, lbSvcIP) + return lbSvcIP, "" + } +} + +// GetPodIP returns IPv6 and IPv4 in vars in order on dual stack respectively and main IP in case of single stack (v4 or v6) in 1st var, and nil in 2nd var +func GetPodIP(oc *exutil.CLI, namespace string, podName string) (string, string) { + ipStack := CheckIPStackType(oc) + if (ipStack == "ipv6single") || (ipStack == "ipv4single") { + podIP, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("pod", "-n", namespace, podName, "-o=jsonpath={.status.podIPs[0].ip}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("The pod %s IP in namespace %s is %q", podName, namespace, podIP) + return podIP, "" + } else if ipStack == "dualstack" { + podIP1, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("pod", "-n", namespace, podName, "-o=jsonpath={.status.podIPs[1].ip}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("The pod's %s 1st IP in namespace %s is %q", podName, namespace, podIP1) + podIP2, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("pod", "-n", namespace, podName, "-o=jsonpath={.status.podIPs[0].ip}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("The pod's %s 2nd IP in namespace %s is %q", podName, namespace, podIP2) + if netutils.IsIPv6String(podIP1) { + e2e.Logf("This is IPv4 primary dual stack cluster") + return podIP1, podIP2 + } + e2e.Logf("This is IPv6 primary dual stack cluster") + return podIP2, podIP1 + } + return "", "" +} + +// CurlPod2PodPass checks connectivity across pods regardless of network addressing type on cluster +func CurlPod2PodPass(oc *exutil.CLI, namespaceSrc string, podNameSrc string, namespaceDst string, podNameDst string) { + podIP1, podIP2 := GetPodIP(oc, namespaceDst, podNameDst) + if podIP2 != "" { + _, err := e2eoutput.RunHostCmd(namespaceSrc, podNameSrc, "curl --connect-timeout 5 -s "+net.JoinHostPort(podIP1, "8080")) + o.Expect(err).NotTo(o.HaveOccurred()) + _, err = e2eoutput.RunHostCmd(namespaceSrc, podNameSrc, "curl --connect-timeout 5 -s "+net.JoinHostPort(podIP2, "8080")) + o.Expect(err).NotTo(o.HaveOccurred()) + } else { + _, err := e2eoutput.RunHostCmd(namespaceSrc, podNameSrc, "curl --connect-timeout 5 -s "+net.JoinHostPort(podIP1, "8080")) + o.Expect(err).NotTo(o.HaveOccurred()) + } +} + +// CurlPod2PodFail ensures no connectivity from a pod to pod regardless of network addressing type on cluster +func CurlPod2PodFail(oc *exutil.CLI, namespaceSrc string, podNameSrc string, namespaceDst string, podNameDst string) { + podIP1, podIP2 := GetPodIP(oc, namespaceDst, podNameDst) + if podIP2 != "" { + _, err := e2eoutput.RunHostCmd(namespaceSrc, podNameSrc, "curl --connect-timeout 5 -s "+net.JoinHostPort(podIP1, "8080")) + o.Expect(err).To(o.HaveOccurred()) + _, err = e2eoutput.RunHostCmd(namespaceSrc, podNameSrc, "curl --connect-timeout 5 -s "+net.JoinHostPort(podIP2, "8080")) + o.Expect(err).To(o.HaveOccurred()) + } else { + _, err := e2eoutput.RunHostCmd(namespaceSrc, podNameSrc, "curl --connect-timeout 5 -s "+net.JoinHostPort(podIP1, "8080")) + o.Expect(err).To(o.HaveOccurred()) + } +} + +// CurlNode2PodPass checks node to pod connectivity regardless of network addressing type on cluster +func CurlNode2PodPass(oc *exutil.CLI, nodeName string, namespace string, podName string) { + //GetPodIP returns IPv6 and IPv4 in order on dual stack in PodIP1 and PodIP2 respectively and main IP in case of single stack (v4 or v6) in PodIP1, and nil in PodIP2 + podIP1, podIP2 := GetPodIP(oc, namespace, podName) + if podIP2 != "" { + podv6URL := net.JoinHostPort(podIP1, "8080") + podv4URL := net.JoinHostPort(podIP2, "8080") + _, err := DebugNode(oc, nodeName, "curl", podv4URL, "-s", "--connect-timeout", "5") + o.Expect(err).NotTo(o.HaveOccurred()) + _, err = DebugNode(oc, nodeName, "curl", podv6URL, "-s", "--connect-timeout", "5") + o.Expect(err).NotTo(o.HaveOccurred()) + } else { + podURL := net.JoinHostPort(podIP1, "8080") + _, err := DebugNode(oc, nodeName, "curl", podURL, "-s", "--connect-timeout", "5") + o.Expect(err).NotTo(o.HaveOccurred()) + } +} + +// CurlNode2PodFail checks node to pod disconnectivity regardless of network addressing type on cluster +func CurlNode2PodFail(oc *exutil.CLI, nodeName string, namespace string, podName string) { + //GetPodIP returns IPv6 and IPv4 in order on dual stack in PodIP1 and PodIP2 respectively and main IP in case of single stack (v4 or v6) in PodIP1, and nil in PodIP2 + podIP1, podIP2 := GetPodIP(oc, namespace, podName) + if podIP2 != "" { + podv6URL := net.JoinHostPort(podIP1, "8080") + podv4URL := net.JoinHostPort(podIP2, "8080") + _, err := DebugNode(oc, nodeName, "curl", podv4URL, "-s", "--connect-timeout", "5") + o.Expect(err).To(o.HaveOccurred()) + _, err = DebugNode(oc, nodeName, "curl", podv6URL, "-s", "--connect-timeout", "5") + o.Expect(err).To(o.HaveOccurred()) + } else { + podURL := net.JoinHostPort(podIP1, "8080") + _, err := DebugNode(oc, nodeName, "curl", podURL, "-s", "--connect-timeout", "5") + o.Expect(err).To(o.HaveOccurred()) + } +} + +// CurlNode2SvcPass checks node to svc connectivity regardless of network addressing type on cluster +func CurlNode2SvcPass(oc *exutil.CLI, nodeName string, namespace string, svcName string) { + svcIP1, svcIP2 := GetSvcIP(oc, namespace, svcName) + if svcIP2 != "" { + svc6URL := net.JoinHostPort(svcIP1, "27017") + svc4URL := net.JoinHostPort(svcIP2, "27017") + _, err := DebugNode(oc, nodeName, "curl", svc4URL, "-s", "--connect-timeout", "5") + o.Expect(err).NotTo(o.HaveOccurred()) + _, err = DebugNode(oc, nodeName, "curl", svc6URL, "-s", "--connect-timeout", "5") + o.Expect(err).NotTo(o.HaveOccurred()) + } else { + svcURL := net.JoinHostPort(svcIP1, "27017") + _, err := DebugNode(oc, nodeName, "curl", svcURL, "-s", "--connect-timeout", "5") + o.Expect(err).NotTo(o.HaveOccurred()) + } +} + +// CurlNode2SvcFail checks node to svc connectivity regardless of network addressing type on cluster +func CurlNode2SvcFail(oc *exutil.CLI, nodeName string, namespace string, svcName string) { + svcIP1, svcIP2 := GetSvcIP(oc, namespace, svcName) + if svcIP2 != "" { + svc6URL := net.JoinHostPort(svcIP1, "27017") + svc4URL := net.JoinHostPort(svcIP2, "27017") + output, _ := DebugNode(oc, nodeName, "curl", svc4URL, "--connect-timeout", "5") + o.Expect(output).To(o.Or(o.ContainSubstring("28"), o.ContainSubstring("Failed"))) + output, _ = DebugNode(oc, nodeName, "curl", svc6URL, "--connect-timeout", "5") + o.Expect(output).To(o.Or(o.ContainSubstring("28"), o.ContainSubstring("Failed"))) + } else { + svcURL := net.JoinHostPort(svcIP1, "27017") + output, _ := DebugNode(oc, nodeName, "curl", svcURL, "--connect-timeout", "5") + o.Expect(output).To(o.Or(o.ContainSubstring("28"), o.ContainSubstring("Failed"))) + } +} + +// CurlPod2SvcPass checks pod to svc connectivity regardless of network addressing type on cluster +func CurlPod2SvcPass(oc *exutil.CLI, namespaceSrc string, namespaceSvc string, podNameSrc string, svcName string) { + svcIP1, svcIP2 := GetSvcIP(oc, namespaceSvc, svcName) + if svcIP2 != "" { + _, err := e2eoutput.RunHostCmdWithRetries(namespaceSrc, podNameSrc, "curl --connect-timeout 5 -s "+net.JoinHostPort(svcIP1, "27017"), 3*time.Second, 15*time.Second) + o.Expect(err).NotTo(o.HaveOccurred()) + _, err = e2eoutput.RunHostCmdWithRetries(namespaceSrc, podNameSrc, "curl --connect-timeout 5 -s "+net.JoinHostPort(svcIP2, "27017"), 3*time.Second, 15*time.Second) + o.Expect(err).NotTo(o.HaveOccurred()) + } else { + _, err := e2eoutput.RunHostCmdWithRetries(namespaceSrc, podNameSrc, "curl --connect-timeout 5 -s "+net.JoinHostPort(svcIP1, "27017"), 3*time.Second, 15*time.Second) + o.Expect(err).NotTo(o.HaveOccurred()) + } +} + +// CurlPod2SvcFail ensures no connectivity from a pod to svc regardless of network addressing type on cluster +func CurlPod2SvcFail(oc *exutil.CLI, namespaceSrc string, namespaceSvc string, podNameSrc string, svcName string) { + svcIP1, svcIP2 := GetSvcIP(oc, namespaceSvc, svcName) + if svcIP2 != "" { + _, err := e2eoutput.RunHostCmd(namespaceSrc, podNameSrc, "curl --connect-timeout 3 -s "+net.JoinHostPort(svcIP1, "27017")) + o.Expect(err).To(o.HaveOccurred()) + _, err = e2eoutput.RunHostCmd(namespaceSrc, podNameSrc, "curl --connect-timeout 3 -s "+net.JoinHostPort(svcIP2, "27017")) + o.Expect(err).To(o.HaveOccurred()) + } else { + _, err := e2eoutput.RunHostCmd(namespaceSrc, podNameSrc, "curl --connect-timeout 3 -s "+net.JoinHostPort(svcIP1, "27017")) + o.Expect(err).To(o.HaveOccurred()) + } +} + +func CheckProxy(oc *exutil.CLI) bool { + httpProxy, err := DoAction(oc, "get", true, true, "proxy", "cluster", "-o=jsonpath={.status.httpProxy}") + o.Expect(err).NotTo(o.HaveOccurred()) + httpsProxy, err := DoAction(oc, "get", true, true, "proxy", "cluster", "-o=jsonpath={.status.httpsProxy}") + o.Expect(err).NotTo(o.HaveOccurred()) + if httpProxy != "" || httpsProxy != "" { + return true + } + return false +} + +// SDNHostwEgressIP find out which egress node has the egressIP +func SDNHostwEgressIP(oc *exutil.CLI, node []string, egressip string) string { + var ip []string + var foundHost string + for i := 0; i < len(node); i++ { + iplist, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("hostsubnet", node[i], "-o=jsonpath={.egressIPs}").Output() + e2e.Logf("iplist for node %v: %v", node, iplist) + if iplist != "" && iplist != "[]" { + ip = strings.Split(iplist[2:len(iplist)-2], "\",\"") + } + if iplist == "" || iplist == "[]" || err != nil { + err = wait.Poll(30*time.Second, 3*time.Minute, func() (bool, error) { + iplist, err = oc.AsAdmin().WithoutNamespace().Run("get").Args("hostsubnet", node[i], "-o=jsonpath={.egressIPs}").Output() + if iplist != "" && iplist != "[]" { + e2e.Logf("Found egressIP list for node %v is: %v", node, iplist) + ip = strings.Split(iplist[2:len(iplist)-2], "\",\"") + return true, nil + } + if err != nil { + e2e.Logf("only got %d egressIP, or have err:%v, and try next round", len(ip), err) + return false, nil + } + return false, nil + }) + } + if IsValueInList(egressip, ip) { + foundHost = node[i] + break + } + } + return foundHost +} + +func IsValueInList(value string, list []string) bool { + for _, v := range list { + if v == value { + return true + } + } + return false +} + +// check if an ip address is added to node's NIC, or removed from node's NIC +func CheckPrimaryNIC(oc *exutil.CLI, nodeName string, ip string, flag bool) { + checkErr := wait.Poll(10*time.Second, 60*time.Second, func() (bool, error) { + output, err := DebugNodeWithChroot(oc, nodeName, "bash", "-c", "/usr/sbin/ip -4 -brief address show") + if err != nil { + e2e.Logf("Cannot get primary NIC interface, errors: %v, try again", err) + return false, nil + } + if flag && !strings.Contains(output, ip) { + e2e.Logf("egressIP has not been added to node's NIC correctly, try again") + return false, nil + } + if !flag && strings.Contains(output, ip) { + e2e.Logf("egressIP has not been removed from node's NIC correctly, try again") + return false, nil + } + return true, nil + }) + o.Expect(checkErr).NotTo(o.HaveOccurred(), fmt.Sprintf("Failed to get NIC on the host:%s", checkErr)) +} + +func CheckEgressIPonSDNHost(oc *exutil.CLI, node string, expectedEgressIP []string) { + checkErr := wait.Poll(10*time.Second, 60*time.Second, func() (bool, error) { + ip, err := GetEgressIPByKind(oc, "hostsubnet", node, len(expectedEgressIP)) + if err != nil { + e2e.Logf("\n got the error: %v\n, try again", err) + return false, nil + } + if !UnorderedEqual(ip, expectedEgressIP) { + e2e.Logf("\n got egressIP as %v while expected egressIP is %v, try again", ip, expectedEgressIP) + return false, nil + } + return true, nil + }) + o.Expect(checkErr).NotTo(o.HaveOccurred(), fmt.Sprintf("Failed to get egressIP on the host:%s", checkErr)) +} + +func UnorderedEqual(first, second []string) bool { + if len(first) != len(second) { + return false + } + for _, value := range first { + if !Contains(second, value) { + return false + } + } + return true +} + +func CheckovnkubeMasterNetworkProgrammingetrics(oc *exutil.CLI, url string, metrics string) { + var metricsOutput []byte + olmToken, err := GetSAToken(oc) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(olmToken).NotTo(o.BeEmpty()) + metricsErr := wait.Poll(5*time.Second, 10*time.Second, func() (bool, error) { + output, err := oc.AsAdmin().WithoutNamespace().Run("exec").Args("-n", "openshift-monitoring", "-c", "prometheus", "prometheus-k8s-0", "--", "curl", "-k", "-H", fmt.Sprintf("Authorization: Bearer %v", olmToken), fmt.Sprintf("%s", url)).OutputToFile("metrics.txt") + if err != nil { + e2e.Logf("Can't get metrics and try again, the error is:%s", err) + return false, nil + } + metricsOutput, _ = exec.Command("bash", "-c", "cat "+output+" | grep "+metrics+" | awk 'NR==2{print $2}'").Output() + metricsValue := strings.TrimSpace(string(metricsOutput)) + if metricsValue != "" { + e2e.Logf("The output of the metrics for %s is : %v", metrics, metricsValue) + } else { + e2e.Logf("Can't get metrics for %s:", metrics) + return false, nil + } + return true, nil + }) + o.Expect(metricsErr).NotTo(o.HaveOccurred(), fmt.Sprintf("Fail to get metric and the error is:%s", metricsErr)) +} + +func GetControllerManagerLeaderIP(oc *exutil.CLI) string { + leaderPodName, leaderErr := oc.AsAdmin().WithoutNamespace().Run("get").Args("lease", "openshift-master-controllers", "-n", "openshift-controller-manager", "-o=jsonpath={.spec.holderIdentity}").Output() + o.Expect(leaderErr).NotTo(o.HaveOccurred()) + o.Expect(leaderPodName).ShouldNot(o.BeEmpty(), "leader pod name is empty") + e2e.Logf("The leader pod name is %s", leaderPodName) + leaderPodIP := GetPodIPv4(oc, "openshift-controller-manager", leaderPodName) + return leaderPodIP +} + +func DescribeCheckEgressIPByKind(oc *exutil.CLI, kind string, kindName string) string { + output, err := oc.AsAdmin().WithoutNamespace().Run("describe").Args(kind, kindName).Output() + + o.Expect(err).NotTo(o.HaveOccurred()) + egressIPReg, _ := regexp.Compile(".*Egress IPs.*") + egressIPStr := egressIPReg.FindString(output) + egressIPArr := strings.Split(egressIPStr, ":") + + //remove whitespace in front of the ip address + ip := strings.TrimSpace(egressIPArr[1]) + e2e.Logf("get egressIP from oc describe %v %v: --->%s<---", kind, kindName, ip) + return ip +} + +func FindUnUsedIPsOnNodeOrFail(oc *exutil.CLI, nodeName, cidr string, expectedNum int) []string { + freeIPs := FindUnUsedIPsOnNode(oc, nodeName, cidr, expectedNum) + if len(freeIPs) != expectedNum { + g.Skip("Did not get enough free IPs for the test, skip the test.") + } + return freeIPs +} + +func (pod *ExternalIPPod) CreateExternalIPPod(oc *exutil.CLI) { + err := wait.Poll(5*time.Second, 20*time.Second, func() (bool, error) { + err1 := ApplyResourceFromTemplate(oc, "--ignore-unknown-parameters=true", "-f", pod.Template, "-p", "NAME="+pod.Name, "NAMESPACE="+pod.Namespace) + if err1 != nil { + e2e.Logf("the err:%v, and try next round", err1) + return false, nil + } + return true, nil + }) + o.Expect(err).NotTo(o.HaveOccurred(), fmt.Sprintf("fail to create the externalIP pod %v", pod.Name)) +} + +func CheckParameter(oc *exutil.CLI, namespace string, kind string, kindName string, parameter string) string { + output, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("-n", namespace, kind, kindName, parameter).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + return output +} + +func PatchReplaceResourceAsAdmin(oc *exutil.CLI, resource, patch string, nameSpace ...string) { + var cargs []string + if len(nameSpace) > 0 { + cargs = []string{resource, "-p", patch, "-n", nameSpace[0], "--type=json"} + } else { + cargs = []string{resource, "-p", patch, "--type=json"} + } + err := oc.AsAdmin().WithoutNamespace().Run("patch").Args(cargs...).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) +} + +// For SingleStack function returns IPv6 or IPv4 hostsubnet in case OVN +// For SDN plugin returns only IPv4 hostsubnet +// Dual stack not supported on openshiftSDN +// IPv6 single stack not supported on openshiftSDN +// network can be "default" for the default network or UDN network name +func GetNodeSubnet(oc *exutil.CLI, nodeName string, network string) string { + + output, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("node", nodeName, "-o=jsonpath={.metadata.annotations.k8s\\.ovn\\.org/node-subnets}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + var data map[string]interface{} + json.Unmarshal([]byte(output), &data) + hostSubnets := data[network].([]interface{}) + hostSubnet := hostSubnets[0].(string) + return hostSubnet + +} + +func GetNodeSubnetDualStack(oc *exutil.CLI, nodeName string, network string) (string, string) { + + output, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("node", nodeName, "-o=jsonpath={.metadata.annotations.k8s\\.ovn\\.org/node-subnets}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("output is %v", output) + var data map[string]interface{} + json.Unmarshal([]byte(output), &data) + hostSubnets := data[network].([]interface{}) + hostSubnetIPv4 := hostSubnets[0].(string) + hostSubnetIPv6 := hostSubnets[1].(string) + + e2e.Logf("Host subnet is %v and %v", hostSubnetIPv4, hostSubnetIPv6) + + return hostSubnetIPv4, hostSubnetIPv6 +} + +func GetIPv4Capacity(oc *exutil.CLI, nodeName string) string { + ipv4Capacity := "" + egressIPConfig, err := oc.WithoutNamespace().AsAdmin().Run("get").Args("node", nodeName, "-o=jsonpath={.metadata.annotations.cloud\\.network\\.openshift\\.io/egress-ipconfig}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("The egressipconfig is %v \n", egressIPConfig) + switch CheckPlatform(oc) { + case "aws": + ipv4Capacity = strings.Split(strings.Split(egressIPConfig, ":")[5], ",")[0] + case "gcp": + ipv4Capacity = strings.Split(egressIPConfig, ":")[5] + ipv4Capacity = ipv4Capacity[:len(ipv4Capacity)-3] + default: + e2e.Logf("Not support cloud provider for auto egressip cases for now.") + g.Skip("Not support cloud provider for auto egressip cases for now.") + } + + return ipv4Capacity +} + +func (AclSettings *AclSettings) GetJSONString() string { + jsonACLSetting, _ := json.Marshal(AclSettings) + annotationString := "k8s.ovn.org/acl-logging=" + string(jsonACLSetting) + return annotationString +} + +func EnableACLOnNamespace(oc *exutil.CLI, namespace, denyLevel, allowLevel string) { + e2e.Logf("Enable ACL looging on the namespace %s", namespace) + aclSettings := AclSettings{DenySetting: denyLevel, AllowSetting: allowLevel} + err1 := oc.AsAdmin().WithoutNamespace().Run("annotate").Args("--overwrite", "ns", namespace, aclSettings.GetJSONString()).Execute() + o.Expect(err1).NotTo(o.HaveOccurred()) +} + +func DisableACLOnNamespace(oc *exutil.CLI, namespace string) { + e2e.Logf("Disable ACL looging on the namespace %s", namespace) + err1 := oc.AsAdmin().WithoutNamespace().Run("annotate").Args("ns", namespace, "k8s.ovn.org/acl-logging-").Execute() + o.Expect(err1).NotTo(o.HaveOccurred()) +} + +func GetNodeMacAddress(oc *exutil.CLI, nodeName string) string { + var macAddress string + output, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("node", nodeName, "-o=jsonpath={.metadata.annotations.k8s\\.ovn\\.org/l3-gateway-config}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + var data map[string]interface{} + json.Unmarshal([]byte(output), &data) + l3GatewayConfigAnnotations := data["default"].(interface{}) + l3GatewayConfigAnnotationsJSON := l3GatewayConfigAnnotations.(map[string]interface{}) + macAddress = l3GatewayConfigAnnotationsJSON["mac-address"].(string) + return macAddress + +} + +// check if an env is in a configmap in specific namespace [usage: CheckConfigMap(oc, namesapce, configmapName, envString)] +func CheckEnvInConfigMap(oc *exutil.CLI, ns, configmapName string, envString string) error { + err := CheckConfigMap(oc, ns, configmapName) + o.Expect(err).NotTo(o.HaveOccurred(), fmt.Sprintf("cm %v is not found in namespace %v", configmapName, ns)) + + checkErr := wait.Poll(10*time.Second, 60*time.Second, func() (bool, error) { + output, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("configmap", "-n", ns, configmapName, "-oyaml").Output() + if err != nil { + e2e.Logf("Failed to get configmap %v, error: %s. Trying again", configmapName, err) + return false, nil + } + if !strings.Contains(output, envString) { + e2e.Logf("Did not find %v in ovnkube-config configmap,try next round.", envString) + return false, nil + } + return true, nil + }) + return checkErr +} + +// check if certain log message is in a pod in specific namespace +func CheckLogMessageInPod(oc *exutil.CLI, namespace string, containerName string, podName string, filter string) (string, error) { + var podLogs string + var err, checkErr error + checkErr = wait.Poll(10*time.Second, 60*time.Second, func() (bool, error) { + podLogs, err = GetSpecificPodLogsCombinedOrNot(oc, namespace, containerName, podName, filter, true) + if len(podLogs) == 0 || err != nil { + e2e.Logf("did not get expected podLogs: %v, or have err:%v, try again", podLogs, err) + return false, nil + } + return true, nil + }) + if checkErr != nil { + return podLogs, fmt.Errorf("fail to get expected log in pod %v, err: %v", podName, err) + } + return podLogs, nil +} + +// get OVN-Kubernetes management interface (ovn-k8s-mp0) IP for the node +func GetOVNK8sNodeMgmtIPv4(oc *exutil.CLI, nodeName string) string { + var output string + var err error + checkErr := wait.Poll(10*time.Second, 60*time.Second, func() (bool, error) { + output, err = DebugNodeWithChroot(oc, nodeName, "bash", "-c", "/usr/sbin/ip -4 -brief address show | grep ovn-k8s-mp0") + if output == "" || err != nil { + e2e.Logf("Did not get node's management interface, errors: %v, try again", err) + return false, nil + } + return true, nil + }) + o.Expect(checkErr).NotTo(o.HaveOccurred(), fmt.Sprintf("fail to get management interface for node %v, err: %v", nodeName, checkErr)) + + e2e.Logf("Match out the OVN-Kubernetes management IP address for the node") + re := regexp.MustCompile(`(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)){3}`) + nodeOVNK8sMgmtIP := re.FindAllString(output, -1)[0] + e2e.Logf("Got ovn-k8s management interface IP for node %v as: %v", nodeName, nodeOVNK8sMgmtIP) + return nodeOVNK8sMgmtIP +} + +// FindLogFromPod will search logs for a specific string in the specific container of the pod or just the pod +func FindLogFromPod(oc *exutil.CLI, searchString string, namespace string, podLabel string, podContainer ...string) bool { + findLog := false + podNames := GetPodName(oc, namespace, podLabel) + var cargs []string + for _, podName := range podNames { + if len(podContainer) > 0 { + cargs = []string{podName, "-c", podContainer[0], "-n", namespace} + } else { + cargs = []string{podName, "-n", namespace} + } + output, err := oc.AsAdmin().WithoutNamespace().Run("logs").Args(cargs...).OutputToFile("podlog") + o.Expect(err).NotTo(o.HaveOccurred()) + grepOutput, err := exec.Command("bash", "-c", "cat "+output+" | grep -i '"+searchString+"' | wc -l").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + grepOutputString := strings.TrimSpace(string(grepOutput)) + if grepOutputString != "0" { + e2e.Logf("Found the '%s' string in %s number of lines.", searchString, grepOutputString) + findLog = true + break + } + } + return findLog +} + +// SearchOVNDBForSpecCmd This is used for lr-policy-list and snat rules check in ovn db. +func SearchOVNDBForSpecCmd(oc *exutil.CLI, cmd, searchKeyword string, times int) error { + ovnPod := GetOVNKMasterOVNkubeNode(oc) + o.Expect(ovnPod).ShouldNot(o.Equal("")) + var cmdOutput string + checkOVNDbErr := wait.Poll(10*time.Second, 2*time.Minute, func() (bool, error) { + output, cmdErr := RemoteShPodWithBash(oc, "openshift-ovn-kubernetes", ovnPod, cmd) + if cmdErr != nil { + e2e.Logf("%v,Waiting for expected result to be synced, try next ...,", cmdErr) + return false, nil + } + cmdOutput = output + if strings.Count(output, searchKeyword) == times { + return true, nil + } + return false, nil + }) + if checkOVNDbErr != nil { + e2e.Logf("The command check result in ovndb is not expected ! See below output \n %s ", cmdOutput) + } + return checkOVNDbErr +} + +// WaitEgressFirewallApplied Wait egressfirewall applied +func WaitEgressFirewallApplied(oc *exutil.CLI, efName, ns string) error { + checkErr := wait.Poll(10*time.Second, 60*time.Second, func() (bool, error) { + output, efErr := oc.AsAdmin().WithoutNamespace().Run("get").Args("egressfirewall", "-n", ns, efName).Output() + if efErr != nil { + e2e.Logf("Failed to get egressfirewall %v, error: %s. Trying again", efName, efErr) + return false, nil + } + if !strings.Contains(output, "EgressFirewall Rules applied") { + e2e.Logf("The egressfirewall was not applied, trying again. \n %s", output) + return false, nil + } + return true, nil + }) + return checkErr +} + +// SwitchOVNGatewayMode will switch to requested mode, shared or local +func SwitchOVNGatewayMode(oc *exutil.CLI, mode string) { + currentMode := GetOVNGatewayMode(oc) + if currentMode == "local" && mode == "shared" { + e2e.Logf("Migrating cluster to shared gateway mode") + PatchResourceAsAdmin(oc, "network.operator/cluster", "{\"spec\":{\"defaultNetwork\":{\"ovnKubernetesConfig\":{\"gatewayConfig\":{\"routingViaHost\": false}}}}}") + } else if currentMode == "shared" && mode == "local" { + e2e.Logf("Migrating cluster to Local gw mode") + PatchResourceAsAdmin(oc, "network.operator/cluster", "{\"spec\":{\"defaultNetwork\":{\"ovnKubernetesConfig\":{\"gatewayConfig\":{\"routingViaHost\": true}}}}}") + } else { + e2e.Logf("Cluster is already on requested gateway mode") + } + _, err := oc.AsAdmin().WithoutNamespace().Run("rollout").Args("status", "-n", "openshift-ovn-kubernetes", "ds", "ovnkube-node").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + //on OVN IC it takes upto 660 seconds for nodes ds to rollout so lets poll with timeout of 700 seconds + WaitForNetworkOperatorState(oc, 100, 18, "True.*False.*False") +} + +// GetOVNGatewayMode will return configured OVN gateway mode, shared or local +func GetOVNGatewayMode(oc *exutil.CLI) string { + nodeList, err := e2enode.GetReadySchedulableNodes(context.TODO(), oc.KubeFramework().ClientSet) + o.Expect(err).NotTo(o.HaveOccurred()) + if len(nodeList.Items) < 1 { + g.Skip("This case requires at least one schedulable node") + } + output, err := oc.AsAdmin().WithoutNamespace().NotShowInfo().Run("describe").Args("node", nodeList.Items[0].Name).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + str := "local" + modeString := strconv.Quote(str) + if strings.Contains(output, modeString) { + e2e.Logf("Cluster is running on OVN Local Gateway Mode") + return str + } + return "shared" +} + +func GetEgressCIDRsForNode(oc *exutil.CLI, nodeName string) string { + var sub1 string + platform := CheckPlatform(oc) + if strings.Contains(platform, "vsphere") || strings.Contains(platform, "baremetal") || strings.Contains(platform, "none") || strings.Contains(platform, "nutanix") || strings.Contains(platform, "powervs") { + defaultSubnetV4, err := GetDefaultSubnet(oc) + o.Expect(err).NotTo(o.HaveOccurred()) + _, ipNet, err1 := net.ParseCIDR(defaultSubnetV4) + o.Expect(err1).NotTo(o.HaveOccurred()) + e2e.Logf("ipnet: %v", ipNet) + sub1 = ipNet.String() + e2e.Logf("\n\n\n sub1 as -->%v<--\n\n\n", sub1) + } else { + sub1 = GetIfaddrFromNode(nodeName, oc) + } + return sub1 +} + +// get routerID by node name +func GetRouterID(oc *exutil.CLI, nodeName string) (string, error) { + // get the ovnkube-node pod on the node + ovnKubePod, podErr := GetOVNKPodOnNode(oc, "openshift-ovn-kubernetes", "app=ovnkube-node", nodeName) + o.Expect(podErr).NotTo(o.HaveOccurred()) + o.Expect(ovnKubePod).ShouldNot(o.Equal("")) + var cmdOutput, routerName, routerID string + var cmdErr error + routerName = "GR_" + nodeName + cmd := "ovn-nbctl show | grep " + routerName + " | grep 'router '|awk '{print $2}'" + checkOVNDbErr := wait.Poll(10*time.Second, 2*time.Minute, func() (bool, error) { + cmdOutput, cmdErr = RemoteShPodWithBash(oc, "openshift-ovn-kubernetes", ovnKubePod, cmd) + if cmdErr != nil { + e2e.Logf("%v,Waiting for expected result to be synced, try again ...,", cmdErr) + return false, nil + } + + // Command output always has first line as: Defaulted container "northd" out of: northd, nbdb, kube-rbac-proxy, sbdb, ovnkube-master, ovn-dbchecker + // Take result from the second line + cmdOutputLines := strings.Split(cmdOutput, "\n") + if len(cmdOutputLines) >= 2 { + routerID = cmdOutputLines[1] + return true, nil + } + e2e.Logf("Waiting for expected result to be synced, try again ...") + return false, nil + }) + if checkOVNDbErr != nil { + e2e.Logf("The command check result in ovndb is not expected ! See below output \n %s ", cmdOutput) + } + return routerID, checkOVNDbErr +} + +func GetSNATofEgressIP(oc *exutil.CLI, nodeName, egressIP string) ([]string, error) { + // get the ovnkube-node pod on the node + ovnKubePod, podErr := GetOVNKPodOnNode(oc, "openshift-ovn-kubernetes", "app=ovnkube-node", nodeName) + o.Expect(podErr).NotTo(o.HaveOccurred()) + o.Expect(ovnKubePod).ShouldNot(o.Equal("")) + var cmdOutput string + var cmdErr error + var snatIP []string + + cmd := "ovn-nbctl --no-headings --column logical_ip --format=table find nat external_ip=" + egressIP + checkOVNDbErr := wait.Poll(10*time.Second, 2*time.Minute, func() (bool, error) { + cmdOutput, cmdErr = RemoteShPodWithBashSpecifyContainer(oc, "openshift-ovn-kubernetes", ovnKubePod, "northd", cmd) + if cmdErr != nil { + e2e.Logf("%v,Waiting for expected result to be synced, try again ...,", cmdErr) + return false, nil + } + + if cmdOutput != "" { + cmdOutputLines := strings.Split(cmdOutput, "\n") + for i := 0; i < len(cmdOutputLines); i++ { + ip := strings.Trim(cmdOutputLines[i], "\"") + snatIP = append(snatIP, ip) + } + return true, nil + } + + e2e.Logf("Waiting for expected result to be synced, try again ...") + return false, nil + }) + if checkOVNDbErr != nil { + e2e.Logf("The command check result in ovndb is not expected ! See below output \n %s ", cmdOutput) + } + return snatIP, checkOVNDbErr +} + +// EnableSCTPModuleOnNode Manual way to enable sctp in a cluster +func EnableSCTPModuleOnNode(oc *exutil.CLI, nodeName, role string) { + e2e.Logf("This is %s worker node: %s", role, nodeName) + checkSCTPCmd := "cat /sys/module/sctp/initstate" + output, err := DebugNodeWithChroot(oc, nodeName, "bash", "-c", checkSCTPCmd) + var installCmd string + if err != nil || !strings.Contains(output, "live") { + e2e.Logf("No sctp module installed, will enable sctp module!!!") + if strings.EqualFold(role, "rhel") { + // command for rhel nodes + installCmd = "yum install -y kernel-modules-extra-`uname -r` && insmod /usr/lib/modules/`uname -r`/kernel/net/sctp/sctp.ko.xz" + } else { + // command for rhcos nodes + installCmd = "modprobe sctp" + } + e2e.Logf("Install command is %s", installCmd) + + // Try 3 times to enable sctp + o.Eventually(func() error { + _, installErr := DebugNodeWithChroot(oc, nodeName, "bash", "-c", installCmd) + if installErr != nil && strings.EqualFold(role, "rhel") { + e2e.Logf("%v", installErr) + g.Skip("Yum insall to enable sctp cannot work in a disconnected cluster, skip the test!!!") + } + return installErr + }, "15s", "5s").ShouldNot(o.HaveOccurred(), fmt.Sprintf("Failed to install sctp module on node %s", nodeName)) + + // Wait for sctp applied + o.Eventually(func() string { + output, err := DebugNodeWithChroot(oc, nodeName, "bash", "-c", checkSCTPCmd) + if err != nil { + e2e.Logf("Wait for sctp applied, %v", err) + } + return output + }, "60s", "10s").Should(o.ContainSubstring("live"), fmt.Sprintf("Failed to load sctp module on node %s", nodeName)) + } else { + e2e.Logf("sctp module is loaded on node %s\n%s", nodeName, output) + } + +} + +func PrepareSCTPModule(oc *exutil.CLI, sctpModule string) { + nodesOutput, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("node").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + if strings.Contains(nodesOutput, "SchedulingDisabled") || strings.Contains(nodesOutput, "NotReady") { + g.Skip("There are already some nodes in NotReady or SchedulingDisabled status in cluster, skip the test!!! ") + } + + nodes, err := GetSchedulableLinuxWorkerNodes(oc) + if err != nil || len(nodes) < 1 { + g.Skip("Can not find any woker nodes in the cluster") + } + + for _, worker := range nodes { + EnableSCTPModuleOnNode(oc, worker, "rhcos") + } + +} + +// GetIPv4Gateway get ipv4 gateway address +func GetIPv4Gateway(oc *exutil.CLI, nodeName string) string { + cmd := "ip -4 route | grep default | awk '{print $3}'" + output, err := DebugNode(oc, nodeName, "bash", "-c", cmd) + o.Expect(err).NotTo(o.HaveOccurred()) + re := regexp.MustCompile(`(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)){3}`) + ips := re.FindAllString(output, -1) + if len(ips) == 0 { + return "" + } + e2e.Logf("The default gateway of node %s is %s", nodeName, ips[0]) + return ips[0] +} + +// GetInterfacePrefix return the prefix of the primary interface IP +func GetInterfacePrefix(oc *exutil.CLI, nodeName string) string { + defInf, err := GetDefaultInterface(oc) + o.Expect(err).NotTo(o.HaveOccurred()) + cmd := fmt.Sprintf("ip -4 -brief a show %s | awk '{print $3}' ", defInf) + output, err := DebugNode(oc, nodeName, "bash", "-c", cmd) + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("IP address for default interface %s is %s", defInf, output) + sli := strings.Split(output, "/") + if len(sli) > 0 { + return strings.Split(sli[1], "\n")[0] + } + return "24" +} + +func ExcludeSriovNodes(oc *exutil.CLI) []string { + // In rdu1 and rdu2 clusters, there are two sriov nodes with mlx nic, by default, egressrouter case cannot run on it + // So here exclude sriov nodes in rdu1 and rdu2 clusters, just use the other common worker nodes + var workers []string + nodeList, err := e2enode.GetReadySchedulableNodes(context.TODO(), oc.KubeFramework().ClientSet) + o.Expect(err).NotTo(o.HaveOccurred()) + for _, node := range nodeList.Items { + _, ok := node.Labels["node-role.kubernetes.io/sriov"] + if !ok { + e2e.Logf("node %s is not sriov node,add it to worker list.", node.Name) + workers = append(workers, node.Name) + } + } + return workers +} + +func GetSriovNodes(oc *exutil.CLI) []string { + // In rdu1 and rdu2 clusters, there are two sriov nodes with mlx nic + var workers string + workers, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("nodes", "-l", "node-role.kubernetes.io/sriov", "--no-headers", "-o=custom-columns=NAME:.metadata.Name").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + return strings.Split(workers, "\n") +} + +func CheckClusterStatus(oc *exutil.CLI, expectedStatus string) { + // get all master nodes + masterNodes, getAllMasterNodesErr := GetClusterNodesBy(oc, "master") + o.Expect(getAllMasterNodesErr).NotTo(o.HaveOccurred()) + o.Expect(masterNodes).NotTo(o.BeEmpty()) + + // check master nodes status, expect Ready status for them + for _, masterNode := range masterNodes { + CheckNodeStatus(oc, masterNode, "Ready") + } + + // get all worker nodes + workerNodes, getAllWorkerNodesErr := GetClusterNodesBy(oc, "worker") + o.Expect(getAllWorkerNodesErr).NotTo(o.HaveOccurred()) + o.Expect(workerNodes).NotTo(o.BeEmpty()) + + // check worker nodes status, expect Ready status for them + for _, workerNode := range masterNodes { + CheckNodeStatus(oc, workerNode, "Ready") + } +} + +func GetOVNKCtrlPlanePodOnHostedCluster(oc *exutil.CLI, namespace, cmName, hyperShiftMgmtNS string) string { + // get leader ovnkube-control-plane pod on hypershift hosted cluster + ovnkCtrlPlanePodLead, leaderErr := oc.AsGuestKubeconf().Run("get").Args("lease", "ovn-kubernetes-master", "-n", "openshift-ovn-kubernetes", "-o=jsonpath={.spec.holderIdentity}").Output() + o.Expect(leaderErr).NotTo(o.HaveOccurred()) + e2e.Logf("ovnkube-control-plane pod of the hosted cluster is %s", ovnkCtrlPlanePodLead) + return ovnkCtrlPlanePodLead +} + +func WaitForPodWithLabelReadyOnHostedCluster(oc *exutil.CLI, ns, label string) error { + return wait.Poll(15*time.Second, 10*time.Minute, func() (bool, error) { + status, err := oc.AsAdmin().AsGuestKubeconf().WithoutNamespace().Run("get").Args("pod", "-n", ns, "-l", label, "-ojsonpath={.items[*].status.conditions[?(@.type==\"Ready\")].status}").Output() + e2e.Logf("the Ready status of pod is %v", status) + if err != nil || status == "" { + e2e.Logf("failed to get pod status: %v, retrying...", err) + return false, nil + } + if strings.Contains(status, "False") { + e2e.Logf("the pod Ready status not met; wanted True but got %v, retrying...", status) + return false, nil + } + return true, nil + }) +} + +func GetPodNameOnHostedCluster(oc *exutil.CLI, namespace, label string) []string { + var podName []string + podNameAll, err := oc.AsAdmin().AsGuestKubeconf().Run("get").Args("-n", namespace, "pod", "-l", label, "-ojsonpath={.items[*].metadata.name}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + podName = strings.Split(podNameAll, " ") + e2e.Logf("The pod(s) are %v ", podName) + return podName +} + +func GetReadySchedulableNodesOnHostedCluster(oc *exutil.CLI) ([]string, error) { + output, err := oc.AsAdmin().AsGuestKubeconf().Run("get").Args("node", "-ojsonpath={.items[*].metadata.name}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + + var nodesOnHostedCluster, schedulableNodes []string + nodesOnHostedCluster = strings.Split(output, " ") + for _, nodeName := range nodesOnHostedCluster { + err := wait.Poll(10*time.Second, 15*time.Minute, func() (bool, error) { + statusOutput, err := oc.AsAdmin().AsGuestKubeconf().Run("get").Args("nodes", nodeName, "-ojsonpath={.status.conditions[-1].status}").Output() + if err != nil { + e2e.Logf("\nGet node status with error : %v", err) + return false, nil + } + if statusOutput != "True" { + return false, nil + } + schedulableNodes = append(schedulableNodes, nodeName) + return true, nil + }) + o.Expect(err).NotTo(o.HaveOccurred(), fmt.Sprintf("Node %s is not in expected status %s", nodeName, "Ready")) + } + e2e.Logf("Scheduleable nodes on hosted cluster are: %v ", schedulableNodes) + return schedulableNodes, nil +} + +func CheckLogMessageInPodOnHostedCluster(oc *exutil.CLI, namespace string, containerName string, podName string, filter string) (string, error) { + var podLogs string + var err, checkErr error + checkErr = wait.Poll(10*time.Second, 60*time.Second, func() (bool, error) { + podLogs, err = GetSpecificPodLogs(oc.AsAdmin().AsGuestKubeconf(), namespace, containerName, podName, filter) + if len(podLogs) == 0 || err != nil { + e2e.Logf("did not get expected podLog: %v, or have err:%v, try again", podLogs, err) + return false, nil + } + return true, nil + }) + o.Expect(checkErr).NotTo(o.HaveOccurred(), fmt.Sprintf("fail to get expected log in pod %v, err: %v", podName, checkErr)) + return podLogs, nil + +} + +// get OVN-Kubernetes management interface (ovn-k8s-mp0) IP for the node on hosted cluster +func GetOVNK8sNodeMgmtIPv4OnHostedCluster(oc *exutil.CLI, nodeName string) string { + var output string + var outputErr error + defer RecoverNamespaceRestricted(oc.AsGuestKubeconf(), "default") + SetNamespacePrivileged(oc.AsGuestKubeconf(), "default") + checkErr := wait.Poll(10*time.Second, 60*time.Second, func() (bool, error) { + output, outputErr = oc.AsGuestKubeconf().WithoutNamespace().Run("debug").Args("-n", "default", "node/"+nodeName, "--", "chroot", "/host", "bash", "-c", "/usr/sbin/ip -4 -brief address show | grep ovn-k8s-mp0").Output() + if output == "" || outputErr != nil { + e2e.Logf("Did not get node's management interface on hosted cluster, errors: %v, try again", outputErr) + return false, nil + } + return true, nil + }) + o.Expect(checkErr).NotTo(o.HaveOccurred(), fmt.Sprintf("fail to get management interface for node %v, err: %v", nodeName, checkErr)) + + e2e.Logf("Match out the OVN-Kubernetes management IP address for the node on hosted cluster") + re := regexp.MustCompile(`(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)){3}`) + nodeOVNK8sMgmtIPOnHostedCluster := re.FindAllString(output, -1)[0] + e2e.Logf("Got ovn-k8s management interface IP for node on hosted cluster %v as: %v", nodeName, nodeOVNK8sMgmtIPOnHostedCluster) + return nodeOVNK8sMgmtIPOnHostedCluster +} + +// execute command on debug node with chroot on node of hosted cluster +func ExecCmdOnDebugNodeOfHostedCluster(oc *exutil.CLI, nodeName string, cmdOptions []string) error { + cargs := []string{"node/" + nodeName, "--", "chroot", "/host"} + if len(cmdOptions) > 0 { + cargs = append(cargs, cmdOptions...) + } + + debugErr := oc.AsGuestKubeconf().WithoutNamespace().Run("debug").Args(cargs...).Execute() + + return debugErr +} + +// check the cronjobs in the openshift-multus namespace +func GetMultusCronJob(oc *exutil.CLI) string { + cronjobLog, cronjobErr := oc.AsAdmin().WithoutNamespace().Run("get").Args("cronjobs", "-n", "openshift-multus").Output() + o.Expect(cronjobErr).NotTo(o.HaveOccurred()) + return cronjobLog +} + +// get name of OVN egressIP object(s) +func GetOVNEgressIPObject(oc *exutil.CLI) []string { + var egressIPObjects = []string{} + egressIPObjectsAll, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("egressip", "-ojsonpath={.items[*].metadata.name}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + if len(egressIPObjectsAll) > 0 { + egressIPObjects = strings.Split(egressIPObjectsAll, " ") + } + e2e.Logf("egressIPObjects are %v ", egressIPObjects) + return egressIPObjects +} + +// get node that hosts the egressIP +func GetHostsubnetByEIP(oc *exutil.CLI, expectedEIP string) string { + var nodeHostsEIP string + nodeList, err := e2enode.GetReadySchedulableNodes(context.TODO(), oc.KubeFramework().ClientSet) + o.Expect(err).NotTo(o.HaveOccurred()) + + for i, v := range nodeList.Items { + ip, err := GetEgressIPByKind(oc, "hostsubnet", nodeList.Items[i].Name, 1) + o.Expect(err).NotTo(o.HaveOccurred()) + if ip[0] == expectedEIP { + e2e.Logf("Found node %v host egressip %v", v.Name, expectedEIP) + nodeHostsEIP = nodeList.Items[i].Name + break + } + } + return nodeHostsEIP +} + +// find the ovn-K cluster manager master pod +func GetOVNKMasterPod(oc *exutil.CLI) string { + leaderCtrlPlanePod, leaderNodeLogerr := oc.AsAdmin().WithoutNamespace().Run("get").Args("lease", "ovn-kubernetes-master", "-n", "openshift-ovn-kubernetes", "-o=jsonpath={.spec.holderIdentity}").Output() + o.Expect(leaderNodeLogerr).NotTo(o.HaveOccurred()) + return leaderCtrlPlanePod +} + +// find the cluster-manager's ovnkube-node for accessing master components +func GetOVNKMasterOVNkubeNode(oc *exutil.CLI) string { + leaderPod, leaderNodeLogerr := oc.AsAdmin().WithoutNamespace().Run("get").Args("lease", "ovn-kubernetes-master", "-n", "openshift-ovn-kubernetes", "-o=jsonpath={.spec.holderIdentity}").Output() + o.Expect(leaderNodeLogerr).NotTo(o.HaveOccurred()) + leaderNodeName, getNodeErr := GetPodNodeName(oc, "openshift-ovn-kubernetes", leaderPod) + o.Expect(getNodeErr).NotTo(o.HaveOccurred()) + ovnKubePod, podErr := GetOVNKPodOnNode(oc, "openshift-ovn-kubernetes", "app=ovnkube-node", leaderNodeName) + o.Expect(podErr).NotTo(o.HaveOccurred()) + return ovnKubePod +} + +// enable multicast on specific namespace +func EnableMulticast(oc *exutil.CLI, ns string) { + _, err := RunOcWithRetry(oc.AsAdmin().WithoutNamespace(), "annotate", "namespace", ns, "k8s.ovn.org/multicast-enabled=true") + o.Expect(err).NotTo(o.HaveOccurred()) +} + +func GetCNOStatusCondition(oc *exutil.CLI) string { + CNOStatusCondition, err := oc.WithoutNamespace().AsAdmin().Run("get").Args("clusteroperators", "network", "-o=jsonpath={.status.conditions}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + return CNOStatusCondition +} + +// return severity, expr and runbook of specific ovn alert in networking-rules +func GetOVNAlertNetworkingRules(oc *exutil.CLI, alertName string) (string, string, string) { + // get all ovn alert names in networking-rules + ns := "openshift-ovn-kubernetes" + allAlerts, nameErr := oc.AsAdmin().Run("get").Args("prometheusrule", "-n", ns, "networking-rules", "-o=jsonpath={.spec.groups[*].rules[*].alert}").Output() + o.Expect(nameErr).NotTo(o.HaveOccurred()) + e2e.Logf("The alert are %v", allAlerts) + + if !strings.Contains(allAlerts, alertName) { + e2e.Failf("Target alert %v is not found", alertName) + return "", "", "" + } else { + var severity, expr string + severity, severityErr := oc.AsAdmin().Run("get").Args("prometheusrule", "-n", ns, "networking-rules", "-o=jsonpath={.spec.groups[*].rules[?(@.alert==\""+alertName+"\")].labels.severity}").Output() + o.Expect(severityErr).NotTo(o.HaveOccurred()) + e2e.Logf("The alert severity is %v", severity) + expr, exprErr := oc.AsAdmin().Run("get").Args("prometheusrule", "-n", ns, "networking-rules", "-o=jsonpath={.spec.groups[*].rules[?(@.alert==\""+alertName+"\")].expr}").Output() + o.Expect(exprErr).NotTo(o.HaveOccurred()) + e2e.Logf("The alert expr is %v", expr) + runbook, runbookErr := oc.AsAdmin().Run("get").Args("prometheusrule", "-n", ns, "networking-rules", "-o=jsonpath={.spec.groups[*].rules[?(@.alert==\""+alertName+"\")].annotations.runbook_url}").Output() + o.Expect(runbookErr).NotTo(o.HaveOccurred()) + e2e.Logf("The alert runbook is %v", runbook) + + return severity, expr, runbook + } +} + +// return severity, expr and runbook of specific ovn alert in master-rules +func GetOVNAlertMasterRules(oc *exutil.CLI, alertName string) (string, string, string) { + // get all ovn alert names in networking-rules + ns := "openshift-ovn-kubernetes" + allAlerts, nameErr := oc.AsAdmin().Run("get").Args("prometheusrule", "-n", ns, "master-rules", "-o=jsonpath={.spec.groups[*].rules[*].alert}").Output() + o.Expect(nameErr).NotTo(o.HaveOccurred()) + e2e.Logf("The alert are %v", allAlerts) + + if !strings.Contains(allAlerts, alertName) { + e2e.Failf("Target alert %v is not found", alertName) + return "", "", "" + } else { + var severity, expr string + severity, severityErr := oc.AsAdmin().Run("get").Args("prometheusrule", "-n", ns, "master-rules", "-o=jsonpath={.spec.groups[*].rules[?(@.alert==\""+alertName+"\")].labels.severity}").Output() + o.Expect(severityErr).NotTo(o.HaveOccurred()) + e2e.Logf("The alert severity is %v", severity) + expr, exprErr := oc.AsAdmin().Run("get").Args("prometheusrule", "-n", ns, "master-rules", "-o=jsonpath={.spec.groups[*].rules[?(@.alert==\""+alertName+"\")].expr}").Output() + o.Expect(exprErr).NotTo(o.HaveOccurred()) + e2e.Logf("The alert expr is %v", expr) + runbook, runbookErr := oc.AsAdmin().Run("get").Args("prometheusrule", "-n", ns, "master-rules", "-o=jsonpath={.spec.groups[*].rules[?(@.alert==\""+alertName+"\")].annotations.runbook_url}").Output() + o.Expect(runbookErr).NotTo(o.HaveOccurred()) + e2e.Logf("The alert runbook is %v", runbook) + + return severity, expr, runbook + } +} + +// returns all the logical routers and switches on all the nodes +func GetOVNConstructs(oc *exutil.CLI, constructType string, nodeNames []string) []string { + var ovnConstructs []string + var matchStr string + //var cmdOutput string + + getCmd := "ovn-nbctl --no-leader-only " + constructType + ovnPod := GetOVNKMasterOVNkubeNode(oc) + o.Expect(ovnPod).ShouldNot(o.Equal("")) + checkOVNDbErr := wait.Poll(10*time.Second, 2*time.Minute, func() (bool, error) { + cmdOutput, cmdErr := RemoteShPodWithBash(oc, "openshift-ovn-kubernetes", ovnPod, getCmd) + if cmdErr != nil { + e2e.Logf("%v,Waiting for expected result to be synced, try again ...,", cmdErr) + return false, nil + } + o.Expect(cmdOutput).ShouldNot(o.Equal("")) + for _, index := range strings.Split(cmdOutput, "\n") { + for _, node := range nodeNames { + if constructType == "ls-list" { + matchStr = fmt.Sprintf("\\((%s\\))", node) + } else { + matchStr = fmt.Sprintf("\\((GR_%s\\))", node) + } + re := regexp.MustCompile(matchStr) + if re.FindString(index) != "" { + ovnConstruct := strings.Fields(index) + ovnConstructs = append(ovnConstructs, ovnConstruct[0]) + } + } + } + return true, nil + }) + if checkOVNDbErr != nil { + e2e.Logf("The result in ovndb is not expected ! See below output \n %s ", checkOVNDbErr) + } + return ovnConstructs +} + +// Returns the logical router or logical switch on a node +func (SvcEndpontDetails *SvcEndpontDetails) GetOVNConstruct(oc *exutil.CLI, constructType string) string { + var ovnConstruct string + var matchStr string + getCmd := "ovn-nbctl " + constructType + checkOVNDbErr := wait.Poll(10*time.Second, 2*time.Minute, func() (bool, error) { + cmdOutput, cmdErr := RemoteShPodWithBash(oc, "openshift-ovn-kubernetes", SvcEndpontDetails.OvnKubeNodePod, getCmd) + if cmdErr != nil { + e2e.Logf("%v,Waiting for expected result to be synced, try again ...,", cmdErr) + return false, nil + } + if cmdOutput == "" { + return true, nil + } + for _, index := range strings.Split(cmdOutput, "\n") { + + if constructType == "ls-list" { + matchStr = fmt.Sprintf("\\((%s\\))", SvcEndpontDetails.NodeName) + } else { + matchStr = fmt.Sprintf("\\((GR_%s\\))", SvcEndpontDetails.NodeName) + } + re := regexp.MustCompile(matchStr) + if re.FindString(index) != "" { + matchedStr := strings.Fields(index) + ovnConstruct = matchedStr[0] + } + } + return true, nil + }) + if checkOVNDbErr != nil { + e2e.Logf("The result in ovndb is not expected ! See below output \n %s ", checkOVNDbErr) + } + return ovnConstruct +} + +// returns load balancer entries created for LB service type on routers or switches on all nodes +func GetOVNLBContructs(oc *exutil.CLI, constructType string, endPoint string, ovnConstruct []string) bool { + var result bool + ovnPod := GetOVNKMasterOVNkubeNode(oc) + o.Expect(ovnPod).ShouldNot(o.Equal("")) + //only if the count for any of output is less than three the success will be false + result = true + for _, construct := range ovnConstruct { + checkOVNDbErr := wait.Poll(10*time.Second, 2*time.Minute, func() (bool, error) { + getCmd := "ovn-nbctl --no-leader-only " + constructType + " " + construct + " | grep " + endPoint + cmdOutput, cmdErr := RemoteShPodWithBashSpecifyContainer(oc, "openshift-ovn-kubernetes", ovnPod, "northd", getCmd) + if cmdErr != nil { + e2e.Logf("%v,Waiting for expected result to be synced, try next ...,", cmdErr) + result = false + return false, nil + } + if len(strings.Split(cmdOutput, "\n")) >= 2 { + e2e.Logf("Required entries %s were created for service on %s", constructType, construct) + result = true + } else { + e2e.Logf("Required entries %s were not created for service on %s", constructType, construct) + result = false + } + return true, nil + }) + if checkOVNDbErr != nil { + e2e.Logf("The command check result in ovndb is not expected ! See below output \n %s ", checkOVNDbErr) + result = false + } + + } + return result +} + +// returns load balancer entries created for LB service type on routers or switches on a single node +func (SvcEndpontDetails *SvcEndpontDetails) GetOVNLBContruct(oc *exutil.CLI, constructType string, construct string) bool { + var result bool + //only if the count for any of output is less than three the success will be false + result = true + checkOVNDbErr := wait.Poll(10*time.Second, 2*time.Minute, func() (bool, error) { + getCmd := "ovn-nbctl " + constructType + " " + construct + " | grep " + SvcEndpontDetails.PodIP + cmdOutput, cmdErr := RemoteShPodWithBashSpecifyContainer(oc, "openshift-ovn-kubernetes", SvcEndpontDetails.OvnKubeNodePod, "northd", getCmd) + if cmdErr != nil { + e2e.Logf("%v,Waiting for expected result to be synced, try next ...,", cmdErr) + result = false + return false, nil + } + if len(strings.Split(cmdOutput, "\n")) >= 2 { + e2e.Logf("Required entries %s were created for service on %s", constructType, construct) + result = true + } else { + e2e.Logf("Required entries %s were not created for service on %s", constructType, construct) + result = false + } + return true, nil + }) + if checkOVNDbErr != nil { + e2e.Logf("The command check result in ovndb is not expected ! See below output \n %s ", checkOVNDbErr) + result = false + } + + return result +} + +func GetServiceEndpoints(oc *exutil.CLI, serviceName string, serviceNamespace string) string { + serviceEndpoint, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("ep", serviceName, "-n", serviceNamespace, "--no-headers").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(serviceEndpoint).ShouldNot(o.BeEmpty()) + e2e.Logf("Service endpoint %v", serviceEndpoint) + result := strings.Fields(serviceEndpoint) + return result[1] +} + +func GetOVNMetricsInSpecificContainer(oc *exutil.CLI, containerName string, podName string, url string, metricName string) string { + var metricValue string + metricsErr := wait.Poll(5*time.Second, 10*time.Second, func() (bool, error) { + output, err := oc.AsAdmin().WithoutNamespace().Run("exec").Args("-n", "openshift-ovn-kubernetes", "-c", containerName, podName, "--", "curl", url).OutputToFile("metrics.txt") + if err != nil { + e2e.Logf("Can't get metrics and try again, the error is:%s", err) + return false, nil + } + metricOutput, getMetricErr := exec.Command("bash", "-c", "cat "+output+" | grep -e '^"+metricName+" ' | awk 'END {print $2}'").Output() + o.Expect(getMetricErr).NotTo(o.HaveOccurred()) + metricValue = strings.TrimSpace(string(metricOutput)) + o.Expect(metricValue).ShouldNot(o.BeEmpty()) + e2e.Logf("The output of the %s is : %v", metricName, metricValue) + return true, nil + + }) + o.Expect(metricsErr).NotTo(o.HaveOccurred(), fmt.Sprintf("Fail to get metric and the error is:%s", metricsErr)) + return metricValue +} + +// CurlNodePortPass checks nodeport svc reacability from a node regardless of network addressing type on cluster +func CurlNodePortPass(oc *exutil.CLI, nodeNameFrom string, nodeNameTo string, nodePort string) { + nodeIP1, nodeIP2 := GetNodeIP(oc, nodeNameTo) + if nodeIP1 != "" { + nodev6URL := net.JoinHostPort(nodeIP1, nodePort) + nodev4URL := net.JoinHostPort(nodeIP2, nodePort) + curlCmd := fmt.Sprintf("curl %s --connect-timeout 5 && curl %s --connect-timeout 5", nodev4URL, nodev6URL) + o.Eventually(func() bool { + output, err := DebugNode(oc, nodeNameFrom, "bash", "-c", curlCmd) + return err == nil && strings.Contains(output, "Hello OpenShift") + }, "30s", "10s").Should(o.BeTrue(), "NodePort Service was not be able to access!") + } else { + nodeURL := net.JoinHostPort(nodeIP2, nodePort) + o.Eventually(func() bool { + output, err := DebugNode(oc, nodeNameFrom, "curl", nodeURL, "-s", "--connect-timeout", "5") + return err == nil && strings.Contains(output, "Hello OpenShift") + }, "30s", "10s").Should(o.BeTrue(), "NodePort Service was not be able to access!") + } +} + +// CurlNodePortFail checks nodeport svc unreacability from a node regardless of network addressing type on cluster +func CurlNodePortFail(oc *exutil.CLI, nodeNameFrom string, nodeNameTo string, nodePort string) { + nodeIP1, nodeIP2 := GetNodeIP(oc, nodeNameTo) + if nodeIP1 != "" { + nodev6URL := net.JoinHostPort(nodeIP1, nodePort) + nodev4URL := net.JoinHostPort(nodeIP2, nodePort) + output, _ := DebugNode(oc, nodeNameFrom, "curl", nodev4URL, "--connect-timeout", "5") + o.Expect(output).To(o.Or(o.ContainSubstring("28"), o.ContainSubstring("timed out"), o.ContainSubstring("Connection refused"))) + output, _ = DebugNode(oc, nodeNameFrom, "curl", nodev6URL, "--connect-timeout", "5") + o.Expect(output).To(o.Or(o.ContainSubstring("28"), o.ContainSubstring("timed out"), o.ContainSubstring("Connection refused"))) + } else { + nodeURL := net.JoinHostPort(nodeIP2, nodePort) + output, _ := DebugNode(oc, nodeNameFrom, "curl", nodeURL, "--connect-timeout", "5") + o.Expect(output).To(o.Or(o.ContainSubstring("28"), o.ContainSubstring("timed out"), o.ContainSubstring("Connection refused"))) + } +} + +func CurlPod2NodePortPass(oc *exutil.CLI, namespaceSrc string, podNameSrc string, nodeNameTo string, nodePort string) { + nodeIP1, nodeIP2 := GetNodeIP(oc, nodeNameTo) + if nodeIP1 != "" { + nodev6URL := net.JoinHostPort(nodeIP1, nodePort) + nodev4URL := net.JoinHostPort(nodeIP2, nodePort) + output, err := e2eoutput.RunHostCmd(namespaceSrc, podNameSrc, "curl "+nodev4URL+" --connect-timeout 5") + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).Should(o.ContainSubstring("Hello OpenShift")) + output, err = e2eoutput.RunHostCmd(namespaceSrc, podNameSrc, "curl "+nodev6URL+" --connect-timeout 5") + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).Should(o.ContainSubstring("Hello OpenShift")) + } else { + nodeURL := net.JoinHostPort(nodeIP2, nodePort) + output, err := e2eoutput.RunHostCmd(namespaceSrc, podNameSrc, "curl "+nodeURL+" --connect-timeout 5") + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).Should(o.ContainSubstring("Hello OpenShift")) + } +} + +func CurlPod2NodePortFail(oc *exutil.CLI, namespaceSrc string, podNameSrc string, nodeNameTo string, nodePort string) { + nodeIP1, nodeIP2 := GetNodeIP(oc, nodeNameTo) + if nodeIP1 != "" { + nodev6URL := net.JoinHostPort(nodeIP1, nodePort) + nodev4URL := net.JoinHostPort(nodeIP2, nodePort) + _, err := e2eoutput.RunHostCmd(namespaceSrc, podNameSrc, "curl "+nodev4URL+" --connect-timeout 5") + o.Expect(err).To(o.HaveOccurred()) + _, err = e2eoutput.RunHostCmd(namespaceSrc, podNameSrc, "curl "+nodev6URL+" --connect-timeout 5") + o.Expect(err).To(o.HaveOccurred()) + } else { + nodeURL := net.JoinHostPort(nodeIP2, nodePort) + _, err := e2eoutput.RunHostCmd(namespaceSrc, podNameSrc, "curl "+nodeURL+" --connect-timeout 5") + o.Expect(err).To(o.HaveOccurred()) + } +} + +// get primary NIC interface name +func GetPrimaryNICname(oc *exutil.CLI) string { + masterNode, getMasterNodeErr := GetFirstMasterNode(oc) + o.Expect(getMasterNodeErr).NotTo(o.HaveOccurred()) + primary_int, err := DebugNodeWithChroot(oc, masterNode, "bash", "-c", "nmcli -g connection.interface-name c show ovs-if-phys0") + o.Expect(err).NotTo(o.HaveOccurred()) + primary_inf_name := strings.Split(primary_int, "\n") + e2e.Logf("Primary Inteface name is : %s", primary_inf_name[0]) + return primary_inf_name[0] +} + +// get file contents to be modified for SCTP +func GetFileContentforSCTP(baseDir string, name string) (fileContent string) { + filePath := filepath.Join(testdata.FixturePath("networking", baseDir), name) + fileOpen, err := os.Open(filePath) + defer fileOpen.Close() + if err != nil { + e2e.Failf("Failed to open file: %s", filePath) + } + fileRead, _ := io.ReadAll(fileOpen) + if err != nil { + e2e.Failf("Failed to read file: %s", filePath) + } + return string(fileRead) +} + +// get generic sctpclient pod yaml file, replace variables as per requirements +func CreateSCTPclientOnNode(oc *exutil.CLI, pod_pmtrs map[string]string) (err error) { + podGenericYaml := GetFileContentforSCTP("sctp", "sctpclientspecificnode.yaml") + for rep, value := range pod_pmtrs { + podGenericYaml = strings.ReplaceAll(podGenericYaml, rep, value) + } + podFileName := "temp-sctp-client-pod-" + GetRandomString() + ".yaml" + defer os.Remove(podFileName) + os.WriteFile(podFileName, []byte(podGenericYaml), 0644) + // create ping pod for Microshift + _, err = oc.AsAdmin().WithoutNamespace().Run("create").Args("-f", podFileName).Output() + return err +} + +// get generic sctpserver pod yaml file, replace variables as per requirements +func CreateSCTPserverOnNode(oc *exutil.CLI, pod_pmtrs map[string]string) (err error) { + podGenericYaml := GetFileContentforSCTP("sctp", "sctpserverspecificnode.yaml") + for rep, value := range pod_pmtrs { + podGenericYaml = strings.ReplaceAll(podGenericYaml, rep, value) + } + podFileName := "temp-sctp-server-pod-" + GetRandomString() + ".yaml" + defer os.Remove(podFileName) + os.WriteFile(podFileName, []byte(podGenericYaml), 0644) + // create ping pod for Microshift + _, err = oc.AsAdmin().WithoutNamespace().Run("create").Args("-f", podFileName).Output() + return err +} + +// configure IPSec at runtime, targetStatus can be full/disabled/external +func ConfigIPSecAtRuntime(oc *exutil.CLI, targetStatus string) (err error) { + var targetConfig, currentStatus string + ipsecState := CheckIPsec(oc) + if ipsecState == "{}" || ipsecState == "Full" { + currentStatus = "full" + } else if ipsecState == "Disabled" { + currentStatus = "disabled" + } else if ipsecState == "External" { + currentStatus = "external" + } + if currentStatus == targetStatus { + e2e.Logf("The IPSec is already in %v state", targetStatus) + return + } else if targetStatus == "full" { + //In 4.15+, enabling/disabling ipsec would require nodes restart + targetConfig = "true" + e2e.Logf("Start to enable ipsec.") + _, err = oc.AsAdmin().WithoutNamespace().Run("patch").Args("networks.operator.openshift.io", "cluster", "-p", "{\"spec\":{\"defaultNetwork\":{\"ovnKubernetesConfig\":{\"ipsecConfig\":{\"mode\":\"Full\"}}}}}", "--type=merge").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("Wait the MC applying getting started") + o.Eventually(func() error { + err := AssertOrCheckMCP(oc, "master", 30*time.Second, 30*time.Second, false) + return err + }, "300s", "30s").ShouldNot(o.BeNil(), "MC applying didn't start yet.") + //Add test points for case OCP-79034 + e2e.Logf("Both IPsec container and host pods will be launched.") + output, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("ds", "-n", "openshift-ovn-kubernetes").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("The ds in openshift-ovn-kubernetes are : \n%s", output) + o.Expect(strings.Contains(output, "ovn-ipsec-containerized")).Should(o.BeTrue()) + o.Expect(strings.Contains(output, "ovn-ipsec-host")).Should(o.BeTrue()) + e2e.Logf("Verify CNO status shows progress state") + CheckCNORenderState(oc) + e2e.Logf("Wait the MC were applied to nodes ") + err = AssertOrCheckMCP(oc, "master", 60*time.Second, 30*time.Minute, false) + o.Expect(err).NotTo(o.HaveOccurred()) + err = AssertOrCheckMCP(oc, "worker", 60*time.Second, 5*time.Minute, false) + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("MC applying done ") + e2e.Logf("Wait ipsec container ds disappeared in openshift-ovn-kubernetes") + o.Eventually(func() bool { + output, err = oc.AsAdmin().WithoutNamespace().Run("get").Args("ds", "-n", "openshift-ovn-kubernetes").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("The ds in openshift-ovn-kubernetes are : \n%s", output) + return !strings.Contains(output, "ovn-ipsec-containerized") + }, "300s", "30s").ShouldNot(o.BeNil(), "Timeout for waiting ovn-ipsec-containerized being removed!") + e2e.Logf("Wait ipsec host pods running in openshift-ovn-kubernetes") + err = WaitForPodWithLabelReady(oc, "openshift-ovn-kubernetes", "app=ovn-ipsec") + o.Expect(err).NotTo(o.HaveOccurred()) + } else if targetStatus == "disabled" { + targetConfig = "false" + _, err = oc.AsAdmin().WithoutNamespace().Run("patch").Args("networks.operator.openshift.io", "cluster", "-p", "{\"spec\":{\"defaultNetwork\":{\"ovnKubernetesConfig\":{\"ipsecConfig\":{\"mode\":\"Disabled\"}}}}}", "--type=merge").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("Wait ovn-ipsec pods disappeared") + err = WaitForPodWithLabelGone(oc, "openshift-ovn-kubernetes", "app=ovn-ipsec") + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("Wait the MC applying getting started ") + o.Eventually(func() error { + err := AssertOrCheckMCP(oc, "master", 30*time.Second, 30*time.Second, false) + return err + }, "300s", "30s").ShouldNot(o.BeNil(), "MC applying didn't start yet.") + e2e.Logf("Verify CNO status shows progress state") + CheckCNORenderState(oc) + e2e.Logf("Wait the MC were applied to nodes ") + err = AssertOrCheckMCP(oc, "master", 60*time.Second, 30*time.Minute, false) + o.Expect(err).NotTo(o.HaveOccurred()) + err = AssertOrCheckMCP(oc, "worker", 60*time.Second, 5*time.Minute, false) + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("MC applying done ") + e2e.Logf("Verify IPsec MC were removed") + output, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("mc").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(strings.Contains(output, "ipsec")).ShouldNot(o.BeTrue()) + } + + checkErr := CheckIPSecInDB(oc, targetConfig) + o.Expect(checkErr).NotTo(o.HaveOccurred(), "check IPSec configuration failed") + + return nil +} + +// check IPSec configuration in northd, targetConfig should be "true" or "false" +func CheckIPSecInDB(oc *exutil.CLI, targetConfig string) error { + ovnLeaderpod := GetOVNKMasterOVNkubeNode(oc) + return wait.Poll(10*time.Second, 60*time.Second, func() (bool, error) { + getIPSec, getErr := ExecCommandInSpecificPod(oc, "openshift-ovn-kubernetes", ovnLeaderpod, "ovn-nbctl --no-leader-only get nb_global . ipsec") + o.Expect(getErr).NotTo(o.HaveOccurred()) + if strings.Contains(getIPSec, targetConfig) { + return true, nil + } + e2e.Logf("Can't get expected ipsec configuration and try again") + return false, nil + }) +} + +// IsIPv4 check if the string is an IPv4 address. +func IsIPv4(str string) bool { + ip := net.ParseIP(str) + return ip != nil && strings.Contains(str, ".") +} + +// IsIPv6 check if the string is an IPv6 address. +func IsIPv6(str string) bool { + ip := net.ParseIP(str) + return ip != nil && strings.Contains(str, ":") +} + +// CheckSCTPResultPASS +func CheckSCTPResultPASS(oc *exutil.CLI, namespace, sctpServerPodName, sctpClientPodname, dstIP, dstPort string) { + g.By("sctpserver pod start to wait for sctp traffic") + _, _, _, err1 := oc.Run("exec").Args("-n", oc.Namespace(), sctpServerPodName, "--", "/usr/bin/ncat", "-l", "30102", "--sctp").Background() + o.Expect(err1).NotTo(o.HaveOccurred()) + time.Sleep(5 * time.Second) + + g.By("check sctp process enabled in the sctp server pod") + msg, err2 := e2eoutput.RunHostCmd(oc.Namespace(), sctpServerPodName, "ps aux | grep sctp") + o.Expect(err2).NotTo(o.HaveOccurred()) + o.Expect(strings.Contains(msg, "/usr/bin/ncat -l 30102 --sctp")).To(o.BeTrue()) + + g.By("sctpclient pod start to send sctp traffic") + _, err3 := e2eoutput.RunHostCmd(oc.Namespace(), sctpClientPodname, "echo 'Test traffic using sctp port from sctpclient to sctpserver' | { ncat -v "+dstIP+" "+dstPort+" --sctp; }") + o.Expect(err3).NotTo(o.HaveOccurred()) + + g.By("server sctp process will end after get sctp traffic from sctp client") + time.Sleep(5 * time.Second) + msg1, err4 := e2eoutput.RunHostCmd(oc.Namespace(), sctpServerPodName, "ps aux | grep sctp") + o.Expect(err4).NotTo(o.HaveOccurred()) + o.Expect(msg1).NotTo(o.ContainSubstring("/usr/bin/ncat -l 30102 --sctp")) +} + +func OvnkubeNodePod(oc *exutil.CLI, nodeName string) string { + ovnNodePod, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("-n", "openshift-ovn-kubernetes", "pod", "-l", "app=ovnkube-node", + "-o=jsonpath={.items[?(@.spec.nodeName==\""+nodeName+"\")].metadata.name}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("The ovnkube-node pod on node %s is %s", nodeName, ovnNodePod) + o.Expect(ovnNodePod).NotTo(o.BeEmpty()) + return ovnNodePod +} + +func WaitForNetworkOperatorState(oc *exutil.CLI, interval int, timeout int, expectedStatus string) { + WaitForClusterOperatorState(oc, "network", interval, timeout, expectedStatus) +} + +func WaitForClusterOperatorState(oc *exutil.CLI, co string, interval int, timeout int, expectedStatus string) { + errCheck := wait.Poll(time.Duration(interval)*time.Second, time.Duration(timeout)*time.Minute, func() (bool, error) { + output, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("co", co).Output() + if err != nil { + e2e.Logf("Fail to get clusteroperator network, error:%s. Trying again", err) + return false, nil + } + if matched, _ := regexp.MatchString(expectedStatus, output); !matched { + e2e.Logf("Network operator state is:%s", output) + return false, nil + } + return true, nil + }) + o.Expect(errCheck).NotTo(o.HaveOccurred(), fmt.Sprintf("Timed out waiting for the expected condition")) +} + +func EnableIPForwardingOnSpecNodeNIC(oc *exutil.CLI, worker, secNIC string) { + cmd := fmt.Sprintf("sysctl net.ipv4.conf.%s.forwarding", secNIC) + output, debugNodeErr := DebugNode(oc, worker, "bash", "-c", cmd) + o.Expect(debugNodeErr).NotTo(o.HaveOccurred()) + if !strings.Contains(output, ".forwarding = 1") { + e2e.Logf("Enable IP forwarding for NIC %s on node %s ...", secNIC, worker) + enableCMD := fmt.Sprintf("sysctl -w net.ipv4.conf.%s.forwarding=1", secNIC) + _, debugNodeErr = DebugNode(oc, worker, "bash", "-c", enableCMD) + o.Expect(debugNodeErr).NotTo(o.HaveOccurred()) + } + e2e.Logf("IP forwarding was enabled for NIC %s on node %s!", secNIC, worker) +} + +func DisableIPForwardingOnSpecNodeNIC(oc *exutil.CLI, worker, secNIC string) { + cmd := fmt.Sprintf("sysctl net.ipv4.conf.%s.forwarding", secNIC) + output, debugNodeErr := DebugNode(oc, worker, "bash", "-c", cmd) + o.Expect(debugNodeErr).NotTo(o.HaveOccurred()) + if strings.Contains(output, ".forwarding = 1") { + e2e.Logf("Disable IP forwarding for NIC %s on node %s ...", secNIC, worker) + disableCMD := fmt.Sprintf("sysctl -w net.ipv4.conf.%s.forwarding=0", secNIC) + _, debugNodeErr = DebugNode(oc, worker, "bash", "-c", disableCMD) + o.Expect(debugNodeErr).NotTo(o.HaveOccurred()) + } + e2e.Logf("IP forwarding was disabled for NIC %s on node %s!", secNIC, worker) +} + +func NbContructToMap(nbConstruct string) map[string]string { + listKeyValues := strings.Split(nbConstruct, "\n") + var tempMap map[string]string + tempMap = make(map[string]string) + for _, keyValPair := range listKeyValues { + keyValItem := strings.SplitN(keyValPair, ":", 2) + key := strings.Trim(keyValItem[0], " ") + val := strings.TrimLeft(keyValItem[1], " ") + tempMap[key] = val + + } + return tempMap +} + +// Create live migration job on Kubevirt cluster +func (migrationjob *MigrationDetails) CreateMigrationJob(oc *exutil.CLI) { + err := wait.Poll(5*time.Second, 20*time.Second, func() (bool, error) { + err1 := ApplyResourceFromTemplateByAdmin(oc, "--ignore-unknown-parameters=true", "-f", migrationjob.Template, "-p", "NAME="+migrationjob.Name, "NAMESPACE="+migrationjob.Namespace, "VMI="+migrationjob.Virtualmachinesintance) + if err1 != nil { + e2e.Logf("the err:%v, and try next round", err1) + return false, nil + } + return true, nil + }) + o.Expect(err).NotTo(o.HaveOccurred(), fmt.Sprintf("fail to create migration job %v", migrationjob.Name)) +} + +// Delete migration job on Kubevirt cluster +func (migrationjob *MigrationDetails) DeleteMigrationJob(oc *exutil.CLI) { + RemoveResource(oc, true, true, "virtualmachineinstancemigration.kubevirt.io", migrationjob.Name, "-n", migrationjob.Namespace) +} + +// Check all cluster operators status on the cluster +func CheckAllClusterOperatorsState(oc *exutil.CLI, interval int, timeout int) { + operatorsString, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("co", "-o=jsonpath={.items[*].metadata.name}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + + var clusterOperators []string + if operatorsString != "" { + clusterOperators = strings.Split(operatorsString, " ") + } + + for _, clusterOperator := range clusterOperators { + errCheck := wait.Poll(time.Duration(interval)*time.Second, time.Duration(timeout)*time.Minute, func() (bool, error) { + output, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("co", clusterOperator).Output() + if err != nil { + e2e.Logf("Fail to get state for operator %s, error:%s. Trying again", clusterOperator, err) + return false, err + } + if matched, _ := regexp.MatchString("True.*False.*False", output); !matched { + e2e.Logf("Operator on hosted cluster is in state: %s", output) + return false, nil + } + return true, nil + }) + o.Expect(errCheck).NotTo(o.HaveOccurred(), "Timed out waiting for the expected condition") + } +} + +// Check OVNK health: OVNK pods health and ovnkube-node DS health +func CheckOVNKState(oc *exutil.CLI) error { + // check all OVNK pods + WaitForPodWithLabelReady(oc, "openshift-ovn-kubernetes", "app=ovnkube-node") + if !IsHypershiftHostedCluster(oc) { + WaitForPodWithLabelReady(oc, "openshift-ovn-kubernetes", "app=ovnkube-control-plane") + } + // check ovnkube-node ds rollout status and confirm if rollout has triggered + return wait.Poll(10*time.Second, 2*time.Minute, func() (bool, error) { + status, err := oc.AsAdmin().WithoutNamespace().Run("rollout").Args("status", "-n", "openshift-ovn-kubernetes", "ds", "ovnkube-node", "--timeout", "5m").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + if strings.Contains(status, "rollout to finish") && strings.Contains(status, "successfully rolled out") { + e2e.Logf("ovnkube rollout was triggerred and rolled out successfully") + return true, nil + } + e2e.Logf("ovnkube rollout trigger hasn't happened yet. Trying again") + return false, nil + }) +} + +func AddDummyInferface(oc *exutil.CLI, nodeName, IP, nicName string) { + e2e.Logf("Add a dummy interface %s on node %s \n", nicName, nodeName) + cmd := fmt.Sprintf("ip link a %s type dummy && ip link set dev %s up && ip a add %s dev %s && ip a show %s", nicName, nicName, IP, nicName, nicName) + output, debugNodeErr := DebugNode(oc, nodeName, "bash", "-c", cmd) + o.Expect(debugNodeErr).NotTo(o.HaveOccurred()) + e2e.Logf("The dummy interface was added. \n %s", output) + +} + +func AddIPtoInferface(oc *exutil.CLI, nodeName, IP, nicName string) { + e2e.Logf("Add IP address %s to interface %s on node %s \n", IP, nicName, nodeName) + cmd := fmt.Sprintf("ip a show %s && ip a add %s dev %s", nicName, IP, nicName) + _, debugNodeErr := DebugNode(oc, nodeName, "bash", "-c", cmd) + o.Expect(debugNodeErr).NotTo(o.HaveOccurred()) +} + +func DelIPFromInferface(oc *exutil.CLI, nodeName, IP, nicName string) { + e2e.Logf("Remove IP address %s from interface %s on node %s \n", IP, nicName, nodeName) + cmd := fmt.Sprintf("ip a show %s && ip a del %s dev %s", nicName, IP, nicName) + _, debugNodeErr := DebugNode(oc, nodeName, "bash", "-c", cmd) + o.Expect(debugNodeErr).NotTo(o.HaveOccurred()) +} + +func RemoveDummyInterface(oc *exutil.CLI, nodeName, nicName string) { + e2e.Logf("Remove a dummy interface %s on node %s \n", nicName, nodeName) + cmd := fmt.Sprintf("ip a show %s && ip link del %s type dummy", nicName, nicName) + output, debugNodeErr := DebugNode(oc, nodeName, "bash", "-c", cmd) + nicNotExistStr := fmt.Sprintf("Device \"%s\" does not exist", nicName) + if debugNodeErr != nil && strings.Contains(output, nicNotExistStr) { + e2e.Logf("The dummy interface %s does not exist on node %s ! \n", nicName, nodeName) + return + } + o.Expect(debugNodeErr).NotTo(o.HaveOccurred()) + e2e.Logf("The dummy interface %s was removed from node %s ! \n", nicName, nodeName) +} + +func (kkPod *KubeletKillerPod) CreateKubeletKillerPodOnNode(oc *exutil.CLI) { + err := wait.Poll(5*time.Second, 20*time.Second, func() (bool, error) { + err1 := ApplyResourceFromTemplateByAdmin(oc, "--ignore-unknown-parameters=true", "-f", kkPod.Template, "-p", "NAME="+kkPod.Name, "NAMESPACE="+kkPod.Namespace, "NODENAME="+kkPod.Nodename) + if err1 != nil { + e2e.Logf("the err:%v, and try next round", err1) + return false, nil + } + return true, nil + }) + o.Expect(err).NotTo(o.HaveOccurred(), fmt.Sprintf("fail to create Kubelet-Killer pod %v", kkPod.Name)) +} + +func GetNodeNameByIPv4(oc *exutil.CLI, nodeIPv4 string) (nodeName string) { + nodeList, err := e2enode.GetReadySchedulableNodes(context.TODO(), oc.KubeFramework().ClientSet) + o.Expect(err).NotTo(o.HaveOccurred()) + for _, node := range nodeList.Items { + _, IPv4 := GetNodeIP(oc, node.Name) + if IPv4 == nodeIPv4 { + nodeName = node.Name + break + } + } + return nodeName +} + +// patch resource in specific namespace, this is useful when patching resource to hosted cluster that is in "-n clusters" namespace +func PatchResourceAsAdminNS(oc *exutil.CLI, ns, resource, patch string) { + err := oc.AsAdmin().WithoutNamespace().Run("patch").Args(resource, "-p", patch, "--type=merge", "-n", ns).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) +} + +// get proxy IP and port of hosted cluster +func GetProxyIPandPortOnHostedCluster(oc *exutil.CLI, hostedClusterName, namespace string) (string, string) { + output, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("hostedclusters", hostedClusterName, "-n", namespace, "-o=jsonpath={.spec.configuration.proxy.httpProxy}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + if len(output) != 0 { + //match out the proxy IP + re := regexp.MustCompile(`(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)){3}`) + proxyIP := re.FindAllString(output, -1)[0] + proxyPort := strings.Split(output, ":")[2] + e2e.Logf("proxy IP is %s, proxy port is %s", proxyIP, proxyPort) + return proxyIP, proxyPort + } else { + return "", "" + } +} + +// GetMachineNamesFromMachineSetOnROSA gets all Machines in a Machinepool on a classic ROSA cluster by label +// This function only appliable to classic ROSA, as there is no "machine" resource on ROSA hosted cluster +func GetMachineNamesFromMachinePoolOnROSA(oc *exutil.CLI, machineSetName string, machineAPINamespace string) []string { + e2e.Logf("Getting all Machines in a Machineset by specific label ...") + machineNames, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("machine", "-o=jsonpath={.items[*].metadata.name}", "-l", "machine.openshift.io/cluster-api-machine-type="+machineSetName, "-n", machineAPINamespace).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + if machineNames != "" { + return strings.Split(machineNames, " ") + } else { + return nil + } +} + +// Wait for machine on a classic ROSA to be ready - this function only appliable to classic ROSA, as there is no "machine" resource on ROSA hosted cluster +func WaitMachineOnROSAReady(oc *exutil.CLI, machineName string, namespace string) error { + return wait.Poll(15*time.Second, 10*time.Minute, func() (bool, error) { + status, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("machine", machineName, "-n", namespace, "-o=jsonpath={.status.phase}").Output() + e2e.Logf("Machine %v status is %v", machineName, status) + if err != nil || status == "" { + e2e.Logf("Failed to get machine status: %v, retrying...", err) + return false, nil + } + if !strings.Contains(status, "Running") { + e2e.Logf("Machine is in %v, not in Running state, retrying...", status) + return false, nil + } + return true, nil + }) +} + +type ApbStaticExternalRoute struct { + Name string + Labelkey string + Labelvalue string + Ip1 string + Ip2 string + Bfd bool + Template string +} + +type ApbDynamicExternalRoute struct { + Name string + LabelKey string + LabelValue string + PodLabelKey string + PodLabelValue string + NamespaceLabelKey string + NamespaceLabelValue string + Bfd bool + Template string +} + +func (sgwpr *ApbStaticExternalRoute) DeleteAPBExternalRoute(oc *exutil.CLI) { + RemoveResource(oc, true, true, "apbexternalroute", sgwpr.Name) +} + +func (sgwpr *ApbStaticExternalRoute) CreateAPBExternalRoute(oc *exutil.CLI) { + err := wait.Poll(5*time.Second, 20*time.Second, func() (bool, error) { + err1 := ApplyResourceFromTemplateByAdmin(oc, "--ignore-unknown-parameters=true", "-f", sgwpr.Template, "-p", "NAME="+sgwpr.Name, "LABELKEY="+sgwpr.Labelkey, "LABELVALUE="+sgwpr.Labelvalue, "IP1="+sgwpr.Ip1, "IP2="+sgwpr.Ip2, "BFD="+strconv.FormatBool(sgwpr.Bfd)) + if err1 != nil { + e2e.Logf("the err:%v, and try next round", err1) + return false, nil + } + return true, nil + }) + o.Expect(err).NotTo(o.HaveOccurred(), fmt.Sprintf("fail to create apbexternalroute %v", sgwpr.Name)) +} + +func (sgwpr *ApbDynamicExternalRoute) CreateAPBDynamicExternalRoute(oc *exutil.CLI) { + err := wait.Poll(5*time.Second, 20*time.Second, func() (bool, error) { + err1 := ApplyResourceFromTemplateByAdmin(oc, "--ignore-unknown-parameters=true", "-f", sgwpr.Template, "-p", "NAME="+sgwpr.Name, "LABELKEY="+sgwpr.LabelKey, "LABELVALUE="+sgwpr.LabelValue, + "PODLABELKEY="+sgwpr.PodLabelKey, "PODLABELVALUE="+sgwpr.PodLabelValue, + "NSLABELKEY="+sgwpr.NamespaceLabelKey, "NSLABELVALUE="+sgwpr.NamespaceLabelValue, + "BFD="+strconv.FormatBool(sgwpr.Bfd)) + if err1 != nil { + e2e.Logf("Could not create due to err:%v, and try next round", err1) + return false, nil + } + return true, nil + }) + o.Expect(err).NotTo(o.HaveOccurred(), fmt.Sprintf("Failed to create APB External Route %s due to %v", sgwpr.Name, err)) +} + +func CheckAPBExternalRouteStatus(oc *exutil.CLI, gwName string, expectedStatus string) error { + checkErr := wait.Poll(10*time.Second, 60*time.Second, func() (bool, error) { + output, gwErr := oc.AsAdmin().WithoutNamespace().Run("get").Args("apbexternalroute", gwName).Output() + if gwErr != nil { + e2e.Logf("Failed to get apbexternalroute %v, error: %s. Trying again", gwName, gwErr) + return false, nil + } + if !strings.Contains(output, expectedStatus) { + e2e.Logf("Expected status is %v, the apbexternalroute status is %v, trying again.", expectedStatus, output) + return false, nil + } + return true, nil + }) + return checkErr +} + +func CheckEgressFWStatus(oc *exutil.CLI, fwName string, ns string, expectedStatus string) error { + checkErr := wait.Poll(10*time.Second, 60*time.Second, func() (bool, error) { + output, fwErr := oc.AsAdmin().WithoutNamespace().Run("get").Args("egressfirewall", "-n", ns, fwName).Output() + if fwErr != nil { + e2e.Logf("Failed to get egressfirewall %v, error: %s. Trying again", fwName, fwErr) + return false, nil + } + if !strings.Contains(output, expectedStatus) { + e2e.Logf("Expected status is %v, the egressfirewall status is %v, trying again.", expectedStatus, output) + return false, nil + } + return true, nil + }) + return checkErr +} + +func CheckNodeIdentityWebhook(oc *exutil.CLI) (string, error) { + webhooks, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("ValidatingWebhookConfiguration", "network-node-identity.openshift.io", "-o=jsonpath={.webhooks[*].Name}").Output() + return webhooks, err +} + +func DisableNodeIdentityWebhook(oc *exutil.CLI, namespace string, cmName string) (string, error) { + _, err := oc.AsAdmin().WithoutNamespace().Run("create").Args("configmap", cmName, "-n", namespace, "--from-literal=enabled=false").Output() + o.Eventually(func() bool { + result := true + _, cmErr := oc.AsAdmin().Run("get").Args("configmap/"+cmName, "-n", namespace).Output() + if cmErr != nil { + e2e.Logf("Wait for configmap/%s to be created", cmName) + result = false + } + return result + }, "60s", "5s").Should(o.BeTrue(), fmt.Sprintf("configmap/%sis not created", cmName)) + return "", err +} + +// get lr-policy-list from logical_router_policy table +func GetlrPolicyList(oc *exutil.CLI, nodeName, tableID string, expected bool) ([]string, error) { + // get the ovnkube-node pod on the node + ovnKubeNodePod, podErr := GetOVNKPodOnNode(oc, "openshift-ovn-kubernetes", "app=ovnkube-node", nodeName) + o.Expect(podErr).NotTo(o.HaveOccurred()) + o.Expect(ovnKubeNodePod).ShouldNot(o.Equal("")) + var lspOutput string + var lspErr error + var lrPolicyList []string + + lspCmd := "ovn-nbctl lr-policy-list ovn_cluster_router | grep '" + tableID + " '" + checkLspErr := wait.Poll(10*time.Second, 2*time.Minute, func() (bool, error) { + lspOutput, lspErr = RemoteShPodWithBashSpecifyContainer(oc, "openshift-ovn-kubernetes", ovnKubeNodePod, "northd", lspCmd) + if lspErr == nil && lspOutput != "" && expected { + cmdOutputLines := strings.Split(lspOutput, "\n") + for i := 0; i < len(cmdOutputLines); i++ { + lrPolicyList = append(lrPolicyList, cmdOutputLines[i]) + } + return true, nil + } + + // check lr-policy-list grep with tableID returned empty, usually there is "command terminated with exit code 1" to lspErr returned, so lspErr is not checked here + if lspOutput != "ip4.src ==" && !expected { + e2e.Logf("lr-policy-list of table %s is cleared up as expected", tableID) + return true, nil + } + + e2e.Logf("Waiting for expected result to be synced, try again ...") + return false, nil + }) + if checkLspErr != nil { + e2e.Logf("The command check result in ovndb is not expected ! See below output \n %s ", lspOutput) + } + return lrPolicyList, checkLspErr +} + +// Create a kubeconfig that impersonates ovnkube-node +func GenerateKubeConfigFileForContext(oc *exutil.CLI, nodeName string, ovnKubeNodePod string, kubeConfigFilePath string, userContext string) bool { + var ( + pemFile = "/etc/ovn/ovnkube-node-certs/ovnkube-client-current.pem" + certFile = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" + clusterName = "default-cluster" + userName = "default-user" + ) + + baseDomain, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("dns/cluster", "-o=jsonpath={.spec.baseDomain}").Output() + if err != nil || baseDomain == "" { + e2e.Logf("Base Domain could not retrieved") + return false + } + e2e.Logf("Base Domain %v", baseDomain) + apiServerFQDN := fmt.Sprintf("api.%s", baseDomain) + + setUpClusterCmd := fmt.Sprintf("export KUBECONFIG=%s; kubectl config set-cluster %s --server=https://%s:6443 --certificate-authority %s --embed-certs", kubeConfigFilePath, clusterName, apiServerFQDN, certFile) + setUserCredentialsCmd := fmt.Sprintf("export KUBECONFIG=%s; kubectl config set-credentials %s --client-key %s --client-certificate %s --embed-certs", kubeConfigFilePath, userName, pemFile, pemFile) + setContextCmd := fmt.Sprintf("export KUBECONFIG=%s; kubectl config set-context %s --cluster %s --user %s", kubeConfigFilePath, userContext, clusterName, userName) + testContextCmd := fmt.Sprintf("export KUBECONFIG=%s; kubectl config use-context %s; oc get nodes", kubeConfigFilePath, userContext) + + cmdOutput, cmdErr := RemoteShPodWithBashSpecifyContainer(oc, "openshift-ovn-kubernetes", ovnKubeNodePod, "ovnkube-controller", setUpClusterCmd) + if cmdErr != nil || !strings.Contains(cmdOutput, "Cluster "+"\""+clusterName+"\""+" set.") { + e2e.Logf("Setting cluster for impersonation failed %v.", cmdErr) + return false + } + e2e.Logf("Cluster set - %v", cmdOutput) + + cmdOutput, cmdErr = RemoteShPodWithBashSpecifyContainer(oc, "openshift-ovn-kubernetes", ovnKubeNodePod, "ovnkube-controller", setUserCredentialsCmd) + if cmdErr != nil || !strings.Contains(cmdOutput, "User "+"\""+userName+"\""+" set.") { + e2e.Logf("Setting user credentials for impersonation failed %v.", cmdErr) + return false + } + e2e.Logf("User credentials set - %v", cmdOutput) + + cmdOutput, cmdErr = RemoteShPodWithBashSpecifyContainer(oc, "openshift-ovn-kubernetes", ovnKubeNodePod, "ovnkube-controller", setContextCmd) + if cmdErr != nil || !strings.Contains(cmdOutput, "Context "+"\""+userContext+"\""+" created.") { + e2e.Logf("Context creation for impersonation failed %v.", cmdErr) + return false + } + e2e.Logf("Context created - %v", cmdOutput) + + cmdOutput, cmdErr = RemoteShPodWithBashSpecifyContainer(oc, "openshift-ovn-kubernetes", ovnKubeNodePod, "ovnkube-controller", testContextCmd) + if cmdErr != nil || !strings.Contains(cmdOutput, "Switched to context "+"\""+userContext+"\"") || !strings.Contains(cmdOutput, nodeName) { + e2e.Logf("Test command for impersonation failed %v.", cmdErr) + return false + } + e2e.Logf("Successfully created and tested kubeconfig for impersonation") + return true +} + +func FindNodesWithSameSubnet(oc *exutil.CLI, nodeList []string) (bool, []string) { + sameSubNode := make(map[string][]string) + for _, node := range nodeList { + subNet := GetNodeSubnet(oc, node, "default") + if _, ok := sameSubNode[subNet]; ok { + sameSubNode[subNet] = append(sameSubNode[subNet], node) + if len(sameSubNode[subNet]) >= 2 { + return true, sameSubNode[subNet] + } + } else { + sameSubNode[subNet] = []string{node} + } + } + return false, nil +} + +// Get endpoints for service:port in northdb of the node +func GetLBListEndpointsbySVCIPPortinNBDB(oc *exutil.CLI, nodeName, svcPort string) ([]string, error) { + // get the ovnkube-node pod of the node + ovnKubePod, podErr := GetOVNKPodOnNode(oc, "openshift-ovn-kubernetes", "app=ovnkube-node", nodeName) + o.Expect(podErr).NotTo(o.HaveOccurred()) + o.Expect(ovnKubePod).ShouldNot(o.Equal("")) + var cmdOutput string + var cmdErr error + var endpoints []string + + lbCmd := "ovn-nbctl lb-list | grep \"" + svcPort + "\" | awk '{print $NF}'" + checkOVNDbErr := wait.Poll(2*time.Second, 2*time.Minute, func() (bool, error) { + cmdOutput, cmdErr = RemoteShPodWithBashSpecifyContainer(oc, "openshift-ovn-kubernetes", ovnKubePod, "northd", lbCmd) + if cmdErr != nil { + e2e.Logf("%v,Waiting for expected result to be synced, try again ...,", cmdErr) + return false, nil + } + + if cmdOutput != "" { + cmdOutputLines := strings.Split(cmdOutput, ",") + for i := 0; i < len(cmdOutputLines); i++ { + endpoints = append(endpoints, cmdOutputLines[i]) + } + return true, nil + } + + e2e.Logf("Waiting for expected result to be synced, try again ...") + return false, nil + }) + if checkOVNDbErr != nil { + e2e.Logf("The command check result in ovndb is not expected ! See below output \n %s ", cmdOutput) + } + return endpoints, checkOVNDbErr +} + +// Get all pods with same label and also are in same state +func GetAllPodsWithLabelAndCertainState(oc *exutil.CLI, namespace string, label string, podState string) []string { + var allPodsWithCertainState []string + allPodsWithLabel, getPodErr := GetAllPodsWithLabel(oc, namespace, label) + o.Expect(getPodErr).NotTo(o.HaveOccurred()) + o.Expect(len(allPodsWithLabel)).ShouldNot(o.Equal(0)) + + for _, eachPod := range allPodsWithLabel { + podStatus, _ := oc.AsAdmin().WithoutNamespace().Run("get").Args("pod", "-n", namespace, eachPod).Output() + if strings.Contains(podStatus, podState) { + allPodsWithCertainState = append(allPodsWithCertainState, eachPod) + } + } + return allPodsWithCertainState +} + +// Get OVN-Kubernetes management interface (ovn-k8s-mp0) IPv6 address for the node +func GetOVNK8sNodeMgmtIPv6(oc *exutil.CLI, nodeName string) string { + var cmdOutput string + var err error + checkErr := wait.Poll(2*time.Second, 10*time.Second, func() (bool, error) { + cmdOutput, err = DebugNodeWithChroot(oc, nodeName, "bash", "-c", "/usr/sbin/ip -o -6 addr show dev ovn-k8s-mp0 | awk '$3 == \"inet6\" && $6 == \"global\" {print $4}' | cut -d'/' -f1") + if cmdOutput == "" || err != nil { + e2e.Logf("Did not get node's IPv6 management interface, errors: %v, try again", err) + return false, nil + } + return true, nil + }) + o.Expect(checkErr).NotTo(o.HaveOccurred(), fmt.Sprintf("Failed to get IPv6 management interface for node %v, err: %v", nodeName, checkErr)) + + nodeOVNK8sMgmtIPv6 := strings.Split(cmdOutput, "\n")[0] + return nodeOVNK8sMgmtIPv6 +} + +// Get joint switch IP(s) by node name +func GetJoinSwitchIPofNode(oc *exutil.CLI, nodeName string) ([]string, []string) { + // get the ovnkube-node pod on the node + ovnKubePod, podErr := GetOVNKPodOnNode(oc, "openshift-ovn-kubernetes", "app=ovnkube-node", nodeName) + o.Expect(podErr).NotTo(o.HaveOccurred()) + o.Expect(ovnKubePod).ShouldNot(o.Equal("")) + var cmdOutput string + var joinSwitchIPv4s, joinSwitchIPv6s []string + var cmdErr error + cmd := "ovn-nbctl get logical_router_port rtoj-GR_" + nodeName + " networks" + checkOVNDbErr := wait.Poll(3*time.Second, 2*time.Minute, func() (bool, error) { + cmdOutput, cmdErr = RemoteShPodWithBashSpecifyContainer(oc, "openshift-ovn-kubernetes", ovnKubePod, "northd", cmd) + if cmdOutput == "" || cmdErr != nil { + e2e.Logf("%v,Waiting for expected result to be synced, try again ...,", cmdErr) + return false, nil + } + return true, nil + }) + o.Expect(checkOVNDbErr).NotTo(o.HaveOccurred(), fmt.Sprintf("Failed to get join switch networks for node %v, err: %v", nodeName, checkOVNDbErr)) + + // output string would be something like: ["100.64.0.8/16", "fd98::8/64"] + rightTrimed := strings.TrimRight(strings.TrimLeft(cmdOutput, "["), "]") //trim left [ and right ] from the output string + outputs := strings.Split(rightTrimed, ", ") + if len(outputs) > 0 { + for _, str := range outputs { + ipv4orv6 := strings.TrimRight(strings.TrimLeft(str, "\""), "\"") // trim left " and right " around IP address string + if IsIPv4(ipv4orv6) { + joinSwitchIPv4s = append(joinSwitchIPv4s, ipv4orv6) + } + if IsIPv6(ipv4orv6) { + joinSwitchIPv6s = append(joinSwitchIPv6s, ipv4orv6) + } + } + } + return joinSwitchIPv4s, joinSwitchIPv6s +} + +// Get host network IPs in NBDB of node +func GetHostNetworkIPsinNBDB(oc *exutil.CLI, nodeName string, externalID string) []string { + // get the ovnkube-node pod on the node + ovnKubePod, podErr := GetOVNKPodOnNode(oc, "openshift-ovn-kubernetes", "app=ovnkube-node", nodeName) + o.Expect(podErr).NotTo(o.HaveOccurred()) + o.Expect(ovnKubePod).ShouldNot(o.Equal("")) + var cmdOutput string + var hostNetworkIPs []string + var cmdErr error + cmd := "ovn-nbctl --column address find address_set " + externalID + checkOVNDbErr := wait.Poll(3*time.Second, 2*time.Minute, func() (bool, error) { + cmdOutput, cmdErr = RemoteShPodWithBashSpecifyContainer(oc, "openshift-ovn-kubernetes", ovnKubePod, "northd", cmd) + if cmdOutput == "" || cmdErr != nil { + e2e.Logf("%v,Waiting for expected result to be synced, try again ...,", cmdErr) + return false, nil + } + return true, nil + }) + o.Expect(checkOVNDbErr).NotTo(o.HaveOccurred(), fmt.Sprintf("Failed to get host network IPs for node %v, err: %v", nodeName, checkOVNDbErr)) + + // two example outputs from "ovn-nbctl --column address find address_set " command + // addresses : ["10.128.0.2", "10.128.2.2", "10.129.0.2", "10.130.0.2", "10.130.2.2", "10.131.2.2", "100.64.0.2"] + // addresses : ["fd01:0:0:1::2", "fd01:0:0:2::2", "fd01:0:0:3::2", "fd01:0:0:5::2", "fd01:0:0:7::2", "fd01:0:0:8::2"] + // match out all IP (v4 or v6) addresses under " " + re := regexp.MustCompile(`"[^",]+"`) + ipStrs := re.FindAllString(cmdOutput, -1) + for _, eachIpString := range ipStrs { + ip := strings.TrimRight(strings.TrimLeft(eachIpString, "\""), "\"") //trim left " and right " from the string to get IP address + hostNetworkIPs = append(hostNetworkIPs, ip) + } + return hostNetworkIPs +} + +// Check if second array is a subset of first array +func UnorderedContains(first, second []string) bool { + set := make(map[string]bool) + + for _, element := range first { + set[element] = true + } + + for _, element := range second { + if !set[element] { + return false + } + } + + return true +} + +// Get all host CIDRs for a cluster node, including those for multiple interefaces +func GetAllHostCIDR(oc *exutil.CLI, nodeName string) ([]string, []string) { + var allNodeIPsv4, allNodeIPsv6 []string + outputString, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("node", nodeName, "-o=jsonpath={.metadata.annotations.k8s\\.ovn\\.org\\/host-cidrs}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + + // sample output from the command: ["172.22.0.237/24","192.168.111.25/24","fd2e:6f44:5dd8:c956::19/128"] + hostCIDRsString := strings.TrimRight(strings.TrimLeft(outputString, "["), "]") // trim the left [ and right ] around the CIDRs string + hostCIDRs := strings.Split(hostCIDRsString, ",") + + if len(hostCIDRs) != 0 { + for _, eachCIDR := range hostCIDRs { + ipString := strings.TrimRight(strings.TrimLeft(eachCIDR, "\""), "\"") // trim the left " and right "" around the IP string + ip := strings.Split(ipString, "/")[0] //remove IP prefix, only get IP address + if IsIPv4(ip) { + allNodeIPsv4 = append(allNodeIPsv4, ip) + } + if IsIPv6(ip) { + allNodeIPsv6 = append(allNodeIPsv6, ip) + } + } + } + e2e.Logf("\n cluster ipStackType: %s, for node %s, got all its v4 CIDRs: %v, v6 CIDRs: %v\n", CheckIPStackType(oc), nodeName, allNodeIPsv4, allNodeIPsv6) + return allNodeIPsv4, allNodeIPsv6 +} + +// Check a node can be accessed from any of its host interface from a pod +func CheckNodeAccessibilityFromAPod(oc *exutil.CLI, nodeName, ns, podName string) bool { + // Get all host IPs of the node + ipStackType := CheckIPStackType(oc) + allNodeIPsv4, allNodeIPsv6 := GetAllHostCIDR(oc, nodeName) + + if ipStackType == "dualstack" || ipStackType == "ipv4single" { + for _, nodeIPv4Addr := range allNodeIPsv4 { + _, err := e2eoutput.RunHostCmd(ns, podName, "ping -c 2 "+nodeIPv4Addr) + if err != nil { + e2e.Logf("Access to node %s failed at interface %s", nodeName, nodeIPv4Addr) + return false + } + } + } + if ipStackType == "dualstack" || ipStackType == "ipv6single" { + for _, nodeIPv6Addr := range allNodeIPsv6 { + _, err := e2eoutput.RunHostCmd(ns, podName, "ping -c 2 "+nodeIPv6Addr) + if err != nil { + e2e.Logf("Access to node %s failed at interface %s", nodeName, nodeIPv6Addr) + return false + } + } + } + return true +} + +func VerifySctpConnPod2IP(oc *exutil.CLI, namespace, sctpServerPodIP, sctpServerPodName, sctpClientPodname string, pass bool) { + e2e.Logf("sctpserver pod start to wait for sctp traffic") + msg, err := e2eoutput.RunHostCmdWithRetries(namespace, sctpServerPodName, "ps aux | grep sctp", 3*time.Second, 30*time.Second) + o.Expect(err).NotTo(o.HaveOccurred()) + if strings.Contains(msg, "/usr/bin/ncat -l 30102 --sctp") { + e2e.Logf("sctpserver pod is already listening on port 30102.") + } else { + cmdNcat, _, _, _ := oc.AsAdmin().Run("exec").Args("-n", namespace, sctpServerPodName, "--", "/usr/bin/ncat", "-l", "30102", "--sctp").Background() + defer cmdNcat.Process.Kill() + e2e.Logf("check sctp process enabled in the sctp server pod") + o.Eventually(func() string { + msg, err := e2eoutput.RunHostCmdWithRetries(namespace, sctpServerPodName, "ps aux | grep sctp", 3*time.Second, 30*time.Second) + o.Expect(err).NotTo(o.HaveOccurred()) + return msg + }, "10s", "5s").Should(o.ContainSubstring("/usr/bin/ncat -l 30102 --sctp"), "No sctp process running on sctp server pod") + } + + e2e.Logf("sctpclient pod start to send sctp traffic") + e2eoutput.RunHostCmd(namespace, sctpClientPodname, "echo 'Test traffic using sctp port from sctpclient to sctpserver' | { ncat -v "+sctpServerPodIP+" 30102 --sctp; }") + + e2e.Logf("server sctp process will end after get sctp traffic from sctp client") + if pass { + o.Eventually(func() string { + msg, err := e2eoutput.RunHostCmdWithRetries(namespace, sctpServerPodName, "ps aux | grep sctp", 3*time.Second, 30*time.Second) + o.Expect(err).NotTo(o.HaveOccurred()) + return msg + }, "10s", "5s").ShouldNot(o.ContainSubstring("/usr/bin/ncat -l 30102 --sctp"), "Sctp process didn't end after get sctp traffic from sctp client") + } else { + msg, err := e2eoutput.RunHostCmd(namespace, sctpServerPodName, "ps aux | grep sctp") + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(msg).Should(o.ContainSubstring("/usr/bin/ncat -l 30102 --sctp"), "Sctp process ended after get sctp traffic from sctp client") + } + +} + +// Get apiVIP or ingessVIP on the cluster (vSphere or BM) +func GetVIPOnCluster(oc *exutil.CLI, platform string, vipType string) []string { + if !strings.Contains(platform, "baremetal") && !strings.Contains(platform, "vsphere") { + g.Skip("Skip for non-vSphere/non-Baremetal cluster") + } + var cmdOutput, jsonpathstr string + var err error + var vips []string + switch vipType { + case "apiVIP": + jsonpathstr = "-o=jsonpath={.status.platformStatus." + platform + ".apiServerInternalIPs}" + case "ingressVIP": + jsonpathstr = "-o=jsonpath={.status.platformStatus." + platform + ".ingressIPs}" + default: + e2e.Failf("VIP Type only can be apiVIP or ingressVIP") + } + + checkErr := wait.PollUntilContextTimeout(context.TODO(), 2*time.Second, 10*time.Second, false, func(ctx context.Context) (bool, error) { + cmdOutput, err = oc.AsAdmin().WithoutNamespace().Run("get").Args("infrastructure", "cluster", jsonpathstr).Output() + if cmdOutput == "" || err != nil { + e2e.Logf("Did not get %s, errors: %v, try again", vipType, err) + return false, nil + } + return true, nil + }) + o.Expect(checkErr).NotTo(o.HaveOccurred(), fmt.Sprintf("Failed to get %s on this platform %v, err: %v", vipType, platform, checkErr)) + + // match out all IP (v4 or v6) addresses under " " + re := regexp.MustCompile(`"[^",]+"`) + ipStrs := re.FindAllString(cmdOutput, -1) + for _, eachIpString := range ipStrs { + ip := strings.TrimRight(strings.TrimLeft(eachIpString, "\""), "\"") //trim left " and right " from the string to get IP address + vips = append(vips, ip) + } + + return vips +} + +// Find apiVIP or ingressVIP node on vSphere or BM +func FindVIPNode(oc *exutil.CLI, vip string) string { + nodeList, err := GetAllNodesbyOSType(oc, "linux") + o.Expect(err).NotTo(o.HaveOccurred()) + defaultInt, _ := GetDefaultInterface(oc) + + for _, node := range nodeList { + output, err := DebugNode(oc, node, "bash", "-c", "ip add show "+defaultInt) + o.Expect(err).NotTo(o.HaveOccurred()) + if strings.Contains(output, vip) { + e2e.Logf("Node %s is VIP node", node) + return node + } + } + return "" +} + +// Return IPv4 address and IPv4 address with prefix +func GetIPv4AndIPWithPrefixForNICOnNode(oc *exutil.CLI, node, nic string) (string, string) { + cmd := fmt.Sprintf("ip -4 -brief a show %s | awk '{print $3}' ", nic) + output, debugNodeErr := DebugNode(oc, node, "bash", "-c", cmd) + o.Expect(debugNodeErr).NotTo(o.HaveOccurred()) + pattern := `(\d+\.\d+\.\d+\.\d+/\d+)` + re := regexp.MustCompile(pattern) + matches := re.FindStringSubmatch(output) + o.Expect(len(matches) > 1).Should(o.BeTrue()) + ipAddressWithPrefix := matches[1] + e2e.Logf("IP address with prefix: %s", ipAddressWithPrefix) + + ipParts := strings.Split(ipAddressWithPrefix, "/") + ipAddress := ipParts[0] + + e2e.Logf("The IPv4 of interface %s on node %s is %s and ipAddressWithPrefix is %s", nic, node, ipAddress, ipAddressWithPrefix) + return ipAddress, ipAddressWithPrefix +} + +// check respective config availability for IPsec NS on external host specific to Beijing BM host. +// this func might be scaled up in future if we comes down to support net2net as well +func ApplyConfigTypeExtHost(leftPublicIP, configType string) error { + switch configType { + case "host2hostTransportRDU2": + err := SshRunCmd(leftPublicIP, "core", "sudo cp /home/core/nstest_host2host_transport.conf.bak.rdu2 /etc/ipsec.d/nstest.conf && sudo systemctl restart ipsec") + if err != nil { + return fmt.Errorf("Could not apply host2host config. Check External Host %v", err) + } + case "host2hostTunnelRDU2": + err := SshRunCmd(leftPublicIP, "core", "sudo cp /home/core/nstest_host2host_tunnel.conf.bak.rdu2 /etc/ipsec.d/nstest.conf && sudo systemctl restart ipsec") + if err != nil { + return fmt.Errorf("Could not apply host2host config. Check External Host %v", err) + } + case "host2netTransportRDU2": + err := SshRunCmd(leftPublicIP, "core", "sudo cp /home/core/nstest_host2net_transport.conf.rdu2 /etc/ipsec.d/nstest.conf && sudo systemctl restart ipsec") + if err != nil { + return fmt.Errorf("Could not apply host2net config. Check External Host %v", err) + } + case "host2netTunnelRDU2": + err := SshRunCmd(leftPublicIP, "core", "sudo cp /home/core/nstest_host2net_tunnel.conf.rdu2 /etc/ipsec.d/nstest.conf && sudo systemctl restart ipsec") + if err != nil { + return fmt.Errorf("Could not apply host2net config. Check External Host %v", err) + } + } + return nil +} + +// get hostname for LB service, this fuction is likely to be useful only for AWS, other public cloud platforms may not give LB service hostname +func GetLBSVCHostname(oc *exutil.CLI, namespace, svc string) string { + var LBSVCHostname string + var cmdErr error + + platform := CheckPlatform(oc) + if !strings.Contains(platform, "aws") { + g.Skip("Skip for non-AWS cluster") + } + + e2e.Logf("Getting the Load Balancer service hostname ...") + getLBSVCHostnameErr := wait.Poll(5*time.Second, 2*time.Minute, func() (bool, error) { + LBSVCHostname, cmdErr = oc.AsAdmin().WithoutNamespace().Run("get").Args("svc", svc, "-n", namespace, "-o=jsonpath={.status.loadBalancer.ingress[0].hostname}").Output() + if cmdErr != nil || LBSVCHostname == "pending" || LBSVCHostname == "" { + e2e.Logf("%v,Waiting for expected result to be synced, try again ...,", cmdErr) + return false, nil + } + return true, nil + }) + o.Expect(getLBSVCHostnameErr).NotTo(o.HaveOccurred(), fmt.Sprintf("Could not get LB service's hostname, err: %v", getLBSVCHostnameErr)) + + return LBSVCHostname +} + +// get IP address of LB service +func GetLBSVCIP(oc *exutil.CLI, namespace string, svcName string) string { + var svcExternalIP string + var cmdErr error + checkErr := wait.Poll(5*time.Second, 300*time.Second, func() (bool, error) { + svcExternalIP, cmdErr = oc.AsAdmin().WithoutNamespace().Run("get").Args("service", "-n", namespace, svcName, "-o=jsonpath={.status.loadBalancer.ingress[0].ip}").Output() + if svcExternalIP == "" || cmdErr != nil { + e2e.Logf("Waiting for lb service IP assignment. Trying again...") + return false, nil + } + return true, nil + }) + o.Expect(checkErr).NotTo(o.HaveOccurred(), fmt.Sprintf("Failed to get externalIP to the externalIP service %s", svcName)) + + return svcExternalIP +} + +func GetNetworkDiagnosticsAvailable(oc *exutil.CLI) string { + statusOutput, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("Network.config.openshift.io/cluster", "-o=jsonpath={.status.conditions[?(@.type == \"NetworkDiagnosticsAvailable\")].status}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + statusOutput = strings.ToLower(statusOutput) + e2e.Logf("NetworkDiagnosticsAvailable status is %s", statusOutput) + return statusOutput +} + +func VerifyDesitnationAccess(oc *exutil.CLI, podName, podNS, domainName string, passOrFail bool) { + curlCmd := fmt.Sprintf("curl -s -I %s --connect-timeout 5 ", domainName) + if passOrFail { + _, err := e2eoutput.RunHostCmdWithRetries(podNS, podName, curlCmd, 10*time.Second, 20*time.Second) + o.Expect(err).NotTo(o.HaveOccurred()) + ipStackType := CheckIPStackType(oc) + if ipStackType == "dualstack" { + curlCmd = fmt.Sprintf("curl -s -6 -I %s --connect-timeout 5", domainName) + _, err := e2eoutput.RunHostCmdWithRetries(podNS, podName, curlCmd, 10*time.Second, 20*time.Second) + o.Expect(err).NotTo(o.HaveOccurred()) + } + + } else { + o.Eventually(func() error { + _, err := e2eoutput.RunHostCmd(podNS, podName, curlCmd) + return err + }, "20s", "10s").Should(o.HaveOccurred()) + } +} + +// First ip is ipv4, secondary is ipv6. +func GetIPFromDnsName(dnsName string) (string, string) { + ips, err := net.LookupIP(dnsName) + o.Expect(err).NotTo(o.HaveOccurred()) + + var ipv4, ipv6 string + for _, ip := range ips { + if ip.To4() != nil && ipv4 == "" { + ipv4 = ip.String() + } else if strings.Contains(ip.String(), ":") && ipv6 == "" { + ipv6 = ip.String() + } + if ipv4 != "" && ipv6 != "" { + break + } + } + e2e.Logf("The resovled IPv4, IPv6 address for dns name %s is %s,%s", dnsName, ipv4, ipv6) + return ipv4, ipv6 +} + +func VerifyDstIPAccess(podName, podNS, ip string, passOrFail bool) { + var curlCmd string + if strings.Contains(ip, ":") { + e2e.Logf("The IP %s is IPv6 address.", ip) + curlCmd = fmt.Sprintf("curl -s -6 -I [%s] --connect-timeout 5 ", ip) + } else { + e2e.Logf("The IP %s is IPv4 address.", ip) + curlCmd = fmt.Sprintf("curl -s -I %s --connect-timeout 5 ", ip) + } + + if passOrFail { + _, err := e2eoutput.RunHostCmdWithRetries(podNS, podName, curlCmd, 10*time.Second, 120*time.Second) + o.Expect(err).NotTo(o.HaveOccurred()) + } else { + o.Eventually(func() error { + _, err := e2eoutput.RunHostCmd(podNS, podName, curlCmd) + return err + }, "20s", "10s").Should(o.HaveOccurred()) + } +} + +// Function to obtain API VIP on BM cluster +func GetAPIVIPOnCluster(oc *exutil.CLI) string { + apiVIP := "" + var err error + o.Eventually(func() error { + apiVIP, err = oc.WithoutNamespace().AsAdmin().Run("get").Args("infrastructure", "cluster", "-o=jsonpath={.status.platformStatus.baremetal.apiServerInternalIP}").Output() + return err + }, "60s", "5s").ShouldNot(o.HaveOccurred()) + + return apiVIP +} + +func (pod *HttpserverPodResourceNode) CreateHttpservePodNodeByAdmin(oc *exutil.CLI) { + err := wait.Poll(5*time.Second, 20*time.Second, func() (bool, error) { + err1 := ApplyResourceFromTemplateByAdmin(oc, "--ignore-unknown-parameters=true", "-f", pod.Template, "-p", "NAME="+pod.Name, "NAMESPACE="+pod.Namespace, "CONTAINERPORT="+strconv.Itoa(int(pod.Containerport)), "HOSTPORT="+strconv.Itoa(int(pod.Hostport)), "NODENAME="+pod.Nodename) + if err1 != nil { + e2e.Logf("the err:%v, and try next round", err1) + return false, nil + } + return true, nil + }) + o.Expect(err).NotTo(o.HaveOccurred(), fmt.Sprintf("fail to create pod %v", pod.Name)) +} + +// CurlPod2NodePass checks connectivity from a pod to node that has httpserverPod on it +func CurlPod2NodePass(oc *exutil.CLI, namespaceSrc, podNameSrc, nodeNameDst, DstHostPort string) { + nodeIP2, nodeIP1 := GetNodeIP(oc, nodeNameDst) + if nodeIP2 != "" { + _, err := e2eoutput.RunHostCmd(namespaceSrc, podNameSrc, "curl -I --connect-timeout 5 -s "+net.JoinHostPort(nodeIP1, DstHostPort)) + o.Expect(err).NotTo(o.HaveOccurred()) + _, err = e2eoutput.RunHostCmd(namespaceSrc, podNameSrc, "curl -I --connect-timeout 5 -s "+net.JoinHostPort(nodeIP2, DstHostPort)) + o.Expect(err).NotTo(o.HaveOccurred()) + } else { + _, err := e2eoutput.RunHostCmd(namespaceSrc, podNameSrc, "curl -I --connect-timeout 5 -s "+net.JoinHostPort(nodeIP1, DstHostPort)) + o.Expect(err).NotTo(o.HaveOccurred()) + } +} + +// CurlPod2PodFail ensures no connectivity from pod to node that has httpserverPod on it +func CurlPod2NodeFail(oc *exutil.CLI, namespaceSrc, podNameSrc, nodeNameDst, DstHostPort string) { + nodeIP2, nodeIP1 := GetNodeIP(oc, nodeNameDst) + if nodeIP2 != "" { + _, err := e2eoutput.RunHostCmd(namespaceSrc, podNameSrc, "curl -I --connect-timeout 5 -s "+net.JoinHostPort(nodeIP1, DstHostPort)) + o.Expect(err).To(o.HaveOccurred()) + _, err = e2eoutput.RunHostCmd(namespaceSrc, podNameSrc, "curl -I --connect-timeout 5 -s "+net.JoinHostPort(nodeIP2, DstHostPort)) + o.Expect(err).To(o.HaveOccurred()) + } else { + _, err := e2eoutput.RunHostCmd(namespaceSrc, podNameSrc, "curl -I --connect-timeout 5 -s "+net.JoinHostPort(nodeIP1, DstHostPort)) + o.Expect(err).To(o.HaveOccurred()) + } +} + +// CurlPod2HostPass checks connectivity from a pod to host that has httpserverPod on it +func CurlPod2HostPass(oc *exutil.CLI, namespaceSrc, podNameSrc, hostip, DstHostPort string) { + _, err := e2eoutput.RunHostCmd(namespaceSrc, podNameSrc, "curl -I --connect-timeout 5 -s "+net.JoinHostPort(hostip, DstHostPort)) + o.Expect(err).NotTo(o.HaveOccurred()) +} + +// CurlPod2HostFail ensures no connectivity from pod to host that has httpserverPod on it +func CurlPod2HostFail(oc *exutil.CLI, namespaceSrc, podNameSrc, hostip, DstHostPort string) { + _, err := e2eoutput.RunHostCmd(namespaceSrc, podNameSrc, "curl -I --connect-timeout 5 -s "+net.JoinHostPort(hostip, DstHostPort)) + o.Expect(err).To(o.HaveOccurred()) +} + +// Check the cluster is fips enabled +func CheckFips(oc *exutil.CLI) bool { + node, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("node", "--selector=node-role.kubernetes.io/worker,kubernetes.io/os=linux", "-o=jsonpath={.items[0].metadata.name}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + fipsInfo, err := DebugNodeWithChroot(oc, node, "bash", "-c", "fips-mode-setup --check") + o.Expect(err).NotTo(o.HaveOccurred()) + if strings.Contains(fipsInfo, "FIPS mode is disabled.") { + e2e.Logf("FIPS is not enabled.") + return false + } + e2e.Logf("FIPS is enabled.") + return true +} + +// Check whether it is able to access public web with IPv6 address +func CheckIPv6PublicAccess(oc *exutil.CLI) bool { + workNode, err := GetFirstWorkerNode(oc) + o.Expect(err).ShouldNot(o.HaveOccurred()) + curlCMD := "curl -6 www.google.com --connect-timeout 5 -I" + output, err := DebugNode(oc, workNode, "bash", "-c", curlCMD) + if !strings.Contains(output, "HTTP") || err != nil { + e2e.Logf("%s", output) + e2e.Logf("Unable to access the public Internet with IPv6 from the cluster.") + return false + } + e2e.Logf("Successfully connected to the public Internet with IPv6 from the cluster.") + return true +} + +func ForceRebootNode(oc *exutil.CLI, nodeName string) { + e2e.Logf("\nRebooting node %s....", nodeName) + runCmd, _, _, runCmdErr := oc.AsAdmin().Run("debug").Args("node/"+nodeName, "--", "chroot", "/host", "reboot", "--force").Background() + defer runCmd.Process.Kill() + o.Expect(runCmdErr).NotTo(o.HaveOccurred()) + WaitForNetworkOperatorState(oc, 100, 15, "True.*False.*False") +} + +// Create resources in the specified namespace from the file (not template) that is expected to fail +func CreateResourceFromFileWithError(oc *exutil.CLI, ns, file string) error { + err := oc.AsAdmin().WithoutNamespace().Run("create").Args("-f", file, "-n", ns).Execute() + return err +} + +// Struct to create pod with customized response +type CustomResponsePodResource struct { + Name string + Namespace string + LabelKey string + LabelVal string + ResponseStr string + Template string +} + +func (pod *CustomResponsePodResource) CreateCustomResponsePod(oc *exutil.CLI) { + err := wait.Poll(5*time.Second, 20*time.Second, func() (bool, error) { + err1 := ApplyResourceFromTemplate(oc, "--ignore-unknown-parameters=true", "-f", pod.Template, "-p", "NAME="+pod.Name, "NAMESPACE="+pod.Namespace, + "LABELKEY="+pod.LabelKey, "LABELVAL="+pod.LabelVal, + "RESPONSESTR="+pod.ResponseStr) + if err1 != nil { + e2e.Logf("the err:%v, and try again...", err1) + return false, nil + } + return true, nil + }) + o.Expect(err).NotTo(o.HaveOccurred(), fmt.Sprintf("Failed to create pod %s due to %v", pod.Name, err)) +} + +// Struct to create service with session ability +type SessionAffinityServiceResource struct { + Name string + Namespace string + IpFamilyPolicy string + SelLabelKey string + SelLabelVal string + Template string +} + +func (svc *SessionAffinityServiceResource) CreateSessionAffiniltyService(oc *exutil.CLI) { + err := wait.Poll(5*time.Second, 20*time.Second, func() (bool, error) { + err1 := ApplyResourceFromTemplate(oc, "--ignore-unknown-parameters=true", "-f", svc.Template, "-p", "NAME="+svc.Name, "NAMESPACE="+svc.Namespace, + "IPFAMILYPOLICY="+svc.IpFamilyPolicy, "SELLABELKEY="+svc.SelLabelKey, "SELLABELVAL="+svc.SelLabelVal) + if err1 != nil { + e2e.Logf("the err:%v, and try again...", err1) + return false, nil + } + return true, nil + }) + o.Expect(err).NotTo(o.HaveOccurred(), fmt.Sprintf("Failed to create pservice %s due to %v", svc.Name, err)) +} + +func GetEnabledFeatureGates(oc *exutil.CLI) ([]string, error) { + enabledFeatureGates, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("featuregate", "cluster", "-o=jsonpath={.status.featureGates[0].enabled[*].name}").Output() + if err != nil { + return nil, err + } + + return strings.Split(enabledFeatureGates, " "), nil +} + +// IsFeaturegateEnabled check whether a featuregate is in enabled or not +func IsFeaturegateEnabled(oc *exutil.CLI, featuregate string) (bool, error) { + enabledFeatureGates, err := GetEnabledFeatureGates(oc) + if err != nil { + return false, err + } + for _, f := range enabledFeatureGates { + if f == featuregate { + return true, nil + } + } + return false, nil +} + +func SkipIfNoFeatureGate(oc *exutil.CLI, featuregate string) { + enabled, err := IsFeaturegateEnabled(oc, featuregate) + o.Expect(err).NotTo(o.HaveOccurred(), "Error getting enabled featuregates") + + if !enabled { + g.Skip(fmt.Sprintf("Featuregate %s is not enabled in this cluster", featuregate)) + } +} + +// Create VF policy through NMstate +func (vrf *VRFResource) CreateVRF(oc *exutil.CLI) error { + err := wait.Poll(5*time.Second, 20*time.Second, func() (bool, error) { + err1 := ApplyResourceFromTemplateByAdmin(oc, "--ignore-unknown-parameters=true", "-f", vrf.Template, "-p", "NAME="+vrf.Name, "INTFNAME="+vrf.Intfname, "NODENAME="+vrf.Nodename, "TABLEID="+strconv.Itoa(int(vrf.Tableid))) + if err1 != nil { + e2e.Logf("Creating VRF on the node failed :%v, and try next round", err1) + return false, nil + } + return true, nil + }) + if err != nil { + return fmt.Errorf("fail to create VRF on the node %v", vrf.Name) + } + return nil +} + +func (namedPortPod *NamedPortPodResource) CreateNamedPortPod(oc *exutil.CLI) { + g.By("Creating named port pod from template") + err := wait.Poll(5*time.Second, 20*time.Second, func() (bool, error) { + err1 := ApplyResourceFromTemplateByAdmin(oc, "--ignore-unknown-parameters=true", "-f", namedPortPod.Template, "-p", "NAME="+namedPortPod.Name, + "NAMESPACE="+namedPortPod.Namespace, "PODLABELKEY="+namedPortPod.PodLabelKey, "PODLABELVAL="+namedPortPod.PodLabelVal, + "PORTNAME="+namedPortPod.Portname, "CONTAINERPORT="+strconv.Itoa(int(namedPortPod.Containerport))) + if err1 != nil { + e2e.Logf("Error creating resource:%v, and trying again", err1) + return false, nil + } + return true, nil + }) + o.Expect(err).NotTo(o.HaveOccurred(), fmt.Sprintf("Failed to create named port pod %v", namedPortPod.Name)) +} + +func GetTcpdumpOnNodeCmdFromPod(oc *exutil.CLI, nodeName, tcpdumpCmd, namespace, podname, cmdOnPod string) string { + g.By("Enable tcpdump on node") + cmdTcpdump, cmdOutput, _, err := oc.AsAdmin().Run("debug").Args("-n", "default", "node/"+nodeName, "--", "bash", "-c", tcpdumpCmd).Background() + defer cmdTcpdump.Process.Kill() + o.Expect(err).NotTo(o.HaveOccurred()) + + //Wait 5 seconds to let the tcpdump ready for capturing traffic + time.Sleep(5 * time.Second) + + g.By("Curl external host:port from test pods") + + var tcpdumpErr error = nil + checkErr := wait.PollUntilContextTimeout(context.Background(), 10*time.Second, 60*time.Second, false, func(cxt context.Context) (bool, error) { + _, curlErr := e2eoutput.RunHostCmd(namespace, podname, cmdOnPod) + if curlErr == nil { + tcpdumpErr = cmdTcpdump.Wait() + e2e.Logf("The captured tcpdump outout is: \n%s\n", cmdOutput.String()) + } + if curlErr != nil || tcpdumpErr != nil { + e2e.Logf("Getting error at executing curl command: %v or at waiting for tcpdump: %v, try again ...", curlErr, tcpdumpErr) + return false, nil + } + if cmdOutput.String() == "" { + e2e.Logf("Did not capture tcpdump packets,try again ...") + return false, nil + } + return true, nil + }) + o.Expect(checkErr).NotTo(o.HaveOccurred(), fmt.Sprintf("Unable to get tcpdump when curling from pod:%s from Namespace: %s", podname, namespace)) + + cmdTcpdump.Process.Kill() + return cmdOutput.String() +} + +func CollectMustGather(oc *exutil.CLI, dstDir string, imageStream string, parameters []string) (string, error) { + args := []string{"must-gather"} + if dstDir != "" { + args = append(args, "--dest-dir="+dstDir) + } + if imageStream != "" { + args = append(args, "--image-stream="+imageStream) + } + if len(parameters) > 0 { + args = append(args, "--") + for _, param := range parameters { + args = append(args, param) + } + } + output, err := oc.AsAdmin().WithoutNamespace().Run("adm").Args(args...).Output() + if err != nil && strings.Contains(output, "ImagePullBackOff") { + e2e.Logf("unable to pull image, try again...") + output, err = oc.AsAdmin().WithoutNamespace().Run("adm").Args(args...).Output() + if err != nil { + e2e.Logf("collect must-gather failed, err: %v", err) + return "", err + } + } + return output, nil +} + +func VerifyPodConnCrossNodes(oc *exutil.CLI) bool { + buildPruningBaseDir := testdata.FixturePath("networking") + helloDaemonset := filepath.Join(buildPruningBaseDir, "hello-pod-daemonset.yaml") + + g.By("Create a temporay project for pods to pods connection checking.") + oc.SetupProject() + ns := oc.Namespace() + + g.By("Create hello-pod-daemonset in namespace.") + CreateResourceFromFile(oc, ns, helloDaemonset) + err := WaitForPodWithLabelReady(oc, ns, "name=hello-pod") + o.Expect(err).NotTo(o.HaveOccurred(), "ipsec pods are not ready after killing pluto") + + g.By("Checking pods connection") + return VerifyPodConnCrossNodesSpecNS(oc, ns, "name=hello-pod") +} + +func WaitForPodsCount(oc *exutil.CLI, namespace, labelSelector string, expectedCount int, interval, timeout time.Duration) error { + return wait.Poll(interval, timeout, func() (bool, error) { + allPods, getPodErr := GetAllPodsWithLabel(oc, namespace, labelSelector) + if getPodErr != nil { + e2e.Logf("Error fetching pods: %v, retrying...", getPodErr) + return false, nil + } + if len(allPods) == expectedCount { + return true, nil // Condition met, exit polling + } + e2e.Logf("Expected %d pods, but found %d. Retrying...", expectedCount, len(allPods)) + return false, nil + }) +} + +func VerifyPodConnCrossNodesSpecNS(oc *exutil.CLI, namespace, podLabel string) bool { + pass := true + pods := GetPodName(oc, namespace, podLabel) + + for _, srcPod := range pods { + for _, targetPod := range pods { + if targetPod != srcPod { + podIP1, podIP2 := GetPodIP(oc, namespace, targetPod) + e2e.Logf("Curling from pod: %s with IP: %s\n", srcPod, podIP1) + _, err := e2eoutput.RunHostCmd(namespace, srcPod, "curl --connect-timeout 10 -s "+net.JoinHostPort(podIP1, "8080")) + if err != nil { + e2e.Logf("pods connection failed from %s to %s:8080", srcPod, podIP1) + srcNode, err := GetPodNodeName(oc, namespace, srcPod) + o.Expect(err).NotTo(o.HaveOccurred()) + dstNode, err := GetPodNodeName(oc, namespace, targetPod) + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("Checking the if the pods's located nodes in SchedulingDisabled or NotReady status.") + output, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("node", srcNode, dstNode).Output() + e2e.Logf("%s", output) + o.Expect(err).NotTo(o.HaveOccurred()) + if strings.Contains(output, "SchedulingDisabled") || strings.Contains(output, "NotReady") { + continue + } + e2e.Logf("pods connection failed between nodes %s and %s", srcNode, dstNode) + pass = false + } + if podIP2 != "" { + e2e.Logf("Curling from pod: %s with IP: %s\n", srcPod, podIP2) + _, err := e2eoutput.RunHostCmd(namespace, srcPod, "curl --connect-timeout 10 -s "+net.JoinHostPort(podIP2, "8080")) + if err != nil { + e2e.Logf("pods connection failed from %s to %s:8080", srcPod, podIP2) + srcNode, err := GetPodNodeName(oc, namespace, srcPod) + o.Expect(err).NotTo(o.HaveOccurred()) + dstNode, err := GetPodNodeName(oc, namespace, targetPod) + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("Checking the if the pods's located nodes in SchedulingDisabled or NotReady status.") + output, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("node", srcNode, dstNode).Output() + e2e.Logf("%s", output) + o.Expect(err).NotTo(o.HaveOccurred()) + if strings.Contains(output, "SchedulingDisabled") || strings.Contains(output, "NotReady") { + continue + } + e2e.Logf("pods connection failed between nodes %s and %s", srcNode, dstNode) + pass = false + } + } + + } + } + } + e2e.Logf("The pods connection pass check is %v ", pass) + return pass +} + +func CheckIPSecNATTEanbled(oc *exutil.CLI) bool { + ovnPod := GetPodName(oc, "openshift-ovn-kubernetes", "app=ovnkube-node")[0] + cmd := "ovn-nbctl --no-leader-only get NB_Global . options" + e2e.Logf("The command is: %v", cmd) + out, err := RemoteShPodWithBashSpecifyContainer(oc, "openshift-ovn-kubernetes", ovnPod, "nbdb", cmd) + if err != nil { + e2e.Logf("Execute command failed with err:%v and output is %v.", err, out) + } + o.Expect(err).NotTo(o.HaveOccurred()) + + if strings.Contains(out, `ipsec_encapsulation="true"`) { + e2e.Logf("The cluster is IPSec enabled NAT-T.") + return true + } else { + e2e.Logf("The cluster is IPSec enabled with ESP.") + return false + } +} + +func VerifyIPSecLoaded(oc *exutil.CLI, nodeName string, num int) { + var expectedNum int + if num == 0 { + expectedNum = 0 + } else { + expectedNum = (num - 1) * 2 + } + cmd := `ipsec status | grep "Total IPsec connections"` + // After node reboot, need to wait the IPSec connections loaded + checkErr := wait.PollUntilContextTimeout(context.Background(), 10*time.Second, 120*time.Second, false, func(cxt context.Context) (bool, error) { + out, err := DebugNodeWithChroot(oc, nodeName, "bash", "-c", cmd) + e2e.Logf("Total IPsec connection is : \n%s", out) + return strings.Contains(out, fmt.Sprintf("active %v", expectedNum)), err + }) + o.Expect(checkErr).NotTo(o.HaveOccurred(), "IPsec connections were not loaded completely!") +} + +func PingNode2PodPass(oc *exutil.CLI, nodeName string, namespaceDst string, podNameDst string) { + podIP1, podIP2 := GetPodIP(oc, namespaceDst, podNameDst) + if podIP2 != "" { + _, err := DebugNode(oc, nodeName, "ping6", "-c", "4", podIP1) + o.Expect(err).NotTo(o.HaveOccurred()) + _, err = DebugNode(oc, nodeName, "ping", "-c", "4", podIP2) + o.Expect(err).NotTo(o.HaveOccurred()) + } else { + if netutils.IsIPv6String(podIP1) { + _, err := DebugNode(oc, nodeName, "ping6", "-c", "4", podIP1) + o.Expect(err).NotTo(o.HaveOccurred()) + } else { + _, err := DebugNode(oc, nodeName, "ping", "-c", "4", podIP1) + o.Expect(err).NotTo(o.HaveOccurred()) + } + } +} + +func PingNode2PodFail(oc *exutil.CLI, nodeName string, namespaceDst string, podNameDst string) { + podIP1, podIP2 := GetPodIP(oc, namespaceDst, podNameDst) + if podIP2 != "" { + _, err := DebugNode(oc, nodeName, "ping6", "-c", "4", podIP1) + o.Expect(err).To(o.HaveOccurred()) + _, err = DebugNode(oc, nodeName, "ping", "-c", "4", podIP2) + o.Expect(err).To(o.HaveOccurred()) + } else { + if netutils.IsIPv6String(podIP1) { + _, err := DebugNode(oc, nodeName, "ping6", "-c", "4", podIP1) + o.Expect(err).To(o.HaveOccurred()) + } else { + _, err := DebugNode(oc, nodeName, "ping", "-c", "4", podIP1) + o.Expect(err).To(o.HaveOccurred()) + } + } +} + +// Check if BaselineCapabilities have been set +func IsBaselineCapsSet(oc *exutil.CLI) bool { + baselineCapabilitySet, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("clusterversion", "version", "-o=jsonpath={.spec.capabilities.baselineCapabilitySet}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("baselineCapabilitySet parameters: %v\n", baselineCapabilitySet) + return len(baselineCapabilitySet) != 0 +} + +// Check if component is listed in clusterversion.status.capabilities.enabledCapabilities +func IsEnabledCapability(oc *exutil.CLI, component string) bool { + enabledCapabilities, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("clusterversion", "-o=jsonpath={.items[*].status.capabilities.enabledCapabilities}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("Cluster enabled capability parameters: %v\n", enabledCapabilities) + return strings.Contains(enabledCapabilities, component) +} + +// ping pod to external connectivity check +func PingPod2ExternalPass(oc *exutil.CLI, namespaceSrc string, podNameSrc string) { + ipStack := CheckIPStackType(oc) + if (ipStack == "ipv4single") || (ipStack == "dualstack") { + output, err := e2eoutput.RunHostCmd(namespaceSrc, podNameSrc, "ping -c10 www.google.com") + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(strings.Contains(output, "0% packet loss")).To(o.BeTrue()) + } else { + output, err := e2eoutput.RunHostCmd(namespaceSrc, podNameSrc, "ping -6 -c10 www.google.com") + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(strings.Contains(output, "0% packet loss")).To(o.BeTrue()) + } +} + +// Get MTU value of br-ex on the node +func GetNodeMTU(oc *exutil.CLI, nodeName string) int { + cmdMTU := `ip a show br-ex |grep mtu` + out, err := DebugNodeWithChroot(oc, nodeName, "bash", "-c", cmdMTU) + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("%s", out) + // Define regex to capute mtu value + re := regexp.MustCompile(`mtu (\d+)`) + match := re.FindStringSubmatch(out) + o.Expect(len(match) > 1).Should(o.BeTrue()) + e2e.Logf("MTU on node %s is : %s", nodeName, match[1]) + mtu, err := strconv.Atoi(match[1]) + if err == nil { + return mtu + } else { + return 0 + } + +} + +// get mcp status as per nodeType +func GetmcpStatus(oc *exutil.CLI, nodeRole string) error { + return wait.Poll(60*time.Second, 15*time.Minute, func() (bool, error) { + status, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("mcp", nodeRole, "-ojsonpath={.status.conditions[?(@.type=='Updating')].status}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("\nCurrent mcp UPDATING Status is %s\n", status) + if strings.Contains(status, "False") { + e2e.Logf("\nmcp updated successfully ") + } else { + e2e.Logf("\nmcp is still in UPDATING state") + return false, nil + } + return true, nil + }) +} + +// Enable/Disable IPsec, network co will reflect the state from mcp. +func CheckCNORenderState(oc *exutil.CLI) { + errCheck := wait.PollUntilContextTimeout(context.Background(), 10*time.Second, 180*time.Second, false, func(cxt context.Context) (bool, error) { + expectedStatus := "True.*True.*False.*machine config pool in progressing state" + output, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("co", "network").Output() + if err != nil { + e2e.Logf("Fail to get clusteroperator network, error:%s. Trying again", err) + return false, nil + } + if matched, _ := regexp.MatchString(expectedStatus, output); !matched { + e2e.Logf("Network operator state is:%s", output) + return false, nil + } + return true, nil + }) + o.Expect(errCheck).NotTo(o.HaveOccurred(), "Timed out waiting for the expected condition") +} + +// IPSec full mode option, support "Always/Auto" +func ConfigIPSecEncyptOption(oc *exutil.CLI, encyptionOption string) { + e2e.Logf("Configure IPSec full mode option as %s", encyptionOption) + _, err := oc.AsAdmin().WithoutNamespace().Run("patch").Args("networks.operator.openshift.io", "cluster", "-p", "{\"spec\":{\"defaultNetwork\":{\"ovnKubernetesConfig\":{\"ipsecConfig\":{\"mode\":\"Full\",\"full\":{\"encapsulation\": \""+encyptionOption+"\"}}}}}}", "--type=merge").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + + // check ovnkube-node ds rollout status and confirm if rollout has triggered + checkErr := wait.PollUntilContextTimeout(context.Background(), 10*time.Second, 120*time.Second, false, func(cxt context.Context) (bool, error) { + status, err := oc.AsAdmin().WithoutNamespace().Run("rollout").Args("status", "-n", "openshift-ovn-kubernetes", "ds", "ovnkube-node", "--timeout", "5m").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + if strings.Contains(status, "rollout to finish") && strings.Contains(status, "successfully rolled out") { + e2e.Logf("ovnkube rollout was triggerred and rolled out successfully") + return true, nil + } + e2e.Logf("ovnkube rollout trigger hasn't happened yet. Trying again") + return false, nil + }) + o.Expect(checkErr).NotTo(o.HaveOccurred(), "Timed out waiting for ovnkube-node ds rollout.") + +} + +// Wait mcp to be expected status. +// Updating:True/False Updated: True/False Degraded:True/False +func WaitMCPExpectedStatus(oc *exutil.CLI, mcp, mcptype, expectedStatus string) { + // check ovnkube-node ds rollout status and confirm if rollout has triggered + checkErr := wait.PollUntilContextTimeout(context.Background(), 10*time.Second, 120*time.Second, false, func(cxt context.Context) (bool, error) { + status, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("mcp", mcp, "-o=jsonpath={.status.conditions[?(@.type==\""+mcptype+"\")].status}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("MCP %s %s expected status is %s, current status is %s", mcp, mcptype, expectedStatus, status) + if strings.Contains(expectedStatus, status) { + return true, nil + } + return false, nil + }) + + o.Expect(checkErr).NotTo(o.HaveOccurred(), "Timed out waiting for mcp to be expected status.") +} + +func VerifyIPSecLoadedInContainers(oc *exutil.CLI, num int) { + e2e.Logf("The cluster is using IPSec containers") + var expectedNum int + if num == 0 { + expectedNum = 0 + } else { + expectedNum = (num - 1) * 2 + } + cmd := `ipsec status | grep "Total IPsec connections"` + ipsecNS := "openshift-ovn-kubernetes" + // After node reboot, need to wait the IPSec connections loaded + ipsecPods := GetPodName(oc, ipsecNS, "app=ovn-ipsec") + o.Expect(len(ipsecPods) > 0).Should(o.BeTrue()) + + checkErr := wait.PollUntilContextTimeout(context.Background(), 10*time.Second, 120*time.Second, false, func(cxt context.Context) (bool, error) { + out, err := RemoteShPodWithBashSpecifyContainer(oc, ipsecNS, ipsecPods[0], "ovn-ipsec", cmd) + e2e.Logf("Total IPsec connection is : \n%s", out) + return strings.Contains(out, fmt.Sprintf("active %v", expectedNum)), err + }) + o.Expect(checkErr).NotTo(o.HaveOccurred(), "IPsec connections were not loaded completely!") +} + +func IsRDUPlatformSuitable(oc *exutil.CLI) bool { + msg, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("routes", "console", "-n", "openshift-console").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + + validClusters := []string{ + "sriov.openshift-qe.sdn.com", + "offload.openshift-qe.sdn.com", + "sdn146.openshift-qe.sdn.com", + "sdn147.openshift-qe.sdn.com", + "sdn148.openshift-qe.sdn.com", + "sdn149.openshift-qe.sdn.com", + "sdn150.openshift-qe.sdn.com", + } + matched := false + for _, cluster := range validClusters { + if strings.Contains(msg, cluster) { + matched = true + break + } + } + if !matched { + g.Skip("This case will only run on QE RDU clusters. Skipping for other environments!") + return false + } + return true +} + +func GetOpenshiftVersion(oc *exutil.CLI) string { + version, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("clusterversion/version", "-ojsonpath={.status.desired.version}").Output() + if err != nil { + return "" + } + re := regexp.MustCompile(`^(\d+\.\d+)`) + matches := re.FindStringSubmatch(version) + if len(matches) < 2 { + return "" + } + return matches[1] +} + +func CreateImageDigestMirrorSet(oc *exutil.CLI, imagedigestmirrorsetname string, imageDigestMirrorSetFile string) error { + pollInterval := 10 * time.Second + waitTimeout := 120 * time.Second + err := oc.AsAdmin().WithoutNamespace().Run("create").Args("-f", imageDigestMirrorSetFile).Execute() + if err != nil { + return fmt.Errorf("Error applying image digest mirror set: %w", err) + } + return wait.Poll(pollInterval, waitTimeout, func() (bool, error) { + err := oc.AsAdmin().WithoutNamespace(). + Run("get").Args("imagedigestmirrorset", imagedigestmirrorsetname).Execute() + return err == nil, nil + }) +} + +func CreateCatalogSource(oc *exutil.CLI, operatorName string, catalogSourceName string, catalogNamespace string, catalogSourceTemplateFile string) error { + pollInterval := 10 * time.Second + waitTimeout := 120 * time.Second + openshiftVersion := GetOpenshiftVersion(oc) + if openshiftVersion == "" { + return fmt.Errorf("Failed to get OpenShift version") + } + image := "quay.io/redhat-user-workloads/ocp-art-tenant/art-fbc:ocp__" + openshiftVersion + "__" + operatorName + "-rhel9-operator" + e2e.Logf("Creating catalog source with name '%s' in namespace '%s'", catalogSourceName, catalogNamespace) + err := ApplyResourceFromTemplateByAdmin(oc, "--ignore-unknown-parameters=true", "-f", catalogSourceTemplateFile, "-p", "CATALOGSOURCENAME="+catalogSourceName, "CATALOGNAMESPACE="+catalogNamespace, "IMAGE="+image) + if err != nil { + return fmt.Errorf("Error applying catalog source: %w", err) + } + + // Wait for CatalogSource to exist and be ready + return wait.Poll(pollInterval, waitTimeout, func() (bool, error) { + // Check if CatalogSource exists + err := oc.AsAdmin().WithoutNamespace().Run("get").Args("catalogsource", catalogSourceName, "-n", catalogNamespace).Execute() + if err != nil { + e2e.Logf("CatalogSource not found yet: %v", err) + return false, nil + } + + // Check if CatalogSource connection state is READY + connectionState, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("catalogsource", catalogSourceName, + "-n", catalogNamespace, "-o=jsonpath={.status.connectionState.lastObservedState}").Output() + if err != nil { + e2e.Logf("Failed to get connection state: %v", err) + return false, nil + } + + if string(connectionState) != "READY" { + e2e.Logf("CatalogSource connection state is '%s', waiting for 'READY'", string(connectionState)) + return false, nil + } + + // Check if registry pod is running and ready + podName, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("pods", "-n", catalogNamespace, + "-l", "olm.catalogSource="+catalogSourceName, + "-o=jsonpath={.items[0].metadata.name}").Output() + if err != nil || len(podName) == 0 { + e2e.Logf("Registry pod not found yet: %v", err) + return false, nil + } + + // Check pod ready condition + podReady, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("pod", string(podName), "-n", catalogNamespace, + "-o=jsonpath={.status.conditions[?(@.type=='Ready')].status}").Output() + if err != nil { + e2e.Logf("Failed to get pod ready status: %v", err) + return false, nil + } + + if string(podReady) != "True" { + e2e.Logf("Registry pod '%s' is not ready yet: %s", string(podName), string(podReady)) + return false, nil + } + e2e.Logf("CatalogSource '%s' is ready with pod '%s'", catalogSourceName, string(podName)) + return true, nil + }) +} + +func GetOperatorCatalogSource(oc *exutil.CLI, catalog string, namespace string) string { + if IsBaselineCapsSet(oc) && !(IsEnabledCapability(oc, "OperatorLifecycleManager")) { + g.Skip("Skipping the test as baselinecaps have been set and OperatorLifecycleManager capability is not enabled!") + } + catalogSourceNames, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("catalogsource", "-n", namespace, "-o=jsonpath={.items[*].metadata.name}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + if strings.Contains(catalogSourceNames, catalog) { + return catalog + } else { + return "" + } +} + +func GetImageDigestMirrorSet(oc *exutil.CLI, imagedigestmirrorsetname string) string { + imageDigestMirrorSetNames, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("imagedigestmirrorset", "-o=jsonpath={.items[*].metadata.name}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + if strings.Contains(imageDigestMirrorSetNames, imagedigestmirrorsetname) { + return imagedigestmirrorsetname + } else { + return "" + } +} + +// SetupOperatorCatalogSource checks if stage-registry-idms exists and sets up the catalog source accordingly. +// If stage-registry-idms exists, it returns "qe-app-registry" as the catalog source name without creating resources. +// Otherwise, it creates the image digest mirror set and catalog source as needed, then returns the catalog source name. +// operatorName: the name of the operator (e.g., "ingress-node-firewall", "kubernetes-nmstate", "metallb") +func SetupOperatorCatalogSource(oc *exutil.CLI, operatorName, defaultCatalogSourceName, imageDigestMirrorSetName, catalogNamespace, imageDigestMirrorSetFile, catalogSourceTemplate string) string { + g.By("Check the image digest mirror set and catalog source") + // Check if stage-registry-idms exists + idmsList, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("idms", "-o=jsonpath={.items[*].metadata.name}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + + if strings.Contains(idmsList, "stage-registry-idms") { + g.By("stage-registry-idms found, using qe-app-registry directly") + return "qe-app-registry" + } + + // Create image digest mirror set if needed + imageDigestMirrorSet := GetImageDigestMirrorSet(oc, imageDigestMirrorSetName) + if imageDigestMirrorSet == "" { + g.By("Creating image digest mirror set") + o.Expect(CreateImageDigestMirrorSet(oc, imageDigestMirrorSetName, imageDigestMirrorSetFile)).NotTo(o.HaveOccurred()) + } + + // Create catalog source if needed + catalogSource := GetOperatorCatalogSource(oc, defaultCatalogSourceName, catalogNamespace) + if catalogSource == "" { + g.By("Creating catalog source") + o.Expect(CreateCatalogSource(oc, operatorName, defaultCatalogSourceName, catalogNamespace, catalogSourceTemplate)).NotTo(o.HaveOccurred()) + } + + return defaultCatalogSourceName +} + + +// --- Helper functions migrated from openshift-tests-private --- + +// SshClient is a simple SSH client struct +type SshClient struct { + User string + Host string + Port int + PrivateKey string +} + +func (c *SshClient) Run(cmd string) error { + sshCmd := fmt.Sprintf("ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i %s -p %d %s@%s '%s'", + c.PrivateKey, c.Port, c.User, c.Host, cmd) + _, err := exec.Command("bash", "-c", sshCmd).CombinedOutput() + return err +} + +func SetNamespacePrivileged(oc *exutil.CLI, namespace string) { + err := oc.AsAdmin().WithoutNamespace().Run("label").Args("ns", namespace, "security.openshift.io/scc.podSecurityLabelSync=false", "--overwrite").Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + err = oc.AsAdmin().WithoutNamespace().Run("label").Args("ns", namespace, "pod-security.kubernetes.io/enforce=privileged", "--overwrite").Execute() + o.Expect(err).NotTo(o.HaveOccurred()) +} + +func RecoverNamespaceRestricted(oc *exutil.CLI, namespace string) { + _ = oc.AsAdmin().WithoutNamespace().Run("label").Args("ns", namespace, "pod-security.kubernetes.io/enforce=restricted", "--overwrite").Execute() + _ = oc.AsAdmin().WithoutNamespace().Run("label").Args("ns", namespace, "security.openshift.io/scc.podSecurityLabelSync=true", "--overwrite").Execute() +} + +func GetSAToken(oc *exutil.CLI) (string, error) { + return oc.AsAdmin().WithoutNamespace().Run("create").Args("token", "prometheus-k8s", "-n", "openshift-monitoring").Output() +} + +func GetAllNodesbyOSType(oc *exutil.CLI, osType string) ([]string, error) { + output, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("nodes", "-l", fmt.Sprintf("kubernetes.io/os=%s", osType), "-o=jsonpath={.items[*].metadata.name}").Output() + if err != nil { + return nil, err + } + if output == "" { + return []string{}, nil + } + return strings.Split(output, " "), nil +} + +func GetOVNKPodOnNode(oc *exutil.CLI, namespace string, label string, nodeName string) (string, error) { + output, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("pods", "-n", namespace, "-l", label, "--field-selector", "spec.nodeName="+nodeName, "-o=jsonpath={.items[0].metadata.name}").Output() + return output, err +} + +func GetPodNodeName(oc *exutil.CLI, namespace string, podName string) (string, error) { + nodeName, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("pod", "-n", namespace, podName, "-o=jsonpath={.spec.nodeName}").Output() + return nodeName, err +} + +func RemoteShPodWithBash(oc *exutil.CLI, namespace string, podName string, command string) (string, error) { + output, err := oc.AsAdmin().WithoutNamespace().Run("exec").Args("-n", namespace, podName, "--", "bash", "-c", command).Output() + return output, err +} + +func RemoteShPodWithBashSpecifyContainer(oc *exutil.CLI, namespace string, podName string, containerName string, command string) (string, error) { + output, err := oc.AsAdmin().WithoutNamespace().Run("exec").Args("-n", namespace, "-c", containerName, podName, "--", "bash", "-c", command).Output() + return output, err +} + +func DebugNodeWithChroot(oc *exutil.CLI, nodeName string, cmd ...string) (string, error) { + args := []string{"-n", "default", "node/" + nodeName, "--"} + chrootCmd := append([]string{"chroot", "/host"}, cmd...) + args = append(args, chrootCmd...) + output, err := oc.AsAdmin().WithoutNamespace().Run("debug").Args(args...).Output() + return output, err +} + +func DebugNodeWithOptionsAndChroot(oc *exutil.CLI, nodeName string, options []string, cmd ...string) (string, error) { + args := []string{"-n", "default", "node/" + nodeName} + args = append(args, options...) + args = append(args, "--") + chrootCmd := append([]string{"chroot", "/host"}, cmd...) + args = append(args, chrootCmd...) + output, err := oc.AsAdmin().WithoutNamespace().Run("debug").Args(args...).Output() + return output, err +} + +func GetSpecificPodLogs(oc *exutil.CLI, namespace string, containerName string, podName string, filter string) (string, error) { + output, err := oc.AsAdmin().WithoutNamespace().Run("logs").Args("-n", namespace, "-c", containerName, podName).Output() + if err != nil { + return "", err + } + var filteredLines []string + for _, line := range strings.Split(output, "\n") { + if strings.Contains(line, filter) { + filteredLines = append(filteredLines, line) + } + } + return strings.Join(filteredLines, "\n"), nil +} + +func LabelPod(oc *exutil.CLI, namespace string, podName string, label string) error { + return oc.AsAdmin().WithoutNamespace().Run("label").Args("pod", podName, "-n", namespace, label).Execute() +} + +func DebugNode(oc *exutil.CLI, nodeName string, cmd ...string) (string, error) { + args := []string{"-n", "default", "node/" + nodeName, "--"} + args = append(args, cmd...) + output, err := oc.AsAdmin().WithoutNamespace().Run("debug").Args(args...).Output() + return output, err +} + +func GetAllPodsWithLabel(oc *exutil.CLI, namespace string, label string) ([]string, error) { + output, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("pods", "-n", namespace, "-l", label, "-o=jsonpath={.items[*].metadata.name}").Output() + if err != nil { + return nil, err + } + if output == "" { + return []string{}, nil + } + return strings.Split(output, " "), nil +} + +func GetClusterNodesBy(oc *exutil.CLI, role string) ([]string, error) { + output, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("nodes", "-l", fmt.Sprintf("node-role.kubernetes.io/%s=", role), "-o=jsonpath={.items[*].metadata.name}").Output() + if err != nil { + return nil, err + } + if output == "" { + return []string{}, nil + } + return strings.Split(output, " "), nil +} + +func GetFirstMasterNode(oc *exutil.CLI) (string, error) { + nodes, err := GetClusterNodesBy(oc, "master") + if err != nil || len(nodes) == 0 { + return "", fmt.Errorf("no master nodes found: %v", err) + } + return nodes[0], nil +} + +func GetFirstWorkerNode(oc *exutil.CLI) (string, error) { + nodes, err := GetClusterNodesBy(oc, "worker") + if err != nil || len(nodes) == 0 { + return "", fmt.Errorf("no worker nodes found: %v", err) + } + return nodes[0], nil +} + +func GetSchedulableLinuxWorkerNodes(oc *exutil.CLI) ([]string, error) { + output, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("nodes", "-l", "kubernetes.io/os=linux,node-role.kubernetes.io/worker=", "--field-selector", "spec.unschedulable!=true", "-o=jsonpath={.items[*].metadata.name}").Output() + if err != nil { + return nil, err + } + if output == "" { + return []string{}, nil + } + return strings.Split(output, " "), nil +} + +func GetSpecificPodLogsCombinedOrNot(oc *exutil.CLI, namespace string, containerName string, podName string, filter string, combined bool) (string, error) { + var output string + var err error + if combined { + output, err = oc.AsAdmin().WithoutNamespace().Run("logs").Args("-n", namespace, "-c", containerName, podName).Output() + } else { + output, err = oc.AsAdmin().WithoutNamespace().Run("logs").Args("-n", namespace, "-c", containerName, podName).Output() + } + if err != nil { + return "", err + } + if filter == "" { + return output, nil + } + var filteredLines []string + for _, line := range strings.Split(output, "\n") { + if strings.Contains(line, filter) { + filteredLines = append(filteredLines, line) + } + } + return strings.Join(filteredLines, "\n"), nil +} + +func IsHypershiftHostedCluster(oc *exutil.CLI) bool { + output, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("infrastructure", "cluster", "-o=jsonpath={.status.controlPlaneTopology}").Output() + if err != nil { + return false + } + return strings.Contains(output, "External") +} + +func AssertOrCheckMCP(oc *exutil.CLI, pool string, interval time.Duration, timeout time.Duration, assert bool) error { + return wait.Poll(interval, timeout, func() (bool, error) { + output, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("mcp", pool, "-o=jsonpath={.status.conditions[?(@.type==\"Updated\")].status}").Output() + if err != nil { + e2e.Logf("Failed to get MCP status: %v", err) + return false, nil + } + if output == "True" { + return true, nil + } + e2e.Logf("MCP %s not updated yet, status: %s", pool, output) + return false, nil + }) +} + +func GetIfaddrFromNode(nodeName string, oc *exutil.CLI) string { + output, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("node", nodeName, "-o=jsonpath={.status.addresses[?(@.type==\"InternalIP\")].address}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + if output == "" { + return "" + } + // Get the subnet from the IP + ip := strings.Split(output, " ")[0] + parts := strings.Split(ip, ".") + if len(parts) == 4 { + return strings.Join(parts[:3], ".") + } + return ip +} + +func EstimateTimeoutForEgressIP(oc *exutil.CLI) time.Duration { + nodeList, err := e2enode.GetReadySchedulableNodes(context.TODO(), oc.KubeFramework().ClientSet) + o.Expect(err).NotTo(o.HaveOccurred()) + // More nodes means longer timeout + timeout := time.Duration(len(nodeList.Items)*30+60) * time.Second + if timeout < 5*time.Minute { + timeout = 5 * time.Minute + } + return timeout +} + +func FindUnUsedIPsOnNode(oc *exutil.CLI, nodeName, cidr string, expectedNum int) []string { + ipRange, _ := Hosts(cidr) + var ipUnused []string + rand.Seed(time.Now().UnixNano()) + rand.Shuffle(len(ipRange), func(i, j int) { ipRange[i], ipRange[j] = ipRange[j], ipRange[i] }) + for _, ip := range ipRange { + if len(ipUnused) < expectedNum { + pingCmd := fmt.Sprintf("ping -c1 -W1 %s", ip) + _, err := DebugNodeWithChroot(oc, nodeName, "bash", "-c", pingCmd) + if err != nil { + e2e.Logf("%s is not used on node %s", ip, nodeName) + ipUnused = append(ipUnused, ip) + } + } else { + break + } + } + return ipUnused +} + +// NetworkPolicyResource is a struct for creating network policies from templates +type NetworkPolicyResource struct { + Name string + Namespace string + Policy string + PolicyType string + Direction1 string + NamespaceSel1 string + NamespaceSelKey1 string + NamespaceSelVal1 string + Template string +} + +func (np *NetworkPolicyResource) CreateNetworkPolicy(oc *exutil.CLI) { + err := wait.Poll(5*time.Second, 20*time.Second, func() (bool, error) { + err1 := ApplyResourceFromTemplateByAdmin(oc, "--ignore-unknown-parameters=true", "-f", np.Template, "-p", + "NAME="+np.Name, "NAMESPACE="+np.Namespace, "POLICY="+np.Policy, "POLICYTYPE="+np.PolicyType, + "DIRECTION1="+np.Direction1, "NAMESPACESEL1="+np.NamespaceSel1, + "NAMESPACESELKEY1="+np.NamespaceSelKey1, "NAMESPACESELVAL1="+np.NamespaceSelVal1) + if err1 != nil { + e2e.Logf("the err:%v, and try next round", err1) + return false, nil + } + return true, nil + }) + o.Expect(err).NotTo(o.HaveOccurred(), fmt.Sprintf("fail to create network policy %v", np.Name)) +} + +func SkipBaselineCaps(oc *exutil.CLI, sets string) { + baselineCapabilitySet, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("clusterversion", "version", "-o=jsonpath={.spec.capabilities.baselineCapabilitySet}").Output() + if err != nil { + e2e.Failf("get baselineCapabilitySet failed err %v .", err) + } + sets = strings.ReplaceAll(sets, " ", "") + for _, s := range strings.Split(sets, ",") { + if strings.Contains(baselineCapabilitySet, s) { + g.Skip("Skip for cluster with baselineCapabilitySet = '" + baselineCapabilitySet + "' matching filter: " + s) + } + } +} diff --git a/openshift/test/otp/network_segmentation.go b/openshift/test/otp/network_segmentation.go new file mode 100644 index 0000000000..c412bf9327 --- /dev/null +++ b/openshift/test/otp/network_segmentation.go @@ -0,0 +1,1129 @@ +package otp + +import ( + "context" + "fmt" + "net" + "path/filepath" + "regexp" + "strconv" + "strings" + "time" + + g "github.com/onsi/ginkgo/v2" + o "github.com/onsi/gomega" + + exutil "github.com/openshift/origin/test/extended/util" + otputils "github.com/ovn-kubernetes/ovn-kubernetes/openshift/pkg/otp/utils" + "github.com/ovn-kubernetes/ovn-kubernetes/openshift/pkg/otp/testdata" + + "k8s.io/apimachinery/pkg/util/wait" + e2e "k8s.io/kubernetes/test/e2e/framework" + e2enode "k8s.io/kubernetes/test/e2e/framework/node" + e2eoutput "k8s.io/kubernetes/test/e2e/framework/pod/output" +) + +var _ = g.Describe("[sig-network][Suite:openshift/ovn-kubernetes] SDN network segmentation", func() { + defer g.GinkgoRecover() + + var oc = exutil.NewCLI("otp-network-segmentation") + + var testDataDirUDN = testdata.FixturePath("networking", "network_segmentation", "udn") + + // helper: check NAD exists in namespace + checkNAD := func(oc *exutil.CLI, ns string, nad string) bool { + nadOutput, nadOutputErr := oc.AsAdmin().WithoutNamespace().Run("get").Args("net-attach-def", "-n", ns).Output() + o.Expect(nadOutputErr).NotTo(o.HaveOccurred()) + return strings.Contains(nadOutput, nad) + } + + // helper: contains for string slice + containsStr := func(s []string, str string) bool { + for _, v := range s { + if v == str { + return true + } + } + return false + } + + // helper: wait for all pods in a namespace to be ready + waitAllPodsReady := func(oc *exutil.CLI, namespace string) { + err := wait.Poll(10*time.Second, 4*time.Minute, func() (bool, error) { + template := "'{{- range .items -}}{{- range .status.conditions -}}{{- if ne .reason \"PodCompleted\" -}}{{- if eq .type \"Ready\" -}}{{- .status}} {{\" \"}}{{- end -}}{{- end -}}{{- end -}}{{- end -}}'" + stdout, err := oc.AsAdmin().Run("get").Args("pods", "-n", namespace).Template(template).Output() + if err != nil { + e2e.Logf("the err:%v, and try next round", err) + return false, nil + } + if strings.Contains(stdout, "False") { + return false, nil + } + return true, nil + }) + o.Expect(err).NotTo(o.HaveOccurred(), fmt.Sprintf("Some Pods are not ready in NS %s!", namespace)) + } + + g.It("[JIRA:Networking][OTP][Serial] 75223-Restarting ovn pods should not break UDN primary network traffic", func() { + var ( + testDataDir = testdata.FixturePath("networking") + udnNadtemplate = filepath.Join(testDataDirUDN, "udn_nad_template.yaml") + testPodFile = filepath.Join(testDataDir, "testpod.yaml") + ) + + ipStackType := otputils.CheckIPStackType(oc) + g.By("1. Create first namespace") + oc.CreateNamespaceUDN() + ns1 := oc.Namespace() + + g.By("2. Create 2nd namespace") + oc.CreateNamespaceUDN() + ns2 := oc.Namespace() + + nadResourcename := []string{"l3-network-" + ns1, "l3-network-" + ns2} + nadNS := []string{ns1, ns2} + var subnet []string + if ipStackType == "ipv4single" { + subnet = []string{"10.150.0.0/16/24", "10.151.0.0/16/24"} + } else { + if ipStackType == "ipv6single" { + subnet = []string{"2010:100:200::0/60", "2011:100:200::0/60"} + } else { + subnet = []string{"10.150.0.0/16/24,2010:100:200::0/60", "10.151.0.0/16/24,2011:100:200::0/60"} + } + } + + nad := make([]otputils.UdnNetDefResource, 2) + for i := 0; i < 2; i++ { + g.By(fmt.Sprintf("create NAD %s in namespace %s", nadResourcename[i], nadNS[i])) + nad[i] = otputils.UdnNetDefResource{ + Nadname: nadResourcename[i], + Namespace: nadNS[i], + NadNetworkName: nadResourcename[i], + Topology: "layer3", + Subnet: subnet[i], + NetAttachDefName: nadNS[i] + "/" + nadResourcename[i], + Role: "primary", + Template: udnNadtemplate, + } + nad[i].CreateUdnNad(oc) + g.By("Verifying the configured NetworkAttachmentDefinition") + if checkNAD(oc, nadNS[i], nadResourcename[i]) { + e2e.Logf("The correct network-attach-defintion: %v is created!", nadResourcename[i]) + } else { + e2e.Failf("The correct network-attach-defintion: %v is not created!", nadResourcename[i]) + } + } + + g.By("Create replica pods in ns1") + otputils.CreateResourceFromFile(oc, ns1, testPodFile) + err := otputils.WaitForPodWithLabelReady(oc, ns1, "name=test-pods") + o.Expect(err).NotTo(o.HaveOccurred(), "this pod with label name=test-pods not ready") + testpodNS1Names := otputils.GetPodName(oc, ns1, "name=test-pods") + otputils.CurlPod2PodPassUDN(oc, ns1, testpodNS1Names[0], ns1, testpodNS1Names[1]) + + g.By("create replica pods in ns2") + otputils.CreateResourceFromFile(oc, ns2, testPodFile) + err = otputils.WaitForPodWithLabelReady(oc, ns2, "name=test-pods") + o.Expect(err).NotTo(o.HaveOccurred(), "this pod with label name=test-pods not ready") + testpodNS2Names := otputils.GetPodName(oc, ns2, "name=test-pods") + otputils.CurlPod2PodPassUDN(oc, ns2, testpodNS2Names[0], ns2, testpodNS2Names[1]) + + g.By("Restart OVN pods") + err = oc.AsAdmin().WithoutNamespace().Run("delete").Args("pod", "--all", "-n", "openshift-ovn-kubernetes").Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + waitAllPodsReady(oc, "openshift-ovn-kubernetes") + + g.By("Verify the connection in UDN primary network not broken.") + otputils.CurlPod2PodPassUDN(oc, ns1, testpodNS1Names[0], ns1, testpodNS1Names[1]) + otputils.CurlPod2PodPassUDN(oc, ns2, testpodNS2Names[0], ns2, testpodNS2Names[1]) + }) + + g.It("[JIRA:Networking][OTP] 75254-Check kubelet probes are allowed via default network LSP for the UDN pods", func() { + var ( + udnCRDdualStack = filepath.Join(testDataDirUDN, "udn_crd_dualstack2_template.yaml") + udnCRDSingleStack = filepath.Join(testDataDirUDN, "udn_crd_singlestack_template.yaml") + udnPodLivenessTemplate = filepath.Join(testDataDirUDN, "udn_test_pod_liveness_template.yaml") + udnPodReadinessTemplate = filepath.Join(testDataDirUDN, "udn_test_pod_readiness_template.yaml") + udnPodStartupTemplate = filepath.Join(testDataDirUDN, "udn_test_pod_startup_template.yaml") + livenessProbePort = 8080 + readinessProbePort = 8081 + startupProbePort = 1234 + ) + + g.By("1. Create privileged namespace") + oc.CreateNamespaceUDN() + ns := oc.Namespace() + otputils.SetNamespacePrivileged(oc, ns) + + g.By("2. Create CRD for UDN") + ipStackType := otputils.CheckIPStackType(oc) + var cidr, ipv4cidr, ipv6cidr string + var prefix, ipv4prefix, ipv6prefix int32 + if ipStackType == "ipv4single" { + cidr = "10.150.0.0/16" + prefix = 24 + } else { + if ipStackType == "ipv6single" { + cidr = "2010:100:200::0/48" + prefix = 64 + } else { + ipv4cidr = "10.150.0.0/16" + ipv4prefix = 24 + ipv6cidr = "2010:100:200::0/48" + ipv6prefix = 64 + } + } + var udncrd otputils.UdnCRDResource + if ipStackType == "dualstack" { + udncrd = otputils.UdnCRDResource{ + Crdname: "udn-network-ds-75254", + Namespace: ns, + Role: "Primary", + IPv4cidr: ipv4cidr, + IPv4prefix: ipv4prefix, + IPv6cidr: ipv6cidr, + IPv6prefix: ipv6prefix, + Template: udnCRDdualStack, + } + udncrd.CreateUdnCRDDualStack(oc) + } else { + udncrd = otputils.UdnCRDResource{ + Crdname: "udn-network-ss-75254", + Namespace: ns, + Role: "Primary", + Cidr: cidr, + Prefix: prefix, + Template: udnCRDSingleStack, + } + udncrd.CreateUdnCRDSingleStack(oc) + } + err := otputils.WaitUDNCRDApplied(oc, ns, udncrd.Crdname) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("3. Create a udn hello pod with liveness probe in ns1") + pod1 := otputils.UdnPodWithProbeResource{ + Name: "hello-pod-ns1-liveness", + Namespace: ns, + Label: "hello-pod", + Port: livenessProbePort, + Failurethreshold: 1, + Periodseconds: 1, + Template: udnPodLivenessTemplate, + } + pod1.CreateUdnPodWithProbe(oc) + otputils.WaitPodReady(oc, pod1.Namespace, pod1.Name) + + g.By("4. Capture packets in pod " + pod1.Name + ", check liveness probe traffic is allowed via default network") + tcpdumpCmd1 := fmt.Sprintf("timeout 5s tcpdump -nni eth0 port %v", pod1.Port) + cmdTcpdump1, cmdOutput1, _, err1 := oc.AsAdmin().WithoutNamespace().Run("exec").Args("-n", ns, pod1.Name, "--", "bash", "-c", tcpdumpCmd1).Background() + defer cmdTcpdump1.Process.Kill() + o.Expect(err1).NotTo(o.HaveOccurred()) + cmdTcpdump1.Wait() + e2e.Logf("The captured packet is %s", cmdOutput1.String()) + expPacket1 := strconv.Itoa(pod1.Port) + ": Flags [S]" + o.Expect(strings.Contains(cmdOutput1.String(), expPacket1)).To(o.BeTrue()) + + g.By("5. Create a udn hello pod with readiness probe in ns1") + pod2 := otputils.UdnPodWithProbeResource{ + Name: "hello-pod-ns1-readiness", + Namespace: ns, + Label: "hello-pod", + Port: readinessProbePort, + Failurethreshold: 1, + Periodseconds: 1, + Template: udnPodReadinessTemplate, + } + pod2.CreateUdnPodWithProbe(oc) + otputils.WaitPodReady(oc, pod2.Namespace, pod2.Name) + + g.By("6. Capture packets in pod " + pod2.Name + ", check readiness probe traffic is allowed via default network") + tcpdumpCmd2 := fmt.Sprintf("timeout 5s tcpdump -nni eth0 port %v", pod2.Port) + cmdTcpdump2, cmdOutput2, _, err2 := oc.AsAdmin().WithoutNamespace().Run("exec").Args("-n", ns, pod2.Name, "--", "bash", "-c", tcpdumpCmd2).Background() + defer cmdTcpdump2.Process.Kill() + o.Expect(err2).NotTo(o.HaveOccurred()) + cmdTcpdump2.Wait() + e2e.Logf("The captured packet is %s", cmdOutput2.String()) + expPacket2 := strconv.Itoa(pod2.Port) + ": Flags [S]" + o.Expect(strings.Contains(cmdOutput2.String(), expPacket2)).To(o.BeTrue()) + + g.By("7. Create a udn hello pod with startup probe in ns1") + pod3 := otputils.UdnPodWithProbeResource{ + Name: "hello-pod-ns1-startup", + Namespace: ns, + Label: "hello-pod", + Port: startupProbePort, + Failurethreshold: 100, + Periodseconds: 2, + Template: udnPodStartupTemplate, + } + pod3.CreateUdnPodWithProbe(oc) + + g.By("8. Start tcpdump before pod becomes Ready to capture startup probe traffic via default network") + o.Eventually(func() string { + phase, _ := oc.AsAdmin().WithoutNamespace().Run("get").Args("pod", pod3.Name, "-n", ns, "-o=jsonpath={.status.phase}").Output() + return phase + }, "60s", "2s").Should(o.Equal("Running")) + tcpdumpCmd3 := fmt.Sprintf("timeout 30s tcpdump -nni eth0 port %v", pod3.Port) + cmdTcpdump3, cmdOutput3, _, err3 := oc.AsAdmin().WithoutNamespace().Run("exec").Args("-n", ns, pod3.Name, "--", "bash", "-c", tcpdumpCmd3).Background() + o.Expect(err3).NotTo(o.HaveOccurred()) + defer func() { + if cmdTcpdump3 != nil && cmdTcpdump3.Process != nil { + _ = cmdTcpdump3.Process.Kill() + } + }() + otputils.WaitPodReady(oc, pod3.Namespace, pod3.Name) + cmdTcpdump3.Wait() + e2e.Logf("The captured packet is %s", cmdOutput3.String()) + expPacket3 := strconv.Itoa(pod3.Port) + ": Flags [S]" + o.Expect(strings.Contains(cmdOutput3.String(), expPacket3)).To(o.BeTrue()) + }) + + g.It("[JIRA:Networking][OTP] 75503-Overlapping pod CIDRs IPs are allowed in different primary NADs", func() { + var ( + testDataDir = testdata.FixturePath("networking") + udnNadtemplate = filepath.Join(testDataDirUDN, "udn_nad_template.yaml") + testPodFile = filepath.Join(testDataDir, "testpod.yaml") + ) + + nodeList, err := e2enode.GetReadySchedulableNodes(context.TODO(), oc.KubeFramework().ClientSet) + o.Expect(err).NotTo(o.HaveOccurred()) + if len(nodeList.Items) < 2 { + g.Skip("This case requires 2 nodes, but the cluster has fewer than two nodes.") + } + + ipStackType := otputils.CheckIPStackType(oc) + g.By("1. Obtain first namespace") + oc.CreateNamespaceUDN() + ns1 := oc.Namespace() + + g.By("2. Obtain 2nd namespace") + oc.CreateNamespaceUDN() + ns2 := oc.Namespace() + + nadResourcename := []string{"l3-network-" + ns1, "l3-network-" + ns2} + nadNS := []string{ns1, ns2} + var subnet []string + if ipStackType == "ipv4single" { + subnet = []string{"10.150.0.0/26/29", "10.150.0.0/26/29"} + } else { + if ipStackType == "ipv6single" { + subnet = []string{"2010:100:200::0/60", "2010:100:200::0/60"} + } else { + subnet = []string{"10.150.0.0/26/29,2010:100:200::0/60", "10.150.0.0/26/29,2010:100:200::0/60"} + } + } + + nad := make([]otputils.UdnNetDefResource, 2) + for i := 0; i < 2; i++ { + g.By(fmt.Sprintf("create NAD %s in namespace %s", nadResourcename[i], nadNS[i])) + nad[i] = otputils.UdnNetDefResource{ + Nadname: nadResourcename[i], + Namespace: nadNS[i], + NadNetworkName: nadResourcename[i], + Topology: "layer3", + Subnet: subnet[i], + NetAttachDefName: nadNS[i] + "/" + nadResourcename[i], + Role: "primary", + Template: udnNadtemplate, + } + nad[i].CreateUdnNad(oc) + g.By("Verifying the configured NetworkAttachmentDefinition") + if checkNAD(oc, nadNS[i], nadResourcename[i]) { + e2e.Logf("The correct network-attach-defintion: %v is created!", nadResourcename[i]) + } else { + e2e.Failf("The correct network-attach-defintion: %v is not created!", nadResourcename[i]) + } + } + + g.By("Create replica pods in ns1") + otputils.CreateResourceFromFile(oc, ns1, testPodFile) + numberOfPods := "8" + err = oc.AsAdmin().WithoutNamespace().Run("scale").Args("rc", "test-rc", "--replicas="+numberOfPods, "-n", ns1).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + err = otputils.WaitForPodWithLabelReady(oc, ns1, "name=test-pods") + o.Expect(err).NotTo(o.HaveOccurred(), "this pod with label name=test-pods not ready") + testpodNS1Names := otputils.GetPodName(oc, ns1, "name=test-pods") + e2e.Logf("Collect all the pods IPs in namespace %s", ns1) + var podsNS1IP1, podsNS1IP2 []string + for i := 0; i < len(testpodNS1Names); i++ { + podIP1, podIP2 := otputils.GetPodIPUDN(oc, ns1, testpodNS1Names[i], "ovn-udn1") + if podIP2 != "" { + podsNS1IP2 = append(podsNS1IP2, podIP2) + } + podsNS1IP1 = append(podsNS1IP1, podIP1) + } + e2e.Logf("The IPs of pods in first namespace %s for UDN:\n %v %v", ns1, podsNS1IP1, podsNS1IP2) + + g.By("create replica pods in ns2") + otputils.CreateResourceFromFile(oc, ns2, testPodFile) + err = oc.AsAdmin().WithoutNamespace().Run("scale").Args("rc", "test-rc", "--replicas="+numberOfPods, "-n", ns2).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + err = otputils.WaitForPodWithLabelReady(oc, ns2, "name=test-pods") + o.Expect(err).NotTo(o.HaveOccurred(), "this pod with label name=test-pods not ready") + testpodNS2Names := otputils.GetPodName(oc, ns2, "name=test-pods") + e2e.Logf("Collect all the pods IPs in namespace %s", ns2) + var podsNS2IP1, podsNS2IP2 []string + for i := 0; i < len(testpodNS2Names); i++ { + podIP1, podIP2 := otputils.GetPodIPUDN(oc, ns2, testpodNS2Names[i], "ovn-udn1") + if podIP2 != "" { + podsNS2IP2 = append(podsNS2IP2, podIP2) + } + podsNS2IP1 = append(podsNS2IP1, podIP1) + } + e2e.Logf("The IPs of pods in second namespace %s for UDN:\n %v %v", ns2, podsNS2IP1, podsNS2IP2) + + testpodNS1NamesLen := len(testpodNS1Names) + podsNS1IP1Len := len(podsNS1IP1) + podsNS1IP2Len := len(podsNS1IP2) + g.By("Verify udn network should be able to access in same network.") + for i := 0; i < testpodNS1NamesLen; i++ { + for j := 0; j < podsNS1IP1Len; j++ { + if podsNS1IP2Len > 0 && podsNS1IP2[j] != "" { + _, err = e2eoutput.RunHostCmd(ns1, testpodNS1Names[i], "curl --connect-timeout 5 -s "+net.JoinHostPort(podsNS1IP2[j], "8080")) + o.Expect(err).NotTo(o.HaveOccurred()) + } + _, err = e2eoutput.RunHostCmd(ns1, testpodNS1Names[i], "curl --connect-timeout 5 -s "+net.JoinHostPort(podsNS1IP1[j], "8080")) + o.Expect(err).NotTo(o.HaveOccurred()) + } + } + + podsNS2IP1Len := len(podsNS2IP1) + podsNS2IP2Len := len(podsNS2IP2) + g.By("Verify udn network should be isolated in different network.") + for i := 0; i < testpodNS1NamesLen; i++ { + for j := 0; j < podsNS2IP1Len; j++ { + if podsNS2IP2Len > 0 && podsNS2IP2[j] != "" { + if containsStr(podsNS1IP2, podsNS2IP2[j]) { + // as the destination IP in ns2 is same as one in NS1, then it will be able to access that IP and has been executed in previous steps. + continue + } else { + _, err = e2eoutput.RunHostCmd(ns1, testpodNS1Names[i], "curl --connect-timeout 5 -s "+net.JoinHostPort(podsNS2IP2[j], "8080")) + o.Expect(err).To(o.HaveOccurred()) + } + } + if containsStr(podsNS1IP1, podsNS2IP1[j]) { + // as the destination IP in ns2 is same as one in NS1, then it will be able to access that IP and has been executed in previous steps.. + continue + } else { + _, err = e2eoutput.RunHostCmd(ns1, testpodNS1Names[i], "curl --connect-timeout 5 -s "+net.JoinHostPort(podsNS2IP1[j], "8080")) + o.Expect(err).To(o.HaveOccurred()) + } + } + } + }) + + g.It("[JIRA:Networking][OTP] 75955-Verify UDN failed message when user defined join subnet overlaps user defined subnet Layer3", func() { + var ( + netsegDir = testdata.FixturePath("networking/network_segmentation") + udnCRDL3dualStack = filepath.Join(netsegDir, "udn/udn_crd_dualstack2_template.yaml") + udnCRDL3SingleStack = filepath.Join(netsegDir, "udn/udn_crd_singlestack_template.yaml") + UserDefinedPrimaryNetworkJoinSubnetV4 = "100.65.0.0/16" + UserDefinedPrimaryNetworkJoinSubnetV6 = "fd99::/48" + ) + + ipStackType := otputils.CheckIPStackType(oc) + g.By("1. Create namespace") + oc.CreateNamespaceUDN() + ns := oc.Namespace() + + g.By("2. Create CRD for UDN") + var udncrd otputils.UdnCRDResource + var cidr string + var prefix, ipv4prefix, ipv6prefix int32 + if ipStackType == "ipv4single" { + cidr = UserDefinedPrimaryNetworkJoinSubnetV4 + prefix = 24 + } else { + if ipStackType == "ipv6single" { + cidr = UserDefinedPrimaryNetworkJoinSubnetV6 + prefix = 64 + } else { + ipv4prefix = 24 + ipv6prefix = 64 + } + } + if ipStackType == "dualstack" { + udncrd = otputils.UdnCRDResource{ + Crdname: "udn-network-75995", + Namespace: ns, + Role: "Primary", + IPv4cidr: UserDefinedPrimaryNetworkJoinSubnetV4, + IPv4prefix: ipv4prefix, + IPv6cidr: UserDefinedPrimaryNetworkJoinSubnetV6, + IPv6prefix: ipv6prefix, + Template: udnCRDL3dualStack, + } + udncrd.CreateUdnCRDDualStack(oc) + } else { + udncrd = otputils.UdnCRDResource{ + Crdname: "udn-network-75995", + Namespace: ns, + Role: "Primary", + Cidr: cidr, + Prefix: prefix, + Template: udnCRDL3SingleStack, + } + udncrd.CreateUdnCRDSingleStack(oc) + } + err := otputils.WaitUDNCRDApplied(oc, ns, udncrd.Crdname) + o.Expect(err).To(o.HaveOccurred()) + + g.By("3. Check UDN failed message") + output, err := oc.AsAdmin().WithoutNamespace().Run("describe").Args("userdefinednetwork.k8s.ovn.org", udncrd.Crdname, "-n", ns).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).Should(o.Or( + o.ContainSubstring(fmt.Sprintf("user defined join subnet \"100.65.0.0/16\" overlaps user defined subnet \"%s\"", UserDefinedPrimaryNetworkJoinSubnetV4)), + o.ContainSubstring(fmt.Sprintf("user defined join subnet \"fd99::/64\" overlaps user defined subnet \"%s\"", UserDefinedPrimaryNetworkJoinSubnetV6)))) + }) + + g.It("[JIRA:Networking][OTP][Serial] 75984-Check udn pods isolation on user defined networks post OVN gateway migration", func() { + var ( + udnNadtemplate = filepath.Join(testDataDirUDN, "udn_nad_template.yaml") + udnPodTemplate = filepath.Join(testDataDirUDN, "udn_test_pod_template.yaml") + ) + + ipStackType := otputils.CheckIPStackType(oc) + + g.By("1. Create first namespace") + oc.CreateNamespaceUDN() + ns1 := oc.Namespace() + + g.By("2. Create 2nd namespace") + oc.CreateNamespaceUDN() + ns2 := oc.Namespace() + g.By("3. Create 3rd namespace") + oc.CreateNamespaceUDN() + ns3 := oc.Namespace() + + g.By("4. Create 4th namespace") + oc.CreateNamespaceUDN() + ns4 := oc.Namespace() + + nadResourcename := []string{"l3-network-" + ns1, "l3-network-" + ns2, "l2-network-" + ns3, "l2-network-" + ns4} + nadNS := []string{ns1, ns2, ns3, ns4} + topo := []string{"layer3", "layer3", "layer2", "layer2"} + + var subnet []string + if ipStackType == "ipv4single" { + subnet = []string{"10.150.0.0/16/24", "10.151.0.0/16/24", "10.152.0.0/16", "10.153.0.0/16"} + } else { + if ipStackType == "ipv6single" { + subnet = []string{"2010:100:200::0/60", "2011:100:200::0/60", "2012:100:200::0/60", "2013:100:200::0/60"} + } else { + subnet = []string{"10.150.0.0/16/24,2010:100:200::0/60", "10.151.0.0/16/24,2011:100:200::0/60", "10.152.0.0/16,2012:100:200::0/60", "10.153.0.0/16,2013:100:200::0/60"} + } + } + + nad := make([]otputils.UdnNetDefResource, 4) + for i := 0; i < 4; i++ { + g.By(fmt.Sprintf("create NAD %s in namespace %s", nadResourcename[i], nadNS[i])) + nad[i] = otputils.UdnNetDefResource{ + Nadname: nadResourcename[i], + Namespace: nadNS[i], + NadNetworkName: nadResourcename[i], + Topology: topo[i], + Subnet: subnet[i], + NetAttachDefName: nadNS[i] + "/" + nadResourcename[i], + Role: "primary", + Template: udnNadtemplate, + } + nad[i].CreateUdnNad(oc) + } + + pod := make([]otputils.UdnPodResource, 4) + for i := 0; i < 4; i++ { + g.By("create a udn hello pods in ns1 ns2 ns3 and ns4") + pod[i] = otputils.UdnPodResource{ + Name: "hello-pod", + Namespace: nadNS[i], + Label: "hello-pod", + Template: udnPodTemplate, + } + pod[i].CreateUdnPod(oc) + otputils.WaitPodReady(oc, pod[i].Namespace, pod[i].Name) + } + + g.By("create another udn hello pod in ns1 to ensure layer3 connectivity post migration among them") + podNs1 := otputils.UdnPodResource{ + Name: "hello-pod-ns1", + Namespace: nadNS[0], + Label: "hello-pod", + Template: udnPodTemplate, + } + podNs1.CreateUdnPod(oc) + otputils.WaitPodReady(oc, podNs1.Namespace, podNs1.Name) + + g.By("create another udn hello pod in ns3 to ensure layer2 connectivity post migration among them") + podNs3 := otputils.UdnPodResource{ + Name: "hello-pod-ns3", + Namespace: nadNS[2], + Label: "hello-pod", + Template: udnPodTemplate, + } + podNs3.CreateUdnPod(oc) + otputils.WaitPodReady(oc, podNs3.Namespace, podNs3.Name) + + // need to find out original mode cluster is on so that we can revert back to same post test + var desiredMode string + origMode := otputils.GetOVNGatewayMode(oc) + if origMode == "local" { + desiredMode = "shared" + } else { + desiredMode = "local" + } + e2e.Logf("Cluster is currently on gateway mode %s", origMode) + e2e.Logf("Desired mode is %s", desiredMode) + + defer otputils.SwitchOVNGatewayMode(oc, origMode) + otputils.SwitchOVNGatewayMode(oc, desiredMode) + + // udn network connectivity for layer3 should be isolated + otputils.CurlPod2PodFailUDN(oc, ns1, pod[0].Name, ns2, pod[1].Name) + // default network connectivity for layer3 should also be isolated + otputils.CurlPod2PodFail(oc, ns1, pod[0].Name, ns2, pod[1].Name) + + // udn network connectivity for layer2 should be isolated + otputils.CurlPod2PodFailUDN(oc, ns3, pod[2].Name, ns4, pod[3].Name) + // default network connectivity for layer2 should also be isolated + otputils.CurlPod2PodFail(oc, ns3, pod[2].Name, ns4, pod[3].Name) + + // ensure udn network connectivity for layer3 should be there + otputils.CurlPod2PodPassUDN(oc, ns1, pod[0].Name, ns1, podNs1.Name) + // ensure udn network connectivity for layer2 should be there + otputils.CurlPod2PodPassUDN(oc, ns3, pod[2].Name, ns3, podNs3.Name) + }) + + g.It("[JIRA:Networking][OTP][Serial] 76939-Check udn pods isolation on a scaled node", func() { + var ( + udnPodTemplate = filepath.Join(testDataDirUDN, "udn_test_pod_template.yaml") + udnPodTemplateNode = filepath.Join(testDataDirUDN, "udn_test_pod_template_node.yaml") + udnCRDSingleStack = filepath.Join(testDataDirUDN, "udn_crd_singlestack_template.yaml") + ) + + ipStackType := otputils.CheckIPStackType(oc) + o.Expect(ipStackType).NotTo(o.BeEmpty()) + if ipStackType != "ipv4single" { + g.Skip("This case requires IPv4 single stack cluster") + } + platform := otputils.CheckPlatform(oc) + supportedPlatforms := []string{"aws", "azure", "gcp", "vsphere", "ibmcloud", "openstack"} + platformSupported := false + for _, p := range supportedPlatforms { + if platform == p { + platformSupported = true + break + } + } + if !platformSupported { + g.Skip(fmt.Sprintf("Skipping test: platform %s is not supported for this test case", platform)) + } + + g.By("1. Create first namespace") + oc.CreateNamespaceUDN() + ns1 := oc.Namespace() + + g.By("2. Create 2nd namespace") + oc.CreateNamespaceUDN() + ns2 := oc.Namespace() + + udnResourcename := []string{"l3-network-" + ns1, "l3-network-" + ns2} + udnNS := []string{ns1, ns2} + var cidr []string + var prefix int32 + + cidr = []string{"10.150.0.0/16", "10.151.0.0/16"} + prefix = 24 + + udncrd := make([]otputils.UdnCRDResource, 2) + for i := 0; i < 2; i++ { + udncrd[i] = otputils.UdnCRDResource{ + Crdname: udnResourcename[i], + Namespace: udnNS[i], + Role: "Primary", + Cidr: cidr[i], + Prefix: prefix, + Template: udnCRDSingleStack, + } + udncrd[i].CreateUdnCRDSingleStack(oc) + err := otputils.WaitUDNCRDApplied(oc, udnNS[i], udncrd[i].Crdname) + o.Expect(err).NotTo(o.HaveOccurred()) + } + + g.By("create a udn hello pod in ns1") + pod1 := otputils.UdnPodResource{ + Name: "hello-pod-ns1", + Namespace: ns1, + Label: "hello-pod", + Template: udnPodTemplate, + } + pod1.CreateUdnPod(oc) + otputils.WaitPodReady(oc, pod1.Namespace, pod1.Name) + + // Scale up a new node on the cluster using a new machineset + g.By("3. Create a new machineset, get the new node created") + infrastructureName, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("infrastructure", "cluster", "-o=jsonpath={.status.infrastructureName}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + machinesetName := infrastructureName + "-76939" + + // Get an existing machineset to use as template + existingMS, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("machineset", "-n", "openshift-machine-api", "-o=jsonpath={.items[0].metadata.name}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(existingMS).NotTo(o.BeEmpty()) + + // Clone the machineset + msJSON, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("machineset", existingMS, "-n", "openshift-machine-api", "-o", "json").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + + // Create a temporary file with modified machineset + msJSON = strings.ReplaceAll(msJSON, existingMS, machinesetName) + + tmpFile := fmt.Sprintf("/tmp/ms-%s.json", otputils.GetRandomString()) + err = oc.AsAdmin().WithoutNamespace().Run("get").Args("machineset", existingMS, "-n", "openshift-machine-api", "-o", "json").Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + + // Use oc to create a new machineset with replicas=1 + err = oc.AsAdmin().WithoutNamespace().Run("process").Args("-f", "-").Execute() + // Fallback approach: scale an existing machineset + _ = msJSON + _ = tmpFile + + // Get all worker nodes before scaling + nodesBefore, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("nodes", "-l", "node-role.kubernetes.io/worker", "-o=jsonpath={.items[*].metadata.name}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + nodesBeforeList := strings.Fields(nodesBefore) + + // Scale existing machineset + existingReplicas, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("machineset", existingMS, "-n", "openshift-machine-api", "-o=jsonpath={.spec.replicas}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + origReplicas, _ := strconv.Atoi(existingReplicas) + newReplicas := origReplicas + 1 + + defer func() { + _ = oc.AsAdmin().WithoutNamespace().Run("scale").Args("machineset", existingMS, "-n", "openshift-machine-api", fmt.Sprintf("--replicas=%d", origReplicas)).Execute() + e2e.Logf("Waiting for scaled node to be removed") + wait.Poll(30*time.Second, 15*time.Minute, func() (bool, error) { + currentNodes, _ := oc.AsAdmin().WithoutNamespace().Run("get").Args("nodes", "-l", "node-role.kubernetes.io/worker", "-o=jsonpath={.items[*].metadata.name}").Output() + return len(strings.Fields(currentNodes)) <= len(nodesBeforeList), nil + }) + }() + + err = oc.AsAdmin().WithoutNamespace().Run("scale").Args("machineset", existingMS, "-n", "openshift-machine-api", fmt.Sprintf("--replicas=%d", newReplicas)).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + + // Wait for new node to appear and become Ready + var nodeName string + err = wait.Poll(30*time.Second, 15*time.Minute, func() (bool, error) { + nodesAfter, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("nodes", "-l", "node-role.kubernetes.io/worker", "-o=jsonpath={.items[*].metadata.name}").Output() + if err != nil { + return false, nil + } + nodesAfterList := strings.Fields(nodesAfter) + for _, n := range nodesAfterList { + found := false + for _, ob := range nodesBeforeList { + if n == ob { + found = true + break + } + } + if !found { + nodeName = n + return true, nil + } + } + return false, nil + }) + o.Expect(err).NotTo(o.HaveOccurred(), "New node did not appear after scaling") + e2e.Logf("Get nodeName: %v", nodeName) + + otputils.CheckNodeStatus(oc, nodeName, "Ready") + + g.By("create a udn hello pod in ns2") + pod2 := otputils.UdnPodResourceNode{ + Name: "hello-pod-ns2", + Namespace: ns2, + Label: "hello-pod", + Nodename: nodeName, + Template: udnPodTemplateNode, + } + + pod2.CreateUdnPodNode(oc) + otputils.WaitPodReady(oc, pod2.Namespace, pod2.Name) + + // udn network connectivity should be isolated + otputils.CurlPod2PodFailUDN(oc, ns1, pod1.Name, ns2, pod2.Name) + // default network connectivity should also be isolated + otputils.CurlPod2PodFail(oc, ns1, pod1.Name, ns2, pod2.Name) + }) + + g.It("[JIRA:Networking][OTP][Serial] 77542-Check default network ports can be exposed on UDN pods layer3", func() { + var ( + testDataDir = testdata.FixturePath("networking") + netsegDir = testdata.FixturePath("networking/network_segmentation") + sctpModule = filepath.Join(netsegDir, "sctp/load-sctp-module.yaml") + statefulSetHelloPod = filepath.Join(testDataDir, "statefulset-hello.yaml") + tcpPort = 8080 + udpPort = 6000 + sctpPort = 30102 + ) + + g.By("Preparing the nodes for SCTP") + otputils.PrepareSCTPModule(oc, sctpModule) + + g.By("1. Create the UDN namespace") + oc.CreateNamespaceUDN() + ns2 := oc.Namespace() + + g.By("2. Create CRD for UDN in ns2") + err := otputils.ApplyL3UDNtoNamespace(oc, ns2, 0) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("3. Create a udn hello pod in ns2 and get the node name") + otputils.CreateResourceFromFile(oc, ns2, statefulSetHelloPod) + pod2Err := otputils.WaitForPodWithLabelReady(oc, ns2, "app=hello") + o.Expect(pod2Err).NotTo(o.HaveOccurred(), "The statefulSet pod is not ready") + pod2Name := otputils.GetPodName(oc, ns2, "app=hello")[0] + + podNodeName, podNodeNameErr := otputils.GetPodNodeName(oc, ns2, pod2Name) + o.Expect(podNodeNameErr).NotTo(o.HaveOccurred()) + o.Expect(podNodeName).NotTo(o.BeEmpty()) + + g.By("4. Create the non UDN namespace") + oc.SetupProject() + ns1 := oc.Namespace() + + g.By("5. Create a hello pod in ns1") + otputils.CreateResourceFromFile(oc, ns1, statefulSetHelloPod) + pod1Err := otputils.WaitForPodWithLabelReady(oc, ns1, "app=hello") + o.Expect(pod1Err).NotTo(o.HaveOccurred(), "The statefulSet pod is not ready") + pod1Name := otputils.GetPodName(oc, ns1, "app=hello")[0] + + g.By("6. Check host isolation from node to UDN pod's IP on default network on TCP/ICMP, should not be able to access") + otputils.CurlNode2PodFail(oc, podNodeName, ns2, pod2Name) + otputils.PingNode2PodFail(oc, podNodeName, ns2, pod2Name) + + g.By("7. Check ICMP/TCP/UDP/SCTP traffic between pods in ns1 and ns2, should not be able to access") + otputils.PingPod2PodFail(oc, ns1, pod1Name, ns2, pod2Name) + otputils.CurlPod2PodFail(oc, ns1, pod1Name, ns2, pod2Name) + otputils.VerifyConnPod2Pod(oc, ns1, pod1Name, ns2, pod2Name, "UDP", udpPort, false) + otputils.VerifyConnPod2Pod(oc, ns1, pod1Name, ns2, pod2Name, "SCTP", sctpPort, false) + + g.By("8. Add annotation to expose default network port on udn pod") + annotationConf := `k8s.ovn.org/open-default-ports=[{"protocol":"icmp"}, {"protocol":"tcp","port":` + strconv.Itoa(tcpPort) + `}, {"protocol":"udp","port":` + strconv.Itoa(udpPort) + `}, {"protocol":"sctp","port":` + strconv.Itoa(sctpPort) + `}]` + err = oc.AsAdmin().WithoutNamespace().Run("annotate").Args("pod", pod2Name, "-n", ns2, "--overwrite", annotationConf).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + time.Sleep(10 * time.Second) + + g.By("9. Check ICMP/TCP/UDP/SCTP traffic between pods in ns1 and ns2, should be able to access") + otputils.PingPod2PodPass(oc, ns1, pod1Name, ns2, pod2Name) + otputils.CurlPod2PodPass(oc, ns1, pod1Name, ns2, pod2Name) + otputils.VerifyConnPod2Pod(oc, ns1, pod1Name, ns2, pod2Name, "UDP", udpPort, true) + otputils.VerifyConnPod2Pod(oc, ns1, pod1Name, ns2, pod2Name, "SCTP", sctpPort, true) + + g.By("10. Check host isolation from node to UDN pod's IP on default network on TCP and ICMP, should be able to access") + otputils.CurlNode2PodPass(oc, podNodeName, ns2, pod2Name) + otputils.PingNode2PodPass(oc, podNodeName, ns2, pod2Name) + }) + + g.It("[JIRA:Networking][OTP][Serial] 77742-Check default network ports can be exposed on UDN pods layer2", func() { + var ( + testDataDir = testdata.FixturePath("networking") + netsegDir = testdata.FixturePath("networking/network_segmentation") + sctpModule = filepath.Join(netsegDir, "sctp/load-sctp-module.yaml") + udnCRDdualStack = filepath.Join(netsegDir, "udn/udn_crd_layer2_dualstack_template.yaml") + udnCRDSingleStack = filepath.Join(netsegDir, "udn/udn_crd_layer2_singlestack_template.yaml") + statefulSetHelloPod = filepath.Join(testDataDir, "statefulset-hello.yaml") + tcpPort = 8080 + udpPort = 6000 + sctpPort = 30102 + ) + + g.By("Preparing the nodes for SCTP") + otputils.PrepareSCTPModule(oc, sctpModule) + + g.By("1. Create UDN namespace") + oc.CreateNamespaceUDN() + ns2 := oc.Namespace() + + g.By("2. Create CRD for UDN in ns2") + var cidr, ipv4cidr, ipv6cidr string + ipStackType := otputils.CheckIPStackType(oc) + if ipStackType == "ipv4single" { + cidr = "10.150.0.0/16" + } else { + if ipStackType == "ipv6single" { + cidr = "2010:100:200::0/48" + } else { + ipv4cidr = "10.150.0.0/16" + ipv6cidr = "2010:100:200::0/48" + } + } + udncrd := otputils.UdnCRDResource{ + Crdname: "udn-l2-network-77742", + Namespace: ns2, + Role: "Primary", + } + if ipStackType == "dualstack" { + udncrd.IPv4cidr = ipv4cidr + udncrd.IPv6cidr = ipv6cidr + udncrd.Template = udnCRDdualStack + udncrd.CreateLayer2DualStackUDNCRD(oc) + } else { + udncrd.Cidr = cidr + udncrd.Template = udnCRDSingleStack + udncrd.CreateLayer2SingleStackUDNCRD(oc) + } + err := otputils.WaitUDNCRDApplied(oc, ns2, udncrd.Crdname) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("3. Create a udn hello pod in ns2 and get node name") + otputils.CreateResourceFromFile(oc, ns2, statefulSetHelloPod) + pod2Err := otputils.WaitForPodWithLabelReady(oc, ns2, "app=hello") + o.Expect(pod2Err).NotTo(o.HaveOccurred(), "The statefulSet pod is not ready") + pod2Name := otputils.GetPodName(oc, ns2, "app=hello")[0] + + podNodeName, podNodeNameErr := otputils.GetPodNodeName(oc, ns2, pod2Name) + o.Expect(podNodeNameErr).NotTo(o.HaveOccurred()) + o.Expect(podNodeName).NotTo(o.BeEmpty()) + + g.By("4. Create non UDN namespace") + oc.SetupProject() + ns1 := oc.Namespace() + + g.By("5. Create a hello pod in ns1") + otputils.CreateResourceFromFile(oc, ns1, statefulSetHelloPod) + pod1Err := otputils.WaitForPodWithLabelReady(oc, ns1, "app=hello") + o.Expect(pod1Err).NotTo(o.HaveOccurred(), "The statefulSet pod is not ready") + pod1Name := otputils.GetPodName(oc, ns1, "app=hello")[0] + + g.By("6. Check host isolation from node to UDN pod's IP on default network on TCP/ICMP, should not be able to access") + otputils.CurlNode2PodFail(oc, podNodeName, ns2, pod2Name) + otputils.PingNode2PodFail(oc, podNodeName, ns2, pod2Name) + + g.By("7. Check ICMP/TCP/UDP/SCTP traffic between pods in ns1 and ns2, should not be able to access") + otputils.PingPod2PodFail(oc, ns1, pod1Name, ns2, pod2Name) + otputils.CurlPod2PodFail(oc, ns1, pod1Name, ns2, pod2Name) + otputils.VerifyConnPod2Pod(oc, ns1, pod1Name, ns2, pod2Name, "UDP", udpPort, false) + otputils.VerifyConnPod2Pod(oc, ns1, pod1Name, ns2, pod2Name, "SCTP", sctpPort, false) + + g.By("8. Add annotation to expose default network port on udn pod") + annotationConf := `k8s.ovn.org/open-default-ports=[{"protocol":"icmp"}, {"protocol":"tcp","port":` + strconv.Itoa(tcpPort) + `}, {"protocol":"udp","port":` + strconv.Itoa(udpPort) + `}, {"protocol":"sctp","port":` + strconv.Itoa(sctpPort) + `}]` + err = oc.AsAdmin().WithoutNamespace().Run("annotate").Args("pod", pod2Name, "-n", ns2, "--overwrite", annotationConf).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + time.Sleep(10 * time.Second) + + g.By("9. Check ICMP/TCP/UDP/SCTP traffic between pods in ns1 and ns2, should be able to access") + otputils.PingPod2PodPass(oc, ns1, pod1Name, ns2, pod2Name) + otputils.CurlPod2PodPass(oc, ns1, pod1Name, ns2, pod2Name) + otputils.VerifyConnPod2Pod(oc, ns1, pod1Name, ns2, pod2Name, "UDP", udpPort, true) + otputils.VerifyConnPod2Pod(oc, ns1, pod1Name, ns2, pod2Name, "SCTP", sctpPort, true) + + g.By("10. Check host isolation from node to UDN pod's IP on default network on TCP/ICMP, should be able to access") + otputils.CurlNode2PodPass(oc, podNodeName, ns2, pod2Name) + otputils.PingNode2PodPass(oc, podNodeName, ns2, pod2Name) + }) + + g.It("[JIRA:Networking][OTP] 78152-Check udn pods to kapi dns traffic should pass", func() { + var ( + testDataDir = testdata.FixturePath("networking") + testPodTemplate = filepath.Join(testDataDir, "ping-for-pod-template.yaml") + serviceTemplate = filepath.Join(testDataDir, "service-generic-template.yaml") + ns string + ipStackType = otputils.CheckIPStackType(oc) + ) + g.By("1. create udn namespace") + oc.CreateNamespaceUDN() + ns = oc.Namespace() + + g.By("2. Create CRD for UDN") + var cidr, ipv4cidr, ipv6cidr string + if ipStackType == "ipv4single" { + cidr = "10.150.0.0/16" + } else { + if ipStackType == "ipv6single" { + cidr = "2010:100:200::0/48" + } else { + ipv4cidr = "10.150.0.0/16" + ipv6cidr = "2010:100:200::0/48" + } + } + + otputils.CreateGeneralUDNCRD(oc, ns, "udn-78152", ipv4cidr, ipv6cidr, cidr, "layer3") + + g.By("3. Create test pods and service") + defer otputils.RemoveResource(oc, true, true, "pod", "testpod", "-n", ns) + err := otputils.ApplyResourceFromTemplateByAdmin(oc, "--ignore-unknown-parameters=true", "-f", testPodTemplate, "-p", "NAME=testpod", "NAMESPACE="+ns, "-n", ns) + o.Expect(err).NotTo(o.HaveOccurred()) + + err = otputils.WaitForPodWithLabelReady(oc, ns, "name=hello-pod") + o.Expect(err).NotTo(o.HaveOccurred(), "pod with label name=hello-pod not ready") + + testSvc := otputils.GenericServiceResource{ + Servicename: "test-service", + Namespace: ns, + Protocol: "TCP", + Selector: "hello-pod", + ServiceType: "ClusterIP", + IpFamilyPolicy: "PreferDualStack", + InternalTrafficPolicy: "Cluster", + ExternalTrafficPolicy: "", + Template: serviceTemplate, + } + defer oc.AsAdmin().WithoutNamespace().Run("delete").Args("svc", testSvc.Servicename, "-n", testSvc.Namespace).Execute() + testSvc.CreateServiceFromParams(oc) + + g.By("4. check kapi traffic from testpod") + cmd := "curl -k https://kubernetes.default:443/healthz" + outPut, err := e2eoutput.RunHostCmd(ns, "testpod", cmd) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(outPut).Should(o.ContainSubstring("ok")) + + g.By("5. check dns traffic from testpod") + svcIP1, svcIP2 := otputils.GetSvcIP(oc, ns, testSvc.Servicename) + cmdDns := "nslookup " + testSvc.Servicename + outPut, err = e2eoutput.RunHostCmd(ns, "testpod", cmdDns) + o.Expect(err).NotTo(o.HaveOccurred()) + re1 := regexp.MustCompile(`Address:\s+` + svcIP1) + res1 := re1.MatchString(outPut) + o.Expect(res1).To(o.BeTrue()) + if svcIP2 != "" { + re2 := regexp.MustCompile(`Address:\s+` + svcIP2) + res2 := re2.MatchString(outPut) + o.Expect(res2).To(o.BeTrue()) + } + }) + + g.It("[JIRA:Networking][OTP] 78381-Check cudn pods to kapi dns traffic layer 2", func() { + var ( + testDataDir = testdata.FixturePath("networking") + testPodTemplate = filepath.Join(testDataDir, "ping-for-pod-template.yaml") + serviceTemplate = filepath.Join(testDataDir, "service-generic-template.yaml") + ns string + ipStackType = otputils.CheckIPStackType(oc) + values = []string{"value-78381-1", "value-78381-2"} + key = "test.cudn.layer2" + ) + g.By("1. create udn namespace") + oc.CreateNamespaceUDN() + ns = oc.Namespace() + + defer oc.AsAdmin().WithoutNamespace().Run("label").Args("ns", ns, fmt.Sprintf("%s-", key)).Execute() + err := oc.AsAdmin().WithoutNamespace().Run("label").Args("ns", ns, fmt.Sprintf("%s=%s", key, values[0])).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("2. Create CRD for CUDN") + var cidr, ipv4cidr, ipv6cidr string + if ipStackType == "ipv4single" { + cidr = "10.150.0.0/16" + } else { + if ipStackType == "ipv6single" { + cidr = "2010:100:200::0/48" + } else { + ipv4cidr = "10.150.0.0/16" + ipv6cidr = "2010:100:200::0/48" + } + } + + defer otputils.RemoveResource(oc, true, true, "clusteruserdefinednetwork", "udn-78381") + _, err = otputils.CreateCUDNCRD(oc, key, "udn-78381", ipv4cidr, ipv6cidr, cidr, "layer2", values) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("3. Create test pods and service") + defer otputils.RemoveResource(oc, true, true, "pod", "testpod", "-n", ns) + err = otputils.ApplyResourceFromTemplateByAdmin(oc, "--ignore-unknown-parameters=true", "-f", testPodTemplate, "-p", "NAME=testpod", "NAMESPACE="+ns, "-n", ns) + o.Expect(err).NotTo(o.HaveOccurred()) + + err = otputils.WaitForPodWithLabelReady(oc, ns, "name=hello-pod") + o.Expect(err).NotTo(o.HaveOccurred(), "pod with label name=hello-pod not ready") + + testSvc := otputils.GenericServiceResource{ + Servicename: "test-service", + Namespace: ns, + Protocol: "TCP", + Selector: "hello-pod", + ServiceType: "ClusterIP", + IpFamilyPolicy: "PreferDualStack", + InternalTrafficPolicy: "Cluster", + ExternalTrafficPolicy: "", + Template: serviceTemplate, + } + defer oc.AsAdmin().WithoutNamespace().Run("delete").Args("svc", testSvc.Servicename, "-n", testSvc.Namespace).Execute() + testSvc.CreateServiceFromParams(oc) + + g.By("4. check kapi traffic from testpod") + cmd := "curl -k https://kubernetes.default:443/healthz" + outPut, err := e2eoutput.RunHostCmd(ns, "testpod", cmd) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(outPut).Should(o.ContainSubstring("ok")) + + g.By("5. check dns traffic from testpod") + svcIP1, svcIP2 := otputils.GetSvcIP(oc, ns, testSvc.Servicename) + cmdDns := "nslookup " + testSvc.Servicename + outPut, err = e2eoutput.RunHostCmd(ns, "testpod", cmdDns) + o.Expect(err).NotTo(o.HaveOccurred()) + re1 := regexp.MustCompile(`Address:\s+` + svcIP1) + res1 := re1.MatchString(outPut) + o.Expect(res1).To(o.BeTrue()) + if svcIP2 != "" { + re2 := regexp.MustCompile(`Address:\s+` + svcIP2) + res2 := re2.MatchString(outPut) + o.Expect(res2).To(o.BeTrue()) + } + }) + + g.It("[JIRA:Networking][OTP] 79095-Verify event is generated for IP exhaustion in user defined network", func() { + var ( + testDataDirUDN = testdata.FixturePath("networking/network_segmentation/udn") + udnNadtemplate = filepath.Join(testDataDirUDN, "udn_nad_template.yaml") + udnPodTemplate = filepath.Join(testDataDirUDN, "udn_test_pod_template.yaml") + testID = "79095" + ) + + ipStackType := otputils.CheckIPStackType(oc) + if ipStackType == "ipv6single" { + g.Skip("This case cannot be run on ipv6 cluster") + } + + g.By("1. Create a namespace") + oc.CreateNamespaceUDN() + ns1 := oc.Namespace() + + g.By("2. Create UDN NAD") + nadResourcename := "l3-network-" + testID + "-" + ns1 + var subnet string + if ipStackType == "ipv4single" { + subnet = "10.200.0.0/16/30" + } else { + subnet = "10.200.0.0/16/30,2011:100:200::/60" + } + + g.By(fmt.Sprintf("create NAD %s in namespace %s", nadResourcename, ns1)) + nad := otputils.UdnNetDefResource{ + Nadname: nadResourcename, + Namespace: ns1, + NadNetworkName: nadResourcename, + Topology: "layer3", + Subnet: subnet, + NetAttachDefName: ns1 + "/" + nadResourcename, + Role: "primary", + Template: udnNadtemplate, + } + nad.CreateUdnNad(oc) + + g.By("3. Create a udn hello pod in ns1") + pod1 := otputils.UdnPodResource{ + Name: "hello-pod-" + testID + "-ns1", + Namespace: ns1, + Label: "hello-pod", + Template: udnPodTemplate, + } + pod1.CreateUdnPod(oc) + + g.By("4. Poll event log for IP allocation failure") + o.Eventually(func() string { + eventOutput, _ := oc.AsAdmin().Run("get").Args("event").Output() + return eventOutput + }, "60s", "5s").Should(o.ContainSubstring("failed to allocate new IPs")) + }) +}) diff --git a/openshift/test/otp/network_segmentation_multicast.go b/openshift/test/otp/network_segmentation_multicast.go new file mode 100644 index 0000000000..0e06c36a89 --- /dev/null +++ b/openshift/test/otp/network_segmentation_multicast.go @@ -0,0 +1,542 @@ +package otp + +import ( + "context" + "fmt" + "path/filepath" + "time" + + g "github.com/onsi/ginkgo/v2" + o "github.com/onsi/gomega" + + exutil "github.com/openshift/origin/test/extended/util" + otputils "github.com/ovn-kubernetes/ovn-kubernetes/openshift/pkg/otp/utils" + "github.com/ovn-kubernetes/ovn-kubernetes/openshift/pkg/otp/testdata" + + e2enode "k8s.io/kubernetes/test/e2e/framework/node" +) + +var _ = g.Describe("[sig-network][Suite:openshift/conformance/serial] SDN network segmentation multicast", func() { + + var ( + oc = exutil.NewCLI("networking-udn") + testDataDirMcast = testdata.FixturePath("networking", "network_segmentation", "multicast") + ) + + g.It("[JIRA:Networking][OTP] 78447-udn pods should/should not receive multicast traffic when enable/disable multicast.", func() { + var ( + mcastPodTemplate = filepath.Join(testDataDirMcast, "multicast-rc.json") + ns string + ipStackType = otputils.CheckIPStackType(oc) + ipv4List []string + ipv6List []string + mcastipv4 = "232.43.211.234" + mcastipv6 = "ff3e::4321:1234" + port = "4321" + ) + + nodeList, nodeErr := e2enode.GetReadySchedulableNodes(context.TODO(), oc.KubeFramework().ClientSet) + o.Expect(nodeErr).NotTo(o.HaveOccurred()) + if len(nodeList.Items) < 2 { + g.Skip("This test requires at least 2 worker nodes which is not fulfilled. ") + } + //cover both udn and default network. when i ==0, test udn, when i ==1, test default + for i := 0; i < 2; i++ { + if i == 0 { + g.By("############# test multicast on pods with udn primary interface") + g.By("1. create udn namespace") + oc.CreateNamespaceUDN() + ns = oc.Namespace() + + g.By("2. Create CRD for UDN") + var cidr, ipv4cidr, ipv6cidr string + if ipStackType == "ipv4single" { + cidr = "10.150.0.0/16" + } else { + if ipStackType == "ipv6single" { + cidr = "2010:100:200::0/48" + } else { + ipv4cidr = "10.150.0.0/16" + ipv6cidr = "2010:100:200::0/48" + } + } + + otputils.CreateGeneralUDNCRD(oc, ns, "udn-78447", ipv4cidr, ipv6cidr, cidr, "layer3") + + } else { + g.By("############# test multicast on pods with default interface") + oc.SetupProject() + ns = oc.Namespace() + } + g.By("3. Create 3 multicast testing pods") + + defer otputils.RemoveResource(oc, true, true, "ReplicationController", "mcastpod-rc", "-n", ns) + err := otputils.ApplyResourceFromTemplateByAdmin(oc, "--ignore-unknown-parameters=true", "-f", mcastPodTemplate, "-p", "RCNAME=mcastpod-rc", "-n", ns) + o.Expect(err).NotTo(o.HaveOccurred()) + + err = otputils.WaitForPodWithLabelReady(oc, ns, "name=mcastpod-rc") + o.Expect(err).NotTo(o.HaveOccurred()) + + mcastPodList := otputils.GetPodName(oc, ns, "name=mcastpod-rc") + + g.By("4. check multicast traffic without enable multicast in ns") + if ipStackType == "ipv4single" || ipStackType == "dualstack" { + if i == 0 { + ipv4List = otputils.GetPodIPv4UDNList(oc, ns, mcastPodList) + } else { + ipv4List = otputils.GetPodIPv4List(oc, ns, mcastPodList) + } + + chkRes1 := otputils.ChkMcastTraffic(oc, ns, mcastPodList, ipv4List, mcastipv4, port) + o.Expect(chkRes1).Should(o.BeFalse()) + } + if ipStackType == "ipv6single" || ipStackType == "dualstack" { + if i == 0 { + ipv6List = otputils.GetPodIPv6UDNList(oc, ns, mcastPodList) + } else { + ipv6List = otputils.GetPodIPv6List(oc, ns, mcastPodList) + } + + chkRes2 := otputils.ChkMcastTraffic(oc, ns, mcastPodList, ipv6List, mcastipv6, port) + o.Expect(chkRes2).Should(o.BeFalse()) + } + + g.By("5. enable multicast and check multicast traffic again") + otputils.EnableMulticast(oc, ns) + if ipStackType == "ipv4single" || ipStackType == "dualstack" { + chkRes1 := otputils.ChkMcastTraffic(oc, ns, mcastPodList, ipv4List, mcastipv4, port) + o.Expect(chkRes1).Should(o.BeTrue()) + } + if ipStackType == "ipv6single" || ipStackType == "dualstack" { + chkRes2 := otputils.ChkMcastTraffic(oc, ns, mcastPodList, ipv6List, mcastipv6, port) + o.Expect(chkRes2).Should(o.BeTrue()) + } + g.By("6. disable multicast and check multicast traffic again") + otputils.DisableMulticast(oc, ns) + if ipStackType == "ipv4single" || ipStackType == "dualstack" { + chkRes1 := otputils.ChkMcastTraffic(oc, ns, mcastPodList, ipv4List, mcastipv4, port) + o.Expect(chkRes1).Should(o.BeFalse()) + } + if ipStackType == "ipv6single" || ipStackType == "dualstack" { + chkRes2 := otputils.ChkMcastTraffic(oc, ns, mcastPodList, ipv6List, mcastipv6, port) + o.Expect(chkRes2).Should(o.BeFalse()) + } + } + + }) + + g.It("[JIRA:Networking][OTP] 78448-udn pods can join different multicast groups at same time.", func() { + var ( + mcastPodTemplate = filepath.Join(testDataDirMcast, "multicast-rc.json") + ns string + ipStackType = otputils.CheckIPStackType(oc) + ipv4List []string + ipv6List []string + mcastipv4 = []string{"232.43.211.234", "232.43.211.235", "232.43.211.236"} + mcastipv6 = []string{"ff3e::4321:1234", "ff3e::4321:1235", "ff3e::4321:1236"} + intf string + port = []string{"4321", "4322", "4323"} + ) + nodeList, nodeErr := e2enode.GetReadySchedulableNodes(context.TODO(), oc.KubeFramework().ClientSet) + o.Expect(nodeErr).NotTo(o.HaveOccurred()) + if len(nodeList.Items) < 2 { + g.Skip("This test requires at least 2 worker nodes which is not fulfilled. ") + } + //cover both udn and default network. when j ==0, test udn, when j ==1, test default + for j := 0; j < 2; j++ { + if j == 0 { + g.By("############# test multicast on pods with udn primary interface") + g.By("###1. create udn namespace") + oc.CreateNamespaceUDN() + ns = oc.Namespace() + intf = "ovn-udn1" + + g.By("###2. Create CRD for UDN") + + var cidr, ipv4cidr, ipv6cidr string + if ipStackType == "ipv4single" { + cidr = "10.150.0.0/16" + } else { + if ipStackType == "ipv6single" { + cidr = "2010:100:200::0/48" + } else { + ipv4cidr = "10.150.0.0/16" + ipv6cidr = "2010:100:200::0/48" + } + } + otputils.CreateGeneralUDNCRD(oc, ns, "udn-78448", ipv4cidr, ipv6cidr, cidr, "layer3") + } else { + g.By("############# test multicast on pods with default interface") + oc.SetupProject() + ns = oc.Namespace() + intf = "eth0" + } + g.By("###3. Create 3 multicast testing pods") + + defer otputils.RemoveResource(oc, true, true, "ReplicationController", "mcastpod-rc", "-n", ns) + err := otputils.ApplyResourceFromTemplateByAdmin(oc, "--ignore-unknown-parameters=true", "-f", mcastPodTemplate, "-p", "RCNAME=mcastpod-rc", "-n", ns) + o.Expect(err).NotTo(o.HaveOccurred()) + + err = otputils.WaitForPodWithLabelReady(oc, ns, "name=mcastpod-rc") + o.Expect(err).NotTo(o.HaveOccurred()) + + mcastPodList := otputils.GetPodName(oc, ns, "name=mcastpod-rc") + if ipStackType == "ipv4single" || ipStackType == "dualstack" { + if j == 0 { + ipv4List = otputils.GetPodIPv4UDNList(oc, ns, mcastPodList) + } else { + ipv4List = otputils.GetPodIPv4List(oc, ns, mcastPodList) + } + } + if ipStackType == "ipv6single" || ipStackType == "dualstack" { + if j == 0 { + ipv6List = otputils.GetPodIPv6UDNList(oc, ns, mcastPodList) + } else { + ipv6List = otputils.GetPodIPv6List(oc, ns, mcastPodList) + } + } + + g.By("###4. enable multicast and check multicast traffic") + otputils.EnableMulticast(oc, ns) + //send multicast traffic to join different multicast group at the same time + pktFile1 := make([]string, len(mcastPodList)) + pktFile2 := make([]string, len(mcastPodList)) + pktFile3 := make([]string, len(mcastPodList)) + + if ipStackType == "ipv4single" || ipStackType == "dualstack" { + for i, podName := range mcastPodList { + pktFile1[i] = "/tmp/" + otputils.GetRandomString() + ".txt" + pktFile2[i] = "/tmp/" + otputils.GetRandomString() + ".txt" + pktFile3[i] = "/tmp/" + otputils.GetRandomString() + ".txt" + otputils.StartMcastTrafficOnPod(oc, ns, podName, ipv4List, pktFile1[i], mcastipv4[0], port[0]) + otputils.StartMcastTrafficOnPod(oc, ns, podName, ipv4List, pktFile2[i], mcastipv4[1], port[1]) + otputils.StartMcastTrafficOnPod(oc, ns, podName, ipv4List, pktFile3[i], mcastipv4[2], port[2]) + } + //add sleep time to make sure traffic started + time.Sleep(5 * time.Second) + //choose one pod to check the multicast ip address + otputils.ChkMcastAddress(oc, ns, mcastPodList[0], intf, mcastipv4[0]) + otputils.ChkMcastAddress(oc, ns, mcastPodList[0], intf, mcastipv4[1]) + otputils.ChkMcastAddress(oc, ns, mcastPodList[0], intf, mcastipv4[2]) + //add sleep time to make sure traffic completed. + time.Sleep(20 * time.Second) + //choose one pod to check the received multicast packets + chkRes1 := otputils.ChkMcatRcvOnPod(oc, ns, mcastPodList[1], ipv4List[1], ipv4List, mcastipv4[0], pktFile1[1]) + chkRes2 := otputils.ChkMcatRcvOnPod(oc, ns, mcastPodList[1], ipv4List[1], ipv4List, mcastipv4[1], pktFile2[1]) + chkRes3 := otputils.ChkMcatRcvOnPod(oc, ns, mcastPodList[1], ipv4List[1], ipv4List, mcastipv4[2], pktFile3[1]) + o.Expect(chkRes1).Should(o.BeTrue()) + o.Expect(chkRes2).Should(o.BeTrue()) + o.Expect(chkRes3).Should(o.BeTrue()) + + } + if ipStackType == "ipv6single" || ipStackType == "dualstack" { + for i, podName := range mcastPodList { + pktFile1[i] = "/tmp/" + otputils.GetRandomString() + ".txt" + pktFile2[i] = "/tmp/" + otputils.GetRandomString() + ".txt" + pktFile3[i] = "/tmp/" + otputils.GetRandomString() + ".txt" + otputils.StartMcastTrafficOnPod(oc, ns, podName, ipv6List, pktFile1[i], mcastipv6[0], port[0]) + otputils.StartMcastTrafficOnPod(oc, ns, podName, ipv6List, pktFile2[i], mcastipv6[1], port[1]) + otputils.StartMcastTrafficOnPod(oc, ns, podName, ipv6List, pktFile3[i], mcastipv6[2], port[2]) + } + //add sleep time to make sure traffic started. + time.Sleep(5 * time.Second) + //choose one pod to check the multicast ipv6 address + otputils.ChkMcastAddress(oc, ns, mcastPodList[2], intf, mcastipv6[0]) + otputils.ChkMcastAddress(oc, ns, mcastPodList[2], intf, mcastipv6[1]) + otputils.ChkMcastAddress(oc, ns, mcastPodList[2], intf, mcastipv6[2]) + //add sleep time to make sure traffic completed. + time.Sleep(20 * time.Second) + + //choose one pod to check the received multicast packets + chkRes1 := otputils.ChkMcatRcvOnPod(oc, ns, mcastPodList[2], ipv6List[2], ipv6List, mcastipv6[0], pktFile1[2]) + chkRes2 := otputils.ChkMcatRcvOnPod(oc, ns, mcastPodList[2], ipv6List[2], ipv6List, mcastipv6[1], pktFile2[2]) + chkRes3 := otputils.ChkMcatRcvOnPod(oc, ns, mcastPodList[2], ipv6List[2], ipv6List, mcastipv6[2], pktFile3[2]) + o.Expect(chkRes1).Should(o.BeTrue()) + o.Expect(chkRes2).Should(o.BeTrue()) + o.Expect(chkRes3).Should(o.BeTrue()) + + } + + } + + }) + + g.It("[JIRA:Networking][OTP] 78450-Same multicast groups can be created in multiple namespaces with udn configured.", func() { + var ( + mcastPodTemplate = filepath.Join(testDataDirMcast, "multicast-rc.json") + ns1 string + ns2 string + ipStackType = otputils.CheckIPStackType(oc) + ipv4List1 []string + ipv6List1 []string + ipv4List2 []string + ipv6List2 []string + mcastipv4 = "232.43.211.234" + mcastipv6 = "ff3e::4321:1234" + port = "4321" + ) + + nodeList, nodeErr := e2enode.GetReadySchedulableNodes(context.TODO(), oc.KubeFramework().ClientSet) + o.Expect(nodeErr).NotTo(o.HaveOccurred()) + if len(nodeList.Items) < 2 { + g.Skip("This test requires at least 2 worker nodes which is not fulfilled. ") + } + //cover both udn and default network. when i ==0, test udn, when i ==1, test default + for i := 0; i < 2; i++ { + if i == 0 { + g.By("############# test multicast on pods with udn primary interface") + g.By("1. create 2 udn namespaces") + oc.CreateNamespaceUDN() + ns1 = oc.Namespace() + + oc.CreateNamespaceUDN() + ns2 = oc.Namespace() + + g.By("2. Create CRD for UDNs") + + var cidr, ipv4cidr, ipv6cidr string + if ipStackType == "ipv4single" { + cidr = "10.150.0.0/16" + } else { + if ipStackType == "ipv6single" { + cidr = "2010:100:200::0/48" + } else { + ipv4cidr = "10.150.0.0/16" + ipv6cidr = "2010:100:200::0/48" + } + } + otputils.CreateGeneralUDNCRD(oc, ns1, "udn-78450-1", ipv4cidr, ipv6cidr, cidr, "layer3") + otputils.CreateGeneralUDNCRD(oc, ns2, "udn-78450-2", ipv4cidr, ipv6cidr, cidr, "layer3") + } else { + g.By("############# test multicast on pods with default interface") + oc.SetupProject() + ns1 = oc.Namespace() + oc.SetupProject() + ns2 = oc.Namespace() + } + g.By("3. Create 3 multicast testing pods") + + defer otputils.RemoveResource(oc, true, true, "ReplicationController", "mcastpod-rc-1", "-n", ns1) + err := otputils.ApplyResourceFromTemplateByAdmin(oc, "--ignore-unknown-parameters=true", "-f", mcastPodTemplate, "-p", "RCNAME=mcastpod-rc-1", "-n", ns1) + o.Expect(err).NotTo(o.HaveOccurred()) + + defer otputils.RemoveResource(oc, true, true, "ReplicationController", "mcastpod-rc-2", "-n", ns2) + err = otputils.ApplyResourceFromTemplateByAdmin(oc, "--ignore-unknown-parameters=true", "-f", mcastPodTemplate, "-p", "RCNAME=mcastpod-rc-2", "-n", ns2) + o.Expect(err).NotTo(o.HaveOccurred()) + + err = otputils.WaitForPodWithLabelReady(oc, ns1, "name=mcastpod-rc-1") + o.Expect(err).NotTo(o.HaveOccurred()) + + err = otputils.WaitForPodWithLabelReady(oc, ns2, "name=mcastpod-rc-2") + o.Expect(err).NotTo(o.HaveOccurred()) + + mcastPodList1 := otputils.GetPodName(oc, ns1, "name=mcastpod-rc-1") + mcastPodList2 := otputils.GetPodName(oc, ns2, "name=mcastpod-rc-2") + + g.By("4. enable multicast and send multicast traffic in different ns to join a same multicast group") + otputils.EnableMulticast(oc, ns1) + otputils.EnableMulticast(oc, ns2) + if ipStackType == "ipv4single" || ipStackType == "dualstack" { + if i == 0 { + ipv4List1 = otputils.GetPodIPv4UDNList(oc, ns1, mcastPodList1) + ipv4List2 = otputils.GetPodIPv4UDNList(oc, ns2, mcastPodList2) + } else { + ipv4List1 = otputils.GetPodIPv4List(oc, ns1, mcastPodList1) + ipv4List2 = otputils.GetPodIPv4List(oc, ns2, mcastPodList2) + } + + chkRes1 := otputils.ChkMcastTraffic(oc, ns1, mcastPodList1, ipv4List1, mcastipv4, port) + o.Expect(chkRes1).Should(o.BeTrue()) + chkRes2 := otputils.ChkMcastTraffic(oc, ns2, mcastPodList2, ipv4List2, mcastipv4, port) + o.Expect(chkRes2).Should(o.BeTrue()) + } + if ipStackType == "ipv6single" || ipStackType == "dualstack" { + if i == 0 { + ipv6List1 = otputils.GetPodIPv6UDNList(oc, ns1, mcastPodList1) + ipv6List2 = otputils.GetPodIPv6UDNList(oc, ns2, mcastPodList2) + } else { + ipv6List1 = otputils.GetPodIPv6List(oc, ns1, mcastPodList1) + ipv6List2 = otputils.GetPodIPv6List(oc, ns2, mcastPodList2) + } + + chkRes3 := otputils.ChkMcastTraffic(oc, ns1, mcastPodList1, ipv6List1, mcastipv6, port) + o.Expect(chkRes3).Should(o.BeTrue()) + chkRes4 := otputils.ChkMcastTraffic(oc, ns2, mcastPodList2, ipv6List2, mcastipv6, port) + o.Expect(chkRes4).Should(o.BeTrue()) + } + + } + + }) + + g.It("[JIRA:Networking][OTP] 78382-check CUDN pods should not be able to receive multicast traffic from other pods in different namespace which sharing a same CUDN (layer 3).", func() { + var ( + mcastPodTemplate = filepath.Join(testDataDirMcast, "multicast-rc.json") + ns1 string + ns2 string + key = "test.cudn.layer3" + ipStackType = otputils.CheckIPStackType(oc) + ipv4List1 []string + ipv6List1 []string + ipv4List2 []string + ipv6List2 []string + mcastipv4 = "232.43.211.234" + mcastipv6 = "ff3e::4321:1234" + port = "4321" + + crdName = "cudn-network-78382" + values = []string{"value-78382-1", "value-78382-2"} + ) + + nodeList, nodeErr := e2enode.GetReadySchedulableNodes(context.TODO(), oc.KubeFramework().ClientSet) + o.Expect(nodeErr).NotTo(o.HaveOccurred()) + if len(nodeList.Items) < 2 { + g.Skip("This test requires at least 2 worker nodes which is not fulfilled. ") + } + //cover both udn and default network. when i ==0, test udn, when i ==1, test default + for i := 0; i < 2; i++ { + if i == 0 { + g.By("###1. create 2 namespaces for CUDN") + oc.CreateNamespaceUDN() + ns1 = oc.Namespace() + oc.CreateNamespaceUDN() + ns2 = oc.Namespace() + + defer oc.AsAdmin().WithoutNamespace().Run("label").Args("ns", ns1, fmt.Sprintf("%s-", key)).Execute() + err := oc.AsAdmin().WithoutNamespace().Run("label").Args("ns", ns1, fmt.Sprintf("%s=%s", key, values[0])).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + + defer oc.AsAdmin().WithoutNamespace().Run("label").Args("ns", ns2, fmt.Sprintf("%s-", key)).Execute() + err = oc.AsAdmin().WithoutNamespace().Run("label").Args("ns", ns2, fmt.Sprintf("%s=%s", key, values[1])).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("####2. create CUDN in cudnNS") + ipStackType := otputils.CheckIPStackType(oc) + var cidr, ipv4cidr, ipv6cidr string + if ipStackType == "ipv4single" { + cidr = "10.150.0.0/16" + } else { + if ipStackType == "ipv6single" { + cidr = "2010:100:200::0/60" + } else { + ipv4cidr = "10.150.0.0/16" + ipv6cidr = "2010:100:200::0/60" + } + } + + defer otputils.RemoveResource(oc, true, true, "clusteruserdefinednetwork", crdName) + _, err = otputils.CreateCUDNCRD(oc, key, crdName, ipv4cidr, ipv6cidr, cidr, "layer3", values) + o.Expect(err).NotTo(o.HaveOccurred()) + + } else { + g.By("############# test multicast on pods with default interface") + oc.SetupProject() + ns1 = oc.Namespace() + oc.SetupProject() + ns2 = oc.Namespace() + } + g.By("####3. Create 3 multicast testing pods") + + defer otputils.RemoveResource(oc, true, true, "ReplicationController", "mcastpod-rc-1", "-n", ns1) + err := otputils.ApplyResourceFromTemplateByAdmin(oc, "--ignore-unknown-parameters=true", "-f", mcastPodTemplate, "-p", "RCNAME=mcastpod-rc-1", "-n", ns1) + o.Expect(err).NotTo(o.HaveOccurred()) + + defer otputils.RemoveResource(oc, true, true, "ReplicationController", "mcastpod-rc-2", "-n", ns2) + err = otputils.ApplyResourceFromTemplateByAdmin(oc, "--ignore-unknown-parameters=true", "-f", mcastPodTemplate, "-p", "RCNAME=mcastpod-rc-2", "-n", ns2) + o.Expect(err).NotTo(o.HaveOccurred()) + + err = otputils.WaitForPodWithLabelReady(oc, ns1, "name=mcastpod-rc-1") + o.Expect(err).NotTo(o.HaveOccurred()) + + err = otputils.WaitForPodWithLabelReady(oc, ns2, "name=mcastpod-rc-2") + o.Expect(err).NotTo(o.HaveOccurred()) + + mcastPodList1 := otputils.GetPodName(oc, ns1, "name=mcastpod-rc-1") + mcastPodList2 := otputils.GetPodName(oc, ns2, "name=mcastpod-rc-2") + + g.By("###4. enable multicast and send multicast traffic in different ns to join a same multicast group") + otputils.EnableMulticast(oc, ns1) + otputils.EnableMulticast(oc, ns2) + if ipStackType == "ipv4single" || ipStackType == "dualstack" { + if i == 0 { + ipv4List1 = otputils.GetPodIPv4UDNList(oc, ns1, mcastPodList1) + ipv4List2 = otputils.GetPodIPv4UDNList(oc, ns2, mcastPodList2) + } else { + ipv4List1 = otputils.GetPodIPv4List(oc, ns1, mcastPodList1) + ipv4List2 = otputils.GetPodIPv4List(oc, ns2, mcastPodList2) + } + + chkRes1 := otputils.ChkMcastTraffic(oc, ns1, mcastPodList1, ipv4List1, mcastipv4, port) + o.Expect(chkRes1).Should(o.BeTrue()) + chkRes2 := otputils.ChkMcastTraffic(oc, ns2, mcastPodList2, ipv4List2, mcastipv4, port) + o.Expect(chkRes2).Should(o.BeTrue()) + } + if ipStackType == "ipv6single" || ipStackType == "dualstack" { + if i == 0 { + ipv6List1 = otputils.GetPodIPv6UDNList(oc, ns1, mcastPodList1) + ipv6List2 = otputils.GetPodIPv6UDNList(oc, ns2, mcastPodList2) + } else { + ipv6List1 = otputils.GetPodIPv6List(oc, ns1, mcastPodList1) + ipv6List2 = otputils.GetPodIPv6List(oc, ns2, mcastPodList2) + } + + chkRes3 := otputils.ChkMcastTraffic(oc, ns1, mcastPodList1, ipv6List1, mcastipv6, port) + o.Expect(chkRes3).Should(o.BeTrue()) + chkRes4 := otputils.ChkMcastTraffic(oc, ns2, mcastPodList2, ipv6List2, mcastipv6, port) + o.Expect(chkRes4).Should(o.BeTrue()) + } + + g.By("###5. send multicast traffic across different ns to join a same multicast group") + + if ipStackType == "ipv4single" || ipStackType == "dualstack" { + var podIPv4_1, podIPv4_2 string + if i == 0 { + podIPv4_1 = otputils.GetPodIPUDNv4(oc, ns1, mcastPodList1[0], "ovn-udn1") + podIPv4_2 = otputils.GetPodIPUDNv4(oc, ns2, mcastPodList2[0], "ovn-udn1") + } else { + podIPv4_1 = otputils.GetPodIPv4(oc, ns1, mcastPodList1[0]) + podIPv4_2 = otputils.GetPodIPv4(oc, ns2, mcastPodList2[0]) + } + ipv4List := []string{podIPv4_1, podIPv4_2} + pktFile1 := "/tmp/" + otputils.GetRandomString() + ".txt" + pktFile2 := "/tmp/" + otputils.GetRandomString() + ".txt" + //send multicast traffic across different ns + otputils.StartMcastTrafficOnPod(oc, ns1, mcastPodList1[0], ipv4List, pktFile1, mcastipv4, port) + otputils.StartMcastTrafficOnPod(oc, ns2, mcastPodList2[0], ipv4List, pktFile2, mcastipv4, port) + //add sleep time to make sure traffic completed. + time.Sleep(30 * time.Second) + + chkRes1 := otputils.ChkMcatRcvOnPod(oc, ns1, mcastPodList1[0], podIPv4_1, ipv4List, mcastipv4, pktFile1) + chkRes2 := otputils.ChkMcatRcvOnPod(oc, ns2, mcastPodList2[0], podIPv4_2, ipv4List, mcastipv4, pktFile2) + o.Expect(chkRes1).Should(o.BeFalse()) + o.Expect(chkRes2).Should(o.BeFalse()) + + } + if ipStackType == "dualstack" || ipStackType == "ipv6single" { + var podIPv6_1, podIPv6_2 string + if i == 0 { + podIPv6_1 = otputils.GetPodIPUDNv6(oc, ns1, mcastPodList1[0], "ovn-udn1") + podIPv6_2 = otputils.GetPodIPUDNv6(oc, ns2, mcastPodList2[0], "ovn-udn1") + } else { + podIPv6_1 = otputils.GetPodIPv6(oc, ns1, mcastPodList1[0], ipStackType) + podIPv6_2 = otputils.GetPodIPv6(oc, ns2, mcastPodList2[0], ipStackType) + } + ipv6List := []string{podIPv6_1, podIPv6_2} + pktFile1 := "/tmp/" + otputils.GetRandomString() + ".txt" + pktFile2 := "/tmp/" + otputils.GetRandomString() + ".txt" + //send multicast traffic across different ns + otputils.StartMcastTrafficOnPod(oc, ns1, mcastPodList1[0], ipv6List, pktFile1, mcastipv6, port) + otputils.StartMcastTrafficOnPod(oc, ns2, mcastPodList2[0], ipv6List, pktFile2, mcastipv6, port) + //add sleep time to make sure traffic completed. + time.Sleep(30 * time.Second) + + chkRes1 := otputils.ChkMcatRcvOnPod(oc, ns1, mcastPodList1[0], podIPv6_1, ipv6List, mcastipv6, pktFile1) + chkRes2 := otputils.ChkMcatRcvOnPod(oc, ns2, mcastPodList2[0], podIPv6_2, ipv6List, mcastipv6, pktFile2) + o.Expect(chkRes1).Should(o.BeFalse()) + o.Expect(chkRes2).Should(o.BeFalse()) + } + } + + }) + +}) diff --git a/openshift/test/otp/network_segmentation_policy.go b/openshift/test/otp/network_segmentation_policy.go new file mode 100644 index 0000000000..f0ffb8ac93 --- /dev/null +++ b/openshift/test/otp/network_segmentation_policy.go @@ -0,0 +1,706 @@ +package otp + +import ( + "fmt" + "path/filepath" + "strconv" + "strings" + + g "github.com/onsi/ginkgo/v2" + o "github.com/onsi/gomega" + + exutil "github.com/openshift/origin/test/extended/util" + otputils "github.com/ovn-kubernetes/ovn-kubernetes/openshift/pkg/otp/utils" + "github.com/ovn-kubernetes/ovn-kubernetes/openshift/pkg/otp/testdata" + + e2e "k8s.io/kubernetes/test/e2e/framework" + e2eoutput "k8s.io/kubernetes/test/e2e/framework/pod/output" +) + +var _ = g.Describe("[sig-network][Suite:openshift/conformance/serial] SDN network segmentation policy", func() { + var ( + oc = exutil.NewCLI("networking-netseg-policy") + ) + + g.It("[JIRA:Networking][OTP] 78292-Validate ingress allow-same-namespace and allow-all-namespaces network policies in Layer 3 NAD", func() { + var ( + testID = "78292" + testDataDir = testdata.FixturePath("networking") + testDataDirUDN = testdata.FixturePath("networking/network_segmentation/udn") + udnNADTemplate = filepath.Join(testDataDirUDN, "udn_nad_template.yaml") + udnPodTemplate = filepath.Join(testDataDirUDN, "udn_test_pod_template.yaml") + ingressDenyFile = filepath.Join(testDataDir, "networkpolicy/default-deny-ingress.yaml") + ingressAllowSameNSFile = filepath.Join(testDataDir, "networkpolicy/allow-from-same-namespace.yaml") + ingressAllowAllNSFile = filepath.Join(testDataDir, "networkpolicy/allow-from-all-namespaces.yaml") + nsPodMap = make(map[string][]string) + nadResourcename = "l3-network-" + topology = "layer3" + ) + ipStackType := otputils.CheckIPStackType(oc) + var nadName string + var nadNS []string = make([]string, 0, 4) + nsDefaultNetwork := oc.Namespace() + nadNetworkName := []string{"l3-network-test-1", "l3-network-test-2"} + + g.By("1.0 Create 4 UDN namespaces") + for i := 0; i < 4; i++ { + oc.CreateNamespaceUDN() + nadNS = append(nadNS, oc.Namespace()) + } + nadNS = append(nadNS, nsDefaultNetwork) + var subnet []string + if ipStackType == "ipv4single" { + subnet = []string{"10.150.0.0/16/24", "10.152.0.0/16/24"} + } else { + if ipStackType == "ipv6single" { + subnet = []string{"2010:100:200::0/60", "2012:100:200::0/60"} + } else { + subnet = []string{"10.150.0.0/16/24,2010:100:200::0/60", "10.152.0.0/16/24,2012:100:200::0/60"} + } + } + g.By("2. Create Layer 3 NAD in first two namespaces") + // Same network name in both namespaces + nad := make([]otputils.UdnNetDefResource, 4) + for i := 0; i < 2; i++ { + nadName = nadResourcename + strconv.Itoa(i) + "-" + testID + if i == 1 { + o.Expect(oc.AsAdmin().WithoutNamespace().Run("label").Args("ns", nadNS[i], "team=ocp").Execute()).NotTo(o.HaveOccurred()) + } + g.By(fmt.Sprintf("Create NAD %s in namespace %s", nadName, nadNS[i])) + nad[i] = otputils.UdnNetDefResource{ + Nadname: nadName, + Namespace: nadNS[i], + NadNetworkName: nadNetworkName[0], + Topology: topology, + Subnet: subnet[0], + NetAttachDefName: nadNS[i] + "/" + nadName, + Role: "primary", + Template: udnNADTemplate, + } + nad[i].CreateUdnNad(oc) + } + g.By("3. Create two pods in each namespace") + pod := make([]otputils.UdnPodResource, 4) + for i := 0; i < 2; i++ { + for j := 0; j < 2; j++ { + pod[j] = otputils.UdnPodResource{ + Name: "hello-pod-" + testID + "-" + strconv.Itoa(i) + "-" + strconv.Itoa(j), + Namespace: nadNS[i], + Label: "hello-pod", + Template: udnPodTemplate, + } + pod[j].CreateUdnPod(oc) + otputils.WaitPodReady(oc, pod[j].Namespace, pod[j].Name) + nsPodMap[pod[j].Namespace] = append(nsPodMap[pod[j].Namespace], pod[j].Name) + } + } + otputils.CurlPod2PodPassUDN(oc, nadNS[1], nsPodMap[nadNS[1]][0], nadNS[0], nsPodMap[nadNS[0]][0]) + otputils.CurlPod2PodPassUDN(oc, nadNS[0], nsPodMap[nadNS[0]][1], nadNS[0], nsPodMap[nadNS[0]][0]) + + g.By("4. Create default deny ingress type networkpolicy in first namespace") + otputils.CreateResourceFromFile(oc, nadNS[0], ingressDenyFile) + output, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("networkpolicy", "-n", nadNS[0]).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("default-deny-ingress")) + + g.By("5. Validate traffic between pods in first namespace and from pods in second namespace") + otputils.CurlPod2PodFailUDN(oc, nadNS[1], nsPodMap[nadNS[1]][0], nadNS[0], nsPodMap[nadNS[0]][0]) + otputils.CurlPod2PodFailUDN(oc, nadNS[0], nsPodMap[nadNS[0]][1], nadNS[0], nsPodMap[nadNS[0]][0]) + + g.By("6. Create allow same namespace ingress type networkpolicy in first namespace") + otputils.CreateResourceFromFile(oc, nadNS[0], ingressAllowSameNSFile) + output, err = oc.AsAdmin().WithoutNamespace().Run("get").Args("networkpolicy", "-n", nadNS[0]).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("allow-from-same-namespace")) + + g.By("7. Validate traffic between pods in first namespace works but traffic from pod in second namespace is blocked") + otputils.CurlPod2PodPassUDN(oc, nadNS[0], nsPodMap[nadNS[0]][1], nadNS[0], nsPodMap[nadNS[0]][0]) + otputils.CurlPod2PodFailUDN(oc, nadNS[1], nsPodMap[nadNS[1]][0], nadNS[0], nsPodMap[nadNS[0]][0]) + + g.By("8. Create allow ingress from all namespaces networkpolicy in first namespace") + otputils.CreateResourceFromFile(oc, nadNS[0], ingressAllowAllNSFile) + output, err = oc.AsAdmin().WithoutNamespace().Run("get").Args("networkpolicy", "-n", nadNS[0]).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("allow-from-all-namespaces")) + + g.By("9. Validate traffic from pods in second namespace") + otputils.CurlPod2PodPassUDN(oc, nadNS[1], nsPodMap[nadNS[1]][0], nadNS[0], nsPodMap[nadNS[0]][0]) + + g.By(fmt.Sprintf("10. Create NAD with same network %s in namespace %s as the first two namespaces and %s (different network) in %s", nadNetworkName[0], nadNS[2], nadNetworkName[1], nadNS[3])) + for i := 2; i < 4; i++ { + nad[i] = otputils.UdnNetDefResource{ + Nadname: nadResourcename + strconv.Itoa(i) + "-" + testID, + Namespace: nadNS[i], + NadNetworkName: nadNetworkName[i-2], + Topology: topology, + Subnet: subnet[i-2], + NetAttachDefName: nadNS[i] + "/" + nadResourcename + strconv.Itoa(i) + "-" + testID, + Role: "primary", + Template: udnNADTemplate, + } + nad[i].CreateUdnNad(oc) + } + + g.By("11. Create one pod each in last three namespaces, last one being without NAD") + pod = make([]otputils.UdnPodResource, 6) + for i := 2; i < 5; i++ { + for j := 0; j < 1; j++ { + pod[j] = otputils.UdnPodResource{ + Name: "hello-pod-" + testID + "-" + strconv.Itoa(i) + "-" + strconv.Itoa(j), + Namespace: nadNS[i], + Label: "hello-pod", + Template: udnPodTemplate, + } + pod[j].CreateUdnPod(oc) + otputils.WaitPodReady(oc, pod[j].Namespace, pod[j].Name) + nsPodMap[pod[j].Namespace] = append(nsPodMap[pod[j].Namespace], pod[j].Name) + } + } + g.By("12. Validate traffic from pods in third and fourth namespace works but not from pod in fifth namespace (default)") + otputils.CurlPod2PodPassUDN(oc, nadNS[0], nsPodMap[nadNS[0]][0], nadNS[2], nsPodMap[nadNS[2]][0]) + otputils.CurlPod2PodFailUDN(oc, nadNS[0], nsPodMap[nadNS[0]][0], nadNS[3], nsPodMap[nadNS[3]][0]) + otputils.CurlPod2PodFail(oc, nadNS[0], nsPodMap[nadNS[0]][0], nadNS[4], nsPodMap[nadNS[4]][0]) + + g.By("13. Update allow-all-namespaces policy with label to allow ingress traffic from pod in second namespace only") + npPatch := `[{"op": "replace", "path": "/spec/ingress/0/from/0/namespaceSelector", "value": {"matchLabels": {"team": "ocp" }}}]` + otputils.PatchReplaceResourceAsAdmin(oc, "networkpolicy/allow-from-all-namespaces", npPatch, nadNS[0]) + npRules, npErr := oc.AsAdmin().WithoutNamespace().Run("get").Args("networkpolicy", "allow-from-all-namespaces", "-n", nadNS[0], "-o=jsonpath={.spec}").Output() + o.Expect(npErr).NotTo(o.HaveOccurred()) + e2e.Logf("\n Network policy after update: %s", npRules) + + g.By("14. Validate traffic from pods in second namespace works but fails from pod in third namespace") + otputils.CurlPod2PodPassUDN(oc, nadNS[1], nsPodMap[nadNS[1]][0], nadNS[0], nsPodMap[nadNS[0]][0]) + otputils.CurlPod2PodFailUDN(oc, nadNS[2], nsPodMap[nadNS[2]][0], nadNS[0], nsPodMap[nadNS[0]][0]) + }) + + g.It("[JIRA:Networking][OTP] 79092-Validate egress allow-same-namespace and allow-all-namespaces network policies in Layer 2 NAD", func() { + var ( + testID = "79092" + testDataDir = testdata.FixturePath("networking") + netsegDir = testdata.FixturePath("networking/network_segmentation") + testDataDirUDN = testdata.FixturePath("networking/network_segmentation/udn") + udnNADTemplate = filepath.Join(testDataDirUDN, "udn_nad_template.yaml") + udnPodTemplate = filepath.Join(testDataDirUDN, "udn_test_pod_template.yaml") + egressDenyFile = filepath.Join(testDataDir, "networkpolicy/default-deny-egress.yaml") + egressAllowSameNSFile = filepath.Join(testDataDir, "networkpolicy/allow-to-same-namespace.yaml") + egressAllowAllNSFile = filepath.Join(netsegDir, "networkpolicy/allow-to-all-namespaces.yaml") + nsPodMap = make(map[string][]string) + nadResourcename = "l2-network-" + topology = "layer2" + ) + ipStackType := otputils.CheckIPStackType(oc) + var nadName string + var nadNS []string = make([]string, 0, 4) + nadNetworkName := []string{"l2-network-test-1", "l2-network-test-2"} + nsDefaultNetwork := oc.Namespace() + + g.By("1.0 Create 4 UDN namespaces") + for i := 0; i < 4; i++ { + oc.CreateNamespaceUDN() + nadNS = append(nadNS, oc.Namespace()) + } + nadNS = append(nadNS, nsDefaultNetwork) + var subnet []string + if ipStackType == "ipv4single" { + subnet = []string{"10.150.0.0/16", "10.152.0.0/16"} + } else { + if ipStackType == "ipv6single" { + subnet = []string{"2010:100:200::0/60", "2012:100:200::0/60"} + } else { + subnet = []string{"10.150.0.0/16,2010:100:200::0/60", "10.152.0.0/16,2012:100:200::0/60"} + } + } + + g.By("2. Create Layer 2 NAD in first two namespaces") + nad := make([]otputils.UdnNetDefResource, 4) + for i := 0; i < 2; i++ { + nadName = nadResourcename + strconv.Itoa(i) + "-" + testID + if i == 1 { + o.Expect(oc.AsAdmin().WithoutNamespace().Run("label").Args("ns", nadNS[i], "team=ocp").Execute()).NotTo(o.HaveOccurred()) + } + g.By(fmt.Sprintf("Create NAD %s in namespace %s", nadName, nadNS[i])) + nad[i] = otputils.UdnNetDefResource{ + Nadname: nadName, + Namespace: nadNS[i], + NadNetworkName: nadNetworkName[0], + Topology: topology, + Subnet: subnet[0], + NetAttachDefName: nadNS[i] + "/" + nadName, + Role: "primary", + Template: udnNADTemplate, + } + nad[i].CreateUdnNad(oc) + } + + g.By("3. Create two pods in each namespace") + pod := make([]otputils.UdnPodResource, 4) + for i := 0; i < 2; i++ { + for j := 0; j < 2; j++ { + pod[j] = otputils.UdnPodResource{ + Name: "hello-pod-" + testID + "-" + strconv.Itoa(i) + "-" + strconv.Itoa(j), + Namespace: nadNS[i], + Label: "hello-pod", + Template: udnPodTemplate, + } + pod[j].CreateUdnPod(oc) + otputils.WaitPodReady(oc, pod[j].Namespace, pod[j].Name) + nsPodMap[pod[j].Namespace] = append(nsPodMap[pod[j].Namespace], pod[j].Name) + } + } + otputils.CurlPod2PodPassUDN(oc, nadNS[0], nsPodMap[nadNS[0]][0], nadNS[1], nsPodMap[nadNS[1]][0]) + otputils.CurlPod2PodPassUDN(oc, nadNS[0], nsPodMap[nadNS[0]][0], nadNS[0], nsPodMap[nadNS[0]][1]) + + g.By("4. Create default deny egress type networkpolicy in first namespace") + otputils.CreateResourceFromFile(oc, nadNS[0], egressDenyFile) + output, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("networkpolicy", "-n", nadNS[0]).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("default-deny-egress")) + + g.By("5. Validate traffic between pods in first namespace and from pods in second namespace") + otputils.CurlPod2PodFailUDN(oc, nadNS[0], nsPodMap[nadNS[0]][0], nadNS[1], nsPodMap[nadNS[1]][0]) + otputils.CurlPod2PodFailUDN(oc, nadNS[0], nsPodMap[nadNS[0]][0], nadNS[0], nsPodMap[nadNS[0]][1]) + + g.By("6. Create allow egress to same namespace networkpolicy in first namespace") + otputils.CreateResourceFromFile(oc, nadNS[0], egressAllowSameNSFile) + output, err = oc.AsAdmin().WithoutNamespace().Run("get").Args("networkpolicy", "-n", nadNS[0]).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("allow-to-same-namespace")) + + g.By("7. Validate traffic between pods in first namespace works but traffic from pod in second namespace is blocked") + otputils.CurlPod2PodPassUDN(oc, nadNS[0], nsPodMap[nadNS[0]][0], nadNS[0], nsPodMap[nadNS[0]][1]) + otputils.CurlPod2PodFailUDN(oc, nadNS[0], nsPodMap[nadNS[0]][0], nadNS[1], nsPodMap[nadNS[1]][0]) + + g.By("8. Create allow all namespaces egress type networkpolicy in first namespace") + otputils.CreateResourceFromFile(oc, nadNS[0], egressAllowAllNSFile) + output, err = oc.AsAdmin().WithoutNamespace().Run("get").Args("networkpolicy", "-n", nadNS[0]).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("allow-to-all-namespaces")) + + g.By("9. Validate traffic to pods in second namespace") + otputils.CurlPod2PodPassUDN(oc, nadNS[0], nsPodMap[nadNS[0]][0], nadNS[1], nsPodMap[nadNS[1]][0]) + + g.By(fmt.Sprintf("10. Create NAD with same network %s in namespace %s as the first two namespaces and %s (different network) in %s", nadNetworkName[0], nadNS[2], nadNetworkName[1], nadNS[3])) + for i := 2; i < 4; i++ { + nad[i] = otputils.UdnNetDefResource{ + Nadname: nadResourcename + strconv.Itoa(i) + "-" + testID, + Namespace: nadNS[i], + NadNetworkName: nadNetworkName[i-2], + Topology: topology, + Subnet: subnet[i-2], + NetAttachDefName: nadNS[i] + "/" + nadResourcename + strconv.Itoa(i) + "-" + testID, + Role: "primary", + Template: udnNADTemplate, + } + nad[i].CreateUdnNad(oc) + } + + g.By("11. Create one pod each in last three namespaces, last one being without NAD") + pod = make([]otputils.UdnPodResource, 6) + for i := 2; i < 5; i++ { + for j := 0; j < 1; j++ { + pod[j] = otputils.UdnPodResource{ + Name: "hello-pod-" + testID + "-" + strconv.Itoa(i) + "-" + strconv.Itoa(j), + Namespace: nadNS[i], + Label: "hello-pod", + Template: udnPodTemplate, + } + pod[j].CreateUdnPod(oc) + otputils.WaitPodReady(oc, pod[j].Namespace, pod[j].Name) + nsPodMap[pod[j].Namespace] = append(nsPodMap[pod[j].Namespace], pod[j].Name) + } + } + + g.By("12. Validate traffic to pods in third and fourth namespace works but not to pod in fifth namespace (default)") + otputils.CurlPod2PodPassUDN(oc, nadNS[2], nsPodMap[nadNS[2]][0], nadNS[0], nsPodMap[nadNS[0]][0]) + otputils.CurlPod2PodFailUDN(oc, nadNS[3], nsPodMap[nadNS[3]][0], nadNS[0], nsPodMap[nadNS[0]][0]) + otputils.CurlPod2PodFail(oc, nadNS[4], nsPodMap[nadNS[4]][0], nadNS[0], nsPodMap[nadNS[0]][0]) + + g.By("13. Update allow-all-namespaces policy with label to allow egress traffic to pod in second namespace only") + npPatch := `[{"op": "replace", "path": "/spec/egress/0/to/0/namespaceSelector", "value": {"matchLabels": {"team": "ocp" }}}]` + otputils.PatchReplaceResourceAsAdmin(oc, "networkpolicy/allow-to-all-namespaces", npPatch, nadNS[0]) + npRules, npErr := oc.AsAdmin().WithoutNamespace().Run("get").Args("networkpolicy", "allow-to-all-namespaces", "-n", nadNS[0], "-o=jsonpath={.spec}").Output() + o.Expect(npErr).NotTo(o.HaveOccurred()) + e2e.Logf("\n Network policy after update: %s", npRules) + + g.By("14. Validate traffic to pods in second namespace works but fails to pod in third namespace") + otputils.CurlPod2PodPassUDN(oc, nadNS[0], nsPodMap[nadNS[0]][0], nadNS[1], nsPodMap[nadNS[1]][0]) + otputils.CurlPod2PodFailUDN(oc, nadNS[0], nsPodMap[nadNS[0]][0], nadNS[2], nsPodMap[nadNS[2]][0]) + }) + + g.It("[JIRA:Networking][OTP] 79093-Validate ingress CIDR block with and without except clause network policies in Layer 3 CUDN", func() { + var ( + testID = "79093" + testDataDir = testdata.FixturePath("networking") + testDataDirUDN = testdata.FixturePath("networking/network_segmentation/udn") + udnPodTemplate = filepath.Join(testDataDirUDN, "udn_test_pod_template.yaml") + udnPodNodeTemplate = filepath.Join(testDataDirUDN, "udn_test_pod_template_node.yaml") + ingressDenyFile = filepath.Join(testDataDir, "networkpolicy/default-deny-ingress.yaml") + ipBlockIngressTemplateDual = filepath.Join(testDataDir, "networkpolicy/ipblock/ipBlock-ingress-dual-CIDRs-template.yaml") + ipBlockIngressTemplateSingle = filepath.Join(testDataDir, "networkpolicy/ipblock/ipBlock-ingress-single-CIDR-template.yaml") + nsPodMap = make(map[string][]string) + topology = "layer3" + matchLabelKey = "test.io" + matchLabelVal = "ns-" + testID + cudnCRDName = "cudn-l3-network-" + testID + udnCRDName = "udn-l3-network-" + testID + "-0" + ) + ipStackType := otputils.CheckIPStackType(oc) + var allNS []string = make([]string, 0, 3) + var ipBlockPolicyName string + var podCount int + nsDefaultNetwork := oc.Namespace() + + g.By("1.0 Create 3 UDN namespaces") + for i := 0; i < 3; i++ { + oc.CreateNamespaceUDN() + ns := oc.Namespace() + allNS = append(allNS, ns) + // Label first two for CUDN + if i < 2 { + defer oc.AsAdmin().WithoutNamespace().Run("label").Args("ns", ns, fmt.Sprintf("%s-", matchLabelKey)).Execute() + err := oc.AsAdmin().WithoutNamespace().Run("label").Args("ns", ns, fmt.Sprintf("%s=%s", matchLabelKey, matchLabelVal)).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + } + } + // Annotate first namespace for ACL logging + otputils.EnableACLOnNamespace(oc, allNS[0], "alert", "alert") + + allNS = append(allNS, nsDefaultNetwork) + var cidr0, ipv4cidr0, ipv6cidr0, cidr1, ipv4cidr1, ipv6cidr1 string + if ipStackType == "ipv4single" { + cidr0 = "10.150.0.0/16" + cidr1 = "10.152.0.0/16" + } else { + if ipStackType == "ipv6single" { + cidr0 = "2010:100:200::0/48" + cidr1 = "2012:100:200::0/48" + } else { + ipv4cidr0 = "10.150.0.0/16" + ipv4cidr1 = "10.152.0.0/16" + ipv6cidr0 = "2010:100:200::0/48" + ipv6cidr1 = "2012:100:200::0/48" + } + } + + g.By("2. Create default deny ingress type networkpolicy in first namespace before UDN is created") + otputils.CreateResourceFromFile(oc, allNS[0], ingressDenyFile) + output, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("networkpolicy", "-n", allNS[0]).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("default-deny-ingress")) + + g.By("3. Create Layer 3 UDN in first two namespaces with CUDN resource and UDN in third") + defer otputils.RemoveResource(oc, true, true, "clusteruserdefinednetwork", cudnCRDName) + _, cudnErr := otputils.ApplyCUDNtoMatchLabelNS(oc, matchLabelKey, matchLabelVal, cudnCRDName, ipv4cidr0, ipv6cidr0, cidr0, topology) + o.Expect(cudnErr).NotTo(o.HaveOccurred()) + defer otputils.RemoveResource(oc, true, true, "userdefinednetwork", udnCRDName) + otputils.CreateGeneralUDNCRD(oc, allNS[2], udnCRDName, ipv4cidr1, ipv6cidr1, cidr1, topology) + + g.By("4. Create two pods in each namespace") + podCount = 2 + pod := make([]otputils.UdnPodResource, 4) + for i := 0; i < len(allNS); i++ { + if i == 2 { + podCount = 1 + } + for j := 0; j < podCount; j++ { + pod[j] = otputils.UdnPodResource{ + Name: "hello-pod-" + testID + "-" + strconv.Itoa(i) + "-" + strconv.Itoa(j), + Namespace: allNS[i], + Label: "hello-pod", + Template: udnPodTemplate, + } + pod[j].CreateUdnPod(oc) + defer otputils.RemoveResource(oc, true, true, "pod", pod[j].Name, "-n", pod[j].Namespace) + otputils.WaitPodReady(oc, pod[j].Namespace, pod[j].Name) + nsPodMap[pod[j].Namespace] = append(nsPodMap[pod[j].Namespace], pod[j].Name) + } + } + + g.By("5. Validate traffic between pods in first namespace and from pods in second namespace") + otputils.CurlPod2PodFailUDN(oc, allNS[1], nsPodMap[allNS[1]][0], allNS[0], nsPodMap[allNS[0]][0]) + otputils.CurlPod2PodFailUDN(oc, allNS[0], nsPodMap[allNS[0]][1], allNS[0], nsPodMap[allNS[0]][0]) + + g.By("6. Get node name and IPs of first pod in first namespace") + podNodeName, podNodeNameErr := otputils.GetPodNodeName(oc, allNS[0], nsPodMap[allNS[0]][0]) + o.Expect(podNodeNameErr).NotTo(o.HaveOccurred()) + o.Expect(podNodeName).NotTo(o.BeEmpty()) + + g.By("7. Validate verdict=drop message") + output, logErr := oc.AsAdmin().WithoutNamespace().Run("adm").Args("node-logs", podNodeName, "--path=ovn/acl-audit-log.log").Output() + o.Expect(logErr).NotTo(o.HaveOccurred()) + o.Expect(strings.Contains(output, "verdict=drop")).To(o.BeTrue()) + + g.By("8. Create IP Block ingress policy to allow traffic from first pod in second namespace to first pod in first") + var cidrIpv4, cidrIpv6, cidr string + if ipStackType == "dualstack" { + g.By(fmt.Sprintf("Create ipBlock Ingress Dual CIDRs Policy in %s", allNS[0])) + pod1ns1IPv6, pod1ns1IPv4 := otputils.GetPodIPUDN(oc, allNS[1], nsPodMap[allNS[1]][0], "ovn-udn1") + cidrIpv4 = pod1ns1IPv4 + "/32" + cidrIpv6 = pod1ns1IPv6 + "/128" + npIPBlockNS1 := otputils.IpBlockCIDRsDual{ + Name: "ipblock-dual-cidrs-ingress", + Template: ipBlockIngressTemplateDual, + CidrIpv4: cidrIpv4, + CidrIpv6: cidrIpv6, + Namespace: allNS[0], + } + npIPBlockNS1.CreateipBlockCIDRObjectDual(oc) + ipBlockPolicyName = npIPBlockNS1.Name + } else { + pod1ns1, _ := otputils.GetPodIPUDN(oc, allNS[1], nsPodMap[allNS[1]][0], "ovn-udn1") + if ipStackType == "ipv6single" { + cidr = pod1ns1 + "/128" + } else { + cidr = pod1ns1 + "/32" + } + npIPBlockNS1 := otputils.IpBlockCIDRsSingle{ + Name: "ipblock-single-cidr-ingress", + Template: ipBlockIngressTemplateSingle, + Cidr: cidr, + Namespace: allNS[0], + } + npIPBlockNS1.CreateipBlockCIDRObjectSingle(oc) + ipBlockPolicyName = npIPBlockNS1.Name + } + + output, err = oc.AsAdmin().WithoutNamespace().Run("get").Args("networkpolicy", "-n", allNS[0]).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring(ipBlockPolicyName)) + + g.By("9. Validate traffic to first pod in first namespace is allowed from first pod in second namespace and verdict=allow in ACL audit log") + otputils.CurlPod2PodPassUDN(oc, allNS[1], nsPodMap[allNS[1]][0], allNS[0], nsPodMap[allNS[0]][0]) + output, logErr = oc.AsAdmin().WithoutNamespace().Run("adm").Args("node-logs", podNodeName, "--path=ovn/acl-audit-log.log").Output() + o.Expect(logErr).NotTo(o.HaveOccurred()) + o.Expect(strings.Contains(output, "verdict=allow")).To(o.BeTrue()) + + g.By("10. Validate ingress traffic is not allowed from second pod in second namespace, pod in third namespace and pod in fourth (default network)") + otputils.CurlPod2PodFailUDN(oc, allNS[1], nsPodMap[allNS[1]][1], allNS[0], nsPodMap[allNS[0]][0]) + otputils.CurlPod2PodFailUDN(oc, allNS[2], nsPodMap[allNS[2]][0], allNS[0], nsPodMap[allNS[0]][0]) + otputils.CurlPod2PodFailUDN(oc, allNS[3], nsPodMap[allNS[3]][0], allNS[0], nsPodMap[allNS[0]][0]) + + g.By("11. Get node name of first pod in second namespace and schedule another pod on same node") + podNodeName, podNodeNameErr = otputils.GetPodNodeName(oc, allNS[1], nsPodMap[allNS[1]][0]) + o.Expect(podNodeNameErr).NotTo(o.HaveOccurred()) + o.Expect(podNodeName).NotTo(o.BeEmpty()) + newPod := otputils.UdnPodResourceNode{ + Name: "hello-pod-" + testID + "-1-2", + Namespace: allNS[1], + Label: "hello-pod", + Nodename: podNodeName, + Template: udnPodNodeTemplate, + } + newPod.CreateUdnPodNode(oc) + defer otputils.RemoveResource(oc, true, true, "pod", newPod.Name, "-n", newPod.Namespace) + otputils.WaitPodReady(oc, newPod.Namespace, newPod.Name) + + g.By(fmt.Sprintf("12. Update the %s policy to include except clause to block the ingress from the first pod in second", ipBlockPolicyName)) + var patchPayload string + if ipStackType == "dualstack" { + hostSubnetIPv4, hostSubnetIPv6 := otputils.GetNodeSubnetDualStack(oc, podNodeName, "cluster_udn_"+cudnCRDName) + patchPayload = fmt.Sprintf("[{\"op\": \"replace\", \"path\":\"/spec/ingress/0/from\", \"value\": [{\"ipBlock\":{\"cidr\":%s,\"except\":[%s]}},{\"ipBlock\":{\"cidr\":%s,\"except\":[%s]}}] }]", hostSubnetIPv4, cidrIpv4, hostSubnetIPv6, cidrIpv6) + } else { + hostSubnetCIDR := otputils.GetNodeSubnet(oc, podNodeName, "cluster_udn_"+cudnCRDName) + patchPayload = fmt.Sprintf("[{\"op\": \"replace\", \"path\":\"/spec/ingress/0/from\", \"value\": [{\"ipBlock\":{\"cidr\":%s,\"except\":[%s]}}] }]", hostSubnetCIDR, cidr) + } + otputils.PatchReplaceResourceAsAdmin(oc, "networkpolicy/"+ipBlockPolicyName, patchPayload, allNS[0]) + npRules, npErr := oc.AsAdmin().WithoutNamespace().Run("get").Args("networkpolicy", ipBlockPolicyName, "-n", allNS[0], "-o=jsonpath={.spec}").Output() + o.Expect(npErr).NotTo(o.HaveOccurred()) + e2e.Logf("\n Network policy after update: %s", npRules) + + otputils.CurlPod2PodFailUDN(oc, allNS[1], nsPodMap[allNS[1]][0], allNS[0], nsPodMap[allNS[0]][0]) + otputils.CurlPod2PodPassUDN(oc, allNS[1], newPod.Name, allNS[0], nsPodMap[allNS[0]][0]) + }) + + g.It("[JIRA:Networking][OTP] 79094-Validate egress CIDR block with and without except clause network policies in Layer 3 CUDN", func() { + var ( + testID = "79094" + testDataDir = testdata.FixturePath("networking") + testDataDirUDN = testdata.FixturePath("networking/network_segmentation/udn") + udnPodTemplate = filepath.Join(testDataDirUDN, "udn_test_pod_template.yaml") + udnPodNodeTemplate = filepath.Join(testDataDirUDN, "udn_test_pod_template_node.yaml") + egressDenyFile = filepath.Join(testDataDir, "networkpolicy/default-deny-egress.yaml") + egressAllowFile = filepath.Join(testDataDir, "networkpolicy/egress-allow-all.yaml") + ipBlockEgressTemplateDual = filepath.Join(testDataDir, "networkpolicy/ipblock/ipBlock-egress-dual-CIDRs-template.yaml") + ipBlockEgressTemplateSingle = filepath.Join(testDataDir, "networkpolicy/ipblock/ipBlock-egress-single-CIDR-template.yaml") + nsPodMap = make(map[string][]string) + topology = "layer3" + matchLabelKey = "test.io" + matchLabelVal = "ns-" + testID + cudnCRDName = "cudn-l3-network-" + testID + udnCRDName = "udn-l3-network-" + testID + "-0" + ) + ipStackType := otputils.CheckIPStackType(oc) + var allNS []string = make([]string, 0, 3) + var ipBlockPolicyName string + var podCount int + + g.By("1. Create 3 UDN namespaces") + for i := 0; i < 3; i++ { + oc.CreateNamespaceUDN() + ns := oc.Namespace() + allNS = append(allNS, ns) + if i < 2 { + defer oc.AsAdmin().WithoutNamespace().Run("label").Args("ns", ns, fmt.Sprintf("%s-", matchLabelKey)).Execute() + err := oc.AsAdmin().WithoutNamespace().Run("label").Args("ns", ns, fmt.Sprintf("%s=%s", matchLabelKey, matchLabelVal)).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + } + } + + var cidr0, ipv4cidr0, ipv6cidr0, cidr1, ipv4cidr1, ipv6cidr1 string + if ipStackType == "ipv4single" { + cidr0 = "10.150.0.0/16" + cidr1 = "10.152.0.0/16" + } else if ipStackType == "ipv6single" { + cidr0 = "2010:100:200::0/48" + cidr1 = "2012:100:200::0/48" + } else { + ipv4cidr0 = "10.150.0.0/16" + ipv4cidr1 = "10.152.0.0/16" + ipv6cidr0 = "2010:100:200::0/48" + ipv6cidr1 = "2012:100:200::0/48" + } + + g.By("2. Create default deny egress type networkpolicy in first namespace before UDN is created") + otputils.CreateResourceFromFile(oc, allNS[0], egressDenyFile) + output, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("networkpolicy", "-n", allNS[0]).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("default-deny-egress")) + + g.By("3. Create Layer 3 UDN in first two namespaces with CUDN resource and UDN in third") + defer otputils.RemoveResource(oc, true, true, "clusteruserdefinednetwork", cudnCRDName) + _, cudnErr := otputils.ApplyCUDNtoMatchLabelNS(oc, matchLabelKey, matchLabelVal, cudnCRDName, ipv4cidr0, ipv6cidr0, cidr0, topology) + o.Expect(cudnErr).NotTo(o.HaveOccurred()) + defer otputils.RemoveResource(oc, true, true, "userdefinednetwork", udnCRDName) + otputils.CreateGeneralUDNCRD(oc, allNS[2], udnCRDName, ipv4cidr1, ipv6cidr1, cidr1, topology) + + g.By("4. Create two pods in each namespace") + podCount = 2 + pod := make([]otputils.UdnPodResource, 4) + for i := 0; i < len(allNS); i++ { + if i == 2 { + podCount = 1 + } + for j := 0; j < podCount; j++ { + pod[j] = otputils.UdnPodResource{ + Name: "hello-pod-" + testID + "-" + strconv.Itoa(i) + "-" + strconv.Itoa(j), + Namespace: allNS[i], + Label: "test-pods", + Template: udnPodTemplate, + } + pod[j].CreateUdnPod(oc) + defer otputils.RemoveResource(oc, true, true, "pod", pod[j].Name, "-n", pod[j].Namespace) + otputils.WaitPodReady(oc, pod[j].Namespace, pod[j].Name) + nsPodMap[pod[j].Namespace] = append(nsPodMap[pod[j].Namespace], pod[j].Name) + } + } + + g.By("5. Validate traffic between pods in first namespace and from pods in second namespace") + otputils.CurlPod2PodFailUDN(oc, allNS[0], nsPodMap[allNS[0]][0], allNS[1], nsPodMap[allNS[1]][0]) + otputils.CurlPod2PodFailUDN(oc, allNS[0], nsPodMap[allNS[0]][0], allNS[0], nsPodMap[allNS[0]][1]) + + g.By("6. Create IP Block egress policy to allow traffic from first pod in first namespace to first pod in second namespace") + var cidrIpv4, cidrIpv6, cidr string + if ipStackType == "dualstack" { + g.By(fmt.Sprintf("Create ipBlock Egress Dual CIDRs Policy in %s", allNS[0])) + pod1ns1IPv6, pod1ns1IPv4 := otputils.GetPodIPUDN(oc, allNS[1], nsPodMap[allNS[1]][0], "ovn-udn1") + cidrIpv4 = pod1ns1IPv4 + "/32" + cidrIpv6 = pod1ns1IPv6 + "/128" + npIPBlockNS1 := otputils.IpBlockCIDRsDual{ + Name: "ipblock-dual-cidrs-egress", + Template: ipBlockEgressTemplateDual, + CidrIpv4: cidrIpv4, + CidrIpv6: cidrIpv6, + Namespace: allNS[0], + } + npIPBlockNS1.CreateipBlockCIDRObjectDual(oc) + ipBlockPolicyName = npIPBlockNS1.Name + } else { + pod1ns1, _ := otputils.GetPodIPUDN(oc, allNS[1], nsPodMap[allNS[1]][0], "ovn-udn1") + if ipStackType == "ipv6single" { + cidr = pod1ns1 + "/128" + } else { + cidr = pod1ns1 + "/32" + } + npIPBlockNS1 := otputils.IpBlockCIDRsSingle{ + Name: "ipblock-single-cidr-egress", + Template: ipBlockEgressTemplateSingle, + Cidr: cidr, + Namespace: allNS[0], + } + npIPBlockNS1.CreateipBlockCIDRObjectSingle(oc) + ipBlockPolicyName = npIPBlockNS1.Name + } + + output, err = oc.AsAdmin().WithoutNamespace().Run("get").Args("networkpolicy", "-n", allNS[0]).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring(ipBlockPolicyName)) + + g.By("7. Validate egress traffic is allowed to first pod in second namespace and not to others") + otputils.CurlPod2PodPassUDN(oc, allNS[0], nsPodMap[allNS[0]][0], allNS[1], nsPodMap[allNS[1]][0]) + otputils.CurlPod2PodFailUDN(oc, allNS[0], nsPodMap[allNS[0]][0], allNS[1], nsPodMap[allNS[1]][1]) + otputils.CurlPod2PodFailUDN(oc, allNS[0], nsPodMap[allNS[0]][0], allNS[2], nsPodMap[allNS[2]][0]) + + g.By("8. Get node name of first pod in second namespace and schedule another pod on same node") + podNodeName, podNodeNameErr := otputils.GetPodNodeName(oc, allNS[1], nsPodMap[allNS[1]][0]) + o.Expect(podNodeNameErr).NotTo(o.HaveOccurred()) + o.Expect(podNodeName).NotTo(o.BeEmpty()) + newPodNS2 := otputils.UdnPodResourceNode{ + Name: "hello-pod-" + testID + "-1-2", + Namespace: allNS[1], + Label: "test-pods", + Nodename: podNodeName, + Template: udnPodNodeTemplate, + } + newPodNS2.CreateUdnPodNode(oc) + defer otputils.RemoveResource(oc, true, true, "pod", newPodNS2.Name, "-n", newPodNS2.Namespace) + otputils.WaitPodReady(oc, newPodNS2.Namespace, newPodNS2.Name) + + g.By(fmt.Sprintf("9. Update the %s policy to include except clause to block the egress to the first pod in second namespace", ipBlockPolicyName)) + var patchPayload string + if ipStackType == "dualstack" { + hostSubnetIPv4, hostSubnetIPv6 := otputils.GetNodeSubnetDualStack(oc, podNodeName, "cluster_udn_"+cudnCRDName) + patchPayload = fmt.Sprintf("[{\"op\": \"replace\", \"path\":\"/spec/egress/0/to\", \"value\": [{\"ipBlock\":{\"cidr\":%s,\"except\":[%s]}},{\"ipBlock\":{\"cidr\":%s,\"except\":[%s]}}] }]", hostSubnetIPv4, cidrIpv4, hostSubnetIPv6, cidrIpv6) + } else { + hostSubnetCIDR := otputils.GetNodeSubnet(oc, podNodeName, "cluster_udn_"+cudnCRDName) + patchPayload = fmt.Sprintf("[{\"op\": \"replace\", \"path\":\"/spec/egress/0/to\", \"value\": [{\"ipBlock\":{\"cidr\":%s,\"except\":[%s]}}] }]", hostSubnetCIDR, cidr) + } + otputils.PatchReplaceResourceAsAdmin(oc, "networkpolicy/"+ipBlockPolicyName, patchPayload, allNS[0]) + npRules, npErr := oc.AsAdmin().WithoutNamespace().Run("get").Args("networkpolicy", ipBlockPolicyName, "-n", allNS[0], "-o=jsonpath={.spec}").Output() + o.Expect(npErr).NotTo(o.HaveOccurred()) + e2e.Logf("\n Network policy after update: %s", npRules) + + otputils.CurlPod2PodFailUDN(oc, allNS[0], nsPodMap[allNS[0]][0], allNS[1], nsPodMap[allNS[1]][0]) + otputils.CurlPod2PodPassUDN(oc, allNS[0], nsPodMap[allNS[0]][0], allNS[1], newPodNS2.Name) + + g.By("10. Validate egress traffic from first namespace to DNS works with allow-all-egress policy only from pod labeled test-pods") + newPodNS1 := otputils.UdnPodResource{ + Name: "hello-pod-" + testID + "-0-2", + Namespace: allNS[0], + Label: "hello-pod", + Template: udnPodTemplate, + } + defer otputils.RemoveResource(oc, true, true, "pod", newPodNS1.Name, "-n", newPodNS1.Namespace) + newPodNS1.CreateUdnPod(oc) + otputils.WaitPodReady(oc, newPodNS1.Namespace, newPodNS1.Name) + + digOutput, digErr := e2eoutput.RunHostCmd(allNS[0], nsPodMap[allNS[0]][0], "dig kubernetes.default") + o.Expect(digErr).To(o.HaveOccurred()) + o.Expect(digOutput).ShouldNot(o.ContainSubstring("Got answer")) + o.Expect(digOutput).Should(o.ContainSubstring("connection timed out")) + + otputils.CreateResourceFromFile(oc, allNS[0], egressAllowFile) + output, err = oc.AsAdmin().WithoutNamespace().Run("get").Args("networkpolicy", "-n", allNS[0]).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("allow-all-egress")) + + digOutput, digErr = e2eoutput.RunHostCmd(allNS[0], nsPodMap[allNS[0]][0], "dig kubernetes.default") + o.Expect(digErr).NotTo(o.HaveOccurred()) + o.Expect(digOutput).Should(o.ContainSubstring("Got answer")) + o.Expect(digOutput).ShouldNot(o.ContainSubstring("connection timed out")) + + digOutput, digErr = e2eoutput.RunHostCmd(allNS[0], newPodNS1.Name, "dig kubernetes.default") + o.Expect(digErr).To(o.HaveOccurred()) + o.Expect(digOutput).ShouldNot(o.ContainSubstring("Got answer")) + o.Expect(digOutput).Should(o.ContainSubstring("connection timed out")) + }) +}) diff --git a/openshift/test/otp/network_segmentation_services.go b/openshift/test/otp/network_segmentation_services.go new file mode 100644 index 0000000000..64a8021f55 --- /dev/null +++ b/openshift/test/otp/network_segmentation_services.go @@ -0,0 +1,1389 @@ +package otp + +import ( + "context" + "fmt" + "net" + "os/exec" + "path/filepath" + "strings" + "time" + + g "github.com/onsi/ginkgo/v2" + o "github.com/onsi/gomega" + + exutil "github.com/openshift/origin/test/extended/util" + otputils "github.com/ovn-kubernetes/ovn-kubernetes/openshift/pkg/otp/utils" + "github.com/ovn-kubernetes/ovn-kubernetes/openshift/pkg/otp/testdata" + + "k8s.io/apimachinery/pkg/util/wait" + e2e "k8s.io/kubernetes/test/e2e/framework" + e2enode "k8s.io/kubernetes/test/e2e/framework/node" + e2eoutput "k8s.io/kubernetes/test/e2e/framework/pod/output" +) + +var _ = g.Describe("[sig-network][Suite:openshift/ovn-kubernetes] SDN network segmentation services", func() { + defer g.GinkgoRecover() + + var ( + oc = exutil.NewCLI("networking-udn") + testDataDirUDN = testdata.FixturePath("networking/network_segmentation/udn") + ) + + g.BeforeEach(func() { + networkType := otputils.CheckNetworkType(oc) + if !strings.Contains(networkType, "ovn") { + g.Skip("Skip testing on non-ovn cluster!!!") + } + }) + + g.It("[JIRA:Networking][OTP] 76017-Service should be able to access for same NAD UDN pods in different namespaces (L3/L2)", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("networking") + udnNadtemplate = filepath.Join(testDataDirUDN, "udn_nad_template.yaml") + udnPodTemplate = filepath.Join(testDataDirUDN, "udn_test_pod_template.yaml") + genericServiceTemplate = filepath.Join(buildPruningBaseDir, "service-generic-template.yaml") + ipFamilyPolicy = "SingleStack" + ) + + ipStackType := otputils.CheckIPStackType(oc) + + g.By("Get first namespace") + var nadNS []string = make([]string, 0, 4) + + g.By("Create another 3 namespaces") + for i := 0; i < 4; i++ { + oc.CreateNamespaceUDN() + nadNS = append(nadNS, oc.Namespace()) + } + + nadResourcename := []string{"l3-network-test", "l2-network-test"} + topo := []string{"layer3", "layer3", "layer2", "layer2"} + + var subnet []string + if ipStackType == "ipv4single" { + subnet = []string{"10.150.0.0/16/24", "10.150.0.0/16/24", "10.152.0.0/16", "10.152.0.0/16"} + } else { + if ipStackType == "ipv6single" { + subnet = []string{"2010:100:200::0/60", "2010:100:200::0/60", "2012:100:200::0/60", "2012:100:200::0/60"} + } else { + subnet = []string{"10.150.0.0/16/24,2010:100:200::0/60", "10.150.0.0/16/24,2010:100:200::0/60", "10.152.0.0/16,2012:100:200::0/60", "10.152.0.0/16,2012:100:200::0/60"} + ipFamilyPolicy = "PreferDualStack" + } + } + + g.By("5. Create same NAD in ns1 ns2 for layer3") + nad := make([]otputils.UdnNetDefResource, 4) + for i := 0; i < 2; i++ { + g.By(fmt.Sprintf("create NAD %s in namespace %s", nadResourcename[0], nadNS[i])) + nad[i] = otputils.UdnNetDefResource{ + Nadname: nadResourcename[0], + Namespace: nadNS[i], + NadNetworkName: nadResourcename[0], + Topology: topo[i], + Subnet: subnet[i], + NetAttachDefName: nadNS[i] + "/" + nadResourcename[0], + Role: "primary", + Template: udnNadtemplate, + } + nad[i].CreateUdnNad(oc) + } + + g.By("6. Create same NAD in ns3 ns4 for layer 2") + for i := 2; i < 4; i++ { + g.By(fmt.Sprintf("create NAD %s in namespace %s", nadResourcename[1], nadNS[i])) + nad[i] = otputils.UdnNetDefResource{ + Nadname: nadResourcename[1], + Namespace: nadNS[i], + NadNetworkName: nadResourcename[1], + Topology: topo[i], + Subnet: subnet[i], + NetAttachDefName: nadNS[i] + "/" + nadResourcename[1], + Role: "primary", + Template: udnNadtemplate, + } + nad[i].CreateUdnNad(oc) + } + + g.By("7. Create one pod in respective namespaces ns1,ns2,ns3,ns4") + pod := make([]otputils.UdnPodResource, 4) + for i := 0; i < 4; i++ { + pod[i] = otputils.UdnPodResource{ + Name: "hello-pod", + Namespace: nadNS[i], + Label: "hello-pod", + Template: udnPodTemplate, + } + pod[i].CreateUdnPod(oc) + otputils.WaitPodReady(oc, pod[i].Namespace, pod[i].Name) + + // add a step to check ovn-udn1 created. + output, err := e2eoutput.RunHostCmd(pod[i].Namespace, pod[i].Name, "ip -o link show") + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).Should(o.ContainSubstring("ovn-udn1")) + } + + g.By("8. Create service in ns2,ns4") + svc1 := otputils.GenericServiceResource{ + Servicename: "test-service", + Namespace: nadNS[1], + Protocol: "TCP", + Selector: "hello-pod", + ServiceType: "ClusterIP", + IpFamilyPolicy: ipFamilyPolicy, + InternalTrafficPolicy: "Cluster", + ExternalTrafficPolicy: "", + Template: genericServiceTemplate, + } + svc1.CreateServiceFromParams(oc) + + svc2 := otputils.GenericServiceResource{ + Servicename: "test-service", + Namespace: nadNS[3], + Protocol: "TCP", + Selector: "hello-pod", + ServiceType: "ClusterIP", + IpFamilyPolicy: ipFamilyPolicy, + InternalTrafficPolicy: "Cluster", + ExternalTrafficPolicy: "", + Template: genericServiceTemplate, + } + svc2.CreateServiceFromParams(oc) + g.By("9. Verify ClusterIP service in ns2 can be accessed from pod in ns1 for layer 3") + otputils.CurlPod2SvcPass(oc, nadNS[0], nadNS[1], pod[0].Name, svc1.Servicename) + g.By("10. Verify ClusterIP service in ns4 can be accessed from pod in ns3 for layer 2") + otputils.CurlPod2SvcPass(oc, nadNS[2], nadNS[3], pod[2].Name, svc2.Servicename) + }) + + g.It("[JIRA:Networking][OTP] 76732-Validate pod2Service/nodePortService for UDN(Layer2)", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("networking") + udnCRDdualStack = filepath.Join(testDataDirUDN, "udn_crd_layer2_dualstack_template.yaml") + udnCRDSingleStack = filepath.Join(testDataDirUDN, "udn_crd_layer2_singlestack_template.yaml") + pingPodTemplate = filepath.Join(buildPruningBaseDir, "ping-for-pod-specific-node-template.yaml") + genericServiceTemplate = filepath.Join(buildPruningBaseDir, "service-generic-template.yaml") + testPodFile = filepath.Join(buildPruningBaseDir, "testpod.yaml") + ipFamilyPolicy = "SingleStack" + ) + + nodeList, nodeErr := e2enode.GetReadySchedulableNodes(context.TODO(), oc.KubeFramework().ClientSet) + o.Expect(nodeErr).NotTo(o.HaveOccurred()) + if len(nodeList.Items) < 3 { + g.Skip("This test requires at least 3 worker nodes which is not fulfilled. ") + } + + g.By("1. Obtain first namespace") + oc.CreateNamespaceUDN() + ns1 := oc.Namespace() + + g.By("2. Create CRD for UDN") + ipStackType := otputils.CheckIPStackType(oc) + var cidr, ipv4cidr, ipv6cidr string + if ipStackType == "ipv4single" { + cidr = "10.150.0.0/16" + } else { + if ipStackType == "ipv6single" { + cidr = "2010:100:200::0/48" + } else { + ipv4cidr = "10.150.0.0/16" + ipv6cidr = "2010:100:200::0/48" + ipFamilyPolicy = "PreferDualStack" + } + } + + g.By("Create CRD for UDN") + var udncrd otputils.UdnCRDResource + if ipStackType == "dualstack" { + udncrd = otputils.UdnCRDResource{ + Crdname: "udn-network-76732", + Namespace: ns1, + Role: "Primary", + IPv4cidr: ipv4cidr, + IPv6cidr: ipv6cidr, + Template: udnCRDdualStack, + } + udncrd.CreateLayer2DualStackUDNCRD(oc) + + } else { + udncrd = otputils.UdnCRDResource{ + Crdname: "udn-network-76732", + Namespace: ns1, + Role: "Primary", + Cidr: cidr, + Template: udnCRDSingleStack, + } + udncrd.CreateLayer2SingleStackUDNCRD(oc) + } + + g.By("3. Create a pod deployed on node0 as backend pod for service.") + pod1ns1 := otputils.PingPodResourceNode{ + Name: "hello-pod-1", + Namespace: ns1, + Nodename: nodeList.Items[0].Name, + Template: pingPodTemplate, + } + pod1ns1.CreatePingPodNode(oc) + otputils.WaitPodReady(oc, pod1ns1.Namespace, pod1ns1.Name) + + g.By("4. create a udn client pod in ns1 on different node as pod1") + clientPod1 := otputils.PingPodResourceNode{ + Name: "client-pod-1", + Namespace: ns1, + Nodename: nodeList.Items[1].Name, + Template: pingPodTemplate, + } + clientPod1.CreatePingPodNode(oc) + otputils.WaitPodReady(oc, clientPod1.Namespace, clientPod1.Name) + // Update label for pod2 to a different one + err := oc.AsAdmin().WithoutNamespace().Run("label").Args("-n", ns1, "pod", clientPod1.Name, "name=client-pod-1", "--overwrite=true").Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("5. create a udn client pod in ns1 on same node as pod1") + clientPod2 := otputils.PingPodResourceNode{ + Name: "client-pod-2", + Namespace: ns1, + Nodename: nodeList.Items[0].Name, + Template: pingPodTemplate, + } + clientPod2.CreatePingPodNode(oc) + otputils.WaitPodReady(oc, clientPod2.Namespace, clientPod2.Name) + // Update label for pod3 to a different one + err = oc.AsAdmin().WithoutNamespace().Run("label").Args("-n", ns1, "pod", clientPod2.Name, "name=client-pod-2", "--overwrite=true").Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("6. create a service in ns1") + svc := otputils.GenericServiceResource{ + Servicename: "test-service", + Namespace: ns1, + Protocol: "TCP", + Selector: "hello-pod", + ServiceType: "ClusterIP", + IpFamilyPolicy: ipFamilyPolicy, + InternalTrafficPolicy: "Cluster", + ExternalTrafficPolicy: "", + Template: genericServiceTemplate, + } + svc.CreateServiceFromParams(oc) + + g.By("7. Verify ClusterIP service can be accessed from both clientPod1 and clientPod2") + otputils.CurlPod2SvcPass(oc, ns1, ns1, clientPod1.Name, svc.Servicename) + otputils.CurlPod2SvcPass(oc, ns1, ns1, clientPod2.Name, svc.Servicename) + + g.By("8. Create a second namespace") + oc.SetupProject() + ns2 := oc.Namespace() + g.By("9. Create service and pods which are on default network.") + otputils.CreateResourceFromFile(oc, ns2, testPodFile) + err = otputils.WaitForPodWithLabelReady(oc, ns2, "name=test-pods") + o.Expect(err).NotTo(o.HaveOccurred(), "this pod with label name=test-pods not ready") + testPodName := otputils.GetPodName(oc, ns2, "name=test-pods") + + g.By("10. Not be able to access udn service from default network.") + otputils.CurlPod2SvcFail(oc, ns2, ns1, testPodName[0], svc.Servicename) + g.By("11. Not be able to access default network service from udn network.") + otputils.CurlPod2SvcFail(oc, ns1, ns2, clientPod1.Name, "test-service") + + g.By("11. Create third namespace for udn pod") + oc.CreateNamespaceUDN() + ns3 := oc.Namespace() + + g.By("12. Create CRD in third namespace") + if ipStackType == "ipv4single" { + cidr = "10.160.0.0/16" + } else { + if ipStackType == "ipv6single" { + cidr = "2010:200:200::0/48" + } else { + ipv4cidr = "10.160.0.0/16" + ipv6cidr = "2010:200:200::0/48" + } + } + var udncrdns3 otputils.UdnCRDResource + if ipStackType == "dualstack" { + udncrdns3 = otputils.UdnCRDResource{ + Crdname: "udn-network-ds-76732-ns3", + Namespace: ns3, + Role: "Primary", + IPv4cidr: ipv4cidr, + IPv6cidr: ipv6cidr, + Template: udnCRDdualStack, + } + udncrdns3.CreateLayer2DualStackUDNCRD(oc) + } else { + udncrdns3 = otputils.UdnCRDResource{ + Crdname: "udn-network-ss-76732-ns3", + Namespace: ns3, + Role: "Primary", + Cidr: cidr, + Template: udnCRDSingleStack, + } + udncrdns3.CreateLayer2SingleStackUDNCRD(oc) + } + err = otputils.WaitUDNCRDApplied(oc, ns3, udncrdns3.Crdname) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("13. Create a udn pod in third namespace") + otputils.CreateResourceFromFile(oc, ns3, testPodFile) + err = otputils.WaitForPodWithLabelReady(oc, ns3, "name=test-pods") + o.Expect(err).NotTo(o.HaveOccurred(), "this pod with label name=test-pods not ready") + testPodNameNS3 := otputils.GetPodName(oc, ns3, "name=test-pods") + + g.By("14. Verify different udn network, service was isolated.") + otputils.CurlPod2SvcFail(oc, ns3, ns1, testPodNameNS3[0], svc.Servicename) + + g.By("15.Update internalTrafficPolicy as Local for udn service in ns1.") + patch := `[{"op": "replace", "path": "/spec/internalTrafficPolicy", "value": "Local"}]` + err = oc.AsAdmin().WithoutNamespace().Run("patch").Args("service/test-service", "-n", ns1, "-p", patch, "--type=json").Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + g.By("15.1. Verify ClusterIP service can be accessed from pod3 which is deployed same node as service back-end pod.") + otputils.CurlPod2SvcPass(oc, ns1, ns1, clientPod2.Name, svc.Servicename) + g.By("15.2. Verify ClusterIP service can NOT be accessed from pod2 which is deployed different node as service back-end pod.") + otputils.CurlPod2SvcFail(oc, ns1, ns1, clientPod1.Name, svc.Servicename) + + g.By("16. Verify nodePort service can be accessed.") + g.By("16.1 Delete testservice from ns1") + otputils.RemoveResource(oc, true, true, "service", "test-service", "-n", ns1) + g.By("16.2 Create testservice with NodePort in ns1") + svc.ServiceType = "NodePort" + svc.CreateServiceFromParams(oc) + + g.By("16.3 From a third node, be able to access node0:nodePort") + nodePort, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("service", "-n", ns1, svc.Servicename, "-o=jsonpath={.spec.ports[*].nodePort}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + thirdNode := nodeList.Items[2].Name + o.Expect(err).NotTo(o.HaveOccurred()) + otputils.CurlNodePortPass(oc, thirdNode, nodeList.Items[0].Name, nodePort) + g.By("16.4 From a third node, be able to access node1:nodePort") + otputils.CurlNodePortPass(oc, thirdNode, nodeList.Items[1].Name, nodePort) + g.By("16.5 From pod node, be able to access nodePort service") + otputils.CurlNodePortPass(oc, nodeList.Items[0].Name, nodeList.Items[0].Name, nodePort) + otputils.CurlNodePortPass(oc, nodeList.Items[0].Name, nodeList.Items[1].Name, nodePort) + + g.By("17.Update externalTrafficPolicy as Local for udn service in ns1.") + patch = `[{"op": "replace", "path": "/spec/externalTrafficPolicy", "value": "Local"}]` + err = oc.AsAdmin().WithoutNamespace().Run("patch").Args("service/test-service", "-n", ns1, "-p", patch, "--type=json").Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + g.By("17.1 From a third node, be able to access node0:nodePort") + otputils.CurlNodePortPass(oc, thirdNode, nodeList.Items[0].Name, nodePort) + g.By("17.2 From a third node, NOT be able to access node1:nodePort") + otputils.CurlNodePortFail(oc, thirdNode, nodeList.Items[1].Name, nodePort) + }) + + g.It("[JIRA:Networking][OTP] 75942-Validate pod2Service/nodePortService for UDN(Layer3)", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("networking") + udnCRDdualStack = filepath.Join(testDataDirUDN, "udn_crd_dualstack2_template.yaml") + udnCRDSingleStack = filepath.Join(testDataDirUDN, "udn_crd_singlestack_template.yaml") + pingPodTemplate = filepath.Join(buildPruningBaseDir, "ping-for-pod-specific-node-template.yaml") + genericServiceTemplate = filepath.Join(buildPruningBaseDir, "service-generic-template.yaml") + testPodFile = filepath.Join(buildPruningBaseDir, "testpod.yaml") + ipFamilyPolicy = "SingleStack" + ) + + nodeList, nodeErr := e2enode.GetReadySchedulableNodes(context.TODO(), oc.KubeFramework().ClientSet) + o.Expect(nodeErr).NotTo(o.HaveOccurred()) + if len(nodeList.Items) < 3 { + g.Skip("This test requires at least 3 worker nodes which is not fulfilled. ") + } + + g.By("1. Obtain first namespace") + oc.CreateNamespaceUDN() + ns1 := oc.Namespace() + + g.By("2. Create CRD for UDN") + ipStackType := otputils.CheckIPStackType(oc) + var cidr, ipv4cidr, ipv6cidr string + var prefix, ipv4prefix, ipv6prefix int32 + if ipStackType == "ipv4single" { + cidr = "10.150.0.0/16" + prefix = 24 + } else { + if ipStackType == "ipv6single" { + cidr = "2010:100:200::0/48" + prefix = 64 + } else { + ipv4cidr = "10.150.0.0/16" + ipv4prefix = 24 + ipv6cidr = "2010:100:200::0/48" + ipv6prefix = 64 + ipFamilyPolicy = "PreferDualStack" + } + } + var udncrd otputils.UdnCRDResource + if ipStackType == "dualstack" { + udncrd = otputils.UdnCRDResource{ + Crdname: "udn-network-ds-75942", + Namespace: ns1, + Role: "Primary", + IPv4cidr: ipv4cidr, + IPv4prefix: ipv4prefix, + IPv6cidr: ipv6cidr, + IPv6prefix: ipv6prefix, + Template: udnCRDdualStack, + } + udncrd.CreateUdnCRDDualStack(oc) + } else { + udncrd = otputils.UdnCRDResource{ + Crdname: "udn-network-ss-75942", + Namespace: ns1, + Role: "Primary", + Cidr: cidr, + Prefix: prefix, + Template: udnCRDSingleStack, + } + udncrd.CreateUdnCRDSingleStack(oc) + } + err := otputils.WaitUDNCRDApplied(oc, ns1, udncrd.Crdname) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("3. Create a pod deployed on node0 as backend pod for service.") + pod1ns1 := otputils.PingPodResourceNode{ + Name: "hello-pod-1", + Namespace: ns1, + Nodename: nodeList.Items[0].Name, + Template: pingPodTemplate, + } + pod1ns1.CreatePingPodNode(oc) + otputils.WaitPodReady(oc, pod1ns1.Namespace, pod1ns1.Name) + + g.By("4. create a udn client pod in ns1 on different node as pod1") + pod2ns1 := otputils.PingPodResourceNode{ + Name: "hello-pod-2", + Namespace: ns1, + Nodename: nodeList.Items[1].Name, + Template: pingPodTemplate, + } + pod2ns1.CreatePingPodNode(oc) + otputils.WaitPodReady(oc, pod2ns1.Namespace, pod2ns1.Name) + // Update label for pod2 to a different one + err = oc.AsAdmin().WithoutNamespace().Run("label").Args("-n", ns1, "pod", pod2ns1.Name, "name=hello-pod-2", "--overwrite=true").Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("5. create a udn client pod in ns1 on same node as pod1") + pod3ns1 := otputils.PingPodResourceNode{ + Name: "hello-pod-3", + Namespace: ns1, + Nodename: nodeList.Items[0].Name, + Template: pingPodTemplate, + } + pod3ns1.CreatePingPodNode(oc) + otputils.WaitPodReady(oc, pod3ns1.Namespace, pod3ns1.Name) + // Update label for pod3 to a different one + err = oc.AsAdmin().WithoutNamespace().Run("label").Args("-n", ns1, "pod", pod3ns1.Name, "name=hello-pod-3", "--overwrite=true").Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("6. create a ClusterIP service in ns1") + svc := otputils.GenericServiceResource{ + Servicename: "test-service", + Namespace: ns1, + Protocol: "TCP", + Selector: "hello-pod", + ServiceType: "ClusterIP", + IpFamilyPolicy: ipFamilyPolicy, + InternalTrafficPolicy: "Cluster", + ExternalTrafficPolicy: "", + Template: genericServiceTemplate, + } + svc.CreateServiceFromParams(oc) + + g.By("7. Verify ClusterIP service can be accessed from both pod2 and pod3") + otputils.CurlPod2SvcPass(oc, ns1, ns1, pod2ns1.Name, svc.Servicename) + otputils.CurlPod2SvcPass(oc, ns1, ns1, pod3ns1.Name, svc.Servicename) + + g.By("8. Create second namespace") + oc.SetupProject() + ns2 := oc.Namespace() + g.By("9. Create service and pods which are on default network.") + otputils.CreateResourceFromFile(oc, ns2, testPodFile) + err = otputils.WaitForPodWithLabelReady(oc, ns2, "name=test-pods") + o.Expect(err).NotTo(o.HaveOccurred(), "this pod with label name=test-pods not ready") + testPodName := otputils.GetPodName(oc, ns2, "name=test-pods") + + g.By("10. Not be able to access udn service from default network.") + otputils.CurlPod2SvcFail(oc, ns2, ns1, testPodName[0], svc.Servicename) + g.By("11. Not be able to access default network service from udn network.") + otputils.CurlPod2SvcFail(oc, ns1, ns2, pod2ns1.Name, "test-service") + + g.By("11. Create third namespace for udn pod") + oc.CreateNamespaceUDN() + ns3 := oc.Namespace() + + g.By("12. Create CRD in third namespace") + if ipStackType == "ipv4single" { + cidr = "10.160.0.0/16" + } else { + if ipStackType == "ipv6single" { + cidr = "2010:200:200::0/48" + prefix = 64 + } else { + ipv4cidr = "10.160.0.0/16" + ipv4prefix = 24 + ipv6cidr = "2010:200:200::0/48" + ipv6prefix = 64 + } + } + var udncrdns3 otputils.UdnCRDResource + if ipStackType == "dualstack" { + udncrdns3 = otputils.UdnCRDResource{ + Crdname: "udn-network-ds-75942-ns3", + Namespace: ns3, + Role: "Primary", + IPv4cidr: ipv4cidr, + IPv4prefix: ipv4prefix, + IPv6cidr: ipv6cidr, + IPv6prefix: ipv6prefix, + Template: udnCRDdualStack, + } + udncrdns3.CreateUdnCRDDualStack(oc) + } else { + udncrdns3 = otputils.UdnCRDResource{ + Crdname: "udn-network-ss-75942-ns3", + Namespace: ns3, + Role: "Primary", + Cidr: cidr, + Prefix: prefix, + Template: udnCRDSingleStack, + } + udncrdns3.CreateUdnCRDSingleStack(oc) + } + err = otputils.WaitUDNCRDApplied(oc, ns3, udncrdns3.Crdname) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("13. Create a udn pod in third namespace") + otputils.CreateResourceFromFile(oc, ns3, testPodFile) + err = otputils.WaitForPodWithLabelReady(oc, ns3, "name=test-pods") + o.Expect(err).NotTo(o.HaveOccurred(), "this pod with label name=test-pods not ready") + testPodNameNS3 := otputils.GetPodName(oc, ns3, "name=test-pods") + + g.By("14. Verify different udn network, service was isolated.") + otputils.CurlPod2SvcFail(oc, ns3, ns1, testPodNameNS3[0], svc.Servicename) + + g.By("15.Update internalTrafficPolicy as Local for udn service in ns1.") + patch := `[{"op": "replace", "path": "/spec/internalTrafficPolicy", "value": "Local"}]` + err = oc.AsAdmin().WithoutNamespace().Run("patch").Args("service/test-service", "-n", ns1, "-p", patch, "--type=json").Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + g.By("15.1. Verify ClusterIP service can be accessed from pod3 which is deployed same node as service back-end pod.") + otputils.CurlPod2SvcPass(oc, ns1, ns1, pod3ns1.Name, svc.Servicename) + g.By("15.2. Verify ClusterIP service can NOT be accessed from pod2 which is deployed different node as service back-end pod.") + otputils.CurlPod2SvcFail(oc, ns1, ns1, pod2ns1.Name, svc.Servicename) + + g.By("16. Verify nodePort service can be accessed.") + g.By("16.1 Delete testservice from ns1") + err = oc.AsAdmin().WithoutNamespace().Run("delete").Args("svc", "test-service", "-n", ns1).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + g.By("16.2 Create testservice with NodePort in ns1") + svc.ServiceType = "NodePort" + svc.CreateServiceFromParams(oc) + + g.By("16.3 From a third node, be able to access node0:nodePort") + nodePort, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("service", "-n", ns1, svc.Servicename, "-o=jsonpath={.spec.ports[*].nodePort}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + thirdNode := nodeList.Items[2].Name + o.Expect(err).NotTo(o.HaveOccurred()) + otputils.CurlNodePortPass(oc, thirdNode, nodeList.Items[0].Name, nodePort) + g.By("16.4 From a third node, be able to access node1:nodePort") + otputils.CurlNodePortPass(oc, thirdNode, nodeList.Items[1].Name, nodePort) + g.By("16.5 From pod node, be able to access nodePort service") + otputils.CurlNodePortPass(oc, nodeList.Items[0].Name, nodeList.Items[0].Name, nodePort) + otputils.CurlNodePortPass(oc, nodeList.Items[0].Name, nodeList.Items[1].Name, nodePort) + + g.By("17.Update externalTrafficPolicy as Local for udn service in ns1.") + patch = `[{"op": "replace", "path": "/spec/externalTrafficPolicy", "value": "Local"}]` + err = oc.AsAdmin().WithoutNamespace().Run("patch").Args("service/test-service", "-n", ns1, "-p", patch, "--type=json").Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + g.By("17.1 From a third node, be able to access node0:nodePort") + otputils.CurlNodePortPass(oc, thirdNode, nodeList.Items[0].Name, nodePort) + g.By("17.2 From a third node, NOT be able to access node1:nodePort") + otputils.CurlNodePortFail(oc, thirdNode, nodeList.Items[1].Name, nodePort) + }) + + g.It("[JIRA:Networking][OTP] 76014-Validate LoadBalancer service for UDN pods (Layer3/Layer2)", func() { + buildPruningBaseDir := testdata.FixturePath("networking") + udnPodTemplate := filepath.Join(testDataDirUDN, "udn_test_pod_template.yaml") + genericServiceTemplate := filepath.Join(buildPruningBaseDir, "service-generic-template.yaml") + udnCRDSingleStack := filepath.Join(testDataDirUDN, "udn_crd_singlestack_template.yaml") + udnL2CRDSingleStack := filepath.Join(testDataDirUDN, "udn_crd_layer2_singlestack_template.yaml") + + platform := otputils.CheckPlatform(oc) + e2e.Logf("platform %s", platform) + acceptedPlatform := strings.Contains(platform, "gcp") || strings.Contains(platform, "azure") || strings.Contains(platform, "aws") + if !acceptedPlatform { + g.Skip("Test cases should be run on connected AWS,GCP, Azure, skip for other platforms or disconnected cluster!!") + } + if otputils.CheckIPStackType(oc) == "ipv6single" { + g.Skip("LoadBalancer UDN service test is only parameterized for IPv4 single-stack today") + } + + g.By("1. Get namespaces and create a new namespace ") + oc.CreateNamespaceUDN() + ns1 := oc.Namespace() + oc.CreateNamespaceUDN() + ns2 := oc.Namespace() + nadNS := []string{ns1, ns2} + + g.By("2. Create CRD for UDN for layer 3") + udncrd := otputils.UdnCRDResource{ + Crdname: "udn-network-l3-76014", + Namespace: nadNS[0], + Role: "Primary", + Cidr: "10.200.0.0/16", + Prefix: 24, + Template: udnCRDSingleStack, + } + udncrd.CreateUdnCRDSingleStack(oc) + err := otputils.WaitUDNCRDApplied(oc, nadNS[0], udncrd.Crdname) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("3. Create CRD for UDN for layer 2") + udnl2crd := otputils.UdnCRDResource{ + Crdname: "udn-network-l2-76014", + Namespace: nadNS[1], + Role: "Primary", + Cidr: "10.210.0.0/16", + Template: udnL2CRDSingleStack, + } + udnl2crd.CreateLayer2SingleStackUDNCRD(oc) + err = otputils.WaitUDNCRDApplied(oc, nadNS[1], udnl2crd.Crdname) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("4. Create a pod for service per namespace.") + pod := make([]otputils.UdnPodResource, 2) + for i := 0; i < 2; i++ { + pod[i] = otputils.UdnPodResource{ + Name: "hello-pod", + Namespace: nadNS[i], + Label: "hello-pod", + Template: udnPodTemplate, + } + pod[i].CreateUdnPod(oc) + otputils.WaitPodReady(oc, pod[i].Namespace, pod[i].Name) + } + + g.By("5. Create LoadBalancer service.") + svc := make([]otputils.GenericServiceResource, 2) + for i := 0; i < 2; i++ { + svc[i] = otputils.GenericServiceResource{ + Servicename: "test-service", + Namespace: nadNS[i], + Protocol: "TCP", + Selector: "hello-pod", + ServiceType: "LoadBalancer", + IpFamilyPolicy: "SingleStack", + InternalTrafficPolicy: "Cluster", + ExternalTrafficPolicy: "Cluster", + Template: genericServiceTemplate, + } + svc[i].CreateServiceFromParams(oc) + svcOutput, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("service", "-n", nadNS[i], svc[i].Servicename).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(svcOutput).Should(o.ContainSubstring(svc[i].Servicename)) + } + + g.By("6. Get LoadBalancer service URL.") + var svcExternalIP [2]string + for i := 0; i < 2; i++ { + if platform == "aws" { + svcExternalIP[i] = otputils.GetLBSVCHostname(oc, nadNS[i], svc[i].Servicename) + } else { + svcExternalIP[i] = otputils.GetLBSVCIP(oc, nadNS[i], svc[i].Servicename) + } + e2e.Logf("Got externalIP service IP: %v from namespace %s", svcExternalIP[i], nadNS[i]) + o.Expect(svcExternalIP[i]).NotTo(o.BeEmpty()) + } + + g.By("7.Curl the service from test runner\n") + var svcURL [2]string + for i := 0; i < 2; i++ { + svcURL[i] = net.JoinHostPort(svcExternalIP[i], "27017") + e2e.Logf("\n svcURL: %v\n", svcURL[i]) + + err = wait.PollUntilContextTimeout(context.TODO(), 10*time.Second, 120*time.Second, false, func(cxt context.Context) (bool, error) { + output, err1 := exec.Command("curl", svcURL[i], "--connect-timeout", "30").Output() + if err1 != nil || !strings.Contains(string(output), "Hello OpenShift") { + e2e.Logf("got err:%v, and try next round", err1) + return false, nil + } + e2e.Logf("The external service %v access passed!", svcURL[i]) + return true, nil + }) + o.Expect(err).NotTo(o.HaveOccurred(), fmt.Sprintf("Fail to curl the externalIP service from test runner %s", svcURL[i])) + } + }) + + g.It("[JIRA:Networking][OTP] 78767-Validate service for CUDN(Layer3)", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("networking") + pingPodTemplate = filepath.Join(buildPruningBaseDir, "ping-for-pod-specific-node-template.yaml") + genericServiceTemplate = filepath.Join(buildPruningBaseDir, "service-generic-template.yaml") + testPodFile = filepath.Join(buildPruningBaseDir, "testpod.yaml") + ipFamilyPolicy = "SingleStack" + key = "test.cudn.layer3" + crdName = "cudn-network-78767" + crdName2 = "cudn-network-78767-2" + values = []string{"value-78767-1", "value-78767-2"} + values2 = []string{"value2-78767-1", "value2-78767-2"} + cudnNS = []string{} + ) + + nodeList, nodeErr := e2enode.GetReadySchedulableNodes(context.TODO(), oc.KubeFramework().ClientSet) + o.Expect(nodeErr).NotTo(o.HaveOccurred()) + if len(nodeList.Items) < 3 { + g.Skip("This test requires at least 3 worker nodes which is not fulfilled. ") + } + + g.By("1. Create CRD for CUDN") + ipStackType := otputils.CheckIPStackType(oc) + var cidr, ipv4cidr, ipv6cidr string + if ipStackType == "ipv4single" { + cidr = "10.150.0.0/16" + } else { + if ipStackType == "ipv6single" { + cidr = "2010:100:200::0/60" + } else { + ipv4cidr = "10.150.0.0/16" + ipv6cidr = "2010:100:200::0/60" + ipFamilyPolicy = "PreferDualStack" + } + } + + defer otputils.RemoveResource(oc, true, true, "clusteruserdefinednetwork", crdName) + _, err := otputils.CreateCUDNCRD(oc, key, crdName, ipv4cidr, ipv6cidr, cidr, "layer3", values) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("2. Create 2 namespaces and add related values.") + for i := 0; i < 2; i++ { + oc.CreateNamespaceUDN() + cudnNS = append(cudnNS, oc.Namespace()) + defer oc.AsAdmin().WithoutNamespace().Run("label").Args("ns", cudnNS[i], fmt.Sprintf("%s-", key)).Execute() + err := oc.AsAdmin().WithoutNamespace().Run("label").Args("ns", cudnNS[i], fmt.Sprintf("%s=%s", key, values[i])).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + } + + g.By("3. Create a pod deployed on node0 as backend pod for service.") + pod1ns1 := otputils.PingPodResourceNode{ + Name: "hello-pod-1", + Namespace: cudnNS[0], + Nodename: nodeList.Items[0].Name, + Template: pingPodTemplate, + } + defer otputils.RemoveResource(oc, true, true, "pod", pod1ns1.Name, "-n", pod1ns1.Namespace) + pod1ns1.CreatePingPodNode(oc) + otputils.WaitPodReady(oc, pod1ns1.Namespace, pod1ns1.Name) + + g.By("4. create a udn client pod in ns1") + pod2ns1 := otputils.PingPodResourceNode{ + Name: "hello-pod-2", + Namespace: cudnNS[0], + Nodename: nodeList.Items[1].Name, + Template: pingPodTemplate, + } + defer otputils.RemoveResource(oc, true, true, "pod", pod2ns1.Name, "-n", pod2ns1.Namespace) + pod2ns1.CreatePingPodNode(oc) + otputils.WaitPodReady(oc, pod2ns1.Namespace, pod2ns1.Name) + // Update label for pod2 to a different one + err = oc.AsAdmin().WithoutNamespace().Run("label").Args("-n", cudnNS[0], "pod", pod2ns1.Name, "name=hello-pod-2", "--overwrite=true").Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("5. create a udn client pod in ns2. ") + pod1ns2 := otputils.PingPodResourceNode{ + Name: "hello-pod-3", + Namespace: cudnNS[1], + Nodename: nodeList.Items[0].Name, + Template: pingPodTemplate, + } + defer otputils.RemoveResource(oc, true, true, "pod", pod1ns2.Name, "-n", pod1ns2.Namespace) + pod1ns2.CreatePingPodNode(oc) + otputils.WaitPodReady(oc, pod1ns2.Namespace, pod1ns2.Name) + + g.By("6. create a ClusterIP service in ns1") + svc := otputils.GenericServiceResource{ + Servicename: "test-service", + Namespace: cudnNS[0], + Protocol: "TCP", + Selector: "hello-pod", + ServiceType: "ClusterIP", + IpFamilyPolicy: ipFamilyPolicy, + InternalTrafficPolicy: "Cluster", + ExternalTrafficPolicy: "", + Template: genericServiceTemplate, + } + svc.CreateServiceFromParams(oc) + + g.By("7. Verify ClusterIP service can be accessed from both pod2 in ns1 and pod3 in ns2") + otputils.CurlPod2SvcPass(oc, cudnNS[0], cudnNS[0], pod2ns1.Name, svc.Servicename) + otputils.CurlPod2SvcPass(oc, cudnNS[1], cudnNS[0], pod1ns2.Name, svc.Servicename) + + g.By("8. Create third namespace") + oc.SetupProject() + cudnNS = append(cudnNS, oc.Namespace()) + + g.By("9. Create service and pods which are on default network.") + otputils.CreateResourceFromFile(oc, cudnNS[2], testPodFile) + err = otputils.WaitForPodWithLabelReady(oc, cudnNS[2], "name=test-pods") + o.Expect(err).NotTo(o.HaveOccurred(), "this pod with label name=test-pods not ready") + testPodName := otputils.GetPodName(oc, cudnNS[2], "name=test-pods") + + g.By("10. Not be able to access cudn service from default network.") + otputils.CurlPod2SvcFail(oc, cudnNS[2], cudnNS[0], testPodName[0], svc.Servicename) + g.By("11. Not be able to access default network service from cudn network.") + otputils.CurlPod2SvcFail(oc, cudnNS[1], cudnNS[2], pod1ns2.Name, "test-service") + + g.By("11. Create fourth namespace for cudn pod") + oc.CreateNamespaceUDN() + cudnNS = append(cudnNS, oc.Namespace()) + err = oc.AsAdmin().WithoutNamespace().Run("label").Args("ns", cudnNS[3], fmt.Sprintf("%s=%s", key, values2[0])).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("12. Create CRD in fourth namespace") + if ipStackType == "ipv4single" { + cidr = "10.151.0.0/16" + } else { + if ipStackType == "ipv6single" { + cidr = "2011:100:200::0/60" + } else { + ipv4cidr = "10.151.0.0/16" + ipv6cidr = "2011:100:200::0/60" + } + } + defer func() { + oc.AsAdmin().WithoutNamespace().Run("label").Args("ns", cudnNS[3], fmt.Sprintf("%s-", key)).Execute() + otputils.RemoveResource(oc, true, true, "namespace", cudnNS[3]) + otputils.RemoveResource(oc, true, true, "clusteruserdefinednetwork", crdName2) + }() + _, err = otputils.CreateCUDNCRD(oc, key, crdName2, ipv4cidr, ipv6cidr, cidr, "layer3", values2) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("13. Create a udn pod in fourth namespace") + otputils.CreateResourceFromFile(oc, cudnNS[3], testPodFile) + err = otputils.WaitForPodWithLabelReady(oc, cudnNS[3], "name=test-pods") + o.Expect(err).NotTo(o.HaveOccurred(), "this pod with label name=test-pods not ready") + testPodNameNS3 := otputils.GetPodName(oc, cudnNS[3], "name=test-pods") + + g.By("14. Verify different cudn network, service was isolated.") + otputils.CurlPod2SvcFail(oc, cudnNS[3], cudnNS[0], testPodNameNS3[0], svc.Servicename) + + g.By("15.Update internalTrafficPolicy as Local for cudn service in ns1.") + patch := `[{"op": "replace", "path": "/spec/internalTrafficPolicy", "value": "Local"}]` + err = oc.AsAdmin().WithoutNamespace().Run("patch").Args("service/test-service", "-n", cudnNS[0], "-p", patch, "--type=json").Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + g.By("15.1. Verify ClusterIP service can be accessed from pod3 which is deployed same node as service back-end pod.") + otputils.CurlPod2SvcPass(oc, cudnNS[1], cudnNS[0], pod1ns2.Name, svc.Servicename) + g.By("15.2. Verify ClusterIP service can NOT be accessed from pod2 which is deployed different node as service back-end pod.") + otputils.CurlPod2SvcFail(oc, cudnNS[0], cudnNS[0], pod2ns1.Name, svc.Servicename) + + g.By("16. Verify nodePort service can be accessed.") + g.By("16.1 Delete testservice from ns1") + err = oc.AsAdmin().WithoutNamespace().Run("delete").Args("svc", "test-service", "-n", cudnNS[0]).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + g.By("16.2 Create testservice with NodePort in ns1") + svc.ServiceType = "NodePort" + svc.CreateServiceFromParams(oc) + + g.By("16.3 From a third node, be able to access node0:nodePort") + nodePort, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("service", "-n", cudnNS[0], svc.Servicename, "-o=jsonpath={.spec.ports[*].nodePort}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + thirdNode := nodeList.Items[2].Name + o.Expect(err).NotTo(o.HaveOccurred()) + otputils.CurlNodePortPass(oc, thirdNode, nodeList.Items[0].Name, nodePort) + g.By("16.4 From a third node, be able to access node1:nodePort") + otputils.CurlNodePortPass(oc, thirdNode, nodeList.Items[1].Name, nodePort) + g.By("16.5 From pod node, be able to access nodePort service") + otputils.CurlNodePortPass(oc, nodeList.Items[0].Name, nodeList.Items[0].Name, nodePort) + otputils.CurlNodePortPass(oc, nodeList.Items[0].Name, nodeList.Items[1].Name, nodePort) + + g.By("17.Update externalTrafficPolicy as Local for udn service in ns1.") + patch = `[{"op": "replace", "path": "/spec/externalTrafficPolicy", "value": "Local"}]` + err = oc.AsAdmin().WithoutNamespace().Run("patch").Args("service/test-service", "-n", cudnNS[0], "-p", patch, "--type=json").Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + g.By("17.1 From a third node, be able to access node0:nodePort") + otputils.CurlNodePortPass(oc, thirdNode, nodeList.Items[0].Name, nodePort) + g.By("17.2 From a third node, NOT be able to access node1:nodePort") + otputils.CurlNodePortFail(oc, thirdNode, nodeList.Items[1].Name, nodePort) + }) + + g.It("[JIRA:Networking][OTP] 78768-Validate service for CUDN(Layer2)", func() { + var ( + buildPruningBaseDir = testdata.FixturePath("networking") + pingPodTemplate = filepath.Join(buildPruningBaseDir, "ping-for-pod-specific-node-template.yaml") + genericServiceTemplate = filepath.Join(buildPruningBaseDir, "service-generic-template.yaml") + testPodFile = filepath.Join(buildPruningBaseDir, "testpod.yaml") + ipFamilyPolicy = "SingleStack" + key = "test.cudn.layer2" + crdName = "cudn-network-78768" + crdName2 = "cudn-network-78768-2" + values = []string{"value-78768-1", "value-78768-2"} + values2 = []string{"value2-78768-1", "value2-78768-2"} + cudnNS = []string{} + ) + + nodeList, nodeErr := e2enode.GetReadySchedulableNodes(context.TODO(), oc.KubeFramework().ClientSet) + o.Expect(nodeErr).NotTo(o.HaveOccurred()) + if len(nodeList.Items) < 3 { + g.Skip("This test requires at least 3 worker nodes which is not fulfilled. ") + } + + g.By("1. Create CRD for CUDN") + ipStackType := otputils.CheckIPStackType(oc) + var cidr, ipv4cidr, ipv6cidr string + if ipStackType == "ipv4single" { + cidr = "10.150.0.0/16" + } else { + if ipStackType == "ipv6single" { + cidr = "2010:100:200::0/60" + } else { + ipv4cidr = "10.150.0.0/16" + ipv6cidr = "2010:100:200::0/60" + ipFamilyPolicy = "PreferDualStack" + } + } + + defer otputils.RemoveResource(oc, true, true, "clusteruserdefinednetwork", crdName) + _, err := otputils.CreateCUDNCRD(oc, key, crdName, ipv4cidr, ipv6cidr, cidr, "layer2", values) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("2. Create 2 namespaces and add related values.") + for i := 0; i < 2; i++ { + oc.CreateNamespaceUDN() + cudnNS = append(cudnNS, oc.Namespace()) + defer oc.AsAdmin().WithoutNamespace().Run("label").Args("ns", cudnNS[i], fmt.Sprintf("%s-", key)).Execute() + err := oc.AsAdmin().WithoutNamespace().Run("label").Args("ns", cudnNS[i], fmt.Sprintf("%s=%s", key, values[i])).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + } + + g.By("3. Create a pod deployed on node0 as backend pod for service.") + pod1ns1 := otputils.PingPodResourceNode{ + Name: "hello-pod-1", + Namespace: cudnNS[0], + Nodename: nodeList.Items[0].Name, + Template: pingPodTemplate, + } + defer otputils.RemoveResource(oc, true, true, "pod", pod1ns1.Name, "-n", pod1ns1.Namespace) + pod1ns1.CreatePingPodNode(oc) + otputils.WaitPodReady(oc, pod1ns1.Namespace, pod1ns1.Name) + + g.By("4. create a udn client pod in ns1") + pod2ns1 := otputils.PingPodResourceNode{ + Name: "hello-pod-2", + Namespace: cudnNS[0], + Nodename: nodeList.Items[1].Name, + Template: pingPodTemplate, + } + defer otputils.RemoveResource(oc, true, true, "pod", pod2ns1.Name, "-n", pod2ns1.Namespace) + pod2ns1.CreatePingPodNode(oc) + otputils.WaitPodReady(oc, pod2ns1.Namespace, pod2ns1.Name) + // Update label for pod2 to a different one + err = oc.AsAdmin().WithoutNamespace().Run("label").Args("-n", cudnNS[0], "pod", pod2ns1.Name, "name=hello-pod-2", "--overwrite=true").Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("5. create a udn client pod in ns2. ") + pod1ns2 := otputils.PingPodResourceNode{ + Name: "hello-pod-3", + Namespace: cudnNS[1], + Nodename: nodeList.Items[0].Name, + Template: pingPodTemplate, + } + defer otputils.RemoveResource(oc, true, true, "pod", pod1ns2.Name, "-n", pod1ns2.Namespace) + pod1ns2.CreatePingPodNode(oc) + otputils.WaitPodReady(oc, pod1ns2.Namespace, pod1ns2.Name) + + g.By("6. create a ClusterIP service in ns1") + svc := otputils.GenericServiceResource{ + Servicename: "test-service", + Namespace: cudnNS[0], + Protocol: "TCP", + Selector: "hello-pod", + ServiceType: "ClusterIP", + IpFamilyPolicy: ipFamilyPolicy, + InternalTrafficPolicy: "Cluster", + ExternalTrafficPolicy: "", + Template: genericServiceTemplate, + } + svc.CreateServiceFromParams(oc) + + g.By("7. Verify ClusterIP service can be accessed from both pod2 in ns1 and pod3 in ns2") + otputils.CurlPod2SvcPass(oc, cudnNS[0], cudnNS[0], pod2ns1.Name, svc.Servicename) + otputils.CurlPod2SvcPass(oc, cudnNS[1], cudnNS[0], pod1ns2.Name, svc.Servicename) + + g.By("8. Create third namespace") + oc.SetupProject() + cudnNS = append(cudnNS, oc.Namespace()) + + g.By("9. Create service and pods which are on default network.") + otputils.CreateResourceFromFile(oc, cudnNS[2], testPodFile) + err = otputils.WaitForPodWithLabelReady(oc, cudnNS[2], "name=test-pods") + o.Expect(err).NotTo(o.HaveOccurred(), "this pod with label name=test-pods not ready") + testPodName := otputils.GetPodName(oc, cudnNS[2], "name=test-pods") + + g.By("10. Not be able to access cudn service from default network.") + otputils.CurlPod2SvcFail(oc, cudnNS[2], cudnNS[0], testPodName[0], svc.Servicename) + g.By("11. Not be able to access default network service from cudn network.") + otputils.CurlPod2SvcFail(oc, cudnNS[1], cudnNS[2], pod1ns2.Name, "test-service") + + g.By("11. Create fourth namespace for cudn pod") + oc.CreateNamespaceUDN() + cudnNS = append(cudnNS, oc.Namespace()) + err = oc.AsAdmin().WithoutNamespace().Run("label").Args("ns", cudnNS[3], fmt.Sprintf("%s=%s", key, values2[0])).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("12. Create CRD in fourth namespace") + if ipStackType == "ipv4single" { + cidr = "10.151.0.0/16" + } else { + if ipStackType == "ipv6single" { + cidr = "2011:100:200::0/60" + } else { + ipv4cidr = "10.151.0.0/16" + ipv6cidr = "2011:100:200::0/60" + } + } + defer func() { + oc.AsAdmin().WithoutNamespace().Run("label").Args("ns", cudnNS[3], fmt.Sprintf("%s-", key)).Execute() + otputils.RemoveResource(oc, true, true, "namespace", cudnNS[3]) + otputils.RemoveResource(oc, true, true, "clusteruserdefinednetwork", crdName2) + }() + _, err = otputils.CreateCUDNCRD(oc, key, crdName2, ipv4cidr, ipv6cidr, cidr, "layer2", values2) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("13. Create a udn pod in fourth namespace") + otputils.CreateResourceFromFile(oc, cudnNS[3], testPodFile) + err = otputils.WaitForPodWithLabelReady(oc, cudnNS[3], "name=test-pods") + o.Expect(err).NotTo(o.HaveOccurred(), "this pod with label name=test-pods not ready") + testPodNameNS3 := otputils.GetPodName(oc, cudnNS[3], "name=test-pods") + + g.By("14. Verify different cudn network, service was isolated.") + otputils.CurlPod2SvcFail(oc, cudnNS[3], cudnNS[0], testPodNameNS3[0], svc.Servicename) + + g.By("15.Update internalTrafficPolicy as Local for cudn service in ns1.") + patch := `[{"op": "replace", "path": "/spec/internalTrafficPolicy", "value": "Local"}]` + err = oc.AsAdmin().WithoutNamespace().Run("patch").Args("service/test-service", "-n", cudnNS[0], "-p", patch, "--type=json").Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + g.By("15.1. Verify ClusterIP service can be accessed from pod3 which is deployed same node as service back-end pod.") + otputils.CurlPod2SvcPass(oc, cudnNS[1], cudnNS[0], pod1ns2.Name, svc.Servicename) + g.By("15.2. Verify ClusterIP service can NOT be accessed from pod2 which is deployed different node as service back-end pod.") + otputils.CurlPod2SvcFail(oc, cudnNS[0], cudnNS[0], pod2ns1.Name, svc.Servicename) + + g.By("16. Verify nodePort service can be accessed.") + g.By("16.1 Delete testservice from ns1") + err = oc.AsAdmin().WithoutNamespace().Run("delete").Args("svc", "test-service", "-n", cudnNS[0]).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + g.By("16.2 Create testservice with NodePort in ns1") + svc.ServiceType = "NodePort" + svc.CreateServiceFromParams(oc) + + g.By("16.3 From a third node, be able to access node0:nodePort") + nodePort, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("service", "-n", cudnNS[0], svc.Servicename, "-o=jsonpath={.spec.ports[*].nodePort}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + thirdNode := nodeList.Items[2].Name + o.Expect(err).NotTo(o.HaveOccurred()) + otputils.CurlNodePortPass(oc, thirdNode, nodeList.Items[0].Name, nodePort) + g.By("16.4 From a third node, be able to access node1:nodePort") + otputils.CurlNodePortPass(oc, thirdNode, nodeList.Items[1].Name, nodePort) + g.By("16.5 From pod node, be able to access nodePort service") + otputils.CurlNodePortPass(oc, nodeList.Items[0].Name, nodeList.Items[0].Name, nodePort) + otputils.CurlNodePortPass(oc, nodeList.Items[0].Name, nodeList.Items[1].Name, nodePort) + + g.By("17.Update externalTrafficPolicy as Local for udn service in ns1.") + patch = `[{"op": "replace", "path": "/spec/externalTrafficPolicy", "value": "Local"}]` + err = oc.AsAdmin().WithoutNamespace().Run("patch").Args("service/test-service", "-n", cudnNS[0], "-p", patch, "--type=json").Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + g.By("17.1 From a third node, be able to access node0:nodePort") + otputils.CurlNodePortPass(oc, thirdNode, nodeList.Items[0].Name, nodePort) + g.By("17.2 From a third node, NOT be able to access node1:nodePort") + otputils.CurlNodePortFail(oc, thirdNode, nodeList.Items[1].Name, nodePort) + }) + + g.It("[JIRA:Networking][OTP][Serial] 44790-Validate ExternalIP service for default and UDN pods - setup", func() { + nodeList, nodeErr := e2enode.GetReadySchedulableNodes(context.TODO(), oc.KubeFramework().ClientSet) + o.Expect(nodeErr).NotTo(o.HaveOccurred()) + if len(nodeList.Items) < 3 { + g.Skip("This test requires at least 3 worker nodes which is not fulfilled. ") + } + + ipStackType := otputils.CheckIPStackType(oc) + var ( + buildPruningBaseDir = testdata.FixturePath("networking") + netsegDir = testdata.FixturePath("networking/network_segmentation") + rcPingPodTemplate = filepath.Join(netsegDir, "rc-ping-for-pod-template.yaml") + genericServiceTemplate = filepath.Join(buildPruningBaseDir, "service-generic-template.yaml") + allNS = []string{"79163-upgrade-ns1", "79163-upgrade-ns2", "79163-upgrade-ns3"} + ipFamilyPolicy = "SingleStack" + serviceName = "test-service" + ) + + g.By("1. Create three namespaces, ns1 and ns2 for udn network testing, ns3 for default network testing") + for i := 0; i < 2; i++ { + oc.CreateSpecificNamespaceUDN(allNS[i]) + } + oc.AsAdmin().WithoutNamespace().Run("create").Args("namespace", allNS[2]).Execute() + + g.By("2. Find externalIP for testing") + var externalIP, externalIPv6 []string + for i := 0; i < 3; i++ { + nodeIP1, nodeIP2 := otputils.GetNodeIP(oc, nodeList.Items[i].Name) + externalIP = append(externalIP, nodeIP2) + if ipStackType == "dualstack" { + externalIPv6 = append(externalIPv6, nodeIP1) + } + } + + g.By("3. Patch network.config to enable externalIP") + allowedCIDRs := `"` + externalIP[0] + `","` + externalIP[1] + `","` + externalIP[2] + `"` + if ipStackType == "dualstack" { + allowedCIDRs = allowedCIDRs + `,"` + externalIPv6[0] + `","` + externalIPv6[1] + `","` + externalIPv6[2] + `"` + } + otputils.PatchResourceAsAdmin(oc, "network/cluster", "{\"spec\":{\"externalIP\":{\"policy\":{\"allowedCIDRs\":["+allowedCIDRs+"]}}}}") + + g.By("4. Create CRD for layer3 UDN in namespace ns1") + var cidr, ipv4cidr, ipv6cidr string + if ipStackType == "ipv4single" { + cidr = "10.150.0.0/16" + } else { + if ipStackType == "ipv6single" { + cidr = "2010:100:200::0/48" + } else { + ipv4cidr = "10.150.0.0/16" + ipv6cidr = "2010:100:200::0/48" + ipFamilyPolicy = "PreferDualStack" + } + } + otputils.CreateGeneralUDNCRD(oc, allNS[0], "udn-network-"+allNS[0], ipv4cidr, ipv6cidr, cidr, "layer3") + + g.By("5. Create CRD for layer2 UDN in namespace ns2") + if ipStackType == "ipv4single" { + cidr = "10.151.0.0/16" + } else { + if ipStackType == "ipv6single" { + cidr = "2011:100:200::0/48" + } else { + ipv4cidr = "10.151.0.0/16" + ipv6cidr = "2011:100:200::0/48" + ipFamilyPolicy = "PreferDualStack" + } + } + otputils.CreateGeneralUDNCRD(oc, allNS[1], "udn-network-"+allNS[1], ipv4cidr, ipv6cidr, cidr, "layer2") + + g.By("6. Create pod as backend pod for service in each ns") + var podsBackendName []string + for i := 0; i < 3; i++ { + podsBackend := otputils.ReplicationControllerPingPodResource{ + Name: "hello-pod-1", + Replicas: 0, + Namespace: allNS[i], + Template: rcPingPodTemplate, + } + podsBackend.CreateReplicaController(oc) + e2e.Logf("schedule backend pod to %s", nodeList.Items[i].Name) + patchErr := oc.AsAdmin().WithoutNamespace().Run("patch").Args("rc/"+podsBackend.Name, "-n", allNS[i], "-p", "{\"spec\":{\"replicas\":1,\"template\":{\"spec\":{\"nodeSelector\":{\"kubernetes.io/hostname\":\""+nodeList.Items[i].Name+"\"}}}}}", "--type=merge").Execute() + o.Expect(patchErr).NotTo(o.HaveOccurred()) + err := otputils.WaitForPodWithLabelReady(oc, podsBackend.Namespace, "name="+podsBackend.Name) + o.Expect(err).NotTo(o.HaveOccurred(), "The backend pod is not ready") + podsBackendName = append(podsBackendName, otputils.GetPodName(oc, allNS[i], "name="+podsBackend.Name)[0]) + } + + g.By("7. Create udn client pod on different node in ns1 and ns2") + var udnClientName []string + for i := 0; i < 2; i++ { + udnClient := otputils.ReplicationControllerPingPodResource{ + Name: "hello-pod-2", + Replicas: 0, + Namespace: allNS[i], + Template: rcPingPodTemplate, + } + udnClient.CreateReplicaController(oc) + e2e.Logf("schedule udn client pod to %s", nodeList.Items[2].Name) + patchErr := oc.AsAdmin().WithoutNamespace().Run("patch").Args("rc/"+udnClient.Name, "-n", allNS[i], "-p", "{\"spec\":{\"replicas\":1,\"template\":{\"spec\":{\"nodeSelector\":{\"kubernetes.io/hostname\":\""+nodeList.Items[2].Name+"\"}}}}}", "--type=merge").Execute() + o.Expect(patchErr).NotTo(o.HaveOccurred()) + err := otputils.WaitForPodWithLabelReady(oc, udnClient.Namespace, "name="+udnClient.Name) + o.Expect(err).NotTo(o.HaveOccurred(), "The udn client pod is not ready") + udnClientName = append(udnClientName, otputils.GetPodName(oc, allNS[i], "name="+udnClient.Name)[0]) + } + + g.By("8. Create a ClusterIP service in each ns") + for i := 0; i < 3; i++ { + svc := otputils.GenericServiceResource{ + Servicename: serviceName, + Namespace: allNS[i], + Protocol: "TCP", + Selector: "hello-pod-1", + ServiceType: "ClusterIP", + IpFamilyPolicy: ipFamilyPolicy, + InternalTrafficPolicy: "Cluster", + ExternalTrafficPolicy: "", + Template: genericServiceTemplate, + } + svc.CreateServiceFromParams(oc) + e2e.Logf("Patch ExternalIP to service") + otputils.PatchResourceAsAdmin(oc, "svc/"+svc.Servicename, fmt.Sprintf("{\"spec\":{\"externalIPs\": [\"%s\"]}}", externalIP[i]), allNS[i]) + svcOutput, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("service", "-n", allNS[i], svc.Servicename).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(svcOutput).Should(o.ContainSubstring(externalIP[i])) + } + + g.By("9. Validate the externalIP service for default network") + _, err := e2eoutput.RunHostCmdWithRetries(allNS[2], podsBackendName[2], "curl --connect-timeout 5 -s "+net.JoinHostPort(externalIP[2], "27017"), 5*time.Second, 15*time.Second) + o.Expect(err).NotTo(o.HaveOccurred()) + + for i := 0; i < 2; i++ { + if i == 0 { + g.By("10. Validate the externalIP service for layer3 UDN") + } else { + g.By("11. Validate the externalIP service for layer2 UDN") + } + g.By("Validate the externalIP service can be accessed from another udn pod") + _, err := e2eoutput.RunHostCmdWithRetries(allNS[i], udnClientName[i], "curl --connect-timeout 5 -s "+net.JoinHostPort(externalIP[i], "27017"), 5*time.Second, 15*time.Second) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Validate the externalIP service can be accessed from same node as service backend pod") + _, err = otputils.DebugNode(oc, nodeList.Items[i].Name, "curl", net.JoinHostPort(externalIP[i], "27017"), "-s", "--connect-timeout", "5") + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Validate the externalIP service can be accessed from different node than service backend pod") + _, err = otputils.DebugNode(oc, nodeList.Items[2].Name, "curl", net.JoinHostPort(externalIP[i], "27017"), "-s", "--connect-timeout", "5") + o.Expect(err).NotTo(o.HaveOccurred()) + } + + if ipStackType == "dualstack" { + g.By("Retest it with IPv6 address in dualstack cluster") + g.By("12. Patch IPv6 ExternalIP to service") + for i := 0; i < 3; i++ { + otputils.PatchResourceAsAdmin(oc, "svc/"+serviceName, fmt.Sprintf("{\"spec\":{\"externalIPs\": [\"%s\",\"%s\"]}}", externalIP[i], externalIPv6[i]), allNS[i]) + svcOutput, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("service", "-n", allNS[i], serviceName).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(svcOutput).Should(o.ContainSubstring(serviceName)) + } + + g.By("13. Validate the externalIP service for default network") + _, err := e2eoutput.RunHostCmdWithRetries(allNS[2], podsBackendName[2], "curl --connect-timeout 5 -s "+net.JoinHostPort(externalIPv6[2], "27017"), 5*time.Second, 15*time.Second) + o.Expect(err).NotTo(o.HaveOccurred()) + + for i := 0; i < 2; i++ { + if i == 0 { + g.By("14. Validate the externalIP service for layer3 UDN - ipv6") + } else { + g.By("15. Validate the externalIP service for layer2 UDN - ipv6") + } + g.By("Validate the externalIP service can be accessed from another udn pod - ipv6") + _, err := e2eoutput.RunHostCmdWithRetries(allNS[i], udnClientName[i], "curl --connect-timeout 5 -s "+net.JoinHostPort(externalIPv6[i], "27017"), 5*time.Second, 15*time.Second) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Validate the externalIP service can be accessed from same node as service backend pod - ipv6") + _, err = otputils.DebugNode(oc, nodeList.Items[i].Name, "curl", net.JoinHostPort(externalIPv6[i], "27017"), "-s", "--connect-timeout", "5") + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Validate the externalIP service can be accessed from different node than service backend pod - ipv6") + _, err = otputils.DebugNode(oc, nodeList.Items[2].Name, "curl", net.JoinHostPort(externalIPv6[i], "27017"), "-s", "--connect-timeout", "5") + o.Expect(err).NotTo(o.HaveOccurred()) + } + } + }) + + g.It("[JIRA:Networking][OTP][Serial] 79163-Validate ExternalIP service for default and UDN pods - verify", func() { + defer otputils.PatchResourceAsAdmin(oc, "network/cluster", "{\"spec\":{\"externalIP\":{\"policy\":{}}}}") + defer otputils.PatchResourceAsAdmin(oc, "network/cluster", "{\"spec\":{\"externalIP\":{\"policy\":{\"allowedCIDRs\":[]}}}}") + nodeList, nodeErr := e2enode.GetReadySchedulableNodes(context.TODO(), oc.KubeFramework().ClientSet) + o.Expect(nodeErr).NotTo(o.HaveOccurred()) + if len(nodeList.Items) < 3 { + g.Skip("This test requires at least 3 worker nodes which is not fulfilled. ") + } + + ipStackType := otputils.CheckIPStackType(oc) + var ( + allNS = []string{"79163-upgrade-ns1", "79163-upgrade-ns2", "79163-upgrade-ns3"} + podBackendLabel = "hello-pod-1" + udnClientLabel = "hello-pod-2" + ) + + g.By("1. Check the three namespaces are carried over") + for i := 0; i < 3; i++ { + nsErr := oc.AsAdmin().WithoutNamespace().Run("get").Args("ns", allNS[i]).Execute() + if nsErr != nil { + g.Skip("Skip the PstChkUpgrade test as namespace " + allNS[i] + " does not exist, PreChkUpgrade test did not run") + } + } + for i := 0; i < 3; i++ { + defer oc.AsAdmin().WithoutNamespace().Run("delete").Args("project", allNS[i], "--ignore-not-found=true").Execute() + } + + g.By("2. Get externalIP from preserved services") + var externalIP, externalIPv6 []string + for i := 0; i < 3; i++ { + rawIPs, svcErr := oc.AsAdmin().WithoutNamespace().Run("get").Args( + "service", "test-service", "-n", allNS[i], "-o=jsonpath={.spec.externalIPs[*]}", + ).Output() + o.Expect(svcErr).NotTo(o.HaveOccurred()) + ips := strings.Fields(strings.TrimSpace(rawIPs)) + o.Expect(ips).NotTo(o.BeEmpty()) + externalIP = append(externalIP, ips[0]) + if ipStackType == "dualstack" { + o.Expect(len(ips)).To(o.BeNumerically(">=", 2)) + externalIPv6 = append(externalIPv6, ips[1]) + } + } + + g.By("3. Get backend pod from preserved namespaces") + var podsBackendName []string + for i := 0; i < 3; i++ { + err := otputils.WaitForPodWithLabelReady(oc, allNS[i], "name="+podBackendLabel) + o.Expect(err).NotTo(o.HaveOccurred(), "The backend pod is not ready") + podsBackendName = append(podsBackendName, otputils.GetPodName(oc, allNS[i], "name="+podBackendLabel)[0]) + } + + g.By("4. Get udn clients from preserved namespaces") + var udnClientName []string + for i := 0; i < 2; i++ { + err := otputils.WaitForPodWithLabelReady(oc, allNS[i], "name="+udnClientLabel) + o.Expect(err).NotTo(o.HaveOccurred(), "The udn client pod is not ready") + udnClientName = append(udnClientName, otputils.GetPodName(oc, allNS[i], "name="+udnClientLabel)[0]) + } + + g.By("5. Validate the externalIP service for default network") + _, err := e2eoutput.RunHostCmdWithRetries(allNS[2], podsBackendName[2], "curl --connect-timeout 5 -s "+net.JoinHostPort(externalIP[2], "27017"), 5*time.Second, 15*time.Second) + o.Expect(err).NotTo(o.HaveOccurred()) + + for i := 0; i < 2; i++ { + if i == 0 { + g.By("6. Validate the externalIP service for layer3 UDN") + } else { + g.By("7. Validate the externalIP service for layer2 UDN") + } + g.By("Validate the externalIP service can be accessed from another udn pod") + _, err := e2eoutput.RunHostCmdWithRetries(allNS[i], udnClientName[i], "curl --connect-timeout 5 -s "+net.JoinHostPort(externalIP[i], "27017"), 5*time.Second, 15*time.Second) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Validate the externalIP service can be accessed from same node as service backend pod") + _, err = otputils.DebugNode(oc, nodeList.Items[i].Name, "curl", net.JoinHostPort(externalIP[i], "27017"), "-s", "--connect-timeout", "5") + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Validate the externalIP service can be accessed from different node than service backend pod") + _, err = otputils.DebugNode(oc, nodeList.Items[2].Name, "curl", net.JoinHostPort(externalIP[i], "27017"), "-s", "--connect-timeout", "5") + o.Expect(err).NotTo(o.HaveOccurred()) + } + + if ipStackType == "dualstack" { + g.By("Retest it with IPv6 address in dualstack cluster") + + g.By("8. Validate the externalIP service for default network") + _, err := e2eoutput.RunHostCmdWithRetries(allNS[2], podsBackendName[2], "curl --connect-timeout 5 -s "+net.JoinHostPort(externalIPv6[2], "27017"), 5*time.Second, 15*time.Second) + o.Expect(err).NotTo(o.HaveOccurred()) + + for i := 0; i < 2; i++ { + if i == 0 { + g.By("9. Validate the externalIP service for layer3 UDN - ipv6") + } else { + g.By("10. Validate the externalIP service for layer2 UDN - ipv6") + } + g.By("Validate the externalIP service can be accessed from another udn pod - ipv6") + _, err := e2eoutput.RunHostCmdWithRetries(allNS[i], udnClientName[i], "curl --connect-timeout 5 -s "+net.JoinHostPort(externalIPv6[i], "27017"), 5*time.Second, 15*time.Second) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Validate the externalIP service can be accessed from same node as service backend pod - ipv6") + _, err = otputils.DebugNode(oc, nodeList.Items[i].Name, "curl", net.JoinHostPort(externalIPv6[i], "27017"), "-s", "--connect-timeout", "5") + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Validate the externalIP service can be accessed from different node than service backend pod - ipv6") + _, err = otputils.DebugNode(oc, nodeList.Items[2].Name, "curl", net.JoinHostPort(externalIPv6[i], "27017"), "-s", "--connect-timeout", "5") + o.Expect(err).NotTo(o.HaveOccurred()) + } + } + }) +})