From 9a3704bf0f2255b456db80664370dd80ebd18beb Mon Sep 17 00:00:00 2001 From: Anurag Saxena Date: Sun, 14 Jun 2026 23:30:11 -0400 Subject: [PATCH] Implement OpenShift Tests Extension (OTE) framework for CNO Co-Authored-By: Claude Opus 4.6 --- Dockerfile | 3 +- Makefile | 5 + go.mod | 52 +++- go.sum | 73 ++++- pkg/network/mtu.go | 10 +- test/Makefile | 44 +++ test/ote/cli.go | 229 ++++++++++++++ test/ote/cmd/main.go | 57 ++++ test/ote/otp.go | 236 ++++++++++++++ test/ote/util.go | 719 +++++++++++++++++++++++++++++++++++++++++++ 10 files changed, 1409 insertions(+), 19 deletions(-) create mode 100644 test/Makefile create mode 100644 test/ote/cli.go create mode 100644 test/ote/cmd/main.go create mode 100644 test/ote/otp.go create mode 100644 test/ote/util.go diff --git a/Dockerfile b/Dockerfile index c227090996..4c13460406 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,12 +1,13 @@ FROM registry.ci.openshift.org/ocp/builder:rhel-9-golang-1.25-openshift-4.22 AS builder WORKDIR /go/src/github.com/openshift/cluster-network-operator COPY . . -RUN hack/build-go.sh +RUN go mod vendor && hack/build-go.sh && make build-e2e-tests && gzip -9 test/bin/cluster-network-operator-tests-ext FROM registry.ci.openshift.org/ocp/4.22:base-rhel9 COPY --from=builder /go/src/github.com/openshift/cluster-network-operator/cluster-network-operator /usr/bin/ COPY --from=builder /go/src/github.com/openshift/cluster-network-operator/cluster-network-check-endpoints /usr/bin/ COPY --from=builder /go/src/github.com/openshift/cluster-network-operator/cluster-network-check-target /usr/bin/ +COPY --from=builder /go/src/github.com/openshift/cluster-network-operator/test/bin/cluster-network-operator-tests-ext.gz /usr/bin/cluster-network-operator-tests-ext.gz COPY manifests /manifests COPY bindata /bindata diff --git a/Makefile b/Makefile index a86bd5ed5b..d2e20f6433 100644 --- a/Makefile +++ b/Makefile @@ -46,4 +46,9 @@ clean: $(RM) cluster-network-operator cluster-network-check-endpoints cluster-network-check-target .PHONY: clean +.PHONY: build-e2e-tests +build-e2e-tests: + @echo "Building cluster-network-operator-tests-ext binary..." + $(MAKE) -C test build + GO_TEST_PACKAGES :=./pkg/... ./cmd/... diff --git a/go.mod b/go.mod index e2affcb690..a5320df263 100644 --- a/go.mod +++ b/go.mod @@ -15,8 +15,8 @@ require ( github.com/spf13/cobra v1.10.2 github.com/spf13/pflag v1.0.10 github.com/stretchr/testify v1.11.1 - github.com/vishvananda/netlink v1.1.0 - github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae // indirect + github.com/vishvananda/netlink v1.3.1 + github.com/vishvananda/netns v0.0.5 // indirect golang.org/x/net v0.51.0 gopkg.in/yaml.v2 v2.4.0 k8s.io/api v0.35.2 @@ -100,12 +100,16 @@ require ( ) require ( + github.com/onsi/ginkgo/v2 v2.28.1 + github.com/openshift-eng/openshift-tests-extension v0.0.0-20260521151256-b5a8f7ec8a38 github.com/openshift/api v0.0.0-20260609121705-d3390bd1109f github.com/openshift/client-go v0.0.0-20260603140539-6892dc3e1ffc github.com/openshift/library-go v0.0.0-20260303171201-5d9eb6295ff6 github.com/openshift/machine-config-operator v0.0.1-0.20250724162154-ab14c8e2843b k8s.io/apiextensions-apiserver v0.35.2 k8s.io/client-go v0.35.2 + k8s.io/kubernetes v1.35.2 + k8s.io/pod-security-admission v0.35.2 sigs.k8s.io/controller-tools v0.20.1 ) @@ -114,6 +118,7 @@ require ( github.com/Masterminds/semver/v3 v3.4.0 // indirect github.com/antlr4-go/antlr/v4 v4.13.1 // indirect github.com/cenkalti/backoff/v5 v5.0.3 // indirect + github.com/distribution/reference v0.6.0 // indirect github.com/emicklei/go-restful/v3 v3.13.0 // indirect github.com/evanphx/json-patch/v5 v5.9.11 // indirect github.com/fatih/color v1.18.0 // indirect @@ -131,16 +136,21 @@ require ( github.com/go-openapi/swag/stringutils v0.25.5 // indirect github.com/go-openapi/swag/typeutils v0.25.5 // indirect github.com/go-openapi/swag/yamlutils v0.25.5 // indirect + github.com/go-task/slim-sprig/v3 v3.0.0 // indirect github.com/gobuffalo/flect v1.0.3 // indirect github.com/google/btree v1.1.3 // indirect github.com/google/cel-go v0.27.0 // indirect github.com/google/gnostic-models v0.7.1 // indirect github.com/google/pprof v0.0.0-20260115054156-294ebfa9ad83 // indirect + github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 // indirect github.com/kylelemons/godebug v1.1.0 // indirect github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-isatty v0.0.20 // indirect - github.com/onsi/ginkgo/v2 v2.28.1 // indirect + github.com/moby/spdystream v0.5.0 // indirect + github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/robfig/cron/v3 v3.0.1 // indirect github.com/shopspring/decimal v1.3.1 // indirect github.com/spf13/cast v1.10.0 // indirect github.com/stretchr/objx v0.5.3 // indirect @@ -155,9 +165,45 @@ require ( google.golang.org/genproto/googleapis/rpc v0.0.0-20260226221140-a57be14db171 // indirect gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect k8s.io/apiserver v0.35.2 // indirect + k8s.io/component-helpers v0.35.2 // indirect + k8s.io/controller-manager v0.32.1 // indirect k8s.io/gengo/v2 v2.0.0-20251215205346-5ee0d033ba5b // indirect k8s.io/kms v0.35.2 // indirect k8s.io/kube-aggregator v0.35.1 // indirect + k8s.io/kubectl v0.32.1 // indirect + k8s.io/kubelet v0.32.1 // indirect sigs.k8s.io/randfill v1.0.0 // indirect sigs.k8s.io/structured-merge-diff/v6 v6.3.2 // indirect ) + +replace ( + github.com/onsi/ginkgo/v2 => github.com/openshift/onsi-ginkgo/v2 v2.6.1-0.20260303184444-1cc650aa0565 + k8s.io/api => k8s.io/api v0.35.2 + k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.35.2 + k8s.io/apimachinery => k8s.io/apimachinery v0.35.2 + k8s.io/apiserver => k8s.io/apiserver v0.35.2 + k8s.io/cli-runtime => k8s.io/cli-runtime v0.35.2 + k8s.io/client-go => k8s.io/client-go v0.35.2 + k8s.io/cloud-provider => k8s.io/cloud-provider v0.35.2 + k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.35.2 + k8s.io/code-generator => k8s.io/code-generator v0.35.2 + k8s.io/component-base => k8s.io/component-base v0.35.2 + k8s.io/component-helpers => k8s.io/component-helpers v0.35.2 + k8s.io/controller-manager => k8s.io/controller-manager v0.35.2 + k8s.io/cri-api => k8s.io/cri-api v0.35.2 + k8s.io/cri-client => k8s.io/cri-client v0.35.2 + k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.35.2 + k8s.io/dynamic-resource-allocation => k8s.io/dynamic-resource-allocation v0.35.2 + k8s.io/endpointslice => k8s.io/endpointslice v0.35.2 + k8s.io/externaljwt => k8s.io/externaljwt v0.35.2 + k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.35.2 + k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.35.2 + k8s.io/kube-proxy => k8s.io/kube-proxy v0.35.2 + k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.35.2 + k8s.io/kubectl => k8s.io/kubectl v0.35.2 + k8s.io/kubelet => k8s.io/kubelet v0.35.2 + k8s.io/metrics => k8s.io/metrics v0.35.2 + k8s.io/mount-utils => k8s.io/mount-utils v0.35.2 + k8s.io/pod-security-admission => k8s.io/pod-security-admission v0.35.2 + k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.35.2 +) diff --git a/go.sum b/go.sum index 911f6043a9..0725a95869 100644 --- a/go.sum +++ b/go.sum @@ -14,6 +14,8 @@ github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cq github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ= github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= @@ -44,6 +46,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= +github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/emicklei/go-restful/v3 v3.13.0 h1:C4Bl2xDndpU6nJ4bc1jXd+uTmYPVUwkD6bFY/oTyCes= @@ -67,6 +71,12 @@ github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sa github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32 h1:Mn26/9ZMNWSw9C9ERFA1PUxfmGpolnw2v0bKOREu5ew= github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32/go.mod h1:GIjDIg/heH5DOkXY3YJ/wNhfHsQHoXGjl8G8amsYQ1I= +github.com/gkampitakis/ciinfo v0.3.2 h1:JcuOPk8ZU7nZQjdUhctuhQofk7BGHuIy0c9Ez8BNhXs= +github.com/gkampitakis/ciinfo v0.3.2/go.mod h1:1NIwaOcFChN4fa/B0hEBdAb6npDlFL8Bwx4dfRLRqAo= +github.com/gkampitakis/go-diff v1.3.2 h1:Qyn0J9XJSDTgnsgHRdz9Zp24RaJeKMUHg2+PDZZdC4M= +github.com/gkampitakis/go-diff v1.3.2/go.mod h1:LLgOrpqleQe26cte8s36HTWcTmMEur6OPYerdAAS9tk= +github.com/gkampitakis/go-snaps v0.5.15 h1:amyJrvM1D33cPHwVrjo9jQxX8g/7E2wYdZ+01KS3zGE= +github.com/gkampitakis/go-snaps v0.5.15/go.mod h1:HNpx/9GoKisdhw9AFOBT1N7DBs9DiHo/hGheFGBZ+mc= github.com/go-bindata/go-bindata v3.1.2+incompatible h1:5vjJMVhowQdPzjE1LdxyFF7YFTXg5IgGVW4gBr5IbvE= github.com/go-bindata/go-bindata v3.1.2+incompatible/go.mod h1:xK8Dsgwmeed+BBsSy2XTopBn/8uK2HWuGSnA11C3Joo= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -110,7 +120,6 @@ github.com/go-openapi/testify/enable/yaml/v2 v2.4.0 h1:7SgOMTvJkM8yWrQlU8Jm18VeD github.com/go-openapi/testify/enable/yaml/v2 v2.4.0/go.mod h1:14iV8jyyQlinc9StD7w1xVPW3CO3q1Gj04Jy//Kw4VM= github.com/go-openapi/testify/v2 v2.4.0 h1:8nsPrHVCWkQ4p8h1EsRVymA2XABB4OT40gcvAu+voFM= github.com/go-openapi/testify/v2 v2.4.0/go.mod h1:HCPmvFFnheKK2BuwSA0TbbdxJ3I16pjwMkYkP4Ywn54= -github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/gobuffalo/flect v1.0.3 h1:xeWBM2nui+qnVvNM4S3foBhCAL2XgPU+a7FdpelbTq4= @@ -118,6 +127,8 @@ github.com/gobuffalo/flect v1.0.3/go.mod h1:A5msMlrHtLqh9umBSnvabjsMrCcCpAyzglnD github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM= github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= github.com/gobwas/ws v1.2.1/go.mod h1:hRKAFb8wOxFROYNsT1bqfWnhX+b5MFeJM9r2ZSwg/KY= +github.com/goccy/go-yaml v1.18.0 h1:8W7wMFS12Pcas7KU+VVkaiCng+kG8QiFeFwzFb+rwuw= +github.com/goccy/go-yaml v1.18.0/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v5 v5.3.0 h1:pv4AsKCKKZuqlgs5sUmn4x8UlGa0kEVt/puTpKx9vvo= @@ -168,6 +179,8 @@ github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLf github.com/jonboulle/clockwork v0.5.0 h1:Hyh9A8u51kptdkR+cqRpT1EebBwTn1oK9YfGYbdFz6I= github.com/jonboulle/clockwork v0.5.0/go.mod h1:3mZlmanh0g2NDKO5TWZVJAfofYk64M7XN3SzBPjZF60= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/joshdk/go-junit v1.0.0 h1:S86cUKIdwBHWwA6xCmFlf3RTLfVXYQfvanM5Uh+K6GE= +github.com/joshdk/go-junit v1.0.0/go.mod h1:TiiV0PqkaNfFXjEiyjWM3XXrhVyCa1K4Zfga6W52ung= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= @@ -182,17 +195,23 @@ github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/maruel/natural v1.1.1 h1:Hja7XhhmvEFhcByqDoHz9QZbkWey+COd9xWfCfn1ioo= +github.com/maruel/natural v1.1.1/go.mod h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg= github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/mfridman/tparse v0.18.0 h1:wh6dzOKaIwkUGyKgOntDW4liXSo37qg5AXbIhkMV3vE= +github.com/mfridman/tparse v0.18.0/go.mod h1:gEvqZTuCgEhPbYk/2lS3Kcxg1GmTxxU7kTC8DvP0i/A= github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/moby/spdystream v0.5.0 h1:7r0J1Si3QO/kjRitvSLVVFUjxMEb/YLj6S9FF62JBCU= +github.com/moby/spdystream v0.5.0/go.mod h1:xBAYlnt/ay+11ShkdFKNAG7LsyK/tmNBVvVOwrfMgdI= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -201,14 +220,18 @@ github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFd github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= -github.com/onsi/ginkgo/v2 v2.28.1 h1:S4hj+HbZp40fNKuLUQOYLDgZLwNUVn19N3Atb98NCyI= -github.com/onsi/ginkgo/v2 v2.28.1/go.mod h1:CLtbVInNckU3/+gC8LzkGUb9oF+e8W8TdUsxPwvdOgE= github.com/onsi/gomega v1.39.1 h1:1IJLAad4zjPn2PsnhH70V4DKRFlrCzGBNrNaru+Vf28= github.com/onsi/gomega v1.39.1/go.mod h1:hL6yVALoTOxeWudERyfppUcZXjMwIMLnuSfruD2lcfg= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/openshift-eng/openshift-tests-extension v0.0.0-20260521151256-b5a8f7ec8a38 h1:UMblx+pkFZ1RA31arHZOWLyWUe6LROdlya4JxTHxGbo= +github.com/openshift-eng/openshift-tests-extension v0.0.0-20260521151256-b5a8f7ec8a38/go.mod h1:pHOS9c6BjZv91OkkHyIHAOWnYhxwcxWQkyYGEvPyUCE= github.com/openshift/api v0.0.0-20260609121705-d3390bd1109f h1:q7vMHwBYipDQO05yj1iU8E39oNkvJxkYPB8VnFMQw/w= github.com/openshift/api v0.0.0-20260609121705-d3390bd1109f/go.mod h1:pyVjK0nZ4sRs4fuQVQ4rubsJdahI1PB94LnQ8sGdvxo= github.com/openshift/build-machinery-go v0.0.0-20251023084048-5d77c1a5e5af h1:UiYYMi/CCV+kwWrXuXfuUSOY2yNXOpWpNVgHc6aLQlE= @@ -219,6 +242,8 @@ github.com/openshift/library-go v0.0.0-20260303171201-5d9eb6295ff6 h1:xjqy0OolrF github.com/openshift/library-go v0.0.0-20260303171201-5d9eb6295ff6/go.mod h1:D797O/ssKTNglbrGchjIguFq+DbyRYdeds5w4/VTrKM= github.com/openshift/machine-config-operator v0.0.1-0.20250724162154-ab14c8e2843b h1:LvoFr/2IEj0BWy7mKBdR7ueAHpMJGju1EkEIZrXa+DM= github.com/openshift/machine-config-operator v0.0.1-0.20250724162154-ab14c8e2843b/go.mod h1:UL1OVkRAUkB4aaFZrLlSvuY0jayfdF+o+ZxKiKaaArc= +github.com/openshift/onsi-ginkgo/v2 v2.6.1-0.20260303184444-1cc650aa0565 h1:3/q8qM4HbFa+Een8wgzpwO8W6mO7Po+MwY6uxiXi/ac= +github.com/openshift/onsi-ginkgo/v2 v2.6.1-0.20260303184444-1cc650aa0565/go.mod h1:ArE1D/XhNXBXCBkKOLkbsb2c81dQHCRcF5zwn/ykDRo= github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -244,6 +269,8 @@ github.com/prometheus/procfs v0.20.1 h1:XwbrGOIplXW/AU3YhIhLODXMJYyC1isLFfYCsTEy github.com/prometheus/procfs v0.20.1/go.mod h1:o9EMBZGRyvDrSPH1RqdxhojkuXstoe4UlK79eF5TGGo= github.com/robfig/cron v1.2.0 h1:ZjScXvvxeQ63Dbyxy76Fj3AT3Ut0aKsyd2/tl3DTMuQ= github.com/robfig/cron v1.2.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k= +github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= +github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -275,13 +302,20 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= +github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= +github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= +github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 h1:6fotK7otjonDflCTK0BCfls4SPy3NcCVb5dqqmbRknE= github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75/go.mod h1:KO6IkyS8Y3j8OdNO85qEYBsRPuteD+YciPomcXdrMnk= -github.com/vishvananda/netlink v1.1.0 h1:1iyaYNBLmP6L0220aDnYQpo1QEV4t4hJ+xEEhhJH8j0= -github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= -github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= -github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae h1:4hwBBUfQCFe3Cym0ZtKyq7L16eZUtYKs+BaHDN6mAns= -github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= +github.com/vishvananda/netlink v1.3.1 h1:3AEMt62VKqz90r0tmNhog0r/PpWKmrEShJU0wJW6bV0= +github.com/vishvananda/netlink v1.3.1/go.mod h1:ARtKouGSTGchR8aMwmkzC0qiNPrrWO5JS/XMVl45+b4= +github.com/vishvananda/netns v0.0.5 h1:DfiHV+j8bA32MFM7bfEunvT8IAqQ/NzSJHtcmW5zdEY= +github.com/vishvananda/netns v0.0.5/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/xiang90/probing v0.0.0-20221125231312-a49e3df8f510 h1:S2dVYn90KE98chqDkyE9Z4N61UnQd+KOfgp5Iu53llk= @@ -376,8 +410,6 @@ golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -387,6 +419,7 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -456,8 +489,8 @@ k8s.io/api v0.35.2 h1:tW7mWc2RpxW7HS4CoRXhtYHSzme1PN1UjGHJ1bdrtdw= k8s.io/api v0.35.2/go.mod h1:7AJfqGoAZcwSFhOjcGM7WV05QxMMgUaChNfLTXDRE60= k8s.io/apiextensions-apiserver v0.35.2 h1:iyStXHoJZsUXPh/nFAsjC29rjJWdSgUmG1XpApE29c0= k8s.io/apiextensions-apiserver v0.35.2/go.mod h1:OdyGvcO1FtMDWQ+rRh/Ei3b6X3g2+ZDHd0MSRGeS8rU= -k8s.io/apimachinery v0.36.0-alpha.2 h1:I3A/nvRsgV/j/AX7VXDn8XjuDz2gsfcdOTVCkKMRLsQ= -k8s.io/apimachinery v0.36.0-alpha.2/go.mod h1:7mgr/dli8ofwAbcIQXetFVX1fbOYsOYojq3AUbybVmQ= +k8s.io/apimachinery v0.35.2 h1:NqsM/mmZA7sHW02JZ9RTtk3wInRgbVxL8MPfzSANAK8= +k8s.io/apimachinery v0.35.2/go.mod h1:jQCgFZFR1F4Ik7hvr2g84RTJSZegBc8yHgFWKn//hns= k8s.io/apiserver v0.35.2 h1:rb52v0CZGEL0FkhjS+I6jHflAp7fZ4MIaKcEHX7wmDk= k8s.io/apiserver v0.35.2/go.mod h1:CROJUAu0tfjZLyYgSeBsBan2T7LUJGh0ucWwTCSSk7g= k8s.io/client-go v0.35.2 h1:YUfPefdGJA4aljDdayAXkc98DnPkIetMl4PrKX97W9o= @@ -466,18 +499,30 @@ k8s.io/code-generator v0.35.2 h1:3874swbO2c26VWTf6lKD4NWGyHIfyBeTCk7caCG3TuU= k8s.io/code-generator v0.35.2/go.mod h1:id4XLCm0yAQq5nlvyfAKibMOKnMjzlesAwGw6kM3Adc= k8s.io/component-base v0.35.2 h1:btgR+qNrpWuRSuvWSnQYsZy88yf5gVwemvz0yw79pGc= k8s.io/component-base v0.35.2/go.mod h1:B1iBJjooe6xIJYUucAxb26RwhAjzx0gHnqO9htWIX+0= +k8s.io/component-helpers v0.35.2 h1:7Ea4CDgHnyOGrl3ZhD8e46SdTyf1itTONnreJ2Q52UM= +k8s.io/component-helpers v0.35.2/go.mod h1:ybIoc8i92FG7xJFrBcEMzB8ul1wlZgfF0I4Z9w0V6VQ= +k8s.io/controller-manager v0.35.2 h1:EpLIwm4bBgoFwXiVULgNxtpUF9cbXUkcGkZ6vUlurYQ= +k8s.io/controller-manager v0.35.2/go.mod h1:CknUpFd1A0S9h+J9eQZXvZt05ssXMnsYCCLRghm/gp4= k8s.io/gengo/v2 v2.0.0-20251215205346-5ee0d033ba5b h1:0YkdvW3rX2vaBWsqCGZAekxPRwaI5NuYNprOsMNVLns= k8s.io/gengo/v2 v2.0.0-20251215205346-5ee0d033ba5b/go.mod h1:yvyl3l9E+UxlqOMUULdKTAYB0rEhsmjr7+2Vb/1pCSo= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= k8s.io/kms v0.35.2 h1:XPlj7QmLBfzm8gGQnc3+Y95hZLiJs3DjA0IyFOV5Z7g= k8s.io/kms v0.35.2/go.mod h1:VT+4ekZAdrZDMgShK37vvlyHUVhwI9t/9tvh0AyCWmQ= -k8s.io/kube-aggregator v0.35.1 h1:LN+btMJ3yp7biqVgT/0LF6SKIKLyfPU0R+JJ1mycs2I= -k8s.io/kube-aggregator v0.35.1/go.mod h1:HQSjPQfOFRzcv7biQ7jV3cEfKHG+bczpLCfh4QfvxZU= +k8s.io/kube-aggregator v0.35.2 h1:bnF7E238wUOVaPpTyKrqGCAEXOAJ6HRTARvJTZ0UIC0= +k8s.io/kube-aggregator v0.35.2/go.mod h1:7Xl9zFJFsFIrPnwBfu7hve+G5QgLsDZRIedc8gA1mq4= k8s.io/kube-openapi v0.0.0-20260127142750-a19766b6e2d4 h1:HhDfevmPS+OalTjQRKbTHppRIz01AWi8s45TMXStgYY= k8s.io/kube-openapi v0.0.0-20260127142750-a19766b6e2d4/go.mod h1:kdmbQkyfwUagLfXIad1y2TdrjPFWp2Q89B3qkRwf/pQ= k8s.io/kube-proxy v0.35.2 h1:mmvaNWEPJGj64TDXcFxbdZXtLNfoam0h+xplcRNLtYM= k8s.io/kube-proxy v0.35.2/go.mod h1:Js1jQe8Rd9OqSVExRXRNts8ZdwFvcoKkZLApX6FYAMQ= +k8s.io/kubectl v0.35.2 h1:aSmqhSOfsoG9NR5oR8OD5eMKpLN9x8oncxfqLHbJJII= +k8s.io/kubectl v0.35.2/go.mod h1:+OJC779UsDJGxNPbHxCwvb4e4w9Eh62v/DNYU2TlsyM= +k8s.io/kubelet v0.35.2 h1:qF9jOe1j6vT4bVQZ6nnTTA5uu5NCnyR10o9IkW8Z0JQ= +k8s.io/kubelet v0.35.2/go.mod h1:2pyCVLDfm7ErNwWZw2mutCloAXX76gfOToIMCHCq/8s= +k8s.io/kubernetes v1.35.2 h1:2HthVDfK3YJYv624imuKXPzUJ17xQop9OT5dgT+IMKE= +k8s.io/kubernetes v1.35.2/go.mod h1:AaPpCpiS8oAqRbEwpY5r3RitLpwpVp5lVXKFkJril58= +k8s.io/pod-security-admission v0.35.2 h1:vzEfL/TpdwwIE25xQiamiRfmWD+FIcNXJYzoMI50AUY= +k8s.io/pod-security-admission v0.35.2/go.mod h1:zrNF0GSYasCR8SHiAD67q2iUTHitVoFQRvTOy/UijyU= k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2 h1:AZYQSJemyQB5eRxqcPky+/7EdBj0xi3g0ZcxxJ7vbWU= k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2/go.mod h1:xDxuJ0whA3d0I4mf/C4ppKHxXynQ+fxnkmQH0vTHnuk= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.34.0 h1:hSfpvjjTQXQY2Fol2CS0QHMNs/WI1MOSGzCm1KhM5ec= diff --git a/pkg/network/mtu.go b/pkg/network/mtu.go index 98a7515ece..a5abe6e0c1 100644 --- a/pkg/network/mtu.go +++ b/pkg/network/mtu.go @@ -5,6 +5,8 @@ package network import ( "fmt" + "net" + "github.com/pkg/errors" "github.com/vishvananda/netlink" ) @@ -30,8 +32,14 @@ func GetDefaultMTU() (int, error) { const maxMTU = 65536 mtu := maxMTU + 1 for _, route := range routes { - // Skip non-default routes + isDefault := route.Dst == nil if route.Dst != nil { + ones, _ := route.Dst.Mask.Size() + ip := route.Dst.IP + isDefault = (ip.Equal(net.IPv4zero) || ip.Equal(net.IPv6zero)) && ones == 0 + } + + if !isDefault { continue } if route.LinkIndex == 0 { diff --git a/test/Makefile b/test/Makefile new file mode 100644 index 0000000000..2414af02e3 --- /dev/null +++ b/test/Makefile @@ -0,0 +1,44 @@ +# Makefile for CNO OTP Test + +# Binary name +BINARY_NAME := cluster-network-operator-tests-ext + +# Build directory +BUILD_DIR := bin + +# Go module and package +GO_MODULE := github.com/openshift/cluster-network-operator + +# Version information +GIT_COMMIT := $(shell git rev-parse --short HEAD 2>/dev/null || echo "unknown") +BUILD_DATE := $(shell date -u +'%Y-%m-%dT%H:%M:%SZ') + +# Go build flags +LDFLAGS := -X '$(GO_MODULE)/pkg/version.commitFromGit=$(GIT_COMMIT)' \ + -X '$(GO_MODULE)/pkg/version.buildDate=$(BUILD_DATE)' \ + -X '$(GO_MODULE)/pkg/version.versionFromGit=$(GIT_COMMIT)' + +# Default target +.PHONY: all +all: build + +# Build the binary +.PHONY: build +build: + @echo "Building $(BINARY_NAME)..." + @mkdir -p $(BUILD_DIR) + cd otp/cmd && go build -o ../../$(BUILD_DIR)/$(BINARY_NAME) -ldflags="$(LDFLAGS)" . + +# Build for Linux (useful for container builds) +.PHONY: build-linux +build-linux: + @echo "Building $(BINARY_NAME) for Linux..." + @mkdir -p $(BUILD_DIR) + cd otp/cmd && CGO_ENABLED=0 GOOS=linux GOARCH=amd64 \ + go build -o ../../$(BUILD_DIR)/$(BINARY_NAME)-linux-amd64 -ldflags="$(LDFLAGS)" . + +# Clean build artifacts +.PHONY: clean +clean: + @echo "Cleaning build artifacts..." + rm -rf $(BUILD_DIR) diff --git a/test/ote/cli.go b/test/ote/cli.go new file mode 100644 index 0000000000..fe4ee13b9b --- /dev/null +++ b/test/ote/cli.go @@ -0,0 +1,229 @@ +package ote + +import ( + "bytes" + "context" + "fmt" + "os" + "os/exec" + "strings" + "time" + + o "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + e2e "k8s.io/kubernetes/test/e2e/framework" + admissionapi "k8s.io/pod-security-admission/api" +) + +type CLI struct { + execPath string + kubeconfig string + namespace string + withoutNamespace bool + asAdmin bool + verb string + args []string + kubeFramework *e2e.Framework + namespacesToDelete []string +} + +func NewCLIWithPodSecurityLevel(baseName string, level admissionapi.Level) *CLI { + cli := &CLI{ + execPath: "oc", + kubeconfig: os.Getenv("KUBECONFIG"), + kubeFramework: &e2e.Framework{ + BaseName: baseName, + SkipNamespaceCreation: false, + NamespacePodSecurityLevel: level, + Options: e2e.Options{ + ClientQPS: 20, + ClientBurst: 50, + }, + Timeouts: e2e.NewTimeoutContext(), + }, + } + + config, err := cli.getConfig() + if err != nil { + e2e.Failf("Failed to get kubeconfig: %v", err) + } + clientset, err := kubernetes.NewForConfig(config) + if err != nil { + e2e.Failf("Failed to create Kubernetes clientset: %v", err) + } + cli.kubeFramework.ClientSet = clientset + + return cli +} + +func (c *CLI) SetupNamespace() { + nsName := fmt.Sprintf("e2e-test-%s-%s", c.kubeFramework.BaseName, getRandomString()) + + _, err := c.asAdminInternal().withoutNamespaceInternal().run("create", "namespace", nsName).output() + o.Expect(err).NotTo(o.HaveOccurred(), "Failed to create namespace") + + c.namespace = nsName + c.namespacesToDelete = append(c.namespacesToDelete, nsName) + + if c.kubeFramework.NamespacePodSecurityLevel != "" { + level := string(c.kubeFramework.NamespacePodSecurityLevel) + _, err = c.asAdminInternal().withoutNamespaceInternal().run("label", "namespace", nsName, + fmt.Sprintf("pod-security.kubernetes.io/enforce=%s", level), + fmt.Sprintf("pod-security.kubernetes.io/warn=%s", level), + fmt.Sprintf("pod-security.kubernetes.io/audit=%s", level), + "security.openshift.io/scc.podSecurityLabelSync=false", + "--overwrite", + ).output() + o.Expect(err).NotTo(o.HaveOccurred(), "Failed to label namespace") + } + + c.kubeFramework.Namespace = &corev1.Namespace{} + c.kubeFramework.Namespace.Name = nsName + + e2e.Logf("Created test namespace: %s", nsName) +} + +func (c *CLI) TeardownNamespace() { + if len(c.namespacesToDelete) == 0 { + return + } + + if !e2e.TestContext.DeleteNamespace { + e2e.Logf("Skipping namespace deletion (DELETE_NAMESPACE=false)") + return + } + + for _, ns := range c.namespacesToDelete { + e2e.Logf("Deleting namespace: %s", ns) + _, err := c.asAdminInternal().withoutNamespaceInternal().run("delete", "namespace", ns, "--wait=false").output() + if err != nil { + e2e.Logf("Warning: failed to delete namespace %s: %v", ns, err) + } + } +} + +func (c *CLI) Namespace() string { + return c.namespace +} + +func (c *CLI) KubeFramework() *e2e.Framework { + return c.kubeFramework +} + +func (c *CLI) AsAdmin() *CLI { + return c.asAdminInternal() +} + +func (c *CLI) asAdminInternal() *CLI { + nc := *c + nc.asAdmin = true + nc.namespacesToDelete = append([]string(nil), c.namespacesToDelete...) + return &nc +} + +func (c *CLI) WithoutNamespace() *CLI { + return c.withoutNamespaceInternal() +} + +func (c *CLI) withoutNamespaceInternal() *CLI { + nc := *c + nc.withoutNamespace = true + nc.namespacesToDelete = append([]string(nil), c.namespacesToDelete...) + return &nc +} + +func (c *CLI) Run(verb string) *CLI { + nc := *c + nc.verb = verb + nc.args = []string{} + nc.namespacesToDelete = append([]string(nil), c.namespacesToDelete...) + return &nc +} + +func (c *CLI) Args(args ...string) *CLI { + nc := *c + nc.args = append([]string(nil), c.args...) + nc.args = append(nc.args, args...) + nc.namespacesToDelete = append([]string(nil), c.namespacesToDelete...) + return &nc +} + +func (c *CLI) Output() (string, error) { + return c.output() +} + +func (c *CLI) Execute() error { + out, err := c.output() + if err != nil { + e2e.Logf("Command failed with output:\n%s", out) + } + return err +} + +func (c *CLI) run(verb string, args ...string) *CLI { + nc := *c + nc.verb = verb + nc.args = args + nc.namespacesToDelete = append([]string(nil), c.namespacesToDelete...) + return &nc +} + +func (c *CLI) output() (string, error) { + var cmdArgs []string + + if c.kubeconfig != "" { + cmdArgs = append(cmdArgs, fmt.Sprintf("--kubeconfig=%s", c.kubeconfig)) + } + + if !c.withoutNamespace && c.namespace != "" { + cmdArgs = append(cmdArgs, fmt.Sprintf("--namespace=%s", c.namespace)) + } + + if c.verb != "" { + cmdArgs = append(cmdArgs, c.verb) + } + cmdArgs = append(cmdArgs, c.args...) + + e2e.Logf("Running: %s %s", c.execPath, strings.Join(cmdArgs, " ")) + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) + defer cancel() + cmd := exec.CommandContext(ctx, c.execPath, cmdArgs...) + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + + err := cmd.Run() + + outStr := strings.TrimSpace(stdout.String()) + errStr := strings.TrimSpace(stderr.String()) + + if err != nil { + return outStr, fmt.Errorf("command failed: %w\nstdout: %s\nstderr: %s", err, outStr, errStr) + } + + return outStr, nil +} + +func (c *CLI) getConfig() (*rest.Config, error) { + kubeconfig := c.kubeconfig + if kubeconfig == "" { + kubeconfig = os.Getenv("KUBECONFIG") + } + if kubeconfig == "" { + return nil, fmt.Errorf("KUBECONFIG not set") + } + + config, err := clientcmd.BuildConfigFromFlags("", kubeconfig) + if err != nil { + return nil, err + } + + config.QPS = 20 + config.Burst = 50 + + return config, nil +} diff --git a/test/ote/cmd/main.go b/test/ote/cmd/main.go new file mode 100644 index 0000000000..d977145648 --- /dev/null +++ b/test/ote/cmd/main.go @@ -0,0 +1,57 @@ +package main + +import ( + "fmt" + "os" + "time" + + "github.com/openshift-eng/openshift-tests-extension/pkg/cmd" + + "github.com/spf13/cobra" + + e "github.com/openshift-eng/openshift-tests-extension/pkg/extension" + et "github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests" + g "github.com/openshift-eng/openshift-tests-extension/pkg/ginkgo" + + _ "github.com/openshift/cluster-network-operator/test/ote" +) + +func main() { + registry := e.NewRegistry() + + ext := e.NewExtension("openshift", "payload", "cluster-network-operator") + testTimeout := 120 * time.Minute + ext.AddSuite(e.Suite{ + Name: "openshift/cluster-network-operator/disruptive", + Parents: []string{ + "openshift/conformance/serial", + }, + Qualifiers: []string{ + "name.contains('[Suite:openshift/cluster-network-operator/disruptive]')", + }, + ClusterStability: e.ClusterStabilityDisruptive, + TestTimeout: &testTimeout, + }) + + specs, err := g.BuildExtensionTestSpecsFromOpenShiftGinkgoSuite() + if err != nil { + panic(fmt.Sprintf("couldn't build extension test specs from ginkgo: %+v", err.Error())) + } + + specs.Walk(func(spec *et.ExtensionTestSpec) { + spec.Lifecycle = et.LifecycleInforming + }) + ext.AddSpecs(specs) + registry.Register(ext) + + root := &cobra.Command{ + Long: "OpenShift Tests Extension for Cluster Network Operator", + } + root.AddCommand(cmd.DefaultExtensionCommands(registry)...) + + if err := func() error { + return root.Execute() + }(); err != nil { + os.Exit(1) + } +} diff --git a/test/ote/otp.go b/test/ote/otp.go new file mode 100644 index 0000000000..be0420a3d4 --- /dev/null +++ b/test/ote/otp.go @@ -0,0 +1,236 @@ +package ote + +import ( + "context" + "fmt" + "os" + "path/filepath" + + g "github.com/onsi/ginkgo/v2" + o "github.com/onsi/gomega" + + admissionapi "k8s.io/pod-security-admission/api" + + e2e "k8s.io/kubernetes/test/e2e/framework" + e2enode "k8s.io/kubernetes/test/e2e/framework/node" +) + +var _ = g.Describe("[sig-network][Suite:openshift/conformance/serial] CNO", func() { + var oc *CLI + + g.BeforeEach(func() { + oc = NewCLIWithPodSecurityLevel("networking-cno", admissionapi.LevelPrivileged) + oc.SetupNamespace() + }) + + g.AfterEach(func() { + oc.TeardownNamespace() + }) + + g.It("[JIRA:Networking][OTP] 72817-Make sure internalJoinSubnet and internalTransitSwitchSubnet is configurable post install as a Day 2 operation", func() { + var ( + pod1Name = "hello-pod1" + pod2Name = "hello-pod2" + podLabel = "hello-pod" + serviceName = "test-service-72817" + servicePort = 27017 + serviceTargetPort = 8080 + ) + ipStackType := checkIPStackType(oc) + o.Expect(ipStackType).NotTo(o.BeEmpty()) + + nodeList, err := e2enode.GetReadySchedulableNodes(context.TODO(), oc.KubeFramework().ClientSet) + o.Expect(err).NotTo(o.HaveOccurred()) + if len(nodeList.Items) < 2 { + g.Skip("This case requires 2 nodes, but the cluster has less than two nodes") + } + + createPingPodOnNode(oc, pod1Name, oc.Namespace(), podLabel, nodeList.Items[0].Name) + createPingPodOnNode(oc, pod2Name, oc.Namespace(), podLabel, nodeList.Items[1].Name) + + var ipFamilyPolicy string + if ipStackType == "ipv4single" { + ipFamilyPolicy = "SingleStack" + } else { + ipFamilyPolicy = "PreferDualStack" + } + createGenericService(oc, serviceName, oc.Namespace(), "TCP", podLabel, "ClusterIP", ipFamilyPolicy, "Cluster", "", servicePort, serviceTargetPort) + + customPatchIPv4 := `{"spec":{"defaultNetwork":{"ovnKubernetesConfig":{"ipv4":{"internalJoinSubnet": "100.99.0.0/16","internalTransitSwitchSubnet": "100.69.0.0/16"}}}}}` + customPatchIPv6 := `{"spec":{"defaultNetwork":{"ovnKubernetesConfig":{"ipv6":{"internalJoinSubnet": "ab98::/64","internalTransitSwitchSubnet": "ab97::/64"}}}}}` + customPatchDualstack := `{"spec":{"defaultNetwork":{"ovnKubernetesConfig":{"ipv4":{"internalJoinSubnet": "100.99.0.0/16","internalTransitSwitchSubnet": "100.69.0.0/16"},"ipv6": {"internalJoinSubnet": "ab98::/64","internalTransitSwitchSubnet": "ab97::/64"}}}}}` + + currentinternalJoinSubnetIPv4Value, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("Network.operator.openshift.io/cluster", "-o=jsonpath={.spec.defaultNetwork.ovnKubernetesConfig.ipv4.internalJoinSubnet}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + currentinternalTransitSwSubnetIPv4Value, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("Network.operator.openshift.io/cluster", "-o=jsonpath={.spec.defaultNetwork.ovnKubernetesConfig.ipv4.internalTransitSwitchSubnet}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + currentinternalJoinSubnetIPv6Value, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("Network.operator.openshift.io/cluster", "-o=jsonpath={.spec.defaultNetwork.ovnKubernetesConfig.ipv6.internalJoinSubnet}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + currentinternalTransitSwSubnetIPv6Value, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("Network.operator.openshift.io/cluster", "-o=jsonpath={.spec.defaultNetwork.ovnKubernetesConfig.ipv6.internalTransitSwitchSubnet}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + + if currentinternalJoinSubnetIPv4Value == "" { + currentinternalJoinSubnetIPv4Value = "100.64.0.0/16" + } + if currentinternalJoinSubnetIPv6Value == "" { + currentinternalJoinSubnetIPv6Value = "fd98::/64" + } + if currentinternalTransitSwSubnetIPv4Value == "" { + currentinternalTransitSwSubnetIPv4Value = "100.88.0.0/16" + } + if currentinternalTransitSwSubnetIPv6Value == "" { + currentinternalTransitSwSubnetIPv6Value = "fd97::/64" + } + + patchIPv4original := `{"spec":{"defaultNetwork":{"ovnKubernetesConfig":{"ipv4":{"internalJoinSubnet": "` + currentinternalJoinSubnetIPv4Value + `","internalTransitSwitchSubnet": "` + currentinternalTransitSwSubnetIPv4Value + `"}}}}}` + patchIPv6original := `{"spec":{"defaultNetwork":{"ovnKubernetesConfig":{"ipv6":{"internalJoinSubnet": "` + currentinternalJoinSubnetIPv6Value + `","internalTransitSwitchSubnet": "` + currentinternalTransitSwSubnetIPv6Value + `"}}}}}` + patchDualstackoriginal := `{"spec":{"defaultNetwork":{"ovnKubernetesConfig":{"ipv4":{"internalJoinSubnet": "` + currentinternalJoinSubnetIPv4Value + `","internalTransitSwitchSubnet": "` + currentinternalTransitSwSubnetIPv4Value + `"},"ipv6": {"internalJoinSubnet": "` + currentinternalJoinSubnetIPv6Value + `","internalTransitSwitchSubnet": "` + currentinternalTransitSwSubnetIPv6Value + `"}}}}}` + + applyPatchWithCleanup := func(customPatch, originalPatch string) { + g.DeferCleanup(func() { + patchResourceAsAdmin(oc, "Network.operator.openshift.io/cluster", originalPatch) + err := checkOVNKState(oc) + o.Expect(err).NotTo(o.HaveOccurred(), "OVNkube didn't rollout successfully after restoring original configuration") + }) + patchResourceAsAdmin(oc, "Network.operator.openshift.io/cluster", customPatch) + } + + switch ipStackType { + case "ipv4single": + applyPatchWithCleanup(customPatchIPv4, patchIPv4original) + case "ipv6single": + applyPatchWithCleanup(customPatchIPv6, patchIPv6original) + default: + applyPatchWithCleanup(customPatchDualstack, patchDualstackoriginal) + } + err = checkOVNKState(oc) + o.Expect(err).NotTo(o.HaveOccurred(), "OVNkube never trigger or rolled out successfully post oc patch") + curlPod2PodPass(oc, oc.Namespace(), pod1Name, oc.Namespace(), pod2Name, serviceTargetPort) + curlPod2SvcPass(oc, oc.Namespace(), oc.Namespace(), pod1Name, serviceName, servicePort) + }) + + g.It("[JIRA:Networking][OTP] 51727-ovsdb-server and northd should not core dump on node restart", func() { + g.By("Get one node to reboot") + workerList, err := e2enode.GetReadySchedulableNodes(context.TODO(), oc.KubeFramework().ClientSet) + o.Expect(err).NotTo(o.HaveOccurred()) + if len(workerList.Items) < 1 { + g.Skip("This case requires 1 node, but the cluster has none") + } + worker := workerList.Items[0].Name + defer checkNodeStatus(oc, worker, "Ready") + rebootNode(oc, worker) + checkNodeStatus(oc, worker, "NotReady") + checkNodeStatus(oc, worker, "Ready") + + g.By("Check the node core dump output") + mustgatherDir := "/tmp/must-gather-51727" + defer os.RemoveAll(mustgatherDir) + _, err = oc.AsAdmin().WithoutNamespace().Run("adm").Args("must-gather", "--dest-dir="+mustgatherDir, "--", "/usr/bin/gather_core_dumps").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + + match, err := filepath.Glob(filepath.Join(mustgatherDir, "*", "node_core_dumps")) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(len(match)).Should(o.Equal(1)) + files, err := os.ReadDir(match[0]) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(files).Should(o.BeEmpty()) + }) + + g.It("[JIRA:Networking][OTP] 72028-Join switch IP and management port IP for newly added node should be synced correctly into NBDB, pod on new node can communicate with old pod on old node", func() { + ipStackType := checkIPStackType(oc) + o.Expect(ipStackType).NotTo(o.BeEmpty()) + + platform := checkPlatform(oc) + if platform == "baremetal" || platform == "none" || platform == "ovirt" { + g.Skip("Skipping on platform " + platform + " - MachineSet creation not supported") + } + + topology := getControlPlaneTopology(oc) + if topology == "SingleReplicaTopologyMode" { + g.Skip("Skipping on SNO - MachineSet scaling not supported") + } + + g.By("Get an existing schedulable node") + currentNodeList, err := e2enode.GetReadySchedulableNodes(context.TODO(), oc.KubeFramework().ClientSet) + o.Expect(err).NotTo(o.HaveOccurred()) + oldNode := currentNodeList.Items[0].Name + + g.By("Create a network policy in the namespace") + createNetworkPolicy(oc, oc.Namespace()) + output, err := oc.Run("get").Args("networkpolicy").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(output).To(o.ContainSubstring("allow-from-all-namespaces")) + + g.By("Create a test pod on the existing node") + createPingPodOnNode(oc, "hello-pod1", oc.Namespace(), "hello-pod", oldNode) + + g.By("Create a new machineset, get the new node") + infrastructureName := getInfrastructureName(oc) + machineSetName := infrastructureName + "-72028" + defer waitForMachinesDisappear(oc, machineSetName) + defer deleteMachineSet(oc, machineSetName) + createMachineSet(oc, machineSetName) + + waitForMachineSetRunning(oc, machineSetName, 1) + newNodeName := getNodeNameFromMachineSet(oc, machineSetName) + e2e.Logf("Get new node name: %s", newNodeName) + + g.By("Create second namespace and another test pod on the new node") + ns2 := fmt.Sprintf("e2e-test-72028-%s", getRandomString()) + _, err = oc.AsAdmin().WithoutNamespace().Run("create").Args("namespace", ns2).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + defer func() { + _, _ = oc.AsAdmin().WithoutNamespace().Run("delete").Args("namespace", ns2, "--wait=false").Output() + }() + _, err = oc.AsAdmin().WithoutNamespace().Run("label").Args("namespace", ns2, + "pod-security.kubernetes.io/enforce=privileged", + "pod-security.kubernetes.io/warn=privileged", + "pod-security.kubernetes.io/audit=privileged", + "security.openshift.io/scc.podSecurityLabelSync=false", + "--overwrite", + ).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + + createPingPodOnNode(oc, "hello-pod2", ns2, "hello-pod", newNodeName) + + g.By("Get management IP(s) and join switch IP(s) for the new node") + var nodeOVNK8sMgmtIPv4, nodeOVNK8sMgmtIPv6 string + if ipStackType == "dualstack" || ipStackType == "ipv6single" { + nodeOVNK8sMgmtIPv6 = getOVNK8sNodeMgmtIPv6(oc, newNodeName) + } + if ipStackType == "dualstack" || ipStackType == "ipv4single" { + nodeOVNK8sMgmtIPv4 = getOVNK8sNodeMgmtIPv4(oc, newNodeName) + } + e2e.Logf("ipStack type: %s, nodeOVNK8sMgmtIPv4: %s, nodeOVNK8sMgmtIPv6: %s", ipStackType, nodeOVNK8sMgmtIPv4, nodeOVNK8sMgmtIPv6) + + joinSwitchIPv4, joinSwitchIPv6 := getJoinSwitchIPofNode(oc, newNodeName) + e2e.Logf("Got joinSwitchIPv4: %v, joinSwitchIPv6: %v", joinSwitchIPv4, joinSwitchIPv6) + + g.By("Check host network addresses in each node's northdb") + allNodeList, nodeErr := getAllNodes(oc) + o.Expect(nodeErr).NotTo(o.HaveOccurred()) + o.Expect(len(allNodeList)).NotTo(o.BeEquivalentTo(0)) + + for _, eachNodeName := range allNodeList { + ovnKubePod, podErr := getPodNameOnNode(oc, "openshift-ovn-kubernetes", "app=ovnkube-node", eachNodeName) + o.Expect(podErr).NotTo(o.HaveOccurred()) + o.Expect(ovnKubePod).ShouldNot(o.Equal("")) + if ipStackType == "dualstack" || ipStackType == "ipv4single" { + hostNetworkIPsv4 := getHostNetworkIPsinNBDB(oc, eachNodeName, "v4") + e2e.Logf("Got hostNetworkIPsv4 for node %s : %v", eachNodeName, hostNetworkIPsv4) + o.Expect(contains(hostNetworkIPsv4, nodeOVNK8sMgmtIPv4)).Should(o.BeTrue(), fmt.Sprintf("New node's mgmt IPv4 is not updated to node %s in NBDB!", eachNodeName)) + o.Expect(unorderedContains(hostNetworkIPsv4, joinSwitchIPv4)).Should(o.BeTrue(), fmt.Sprintf("New node's join switch IPv4 is not updated to node %s in NBDB!", eachNodeName)) + } + if ipStackType == "dualstack" || ipStackType == "ipv6single" { + hostNetworkIPsv6 := getHostNetworkIPsinNBDB(oc, eachNodeName, "v6") + e2e.Logf("Got hostNetworkIPsv6 for node %s : %v", eachNodeName, hostNetworkIPsv6) + o.Expect(contains(hostNetworkIPsv6, nodeOVNK8sMgmtIPv6)).Should(o.BeTrue(), fmt.Sprintf("New node's mgmt IPv6 is not updated to node %s in NBDB!", eachNodeName)) + o.Expect(unorderedContains(hostNetworkIPsv6, joinSwitchIPv6)).Should(o.BeTrue(), fmt.Sprintf("New node's join switch IPv6 is not updated to node %s in NBDB!", eachNodeName)) + } + } + + g.By("Verify that new pod on new node can communicate with old pod on old node") + curlPod2PodPass(oc, oc.Namespace(), "hello-pod1", ns2, "hello-pod2", 8080) + curlPod2PodPass(oc, ns2, "hello-pod2", oc.Namespace(), "hello-pod1", 8080) + }) +}) diff --git a/test/ote/util.go b/test/ote/util.go new file mode 100644 index 0000000000..b8ed6b80ae --- /dev/null +++ b/test/ote/util.go @@ -0,0 +1,719 @@ +package ote + +import ( + "context" + "fmt" + "math/rand" + "net" + "os" + "path/filepath" + "regexp" + "strconv" + "strings" + "time" + + g "github.com/onsi/ginkgo/v2" + o "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/util/wait" + e2e "k8s.io/kubernetes/test/e2e/framework" + e2eoutput "k8s.io/kubernetes/test/e2e/framework/pod/output" + netutils "k8s.io/utils/net" +) + +func init() { + if kubeconfig := os.Getenv("KUBECONFIG"); kubeconfig != "" { + e2e.TestContext.KubeConfig = kubeconfig + } + e2e.TestContext.KubectlPath = "kubectl" + e2e.TestContext.CloudConfig.Provider = e2e.NullProvider{} + e2e.TestContext.DeleteNamespace = os.Getenv("DELETE_NAMESPACE") != "false" +} + +func getRandomString() string { + chars := "abcdefghijklmnopqrstuvwxyz0123456789" + buffer := make([]byte, 8) + for index := range buffer { + buffer[index] = chars[rand.Intn(len(chars))] + } + return string(buffer) +} + +func checkIPStackType(oc *CLI) string { + svcNetwork, err := oc.WithoutNamespace().AsAdmin().Run("get").Args("network.operator", "cluster", "-o=jsonpath={.spec.serviceNetwork}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + if strings.Count(svcNetwork, ":") >= 2 && strings.Count(svcNetwork, ".") >= 2 { + return "dualstack" + } else if strings.Count(svcNetwork, ":") >= 2 { + return "ipv6single" + } else if strings.Count(svcNetwork, ".") >= 2 { + return "ipv4single" + } + return "" +} + +func getPodStatus(oc *CLI, namespace string, podName string) (string, error) { + podStatus, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("pod", "-n", namespace, podName, "-o=jsonpath={.status.phase}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("The pod %s status in namespace %s is %q", podName, namespace, podStatus) + return podStatus, err +} + +func checkPodReady(oc *CLI, namespace string, podName string) (bool, error) { + podOutPut, err := getPodStatus(oc, namespace, podName) + status := []string{"Running", "Ready", "Complete", "Succeeded"} + return contains(status, podOutPut), err +} + +func contains(s []string, str string) bool { + for _, v := range s { + if v == str { + return true + } + } + return false +} + +func waitPodReady(oc *CLI, namespace string, podName string) { + err := wait.PollUntilContextTimeout(context.Background(), 5*time.Second, 60*time.Second, true, func(ctx context.Context) (bool, error) { + status, err1 := checkPodReady(oc, namespace, podName) + if err1 != nil { + e2e.Logf("the err:%v, wait for pod %v to become ready.", err1, podName) + return status, err1 + } + if !status { + return status, nil + } + return status, nil + }) + + if err != nil { + podDescribe := describePod(oc, namespace, podName) + e2e.Logf("oc describe pod %v.", podName) + e2e.Logf("%s", podDescribe) + } + o.Expect(err).NotTo(o.HaveOccurred(), fmt.Sprintf("pod %v is not ready", podName)) +} + +func describePod(oc *CLI, namespace string, podName string) string { + podDescribe, err := oc.WithoutNamespace().Run("describe").Args("pod", "-n", namespace, podName).Output() + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("The pod %s status is %q", podName, podDescribe) + return podDescribe +} + +func patchResourceAsAdmin(oc *CLI, resource, patch string, nameSpace ...string) { + var cargs []string + if len(nameSpace) > 0 { + cargs = []string{resource, "-p", patch, "-n", nameSpace[0], "--type=merge"} + } else { + cargs = []string{resource, "-p", patch, "--type=merge"} + } + err := oc.AsAdmin().WithoutNamespace().Run("patch").Args(cargs...).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) +} + +func isHypershiftHostedCluster(oc *CLI) bool { + topology, err := oc.WithoutNamespace().AsAdmin().Run("get").Args("infrastructures.config.openshift.io", "cluster", "-o=jsonpath={.status.controlPlaneTopology}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("topology is %s", topology) + if topology == "" { + status, _ := oc.WithoutNamespace().AsAdmin().Run("get").Args("infrastructures.config.openshift.io", "cluster", "-o=jsonpath={.status}").Output() + e2e.Logf("cluster status %s", status) + e2e.Failf("failure: controlPlaneTopology returned empty") + } + return strings.Compare(topology, "External") == 0 +} + +func checkOVNKState(oc *CLI) error { + err := waitForPodWithLabelReady(oc, "openshift-ovn-kubernetes", "app=ovnkube-node") + o.Expect(err).NotTo(o.HaveOccurred()) + + if !isHypershiftHostedCluster(oc) { + err = waitForPodWithLabelReady(oc, "openshift-ovn-kubernetes", "app=ovnkube-control-plane") + o.Expect(err).NotTo(o.HaveOccurred()) + } + return wait.PollUntilContextTimeout(context.Background(), 10*time.Second, 2*time.Minute, true, func(ctx context.Context) (bool, error) { + status, err := oc.AsAdmin().WithoutNamespace().Run("rollout").Args("status", "-n", "openshift-ovn-kubernetes", "ds", "ovnkube-node", "--timeout", "5m").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + if strings.Contains(status, "successfully rolled out") { + e2e.Logf("ovnkube rollout was triggerred and rolled out successfully") + return true, nil + } + e2e.Logf("ovnkube rollout trigger hasn't happened yet. Trying again") + return false, nil + }) +} + +func waitForPodWithLabelReady(oc *CLI, ns, label string) error { + return wait.PollUntilContextTimeout(context.Background(), 5*time.Second, 5*time.Minute, true, func(ctx context.Context) (bool, error) { + status, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("pod", "-n", ns, "-l", label, "-ojsonpath={.items[*].status.conditions[?(@.type==\"Ready\")].status}").Output() + e2e.Logf("the Ready status of pod is %v", status) + if err != nil || status == "" { + e2e.Logf("failed to get pod status: %v, retrying...", err) + return false, nil + } + if strings.Contains(status, "False") { + e2e.Logf("the pod Ready status not met; wanted True but got %v, retrying...", status) + return false, nil + } + return true, nil + }) +} + +func curlPod2PodPass(oc *CLI, namespaceSrc string, podNameSrc string, namespaceDst string, podNameDst string, podPort int) { + podIP1, podIP2 := getPodIP(oc, namespaceDst, podNameDst) + if podIP2 != "" { + _, err := e2eoutput.RunHostCmd(namespaceSrc, podNameSrc, "curl --connect-timeout 5 -s "+net.JoinHostPort(podIP1, strconv.Itoa(podPort))) + o.Expect(err).NotTo(o.HaveOccurred()) + _, err = e2eoutput.RunHostCmd(namespaceSrc, podNameSrc, "curl --connect-timeout 5 -s "+net.JoinHostPort(podIP2, strconv.Itoa(podPort))) + o.Expect(err).NotTo(o.HaveOccurred()) + } else { + _, err := e2eoutput.RunHostCmd(namespaceSrc, podNameSrc, "curl --connect-timeout 5 -s "+net.JoinHostPort(podIP1, strconv.Itoa(podPort))) + o.Expect(err).NotTo(o.HaveOccurred()) + } +} + +func getPodIP(oc *CLI, namespace string, podName string) (string, string) { + ipStack := checkIPStackType(oc) + switch ipStack { + case "ipv6single", "ipv4single": + podIP, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("pod", "-n", namespace, podName, "-o=jsonpath={.status.podIPs[0].ip}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("The pod %s IP in namespace %s is %q", podName, namespace, podIP) + return podIP, "" + case "dualstack": + podIP1, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("pod", "-n", namespace, podName, "-o=jsonpath={.status.podIPs[1].ip}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("The pod's %s 1st IP in namespace %s is %q", podName, namespace, podIP1) + podIP2, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("pod", "-n", namespace, podName, "-o=jsonpath={.status.podIPs[0].ip}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("The pod's %s 2nd IP in namespace %s is %q", podName, namespace, podIP2) + if netutils.IsIPv6String(podIP1) { + e2e.Logf("This is IPv4 primary dual stack cluster") + return podIP1, podIP2 + } + e2e.Logf("This is IPv6 primary dual stack cluster") + return podIP2, podIP1 + } + return "", "" +} + +func curlPod2SvcPass(oc *CLI, namespaceSrc string, namespaceSvc string, podNameSrc string, svcName string, svcPort int) { + svcIP1, svcIP2 := getSvcIP(oc, namespaceSvc, svcName) + if svcIP2 != "" { + _, err := e2eoutput.RunHostCmdWithRetries(namespaceSrc, podNameSrc, "curl --connect-timeout 5 -s "+net.JoinHostPort(svcIP1, strconv.Itoa(svcPort)), 3*time.Second, 15*time.Second) + o.Expect(err).NotTo(o.HaveOccurred()) + _, err = e2eoutput.RunHostCmdWithRetries(namespaceSrc, podNameSrc, "curl --connect-timeout 5 -s "+net.JoinHostPort(svcIP2, strconv.Itoa(svcPort)), 3*time.Second, 15*time.Second) + o.Expect(err).NotTo(o.HaveOccurred()) + } else { + _, err := e2eoutput.RunHostCmdWithRetries(namespaceSrc, podNameSrc, "curl --connect-timeout 5 -s "+net.JoinHostPort(svcIP1, strconv.Itoa(svcPort)), 3*time.Second, 15*time.Second) + o.Expect(err).NotTo(o.HaveOccurred()) + } +} + +func getSvcIP(oc *CLI, namespace string, svcName string) (string, string) { + ipStack := checkIPStackType(oc) + svctype, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("service", "-n", namespace, svcName, "-o=jsonpath={.spec.type}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + ipFamilyType, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("service", "-n", namespace, svcName, "-o=jsonpath={.spec.ipFamilyPolicy}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + if (svctype == "ClusterIP") || (svctype == "NodePort") { + if (ipStack == "ipv6single") || (ipStack == "ipv4single") { + svcIP, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("service", "-n", namespace, svcName, "-o=jsonpath={.spec.clusterIPs[0]}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + if svctype == "ClusterIP" { + e2e.Logf("The service %s IP in namespace %s is %q", svcName, namespace, svcIP) + return svcIP, "" + } + nodePort, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("service", "-n", namespace, svcName, "-o=jsonpath={.spec.ports[*].nodePort}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("The NodePort service %s IP and NodePort in namespace %s is %s %s", svcName, namespace, svcIP, nodePort) + return svcIP, nodePort + + } else if (ipStack == "dualstack" && ipFamilyType == "PreferDualStack") || (ipStack == "dualstack" && ipFamilyType == "RequireDualStack") { + ipFamilyPrecedence, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("service", "-n", namespace, svcName, "-o=jsonpath={.spec.ipFamilies[0]}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + svcIPv4, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("service", "-n", namespace, svcName, "-o=jsonpath={.spec.clusterIPs[0]}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("The service %s IP in namespace %s is %q", svcName, namespace, svcIPv4) + svcIPv6, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("service", "-n", namespace, svcName, "-o=jsonpath={.spec.clusterIPs[1]}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("The service %s IP in namespace %s is %q", svcName, namespace, svcIPv6) + if ipFamilyPrecedence == "IPv4" { + e2e.Logf("The ipFamilyPrecedence is Ipv4, Ipv6") + switch svctype { + case "NodePort": + nodePort, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("service", "-n", namespace, svcName, "-o=jsonpath={.spec.ports[*].nodePort}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("The Dual Stack NodePort service %s IP and NodePort in namespace %s is %s %s", svcName, namespace, svcIPv4, nodePort) + return svcIPv4, nodePort + default: + return svcIPv6, svcIPv4 + } + } else { + e2e.Logf("The ipFamilyPrecedence is Ipv6, Ipv4") + switch svctype { + case "NodePort": + nodePort, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("service", "-n", namespace, svcName, "-o=jsonpath={.spec.ports[*].nodePort}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("The Dual Stack NodePort service %s IP and NodePort in namespace %s is %s %s", svcName, namespace, svcIPv6, nodePort) + return svcIPv6, nodePort + default: + svcIPv4, svcIPv6 = svcIPv6, svcIPv4 + return svcIPv6, svcIPv4 + } + } + } else { + svcIP, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("service", "-n", namespace, svcName, "-o=jsonpath={.spec.clusterIPs[0]}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("The service %s IP in namespace %s is %q", svcName, namespace, svcIP) + return svcIP, "" + } + } else { + e2e.Logf("The serviceType is LoadBalancer") + platform := checkPlatform(oc) + var jsonString string + if platform == "aws" { + jsonString = "-o=jsonpath={.status.loadBalancer.ingress[0].hostname}" + } else { + jsonString = "-o=jsonpath={.status.loadBalancer.ingress[0].ip}" + } + + err := wait.PollUntilContextTimeout(context.Background(), 30*time.Second, 300*time.Second, true, func(ctx context.Context) (bool, error) { + svcIP, er := oc.AsAdmin().WithoutNamespace().Run("get").Args("service", "-n", namespace, svcName, jsonString).Output() + o.Expect(er).NotTo(o.HaveOccurred()) + if svcIP == "" { + e2e.Logf("Waiting for lb service IP assignment. Trying again...") + return false, nil + } + return true, nil + }) + o.Expect(err).NotTo(o.HaveOccurred(), fmt.Sprintf("fail to assign lb svc IP to %v", svcName)) + lbSvcIP, _ := oc.AsAdmin().WithoutNamespace().Run("get").Args("service", "-n", namespace, svcName, jsonString).Output() + e2e.Logf("The %s lb service Ingress VIP in namespace %s is %q", svcName, namespace, lbSvcIP) + return lbSvcIP, "" + } +} + +func checkPlatform(oc *CLI) string { + output, _ := oc.AsAdmin().WithoutNamespace().Run("get").Args("infrastructure", "cluster", "-o=jsonpath={.status.platformStatus.type}").Output() + return strings.ToLower(output) +} + +func getControlPlaneTopology(oc *CLI) string { + output, _ := oc.AsAdmin().WithoutNamespace().Run("get").Args("infrastructures.config.openshift.io", "cluster", "-o=jsonpath={.status.controlPlaneTopology}").Output() + return output +} + +func createPingPodOnNode(oc *CLI, podName, namespace, label, nodeName string) { + podJSON := fmt.Sprintf(`{ + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "name": "%s", + "namespace": "%s", + "labels": { + "name": "%s" + } + }, + "spec": { + "nodeName": "%s", + "containers": [{ + "name": "hello-pod", + "image": "quay.io/openshifttest/hello-sdn@sha256:c89445416459e7adea9a5a416b3365ed3d74f2491beb904d61dc8d1eb89a72a4", + "ports": [{ + "containerPort": 8080 + }] + }], + "restartPolicy": "Never" + } + }`, podName, namespace, label, nodeName) + + tmpFile := fmt.Sprintf("/tmp/pod-%s-%s.json", podName, getRandomString()) + err := os.WriteFile(tmpFile, []byte(podJSON), 0644) + o.Expect(err).NotTo(o.HaveOccurred()) + + defer func() { + if err := os.Remove(tmpFile); err != nil { + e2e.Logf("warning: failed to remove temporary file %s: %v", tmpFile, err) + } + }() + + g.By(fmt.Sprintf("Creating pod %s on node %s", podName, nodeName)) + err = wait.PollUntilContextTimeout(context.Background(), 3*time.Second, 20*time.Second, true, func(ctx context.Context) (bool, error) { + err1 := oc.AsAdmin().WithoutNamespace().Run("apply").Args("-f", tmpFile).Execute() + if err1 != nil { + e2e.Logf("Failed to create pod: %v, retrying...", err1) + return false, nil + } + return true, nil + }) + o.Expect(err).NotTo(o.HaveOccurred(), fmt.Sprintf("failed to create pod %v", podName)) + + waitPodReady(oc, namespace, podName) + e2e.Logf("Pod %s is ready on node %s", podName, nodeName) +} + +func createGenericService(oc *CLI, serviceName, namespace, protocol, selector, serviceType, ipFamilyPolicy, internalTrafficPolicy, externalTrafficPolicy string, servicePort, serviceTargetPort int) { + var ipFamilyPolicyJSON string + if ipFamilyPolicy != "" { + ipFamilyPolicyJSON = fmt.Sprintf(`"ipFamilyPolicy": "%s",`, ipFamilyPolicy) + } + + var internalTrafficPolicyJSON string + if internalTrafficPolicy != "" { + internalTrafficPolicyJSON = fmt.Sprintf(`"internalTrafficPolicy": "%s",`, internalTrafficPolicy) + } + + var externalTrafficPolicyJSON string + if externalTrafficPolicy != "" { + externalTrafficPolicyJSON = fmt.Sprintf(`"externalTrafficPolicy": "%s",`, externalTrafficPolicy) + } + + serviceJSON := fmt.Sprintf(`{ + "apiVersion": "v1", + "kind": "Service", + "metadata": { + "name": "%s", + "namespace": "%s" + }, + "spec": { + "type": "%s", + %s + %s + %s + "selector": { + "name": "%s" + }, + "ports": [{ + "protocol": "%s", + "port": %d, + "targetPort": %d + }] + } + }`, serviceName, namespace, serviceType, ipFamilyPolicyJSON, internalTrafficPolicyJSON, externalTrafficPolicyJSON, selector, protocol, servicePort, serviceTargetPort) + + tmpFile := fmt.Sprintf("/tmp/service-%s-%s.json", serviceName, getRandomString()) + err := os.WriteFile(tmpFile, []byte(serviceJSON), 0644) + o.Expect(err).NotTo(o.HaveOccurred()) + + defer func() { + if err := os.Remove(tmpFile); err != nil { + e2e.Logf("warning: failed to remove temporary file %s: %v", tmpFile, err) + } + }() + + g.By(fmt.Sprintf("Creating service %s in namespace %s", serviceName, namespace)) + err = wait.PollUntilContextTimeout(context.Background(), 3*time.Second, 20*time.Second, true, func(ctx context.Context) (bool, error) { + err1 := oc.AsAdmin().WithoutNamespace().Run("apply").Args("-f", tmpFile).Execute() + if err1 != nil { + e2e.Logf("Failed to create service: %v, retrying...", err1) + return false, nil + } + return true, nil + }) + o.Expect(err).NotTo(o.HaveOccurred(), fmt.Sprintf("failed to create service %v", serviceName)) + e2e.Logf("Service %s created successfully in namespace %s", serviceName, namespace) +} + +func rebootNode(oc *CLI, nodeName string) { + e2e.Logf("Rebooting node %s....", nodeName) + err := oc.AsAdmin().Run("debug").Args("node/"+nodeName, "--", "chroot", "/host", "shutdown", "-r", "+1").Execute() + o.Expect(err).NotTo(o.HaveOccurred()) +} + +func checkNodeStatus(oc *CLI, nodeName string, expectedStatus string) { + var expectedValue string + if expectedStatus == "Ready" { + expectedValue = "True" + } else if expectedStatus == "NotReady" { + expectedValue = "Unknown" + } else { + o.Expect(fmt.Errorf("unsupported node status: %s", expectedStatus)).NotTo(o.HaveOccurred()) + } + err := wait.PollUntilContextTimeout(context.Background(), 5*time.Second, 15*time.Minute, true, func(ctx context.Context) (bool, error) { + statusOutput, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("nodes", nodeName, "-ojsonpath={.status.conditions[-1].status}").Output() + if err != nil { + e2e.Logf("Get node status with error : %v", err) + return false, nil + } + e2e.Logf("Expect Node %s in state %v, kubelet status is %s", nodeName, expectedStatus, statusOutput) + if statusOutput != expectedValue { + return false, nil + } + return true, nil + }) + o.Expect(err).NotTo(o.HaveOccurred(), fmt.Sprintf("Node %s is not in expected status %s", nodeName, expectedStatus)) +} + +func getAllNodes(oc *CLI) ([]string, error) { + output, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("nodes", "-o=jsonpath={.items[*].metadata.name}").Output() + if err != nil { + return nil, err + } + nodes := strings.Fields(output) + return nodes, nil +} + +func getPodNameOnNode(oc *CLI, ns, label, nodeName string) (string, error) { + output, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("pod", "-n", ns, "-l", label, + "--field-selector=spec.nodeName="+nodeName, "-o=jsonpath={.items[0].metadata.name}").Output() + if err != nil { + return "", err + } + return output, nil +} + +func execInPodContainer(oc *CLI, ns, podName, container, cmd string) (string, error) { + return oc.AsAdmin().WithoutNamespace().Run("exec").Args(podName, "-n", ns, "-c", container, "--", "bash", "-c", cmd).Output() +} + +func getOVNK8sNodeMgmtIPv4(oc *CLI, nodeName string) string { + var output string + err := wait.PollUntilContextTimeout(context.Background(), 10*time.Second, 60*time.Second, true, func(ctx context.Context) (bool, error) { + out, cmdErr := oc.AsAdmin().Run("debug").Args("node/"+nodeName, "--", "chroot", "/host", "bash", "-c", "/usr/sbin/ip -4 -brief address show | grep ovn-k8s-mp0").Output() + if out == "" || cmdErr != nil { + e2e.Logf("Did not get node's management interface, errors: %v, try again", cmdErr) + return false, nil + } + output = out + return true, nil + }) + o.Expect(err).NotTo(o.HaveOccurred(), fmt.Sprintf("fail to get management interface for node %v", nodeName)) + + re := regexp.MustCompile(`(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)){3}`) + nodeOVNK8sMgmtIP := re.FindAllString(output, -1)[0] + e2e.Logf("Got ovn-k8s management interface IP for node %v as: %v", nodeName, nodeOVNK8sMgmtIP) + return nodeOVNK8sMgmtIP +} + +func getOVNK8sNodeMgmtIPv6(oc *CLI, nodeName string) string { + var cmdOutput string + err := wait.PollUntilContextTimeout(context.Background(), 2*time.Second, 10*time.Second, true, func(ctx context.Context) (bool, error) { + out, cmdErr := oc.AsAdmin().Run("debug").Args("node/"+nodeName, "--", "chroot", "/host", "bash", "-c", + `/usr/sbin/ip -o -6 addr show dev ovn-k8s-mp0 | awk '$3 == "inet6" && $6 == "global" {print $4}' | cut -d'/' -f1`).Output() + if out == "" || cmdErr != nil { + e2e.Logf("Did not get node's IPv6 management interface, errors: %v, try again", cmdErr) + return false, nil + } + cmdOutput = out + return true, nil + }) + o.Expect(err).NotTo(o.HaveOccurred(), fmt.Sprintf("Failed to get IPv6 management interface for node %v", nodeName)) + return strings.Split(cmdOutput, "\n")[0] +} + +func getJoinSwitchIPofNode(oc *CLI, nodeName string) ([]string, []string) { + ovnKubePod, podErr := getPodNameOnNode(oc, "openshift-ovn-kubernetes", "app=ovnkube-node", nodeName) + o.Expect(podErr).NotTo(o.HaveOccurred()) + o.Expect(ovnKubePod).ShouldNot(o.Equal("")) + + var cmdOutput string + var joinSwitchIPv4s, joinSwitchIPv6s []string + cmd := "ovn-nbctl get logical_router_port rtoj-GR_" + nodeName + " networks" + err := wait.PollUntilContextTimeout(context.Background(), 3*time.Second, 2*time.Minute, true, func(ctx context.Context) (bool, error) { + out, cmdErr := execInPodContainer(oc, "openshift-ovn-kubernetes", ovnKubePod, "northd", cmd) + if out == "" || cmdErr != nil { + e2e.Logf("%v, Waiting for expected result to be synced, try again ...", cmdErr) + return false, nil + } + cmdOutput = out + return true, nil + }) + o.Expect(err).NotTo(o.HaveOccurred(), fmt.Sprintf("Failed to get join switch networks for node %v", nodeName)) + + rightTrimed := strings.TrimRight(strings.TrimLeft(cmdOutput, "["), "]") + outputs := strings.Split(rightTrimed, ", ") + for _, str := range outputs { + ipv4orv6 := strings.Trim(str, "\"") + ipOnly := strings.Split(ipv4orv6, "/")[0] + if isIPv4(ipv4orv6) { + joinSwitchIPv4s = append(joinSwitchIPv4s, ipOnly) + } + if isIPv6(ipv4orv6) { + joinSwitchIPv6s = append(joinSwitchIPv6s, ipOnly) + } + } + return joinSwitchIPv4s, joinSwitchIPv6s +} + +func isIPv4(addr string) bool { + ip := net.ParseIP(strings.Split(addr, "/")[0]) + return ip != nil && ip.To4() != nil +} + +func isIPv6(addr string) bool { + ip := net.ParseIP(strings.Split(addr, "/")[0]) + return ip != nil && ip.To4() == nil +} + +// getHostNetworkIPsinNBDB discovers the host-network address set in NBDB and returns its IPs. +// ipFamily must be "v4" or "v6". Handles both pre-5.0 (Namespace-based) and 5.0+ (PodSelector-based) naming. +func getHostNetworkIPsinNBDB(oc *CLI, nodeName string, ipFamily string) []string { + ovnKubePod, podErr := getPodNameOnNode(oc, "openshift-ovn-kubernetes", "app=ovnkube-node", nodeName) + o.Expect(podErr).NotTo(o.HaveOccurred()) + o.Expect(ovnKubePod).ShouldNot(o.Equal("")) + + candidateExternalIDs := []string{ + `'external_ids:"k8s.ovn.org/id"="default-network-controller:PodSelector:LS{ML:{policy-group.network.openshift.io/host-network: ,},}_LS{}_LNM:` + ipFamily + `"'`, + `'external_ids:"k8s.ovn.org/id"="default-network-controller:Namespace:openshift-host-network:` + ipFamily + `"'`, + } + + var cmdOutput string + var hostNetworkIPs []string + err := wait.PollUntilContextTimeout(context.Background(), 3*time.Second, 3*time.Minute, true, func(ctx context.Context) (bool, error) { + for _, extID := range candidateExternalIDs { + cmd := "ovn-nbctl --column address find address_set " + extID + out, cmdErr := execInPodContainer(oc, "openshift-ovn-kubernetes", ovnKubePod, "northd", cmd) + if out != "" && cmdErr == nil { + e2e.Logf("Found host-network address set (%s) on pod %s using: %s", ipFamily, ovnKubePod, extID) + cmdOutput = out + return true, nil + } + } + e2e.Logf("host-network address set (%s) not found yet on pod %s (node %s) — retrying...", ipFamily, ovnKubePod, nodeName) + return false, nil + }) + o.Expect(err).NotTo(o.HaveOccurred(), fmt.Sprintf("Failed to get host network IPs (%s) for node %v", ipFamily, nodeName)) + + re := regexp.MustCompile(`"[^",]+"`) + ipStrs := re.FindAllString(cmdOutput, -1) + for _, eachIpString := range ipStrs { + ip := strings.Trim(eachIpString, "\"") + hostNetworkIPs = append(hostNetworkIPs, ip) + } + return hostNetworkIPs +} + +func unorderedContains(first, second []string) bool { + set := make(map[string]bool) + for _, element := range first { + set[element] = true + } + for _, element := range second { + if !set[element] { + return false + } + } + return true +} + +func createNetworkPolicy(oc *CLI, namespace string) { + npJSON := fmt.Sprintf(`{ + "apiVersion": "networking.k8s.io/v1", + "kind": "NetworkPolicy", + "metadata": { + "name": "allow-from-all-namespaces", + "namespace": "%s" + }, + "spec": { + "ingress": [{ + "from": [{ + "namespaceSelector": {} + }] + }], + "podSelector": {} + } + }`, namespace) + + tmpFile := fmt.Sprintf("/tmp/np-%s-%s.json", namespace, getRandomString()) + err := os.WriteFile(tmpFile, []byte(npJSON), 0644) + o.Expect(err).NotTo(o.HaveOccurred()) + defer os.Remove(tmpFile) + + err = oc.AsAdmin().WithoutNamespace().Run("apply").Args("-f", tmpFile).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) +} + +func getInfrastructureName(oc *CLI) string { + output, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("infrastructure", "cluster", "-o=jsonpath={.status.infrastructureName}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + return output +} + +func createMachineSet(oc *CLI, machineSetName string) { + existingMS, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("machineset", "-n", "openshift-machine-api", "-o=jsonpath={.items[0].metadata.name}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(existingMS).NotTo(o.BeEmpty(), "No existing MachineSet found to clone") + + msJSON, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("machineset", existingMS, "-n", "openshift-machine-api", "-o=json").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + + msJSON = strings.ReplaceAll(msJSON, existingMS, machineSetName) + msJSON = regexp.MustCompile(`"uid":\s*"[^"]*"`).ReplaceAllString(msJSON, `"uid": ""`) + msJSON = regexp.MustCompile(`"resourceVersion":\s*"[^"]*"`).ReplaceAllString(msJSON, `"resourceVersion": ""`) + msJSON = regexp.MustCompile(`"replicas":\s*\d+`).ReplaceAllString(msJSON, `"replicas": 1`) + msJSON = regexp.MustCompile(`"creationTimestamp":\s*"[^"]*"`).ReplaceAllString(msJSON, `"creationTimestamp": null`) + + tmpFile := fmt.Sprintf("/tmp/ms-%s-%s.json", machineSetName, getRandomString()) + err = os.WriteFile(tmpFile, []byte(msJSON), 0644) + o.Expect(err).NotTo(o.HaveOccurred()) + defer os.Remove(tmpFile) + + err = oc.AsAdmin().WithoutNamespace().Run("create").Args("-f", tmpFile).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + e2e.Logf("MachineSet %s created successfully", machineSetName) +} + +func deleteMachineSet(oc *CLI, machineSetName string) { + e2e.Logf("Deleting MachineSet %s", machineSetName) + err := oc.AsAdmin().WithoutNamespace().Run("delete").Args("machineset", machineSetName, "-n", "openshift-machine-api", "--wait=true").Execute() + if err != nil { + e2e.Logf("Warning: failed to delete machineset %s: %v", machineSetName, err) + } +} + +func waitForMachineSetRunning(oc *CLI, machineSetName string, replicas int) { + err := wait.PollUntilContextTimeout(context.Background(), 30*time.Second, 15*time.Minute, true, func(ctx context.Context) (bool, error) { + readyReplicas, cmdErr := oc.AsAdmin().WithoutNamespace().Run("get").Args("machineset", machineSetName, "-n", "openshift-machine-api", "-o=jsonpath={.status.readyReplicas}").Output() + if cmdErr != nil { + e2e.Logf("Error getting machineset status: %v", cmdErr) + return false, nil + } + if readyReplicas == strconv.Itoa(replicas) { + e2e.Logf("MachineSet %s has %s ready replicas", machineSetName, readyReplicas) + return true, nil + } + e2e.Logf("Waiting for MachineSet %s: readyReplicas=%s, want=%d", machineSetName, readyReplicas, replicas) + return false, nil + }) + o.Expect(err).NotTo(o.HaveOccurred(), fmt.Sprintf("MachineSet %s did not reach %d ready replicas", machineSetName, replicas)) +} + +func getNodeNameFromMachineSet(oc *CLI, machineSetName string) string { + machineName, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("machine", "-n", "openshift-machine-api", + "-l", "machine.openshift.io/cluster-api-machineset="+machineSetName, "-o=jsonpath={.items[0].metadata.name}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(machineName).NotTo(o.BeEmpty()) + + nodeName, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("machine", machineName, "-n", "openshift-machine-api", + "-o=jsonpath={.status.nodeRef.name}").Output() + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(nodeName).NotTo(o.BeEmpty(), fmt.Sprintf("Machine %s has no nodeRef", machineName)) + e2e.Logf("Got node name %s from machineset %s", nodeName, machineSetName) + return nodeName +} + +func waitForMachinesDisappear(oc *CLI, machineSetName string) { + err := wait.PollUntilContextTimeout(context.Background(), 15*time.Second, 10*time.Minute, true, func(ctx context.Context) (bool, error) { + output, cmdErr := oc.AsAdmin().WithoutNamespace().Run("get").Args("machine", "-n", "openshift-machine-api", + "-l", "machine.openshift.io/cluster-api-machineset="+machineSetName, "-o=jsonpath={.items}").Output() + if cmdErr != nil { + e2e.Logf("Error getting machines: %v", cmdErr) + return false, nil + } + if output == "[]" || output == "" { + e2e.Logf("All machines from machineset %s have been removed", machineSetName) + return true, nil + } + e2e.Logf("Waiting for machines from machineset %s to disappear...", machineSetName) + return false, nil + }) + if err != nil { + e2e.Logf("Warning: machines from machineset %s did not fully disappear: %v", machineSetName, err) + } +} + +// Suppress unused import warnings - these are used in the test files +var _ = filepath.Join +var _ = regexp.MustCompile