From fbeeaea71ce6e7bd8042a59137e115cb544198c1 Mon Sep 17 00:00:00 2001 From: Mykola Yurchenko Date: Tue, 12 May 2026 19:34:25 -0700 Subject: [PATCH 01/51] dist, helm, contrib: drop manual zone-name labeling We only support single-node-zone so drop zone-name labeling. Signed-off-by: Mykola Yurchenko --- contrib/kind-common.sh | 8 ------ contrib/kind-helm.sh | 1 - dist/images/ovnkube.sh | 55 ++++-------------------------------------- helm/basic-deploy.sh | 4 --- 4 files changed, 5 insertions(+), 63 deletions(-) diff --git a/contrib/kind-common.sh b/contrib/kind-common.sh index 63b01419ef..4cbc593aba 100644 --- a/contrib/kind-common.sh +++ b/contrib/kind-common.sh @@ -1799,13 +1799,6 @@ remove_no_schedule_taint() { done } -label_ovn_single_node_zones() { - KIND_NODES=$(kind_get_nodes) - for n in $KIND_NODES; do - kubectl label node "${n}" k8s.ovn.org/zone-name=${n} --overwrite - done -} - OPENSSL="" set_openssl_binary() { for s in openssl openssl3; do @@ -1830,7 +1823,6 @@ scale_kind_cluster() { # change this to https://github.com/lobuhi/kindscaler once PR https://github.com/lobuhi/kindscaler/pull/1 is accepted git clone https://github.com/trozet/kindscaler /tmp/kindscaler /tmp/kindscaler/kindscaler.sh ${KIND_CLUSTER_NAME} -r worker -c ${KIND_NUM_WORKER} - label_ovn_single_node_zones if [ "$OVN_IMAGE" == local ]; then set_ovn_image fi diff --git a/contrib/kind-helm.sh b/contrib/kind-helm.sh index 2304b044cc..37d63f1c1f 100755 --- a/contrib/kind-helm.sh +++ b/contrib/kind-helm.sh @@ -465,7 +465,6 @@ helm_prereqs() { create_ovn_kubernetes() { cd ${DIR}/../helm/ovn-kubernetes - label_ovn_single_node_zones value_file="values-single-node-zone.yaml" echo "value_file=${value_file}" # For multi-pod-subnet case, NET_CIDR_IPV4 is a list of CIDRs separated by comma. diff --git a/dist/images/ovnkube.sh b/dist/images/ovnkube.sh index 283d8c5f6f..da78bbf8ce 100755 --- a/dist/images/ovnkube.sh +++ b/dist/images/ovnkube.sh @@ -946,56 +946,11 @@ function memory_trim_on_compaction_supported { } function get_node_zone() { - createKubeconfig=false - # DPU might have K8S_TOKEN/K8S_TOKENFILE and K8S_CACERT/K8S_CACERT_DATA provided to access DPU host (K8S_NODE here) - # which is in a different cluster. So we might need to create kubeconfig accordingly. - if [[ ${ovnkube_node_mode} == "dpu" ]]; then - if [[ -n ${K8S_TOKEN} ]] || [[ -n ${K8S_TOKEN_FILE} ]] || [[ -n ${K8S_CACERT_DATA} ]] || [[ -n ${K8S_CACERT} ]]; then - createKubeconfig=true - fi - fi - - if [[ ${createKubeconfig} == "false" ]]; then - zone=$(kubectl --server=${K8S_APISERVER} --token=${k8s_token} --certificate-authority=${k8s_cacert} \ - get node ${K8S_NODE} -o=jsonpath={'.metadata.labels.k8s\.ovn\.org/zone-name'}) - else - local dpuhost_k8s_apiserver=${K8S_APISERVER} - local dpuhost_k8s_token=${K8S_TOKEN} - local dpuhost_ca_config="" - if [[ -n ${K8S_TOKEN_FILE} ]]; then - dpuhost_k8s_token=$(cat ${K8S_TOKEN_FILE}) - fi - if [[ -n ${K8S_CACERT_DATA} ]]; then - dpuhost_ca_config="certificate-authority-data: ${K8S_CACERT_DATA}" - elif [[ -n ${K8S_CACERT} ]]; then - dpuhost_ca_config="certificate-authority: ${K8S_CACERT}" - fi - zone=$(kubectl --kubeconfig=<(cat < Date: Tue, 12 May 2026 19:35:40 -0700 Subject: [PATCH 02/51] test/e2e/kubevirt: simplify worker selection for live-migration With multi-node-per-zone removed the selection is simplified Signed-off-by: Mykola Yurchenko --- test/e2e/kubevirt.go | 35 ++--------------------------------- 1 file changed, 2 insertions(+), 33 deletions(-) diff --git a/test/e2e/kubevirt.go b/test/e2e/kubevirt.go index 07bd2f9de4..6dbadbca2d 100644 --- a/test/e2e/kubevirt.go +++ b/test/e2e/kubevirt.go @@ -1482,39 +1482,8 @@ passwd: namespace = fr.Namespace.Name workerNodeList, err := fr.ClientSet.CoreV1().Nodes().List(context.Background(), metav1.ListOptions{LabelSelector: labels.FormatLabels(map[string]string{"node-role.kubernetes.io/worker": ""})}) Expect(err).NotTo(HaveOccurred()) - nodesByOVNZone := map[string][]corev1.Node{} - for _, workerNode := range workerNodeList.Items { - ovnZone, ok := workerNode.Labels["k8s.ovn.org/zone-name"] - if !ok { - ovnZone = "global" - } - _, ok = nodesByOVNZone[ovnZone] - if !ok { - nodesByOVNZone[ovnZone] = []corev1.Node{} - } - nodesByOVNZone[ovnZone] = append(nodesByOVNZone[ovnZone], workerNode) - } - - selectedNodes = []corev1.Node{} - // If there is one global zone select the first three for the - // migration - if len(nodesByOVNZone) == 1 { - selectedNodes = []corev1.Node{ - workerNodeList.Items[0], - workerNodeList.Items[1], - workerNodeList.Items[2], - } - // Otherwise select a pair of nodes from different OVN zones - } else { - for _, nodes := range nodesByOVNZone { - selectedNodes = append(selectedNodes, nodes[0]) - if len(selectedNodes) == 3 { - break // we want just three of them - } - } - } - - Expect(selectedNodes).To(HaveLen(3), "at least three nodes in different zones are needed for interconnect scenarios") + Expect(len(workerNodeList.Items)).To(BeNumerically(">=", 3), "at least three worker nodes are needed for interconnect live-migration scenarios") + selectedNodes = workerNodeList.Items[:3] // Label the selected nodes with the generated namespaces, so we can // configure VM nodeSelector with it and live migration will take only From 1e20211e007afe4981d61eeef38a3316fff8970b Mon Sep 17 00:00:00 2001 From: Mykola Yurchenko Date: Tue, 12 May 2026 19:37:12 -0700 Subject: [PATCH 03/51] docs/installation: drop manual zone-name labeling step zone-name label is no longer read by anything. kube node name is used directly Signed-off-by: Mykola Yurchenko --- docs/installation/INSTALL.KUBEADM.md | 5 ----- docs/installation/launching-ovn-kubernetes-with-dpu.md | 3 +-- docs/installation/launching-ovn-kubernetes-with-helm.md | 7 ------- 3 files changed, 1 insertion(+), 14 deletions(-) diff --git a/docs/installation/INSTALL.KUBEADM.md b/docs/installation/INSTALL.KUBEADM.md index be29c39c65..2daded6429 100644 --- a/docs/installation/INSTALL.KUBEADM.md +++ b/docs/installation/INSTALL.KUBEADM.md @@ -469,11 +469,6 @@ Before starting OVN-Kubernetes, work around an issue where `br-int` is added by ovs-vsctl add-br br-int ~~~ -Single-node-zone interconnect requires every node to be labeled with its zone name **before** the Helm install: -~~~ -for n in node1 node2 node3; do kubectl label node $n k8s.ovn.org/zone-name=$n --overwrite; done -~~~ - Next, install OVN-Kubernetes with the Helm chart, pointing `global.image.repository` at the image you just pushed (or use the public `ghcr.io/ovn-kubernetes/ovn-kubernetes/ovn-kube-ubuntu` image to skip the build steps above). If `helm` isn't already on the master, install it with `curl -fsSL https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash`. ~~~ cd $HOME/work/src/github.com/ovn-kubernetes/ovn-kubernetes/helm/ovn-kubernetes diff --git a/docs/installation/launching-ovn-kubernetes-with-dpu.md b/docs/installation/launching-ovn-kubernetes-with-dpu.md index be7612971f..938188ad70 100644 --- a/docs/installation/launching-ovn-kubernetes-with-dpu.md +++ b/docs/installation/launching-ovn-kubernetes-with-dpu.md @@ -22,11 +22,10 @@ A single VF net-device or a group of VF net-devices (configured as SR-IOV device ## K8s Settings on DPU Host -The following node labels must be set on the DPU Host prior to installing OVN K8s CNI +The following node label must be set on the DPU Host prior to installing OVN K8s CNI ```yaml k8s.ovn.org/dpu-host= -k8s.ovn.org/zone-name="dpu-host node name" ``` ## Launching OVN K8s DPU Host cluster using helm diff --git a/docs/installation/launching-ovn-kubernetes-with-helm.md b/docs/installation/launching-ovn-kubernetes-with-helm.md index 104f9e6ce6..3e8686cdda 100644 --- a/docs/installation/launching-ovn-kubernetes-with-helm.md +++ b/docs/installation/launching-ovn-kubernetes-with-helm.md @@ -61,13 +61,6 @@ The chart uses `values-single-node-zone.yaml` by default. ## Step-by-step install -- Set the zone label on each node (required for interconnect mode): -``` -for n in $(kubectl get nodes -o jsonpath='{.items[*].metadata.name}'); do - kubectl label node "${n}" k8s.ovn.org/zone-name=${n} --overwrite -done -``` - - Run `helm install` with the appropriate `k8sAPIServer`, image repo and tag: ``` # cd helm/ovn-kubernetes From be80cc06afa6a79b455d2065b6bb0808ed6ca913 Mon Sep 17 00:00:00 2001 From: Mykola Yurchenko Date: Tue, 12 May 2026 20:20:11 -0700 Subject: [PATCH 04/51] test/e2e: drop dead multi-node-per-zone branching drop the helper as we only run single node per zone Signed-off-by: Mykola Yurchenko --- test/e2e/e2e.go | 7 +------ test/e2e/util.go | 23 +---------------------- 2 files changed, 2 insertions(+), 28 deletions(-) diff --git a/test/e2e/e2e.go b/test/e2e/e2e.go index 33a45406f9..08199165c3 100644 --- a/test/e2e/e2e.go +++ b/test/e2e/e2e.go @@ -674,12 +674,7 @@ var _ = ginkgo.Describe("e2e control plane", func() { } controlPlanePodName = "ovnkube-control-plane" - // in "one node per zone" config, ovnkube-controller doesn't create leader election lease - if !singleNodePerZone() { - controlPlaneLeaseName = "ovn-kubernetes-master-ovn-control-plane" - } else { - controlPlaneLeaseName = "ovn-kubernetes-master" - } + controlPlaneLeaseName = "ovn-kubernetes-master" controlPlanePods, err := f.ClientSet.CoreV1().Pods(deploymentconfig.Get().OVNKubernetesNamespace()).List(context.Background(), metav1.ListOptions{ LabelSelector: "name=" + controlPlanePodName, diff --git a/test/e2e/util.go b/test/e2e/util.go index 474f544944..e7a14b304c 100644 --- a/test/e2e/util.go +++ b/test/e2e/util.go @@ -57,8 +57,6 @@ const ( ovnGatewayMTUSupport = "k8s.ovn.org/gateway-mtu-support" ) -var singleNodePerZoneResult *bool - type IpNeighbor struct { Dst string `json:"dst"` Lladdr string `json:"lladdr"` @@ -1457,27 +1455,8 @@ func isPreConfiguredUdnAddressesEnabled() bool { return val == "true" } -func singleNodePerZone() bool { - if singleNodePerZoneResult == nil { - args := []string{"get", "pods", "--selector=app=ovnkube-node", "-o", "jsonpath={.items[0].spec.containers[*].name}"} - containerNames := e2ekubectl.RunKubectlOrDie(deploymentconfig.Get().OVNKubernetesNamespace(), args...) - result := true - for _, containerName := range strings.Split(containerNames, " ") { - if containerName == "ovnkube-node" { - result = false - break - } - } - singleNodePerZoneResult = &result - } - return *singleNodePerZoneResult -} - func getNodeContainerName() string { - if singleNodePerZone() { - return "ovnkube-controller" - } - return "ovnkube-node" + return "ovnkube-controller" } // getNodeZone returns the node's zone From 7525f9ce154aff29021d2e45bbd843b862ca1ef4 Mon Sep 17 00:00:00 2001 From: Riccardo Ravaioli Date: Tue, 2 Jun 2026 17:57:04 +0200 Subject: [PATCH 05/51] e2e: wait for localnet multi network policy convergence The localnet multihoming tests create and delete MultiNetworkPolicies, then immediately check connectivity. Policy programming is asynchronous, so those immediate checks can observe the old datapath state and make the test flaky. Wait for the expected connectivity state after policy creation and deletion before failing the test. Signed-off-by: Riccardo Ravaioli --- test/e2e/multihoming.go | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/test/e2e/multihoming.go b/test/e2e/multihoming.go index dd5d7bf8df..3af1e08135 100644 --- a/test/e2e/multihoming.go +++ b/test/e2e/multihoming.go @@ -1423,7 +1423,9 @@ var _ = Describe("Multi Homing", feature.MultiHoming, func() { Expect(err).NotTo(HaveOccurred()) By("asserting the *client* pod can contact the underlay service after deleting the policy") - Expect(connectToServer(clientPodConfig, underlayServiceIP, servicePort)).To(Succeed()) + Eventually(func() error { + return connectToServer(clientPodConfig, underlayServiceIP, servicePort) + }, 30*time.Second, time.Second).Should(Succeed()) }) It("can not communicate over a localnet secondary network from pod to the underlay service", func() { @@ -1446,7 +1448,9 @@ var _ = Describe("Multi Homing", feature.MultiHoming, func() { Expect(err).NotTo(HaveOccurred()) By("asserting the *client* pod cannot contact the underlay service after creating the policy") - Expect(connectToServer(clientPodConfig, underlayServiceIP, servicePort)).To(MatchError(ContainSubstring("exit code 28"))) + Eventually(func() error { + return connectToServer(clientPodConfig, underlayServiceIP, servicePort) + }, 30*time.Second, time.Second).Should(MatchError(ContainSubstring("exit code 28"))) }) }) @@ -1480,9 +1484,13 @@ var _ = Describe("Multi Homing", feature.MultiHoming, func() { Expect(err).NotTo(HaveOccurred()) if mnp.Name != denyAllEgress { - Expect(connectToServer(clientPodConfig, underlayServiceIP, servicePort)).To(Succeed()) + Eventually(func() error { + return connectToServer(clientPodConfig, underlayServiceIP, servicePort) + }, 30*time.Second, time.Second).Should(Succeed()) } else { - Expect(connectToServer(clientPodConfig, underlayServiceIP, servicePort)).To(Not(Succeed())) + Eventually(func() error { + return connectToServer(clientPodConfig, underlayServiceIP, servicePort) + }, 30*time.Second, time.Second).Should(MatchError(ContainSubstring("exit code 28"))) } By("deleting the multi-network policy") @@ -1493,7 +1501,9 @@ var _ = Describe("Multi Homing", feature.MultiHoming, func() { )).To(Succeed()) By("asserting the *client* pod can contact the underlay service after deleting the policy") - Expect(connectToServer(clientPodConfig, underlayServiceIP, servicePort)).To(Succeed()) + Eventually(func() error { + return connectToServer(clientPodConfig, underlayServiceIP, servicePort) + }, 30*time.Second, time.Second).Should(Succeed()) }, XEntry( "ingress denyall, ingress policy should have no impact on egress", @@ -1711,7 +1721,9 @@ ip a add %[4]s/24 dev %[2]s }, 2*time.Minute, 6*time.Second).Should(Succeed()) By("asserting the *blocked-client* pod **cannot** contact the server pod exposed endpoint") - Expect(connectToServer(blockedClientPodConfig, serverIP, port)).To(MatchError(ContainSubstring("exit code 28"))) + Eventually(func() error { + return connectToServer(blockedClientPodConfig, serverIP, port) + }, 2*time.Minute, 6*time.Second).Should(MatchError(ContainSubstring("exit code 28"))) }, Entry( "using pod selectors for a pure L2 overlay", From 1d8db6ee35ea5527991df48383391ddfcf68ae23 Mon Sep 17 00:00:00 2001 From: Tim Rozet Date: Thu, 4 Jun 2026 09:24:12 -0400 Subject: [PATCH 06/51] Bump frr-k8s for next-hop API Update the vendored frr-k8s dependency to b43efcb, which adds typed next-hop support on advertised prefixes. Refresh the generated frr-k8s clients and transitive vendored dependencies required by the module update. Signed-off-by: Tim Rozet --- .../LICENSE | 202 ++ .../otel/metric/{v1.39.0 => v1.40.0}/LICENSE | 0 .../otel/sdk/{v1.39.0 => v1.40.0}/LICENSE | 0 .../otel/trace/{v1.39.0 => v1.40.0}/LICENSE | 0 .../otel/{v1.39.0 => v1.40.0}/LICENSE | 0 .../x/sys/{v0.39.0 => v0.40.0}/LICENSE | 0 go-controller/go.mod | 8 +- go-controller/go.sum | 24 +- .../api/v1beta1/frrconfiguration_types.go | 44 +- .../api/v1beta1/frrk8sconfiguration.go | 62 + .../api/v1beta1/zz_generated.deepcopy.go | 105 + .../client/clientset/versioned/clientset.go | 4 +- .../versioned/fake/clientset_generated.go | 7 +- .../versioned/typed/api/v1beta1/api_client.go | 27 +- .../typed/api/v1beta1/fake/fake_api_client.go | 6 +- .../api/v1beta1/fake/fake_frrconfiguration.go | 141 +- .../v1beta1/fake/fake_frrk8sconfiguration.go | 38 + .../typed/api/v1beta1/frrconfiguration.go | 25 +- .../typed/api/v1beta1/frrk8sconfiguration.go | 56 + .../typed/api/v1beta1/generated_expansion.go | 2 + .../api/v1beta1/frrconfiguration.go | 32 +- .../api/v1beta1/frrk8sconfiguration.go | 88 + .../externalversions/api/v1beta1/interface.go | 7 + .../informers/externalversions/generic.go | 4 +- .../api/v1beta1/expansion_generated.go | 8 + .../listers/api/v1beta1/frrconfiguration.go | 22 +- .../api/v1beta1/frrk8sconfiguration.go | 56 + .../otel/attribute/internal/attribute.go | 12 +- .../go.opentelemetry.io/otel/attribute/set.go | 2 +- .../otel/attribute/value.go | 3 +- .../otel/semconv/v1.37.0/MIGRATION.md | 41 - .../otel/semconv/v1.37.0/README.md | 3 - .../otel/semconv/v1.39.0/MIGRATION.md | 78 + .../otel/semconv/v1.39.0/README.md | 3 + .../{v1.37.0 => v1.39.0}/attribute_group.go | 2002 +++++++++++++---- .../otel/semconv/{v1.37.0 => v1.39.0}/doc.go | 4 +- .../{v1.37.0 => v1.39.0}/error_type.go | 2 +- .../semconv/{v1.37.0 => v1.39.0}/exception.go | 2 +- .../semconv/{v1.37.0 => v1.39.0}/schema.go | 4 +- .../go.opentelemetry.io/otel/trace/auto.go | 2 +- .../vendor/golang.org/x/sys/cpu/cpu_x86.go | 174 +- go-controller/vendor/modules.txt | 12 +- 42 files changed, 2527 insertions(+), 785 deletions(-) create mode 100644 LICENSES/packages/github.com/metallb/frr-k8s/v0.0.0-20260603082256-b43efcb206be/LICENSE rename LICENSES/packages/go.opentelemetry.io/otel/metric/{v1.39.0 => v1.40.0}/LICENSE (100%) rename LICENSES/packages/go.opentelemetry.io/otel/sdk/{v1.39.0 => v1.40.0}/LICENSE (100%) rename LICENSES/packages/go.opentelemetry.io/otel/trace/{v1.39.0 => v1.40.0}/LICENSE (100%) rename LICENSES/packages/go.opentelemetry.io/otel/{v1.39.0 => v1.40.0}/LICENSE (100%) rename LICENSES/packages/golang.org/x/sys/{v0.39.0 => v0.40.0}/LICENSE (100%) create mode 100644 go-controller/vendor/github.com/metallb/frr-k8s/api/v1beta1/frrk8sconfiguration.go create mode 100644 go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/clientset/versioned/typed/api/v1beta1/fake/fake_frrk8sconfiguration.go create mode 100644 go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/clientset/versioned/typed/api/v1beta1/frrk8sconfiguration.go create mode 100644 go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/informers/externalversions/api/v1beta1/frrk8sconfiguration.go create mode 100644 go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/listers/api/v1beta1/frrk8sconfiguration.go delete mode 100644 go-controller/vendor/go.opentelemetry.io/otel/semconv/v1.37.0/MIGRATION.md delete mode 100644 go-controller/vendor/go.opentelemetry.io/otel/semconv/v1.37.0/README.md create mode 100644 go-controller/vendor/go.opentelemetry.io/otel/semconv/v1.39.0/MIGRATION.md create mode 100644 go-controller/vendor/go.opentelemetry.io/otel/semconv/v1.39.0/README.md rename go-controller/vendor/go.opentelemetry.io/otel/semconv/{v1.37.0 => v1.39.0}/attribute_group.go (90%) rename go-controller/vendor/go.opentelemetry.io/otel/semconv/{v1.37.0 => v1.39.0}/doc.go (96%) rename go-controller/vendor/go.opentelemetry.io/otel/semconv/{v1.37.0 => v1.39.0}/error_type.go (99%) rename go-controller/vendor/go.opentelemetry.io/otel/semconv/{v1.37.0 => v1.39.0}/exception.go (98%) rename go-controller/vendor/go.opentelemetry.io/otel/semconv/{v1.37.0 => v1.39.0}/schema.go (85%) diff --git a/LICENSES/packages/github.com/metallb/frr-k8s/v0.0.0-20260603082256-b43efcb206be/LICENSE b/LICENSES/packages/github.com/metallb/frr-k8s/v0.0.0-20260603082256-b43efcb206be/LICENSE new file mode 100644 index 0000000000..d645695673 --- /dev/null +++ b/LICENSES/packages/github.com/metallb/frr-k8s/v0.0.0-20260603082256-b43efcb206be/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/LICENSES/packages/go.opentelemetry.io/otel/metric/v1.39.0/LICENSE b/LICENSES/packages/go.opentelemetry.io/otel/metric/v1.40.0/LICENSE similarity index 100% rename from LICENSES/packages/go.opentelemetry.io/otel/metric/v1.39.0/LICENSE rename to LICENSES/packages/go.opentelemetry.io/otel/metric/v1.40.0/LICENSE diff --git a/LICENSES/packages/go.opentelemetry.io/otel/sdk/v1.39.0/LICENSE b/LICENSES/packages/go.opentelemetry.io/otel/sdk/v1.40.0/LICENSE similarity index 100% rename from LICENSES/packages/go.opentelemetry.io/otel/sdk/v1.39.0/LICENSE rename to LICENSES/packages/go.opentelemetry.io/otel/sdk/v1.40.0/LICENSE diff --git a/LICENSES/packages/go.opentelemetry.io/otel/trace/v1.39.0/LICENSE b/LICENSES/packages/go.opentelemetry.io/otel/trace/v1.40.0/LICENSE similarity index 100% rename from LICENSES/packages/go.opentelemetry.io/otel/trace/v1.39.0/LICENSE rename to LICENSES/packages/go.opentelemetry.io/otel/trace/v1.40.0/LICENSE diff --git a/LICENSES/packages/go.opentelemetry.io/otel/v1.39.0/LICENSE b/LICENSES/packages/go.opentelemetry.io/otel/v1.40.0/LICENSE similarity index 100% rename from LICENSES/packages/go.opentelemetry.io/otel/v1.39.0/LICENSE rename to LICENSES/packages/go.opentelemetry.io/otel/v1.40.0/LICENSE diff --git a/LICENSES/packages/golang.org/x/sys/v0.39.0/LICENSE b/LICENSES/packages/golang.org/x/sys/v0.40.0/LICENSE similarity index 100% rename from LICENSES/packages/golang.org/x/sys/v0.39.0/LICENSE rename to LICENSES/packages/golang.org/x/sys/v0.40.0/LICENSE diff --git a/go-controller/go.mod b/go-controller/go.mod index 6caf5263a9..02962ab37a 100644 --- a/go-controller/go.mod +++ b/go-controller/go.mod @@ -29,7 +29,7 @@ require ( github.com/mdlayher/arp v0.0.0-20220512170110-6706a2966875 github.com/mdlayher/ndp v1.0.1 github.com/mdlayher/socket v0.5.1 - github.com/metallb/frr-k8s v0.0.21 + github.com/metallb/frr-k8s v0.0.0-20260603082256-b43efcb206be github.com/miekg/dns v1.1.31 github.com/mitchellh/copystructure v1.2.0 github.com/moby/sys/userns v0.1.0 @@ -49,7 +49,7 @@ require ( golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 golang.org/x/net v0.48.0 golang.org/x/sync v0.19.0 - golang.org/x/sys v0.39.0 + golang.org/x/sys v0.40.0 golang.org/x/time v0.11.0 google.golang.org/grpc v1.79.3 google.golang.org/grpc/security/advancedtls v0.0.0-20240425232638-1e8b9b7fc655 @@ -135,8 +135,8 @@ require ( github.com/x448/float16 v0.8.4 // indirect github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 // indirect go.opencensus.io v0.24.0 // indirect - go.opentelemetry.io/otel v1.39.0 // indirect - go.opentelemetry.io/otel/trace v1.39.0 // indirect + go.opentelemetry.io/otel v1.40.0 // indirect + go.opentelemetry.io/otel/trace v1.40.0 // indirect go.yaml.in/yaml/v2 v2.4.3 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/crypto v0.46.0 // indirect diff --git a/go-controller/go.sum b/go-controller/go.sum index 7f4f23f451..4978069424 100644 --- a/go-controller/go.sum +++ b/go-controller/go.sum @@ -551,8 +551,8 @@ github.com/mdlayher/packet v1.1.2/go.mod h1:GEu1+n9sG5VtiRE4SydOmX5GTwyyYlteZiFU github.com/mdlayher/socket v0.2.1/go.mod h1:QLlNPkFR88mRUNQIzRBMfXxwKal8H7u1h3bL1CV+f0E= github.com/mdlayher/socket v0.5.1 h1:VZaqt6RkGkt2OE9l3GcC6nZkqD3xKeQLyfleW/uBcos= github.com/mdlayher/socket v0.5.1/go.mod h1:TjPLHI1UgwEv5J1B5q0zTZq12A/6H7nKmtTanQE37IQ= -github.com/metallb/frr-k8s v0.0.21 h1:JLlCeXVlW5BLVdPy2u5sS9UCVlnK9x2vzWbIkxb8Atk= -github.com/metallb/frr-k8s v0.0.21/go.mod h1:VMnCZUVXYy7k0Fsa2L3XKwISFs3Thv0Uord7rSZPQZw= +github.com/metallb/frr-k8s v0.0.0-20260603082256-b43efcb206be h1:F+uE81Y/eaGaulFiw+8pa7MlDNHKENWc6Ix9zFUL4lw= +github.com/metallb/frr-k8s v0.0.0-20260603082256-b43efcb206be/go.mod h1:qQKYV1yLbExx5IkDxkt/4kU7Gx/VRvLN9JZtbRha0RU= github.com/mfridman/tparse v0.18.0 h1:wh6dzOKaIwkUGyKgOntDW4liXSo37qg5AXbIhkMV3vE= github.com/mfridman/tparse v0.18.0/go.mod h1:gEvqZTuCgEhPbYk/2lS3Kcxg1GmTxxU7kTC8DvP0i/A= github.com/miekg/dns v1.1.31 h1:sJFOl9BgwbYAWOGEwr61FU28pqsBNdpRBnhGXtO06Oo= @@ -833,16 +833,16 @@ go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= -go.opentelemetry.io/otel v1.39.0 h1:8yPrr/S0ND9QEfTfdP9V+SiwT4E0G7Y5MO7p85nis48= -go.opentelemetry.io/otel v1.39.0/go.mod h1:kLlFTywNWrFyEdH0oj2xK0bFYZtHRYUdv1NklR/tgc8= -go.opentelemetry.io/otel/metric v1.39.0 h1:d1UzonvEZriVfpNKEVmHXbdf909uGTOQjA0HF0Ls5Q0= -go.opentelemetry.io/otel/metric v1.39.0/go.mod h1:jrZSWL33sD7bBxg1xjrqyDjnuzTUB0x1nBERXd7Ftcs= -go.opentelemetry.io/otel/sdk v1.39.0 h1:nMLYcjVsvdui1B/4FRkwjzoRVsMK8uL/cj0OyhKzt18= -go.opentelemetry.io/otel/sdk v1.39.0/go.mod h1:vDojkC4/jsTJsE+kh+LXYQlbL8CgrEcwmt1ENZszdJE= +go.opentelemetry.io/otel v1.40.0 h1:oA5YeOcpRTXq6NN7frwmwFR0Cn3RhTVZvXsP4duvCms= +go.opentelemetry.io/otel v1.40.0/go.mod h1:IMb+uXZUKkMXdPddhwAHm6UfOwJyh4ct1ybIlV14J0g= +go.opentelemetry.io/otel/metric v1.40.0 h1:rcZe317KPftE2rstWIBitCdVp89A2HqjkxR3c11+p9g= +go.opentelemetry.io/otel/metric v1.40.0/go.mod h1:ib/crwQH7N3r5kfiBZQbwrTge743UDc7DTFVZrrXnqc= +go.opentelemetry.io/otel/sdk v1.40.0 h1:KHW/jUzgo6wsPh9At46+h4upjtccTmuZCFAc9OJ71f8= +go.opentelemetry.io/otel/sdk v1.40.0/go.mod h1:Ph7EFdYvxq72Y8Li9q8KebuYUr2KoeyHx0DRMKrYBUE= go.opentelemetry.io/otel/sdk/metric v1.39.0 h1:cXMVVFVgsIf2YL6QkRF4Urbr/aMInf+2WKg+sEJTtB8= go.opentelemetry.io/otel/sdk/metric v1.39.0/go.mod h1:xq9HEVH7qeX69/JnwEfp6fVq5wosJsY1mt4lLfYdVew= -go.opentelemetry.io/otel/trace v1.39.0 h1:2d2vfpEDmCJ5zVYz7ijaJdOF59xLomrvj7bjt6/qCJI= -go.opentelemetry.io/otel/trace v1.39.0/go.mod h1:88w4/PnZSazkGzz/w84VHpQafiU4EtqqlVdxWy+rNOA= +go.opentelemetry.io/otel/trace v1.40.0 h1:WA4etStDttCSYuhwvEa8OP8I5EWu24lkOzp+ZYblVjw= +go.opentelemetry.io/otel/trace v1.40.0/go.mod h1:zeAhriXecNGP/s2SEG3+Y8X9ujcJOTqQ5RgdEJcawiA= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= @@ -1049,8 +1049,8 @@ golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= -golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ= +golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= diff --git a/go-controller/vendor/github.com/metallb/frr-k8s/api/v1beta1/frrconfiguration_types.go b/go-controller/vendor/github.com/metallb/frr-k8s/api/v1beta1/frrconfiguration_types.go index ec725e3444..8333be3b53 100644 --- a/go-controller/vendor/github.com/metallb/frr-k8s/api/v1beta1/frrconfiguration_types.go +++ b/go-controller/vendor/github.com/metallb/frr-k8s/api/v1beta1/frrconfiguration_types.go @@ -40,6 +40,10 @@ type FRRConfigurationSpec struct { // RawConfig is a snippet of raw frr configuration that gets appended to the // rendered configuration. +// +// WARNING: The RawConfig feature is UNSUPPORTED and intended ONLY FOR EXPERIMENTATION. +// It should not be used in production environments. This feature is provided as-is without any +// guarantees of stability, compatibility, or support. Use at your own risk. type RawConfig struct { // Priority is the order with this configuration is appended to the // bottom of the rendered configuration. A higher value means the @@ -55,7 +59,7 @@ type RawConfig struct { type BGPConfig struct { // Routers is the list of routers we want FRR to configure (one per VRF). // +optional - Routers []Router `json:"routers"` + Routers []Router `json:"routers,omitempty"` // BFDProfiles is the list of bfd profiles to be used when configuring the neighbors. // +optional BFDProfiles []BFDProfile `json:"bfdProfiles,omitempty"` @@ -66,6 +70,7 @@ type Router struct { // ASN is the AS number to use for the local end of the session. // +kubebuilder:validation:Minimum=0 // +kubebuilder:validation:Maximum=4294967295 + // +kubebuilder:validation:Format=int64 ASN uint32 `json:"asn"` // ID is the BGP router ID // +optional @@ -98,6 +103,7 @@ type Neighbor struct { // ASN and DynamicASN are mutually exclusive and one of them must be specified. // +kubebuilder:validation:Minimum=0 // +kubebuilder:validation:Maximum=4294967295 + // +kubebuilder:validation:Format=int64 // +optional ASN uint32 `json:"asn,omitempty"` @@ -191,8 +197,10 @@ type Neighbor struct { // +optional ToReceive Receive `json:"toReceive,omitempty"` - // To set if we want to disable MP BGP that will separate IPv4 and IPv6 route exchanges into distinct BGP sessions. - // Deprecated: DisableMP is deprecated in favor of dualStackAddressFamily. + // DisableMP is no longer used and has no effect. + // Use DualStackAddressFamily instead to enable the neighbor for both IPv4 and IPv6 address families. + // + // Deprecated: This field is ignored. Use DualStackAddressFamily instead. // +optional // +kubebuilder:default:=false DisableMP bool `json:"disableMP,omitempty"` @@ -202,6 +210,18 @@ type Neighbor struct { // +optional // +kubebuilder:default:=false DualStackAddressFamily bool `json:"dualStackAddressFamily,omitempty"` + + // LocalASN allows advertising a different AS number to the peer using BGP's + // local-as feature. When set, FRR will advertise this ASN to the peer + // via "neighbor local-as no-prepend replace-as", overriding + // the router-level ASN for this specific session. + // Note: this field is only applicable to eBGP sessions (where the peer ASN differs + // from the router ASN). Setting it on an iBGP session is rejected. + // +optional + // +kubebuilder:validation:Minimum=1 + // +kubebuilder:validation:Maximum=4294967295 + // +kubebuilder:validation:Format=int64 + LocalASN uint32 `json:"localASN,omitempty"` } // Advertise represents a list of prefixes to advertise to the given neighbor. @@ -212,6 +232,11 @@ type Advertise struct { // this neighbor. They must match the prefixes defined in the router. Allowed AllowedOutPrefixes `json:"allowed,omitempty"` + // NextHop sets the BGP next-hop address to advertise with prefixes + // sent to this neighbor. + // +optional + NextHop NextHop `json:"nextHop,omitempty"` + // PrefixesWithLocalPref is a list of prefixes that are associated to a local // preference when being advertised. The prefixes associated to a given local pref // must be in the prefixes allowed to be advertised. @@ -225,6 +250,19 @@ type Advertise struct { PrefixesWithCommunity []CommunityPrefixes `json:"withCommunity,omitempty"` } +// NextHop sets the BGP next-hop address for advertised prefixes. +type NextHop struct { + // IPv4 is the next-hop address to advertise with IPv4 prefixes. + // +optional + // +kubebuilder:validation:Format=ipv4 + IPv4 string `json:"ipv4,omitempty"` + + // IPv6 is the next-hop address to advertise with IPv6 prefixes. + // +optional + // +kubebuilder:validation:Format=ipv6 + IPv6 string `json:"ipv6,omitempty"` +} + // Receive represents a list of prefixes to receive from the given neighbor. type Receive struct { // Allowed is the list of prefixes allowed to be received from diff --git a/go-controller/vendor/github.com/metallb/frr-k8s/api/v1beta1/frrk8sconfiguration.go b/go-controller/vendor/github.com/metallb/frr-k8s/api/v1beta1/frrk8sconfiguration.go new file mode 100644 index 0000000000..434ce52819 --- /dev/null +++ b/go-controller/vendor/github.com/metallb/frr-k8s/api/v1beta1/frrk8sconfiguration.go @@ -0,0 +1,62 @@ +/* +Copyright 2023. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// FRRK8sConfigurationSpec defines the desired state of FRRK8sConfiguration. +type FRRK8sConfigurationSpec struct { + // LogLevel sets the logging verbosity for the FRR-K8s components at runtime. + // When configured, this value overrides the defaults established by the --log-level CLI flag. + // Valid values are: all, debug, info, warn, error, none. + // +kubebuilder:validation:Enum=all;debug;info;warn;error;none + // +optional + LogLevel string `json:"logLevel,omitempty"` +} + +// FRRK8sConfigurationStatus defines the observed state of FRRK8sConfiguration. +type FRRK8sConfigurationStatus struct { + // INSERT ADDITIONAL STATUS FIELD - define observed state of cluster + // Important: Run "make" to regenerate code after modifying this file +} + +//+kubebuilder:object:root=true +//+kubebuilder:subresource:status +//nolint +//+genclient + +// FRRK8sConfiguration holds the FRR Operator configuration with global +// settings for the K8s and FRR. +type FRRK8sConfiguration struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec FRRK8sConfigurationSpec `json:"spec,omitempty"` + Status FRRK8sConfigurationStatus `json:"status,omitempty"` +} + +//+kubebuilder:object:root=true + +// FRRK8sConfigurationList contains a list of FRRK8sConfiguration. +type FRRK8sConfigurationList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []FRRK8sConfiguration `json:"items"` +} + +func init() { + SchemeBuilder.Register(&FRRK8sConfiguration{}, &FRRK8sConfigurationList{}) +} diff --git a/go-controller/vendor/github.com/metallb/frr-k8s/api/v1beta1/zz_generated.deepcopy.go b/go-controller/vendor/github.com/metallb/frr-k8s/api/v1beta1/zz_generated.deepcopy.go index 4670081537..13a69b5d87 100644 --- a/go-controller/vendor/github.com/metallb/frr-k8s/api/v1beta1/zz_generated.deepcopy.go +++ b/go-controller/vendor/github.com/metallb/frr-k8s/api/v1beta1/zz_generated.deepcopy.go @@ -15,6 +15,7 @@ import ( func (in *Advertise) DeepCopyInto(out *Advertise) { *out = *in in.Allowed.DeepCopyInto(&out.Allowed) + out.NextHop = in.NextHop if in.PrefixesWithLocalPref != nil { in, out := &in.PrefixesWithLocalPref, &out.PrefixesWithLocalPref *out = make([]LocalPrefPrefixes, len(*in)) @@ -361,6 +362,95 @@ func (in *FRRConfigurationStatus) DeepCopy() *FRRConfigurationStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *FRRK8sConfiguration) DeepCopyInto(out *FRRK8sConfiguration) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + out.Spec = in.Spec + out.Status = in.Status +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FRRK8sConfiguration. +func (in *FRRK8sConfiguration) DeepCopy() *FRRK8sConfiguration { + if in == nil { + return nil + } + out := new(FRRK8sConfiguration) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *FRRK8sConfiguration) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *FRRK8sConfigurationList) DeepCopyInto(out *FRRK8sConfigurationList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]FRRK8sConfiguration, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FRRK8sConfigurationList. +func (in *FRRK8sConfigurationList) DeepCopy() *FRRK8sConfigurationList { + if in == nil { + return nil + } + out := new(FRRK8sConfigurationList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *FRRK8sConfigurationList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *FRRK8sConfigurationSpec) DeepCopyInto(out *FRRK8sConfigurationSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FRRK8sConfigurationSpec. +func (in *FRRK8sConfigurationSpec) DeepCopy() *FRRK8sConfigurationSpec { + if in == nil { + return nil + } + out := new(FRRK8sConfigurationSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *FRRK8sConfigurationStatus) DeepCopyInto(out *FRRK8sConfigurationStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FRRK8sConfigurationStatus. +func (in *FRRK8sConfigurationStatus) DeepCopy() *FRRK8sConfigurationStatus { + if in == nil { + return nil + } + out := new(FRRK8sConfigurationStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *FRRNodeState) DeepCopyInto(out *FRRNodeState) { *out = *in @@ -523,6 +613,21 @@ func (in *Neighbor) DeepCopy() *Neighbor { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NextHop) DeepCopyInto(out *NextHop) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NextHop. +func (in *NextHop) DeepCopy() *NextHop { + if in == nil { + return nil + } + out := new(NextHop) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PrefixSelector) DeepCopyInto(out *PrefixSelector) { *out = *in diff --git a/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/clientset/versioned/clientset.go b/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/clientset/versioned/clientset.go index 5c92e69505..43321a7622 100644 --- a/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/clientset/versioned/clientset.go +++ b/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/clientset/versioned/clientset.go @@ -5,8 +5,8 @@ package versioned import ( - "fmt" - "net/http" + fmt "fmt" + http "net/http" apiv1beta1 "github.com/metallb/frr-k8s/pkg/client/clientset/versioned/typed/api/v1beta1" discovery "k8s.io/client-go/discovery" diff --git a/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/clientset/versioned/fake/clientset_generated.go b/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/clientset/versioned/fake/clientset_generated.go index d6e24860a6..2aca8501f8 100644 --- a/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/clientset/versioned/fake/clientset_generated.go +++ b/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/clientset/versioned/fake/clientset_generated.go @@ -8,6 +8,7 @@ import ( clientset "github.com/metallb/frr-k8s/pkg/client/clientset/versioned" apiv1beta1 "github.com/metallb/frr-k8s/pkg/client/clientset/versioned/typed/api/v1beta1" fakeapiv1beta1 "github.com/metallb/frr-k8s/pkg/client/clientset/versioned/typed/api/v1beta1/fake" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/watch" "k8s.io/client-go/discovery" @@ -35,9 +36,13 @@ func NewSimpleClientset(objects ...runtime.Object) *Clientset { cs.discovery = &fakediscovery.FakeDiscovery{Fake: &cs.Fake} cs.AddReactor("*", "*", testing.ObjectReaction(o)) cs.AddWatchReactor("*", func(action testing.Action) (handled bool, ret watch.Interface, err error) { + var opts metav1.ListOptions + if watchActcion, ok := action.(testing.WatchActionImpl); ok { + opts = watchActcion.ListOptions + } gvr := action.GetResource() ns := action.GetNamespace() - watch, err := o.Watch(gvr, ns) + watch, err := o.Watch(gvr, ns, opts) if err != nil { return false, nil, err } diff --git a/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/clientset/versioned/typed/api/v1beta1/api_client.go b/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/clientset/versioned/typed/api/v1beta1/api_client.go index 6aebb4137d..665daf2beb 100644 --- a/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/clientset/versioned/typed/api/v1beta1/api_client.go +++ b/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/clientset/versioned/typed/api/v1beta1/api_client.go @@ -5,16 +5,17 @@ package v1beta1 import ( - "net/http" + http "net/http" - v1beta1 "github.com/metallb/frr-k8s/api/v1beta1" - "github.com/metallb/frr-k8s/pkg/client/clientset/versioned/scheme" + apiv1beta1 "github.com/metallb/frr-k8s/api/v1beta1" + scheme "github.com/metallb/frr-k8s/pkg/client/clientset/versioned/scheme" rest "k8s.io/client-go/rest" ) type ApiV1beta1Interface interface { RESTClient() rest.Interface FRRConfigurationsGetter + FRRK8sConfigurationsGetter } // ApiV1beta1Client is used to interact with features provided by the api group. @@ -26,14 +27,16 @@ func (c *ApiV1beta1Client) FRRConfigurations(namespace string) FRRConfigurationI return newFRRConfigurations(c, namespace) } +func (c *ApiV1beta1Client) FRRK8sConfigurations(namespace string) FRRK8sConfigurationInterface { + return newFRRK8sConfigurations(c, namespace) +} + // NewForConfig creates a new ApiV1beta1Client for the given config. // NewForConfig is equivalent to NewForConfigAndClient(c, httpClient), // where httpClient was generated with rest.HTTPClientFor(c). func NewForConfig(c *rest.Config) (*ApiV1beta1Client, error) { config := *c - if err := setConfigDefaults(&config); err != nil { - return nil, err - } + setConfigDefaults(&config) httpClient, err := rest.HTTPClientFor(&config) if err != nil { return nil, err @@ -45,9 +48,7 @@ func NewForConfig(c *rest.Config) (*ApiV1beta1Client, error) { // Note the http client provided takes precedence over the configured transport values. func NewForConfigAndClient(c *rest.Config, h *http.Client) (*ApiV1beta1Client, error) { config := *c - if err := setConfigDefaults(&config); err != nil { - return nil, err - } + setConfigDefaults(&config) client, err := rest.RESTClientForConfigAndClient(&config, h) if err != nil { return nil, err @@ -70,17 +71,15 @@ func New(c rest.Interface) *ApiV1beta1Client { return &ApiV1beta1Client{c} } -func setConfigDefaults(config *rest.Config) error { - gv := v1beta1.SchemeGroupVersion +func setConfigDefaults(config *rest.Config) { + gv := apiv1beta1.SchemeGroupVersion config.GroupVersion = &gv config.APIPath = "/apis" - config.NegotiatedSerializer = scheme.Codecs.WithoutConversion() + config.NegotiatedSerializer = rest.CodecFactoryForGeneratedClient(scheme.Scheme, scheme.Codecs).WithoutConversion() if config.UserAgent == "" { config.UserAgent = rest.DefaultKubernetesUserAgent() } - - return nil } // RESTClient returns a RESTClient that is used to communicate diff --git a/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/clientset/versioned/typed/api/v1beta1/fake/fake_api_client.go b/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/clientset/versioned/typed/api/v1beta1/fake/fake_api_client.go index 21b5048ff3..5ff0c6f65f 100644 --- a/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/clientset/versioned/typed/api/v1beta1/fake/fake_api_client.go +++ b/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/clientset/versioned/typed/api/v1beta1/fake/fake_api_client.go @@ -15,7 +15,11 @@ type FakeApiV1beta1 struct { } func (c *FakeApiV1beta1) FRRConfigurations(namespace string) v1beta1.FRRConfigurationInterface { - return &FakeFRRConfigurations{c, namespace} + return newFakeFRRConfigurations(c, namespace) +} + +func (c *FakeApiV1beta1) FRRK8sConfigurations(namespace string) v1beta1.FRRK8sConfigurationInterface { + return newFakeFRRK8sConfigurations(c, namespace) } // RESTClient returns a RESTClient that is used to communicate diff --git a/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/clientset/versioned/typed/api/v1beta1/fake/fake_frrconfiguration.go b/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/clientset/versioned/typed/api/v1beta1/fake/fake_frrconfiguration.go index 70a9e8d0e9..f848bf9c33 100644 --- a/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/clientset/versioned/typed/api/v1beta1/fake/fake_frrconfiguration.go +++ b/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/clientset/versioned/typed/api/v1beta1/fake/fake_frrconfiguration.go @@ -5,129 +5,34 @@ package fake import ( - "context" - v1beta1 "github.com/metallb/frr-k8s/api/v1beta1" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - labels "k8s.io/apimachinery/pkg/labels" - types "k8s.io/apimachinery/pkg/types" - watch "k8s.io/apimachinery/pkg/watch" - testing "k8s.io/client-go/testing" + apiv1beta1 "github.com/metallb/frr-k8s/pkg/client/clientset/versioned/typed/api/v1beta1" + gentype "k8s.io/client-go/gentype" ) -// FakeFRRConfigurations implements FRRConfigurationInterface -type FakeFRRConfigurations struct { +// fakeFRRConfigurations implements FRRConfigurationInterface +type fakeFRRConfigurations struct { + *gentype.FakeClientWithList[*v1beta1.FRRConfiguration, *v1beta1.FRRConfigurationList] Fake *FakeApiV1beta1 - ns string -} - -var frrconfigurationsResource = v1beta1.SchemeGroupVersion.WithResource("frrconfigurations") - -var frrconfigurationsKind = v1beta1.SchemeGroupVersion.WithKind("FRRConfiguration") - -// Get takes name of the fRRConfiguration, and returns the corresponding fRRConfiguration object, and an error if there is any. -func (c *FakeFRRConfigurations) Get(ctx context.Context, name string, options v1.GetOptions) (result *v1beta1.FRRConfiguration, err error) { - emptyResult := &v1beta1.FRRConfiguration{} - obj, err := c.Fake. - Invokes(testing.NewGetActionWithOptions(frrconfigurationsResource, c.ns, name, options), emptyResult) - - if obj == nil { - return emptyResult, err - } - return obj.(*v1beta1.FRRConfiguration), err -} - -// List takes label and field selectors, and returns the list of FRRConfigurations that match those selectors. -func (c *FakeFRRConfigurations) List(ctx context.Context, opts v1.ListOptions) (result *v1beta1.FRRConfigurationList, err error) { - emptyResult := &v1beta1.FRRConfigurationList{} - obj, err := c.Fake. - Invokes(testing.NewListActionWithOptions(frrconfigurationsResource, frrconfigurationsKind, c.ns, opts), emptyResult) - - if obj == nil { - return emptyResult, err - } - - label, _, _ := testing.ExtractFromListOptions(opts) - if label == nil { - label = labels.Everything() - } - list := &v1beta1.FRRConfigurationList{ListMeta: obj.(*v1beta1.FRRConfigurationList).ListMeta} - for _, item := range obj.(*v1beta1.FRRConfigurationList).Items { - if label.Matches(labels.Set(item.Labels)) { - list.Items = append(list.Items, item) - } - } - return list, err -} - -// Watch returns a watch.Interface that watches the requested fRRConfigurations. -func (c *FakeFRRConfigurations) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) { - return c.Fake. - InvokesWatch(testing.NewWatchActionWithOptions(frrconfigurationsResource, c.ns, opts)) - -} - -// Create takes the representation of a fRRConfiguration and creates it. Returns the server's representation of the fRRConfiguration, and an error, if there is any. -func (c *FakeFRRConfigurations) Create(ctx context.Context, fRRConfiguration *v1beta1.FRRConfiguration, opts v1.CreateOptions) (result *v1beta1.FRRConfiguration, err error) { - emptyResult := &v1beta1.FRRConfiguration{} - obj, err := c.Fake. - Invokes(testing.NewCreateActionWithOptions(frrconfigurationsResource, c.ns, fRRConfiguration, opts), emptyResult) - - if obj == nil { - return emptyResult, err - } - return obj.(*v1beta1.FRRConfiguration), err -} - -// Update takes the representation of a fRRConfiguration and updates it. Returns the server's representation of the fRRConfiguration, and an error, if there is any. -func (c *FakeFRRConfigurations) Update(ctx context.Context, fRRConfiguration *v1beta1.FRRConfiguration, opts v1.UpdateOptions) (result *v1beta1.FRRConfiguration, err error) { - emptyResult := &v1beta1.FRRConfiguration{} - obj, err := c.Fake. - Invokes(testing.NewUpdateActionWithOptions(frrconfigurationsResource, c.ns, fRRConfiguration, opts), emptyResult) - - if obj == nil { - return emptyResult, err - } - return obj.(*v1beta1.FRRConfiguration), err -} - -// UpdateStatus was generated because the type contains a Status member. -// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). -func (c *FakeFRRConfigurations) UpdateStatus(ctx context.Context, fRRConfiguration *v1beta1.FRRConfiguration, opts v1.UpdateOptions) (result *v1beta1.FRRConfiguration, err error) { - emptyResult := &v1beta1.FRRConfiguration{} - obj, err := c.Fake. - Invokes(testing.NewUpdateSubresourceActionWithOptions(frrconfigurationsResource, "status", c.ns, fRRConfiguration, opts), emptyResult) - - if obj == nil { - return emptyResult, err - } - return obj.(*v1beta1.FRRConfiguration), err -} - -// Delete takes name of the fRRConfiguration and deletes it. Returns an error if one occurs. -func (c *FakeFRRConfigurations) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { - _, err := c.Fake. - Invokes(testing.NewDeleteActionWithOptions(frrconfigurationsResource, c.ns, name, opts), &v1beta1.FRRConfiguration{}) - - return err } -// DeleteCollection deletes a collection of objects. -func (c *FakeFRRConfigurations) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error { - action := testing.NewDeleteCollectionActionWithOptions(frrconfigurationsResource, c.ns, opts, listOpts) - - _, err := c.Fake.Invokes(action, &v1beta1.FRRConfigurationList{}) - return err -} - -// Patch applies the patch and returns the patched fRRConfiguration. -func (c *FakeFRRConfigurations) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1beta1.FRRConfiguration, err error) { - emptyResult := &v1beta1.FRRConfiguration{} - obj, err := c.Fake. - Invokes(testing.NewPatchSubresourceActionWithOptions(frrconfigurationsResource, c.ns, name, pt, data, opts, subresources...), emptyResult) - - if obj == nil { - return emptyResult, err +func newFakeFRRConfigurations(fake *FakeApiV1beta1, namespace string) apiv1beta1.FRRConfigurationInterface { + return &fakeFRRConfigurations{ + gentype.NewFakeClientWithList[*v1beta1.FRRConfiguration, *v1beta1.FRRConfigurationList]( + fake.Fake, + namespace, + v1beta1.SchemeGroupVersion.WithResource("frrconfigurations"), + v1beta1.SchemeGroupVersion.WithKind("FRRConfiguration"), + func() *v1beta1.FRRConfiguration { return &v1beta1.FRRConfiguration{} }, + func() *v1beta1.FRRConfigurationList { return &v1beta1.FRRConfigurationList{} }, + func(dst, src *v1beta1.FRRConfigurationList) { dst.ListMeta = src.ListMeta }, + func(list *v1beta1.FRRConfigurationList) []*v1beta1.FRRConfiguration { + return gentype.ToPointerSlice(list.Items) + }, + func(list *v1beta1.FRRConfigurationList, items []*v1beta1.FRRConfiguration) { + list.Items = gentype.FromPointerSlice(items) + }, + ), + fake, } - return obj.(*v1beta1.FRRConfiguration), err } diff --git a/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/clientset/versioned/typed/api/v1beta1/fake/fake_frrk8sconfiguration.go b/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/clientset/versioned/typed/api/v1beta1/fake/fake_frrk8sconfiguration.go new file mode 100644 index 0000000000..a5bac59221 --- /dev/null +++ b/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/clientset/versioned/typed/api/v1beta1/fake/fake_frrk8sconfiguration.go @@ -0,0 +1,38 @@ +// SPDX-License-Identifier:Apache-2.0 + +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + v1beta1 "github.com/metallb/frr-k8s/api/v1beta1" + apiv1beta1 "github.com/metallb/frr-k8s/pkg/client/clientset/versioned/typed/api/v1beta1" + gentype "k8s.io/client-go/gentype" +) + +// fakeFRRK8sConfigurations implements FRRK8sConfigurationInterface +type fakeFRRK8sConfigurations struct { + *gentype.FakeClientWithList[*v1beta1.FRRK8sConfiguration, *v1beta1.FRRK8sConfigurationList] + Fake *FakeApiV1beta1 +} + +func newFakeFRRK8sConfigurations(fake *FakeApiV1beta1, namespace string) apiv1beta1.FRRK8sConfigurationInterface { + return &fakeFRRK8sConfigurations{ + gentype.NewFakeClientWithList[*v1beta1.FRRK8sConfiguration, *v1beta1.FRRK8sConfigurationList]( + fake.Fake, + namespace, + v1beta1.SchemeGroupVersion.WithResource("frrk8sconfigurations"), + v1beta1.SchemeGroupVersion.WithKind("FRRK8sConfiguration"), + func() *v1beta1.FRRK8sConfiguration { return &v1beta1.FRRK8sConfiguration{} }, + func() *v1beta1.FRRK8sConfigurationList { return &v1beta1.FRRK8sConfigurationList{} }, + func(dst, src *v1beta1.FRRK8sConfigurationList) { dst.ListMeta = src.ListMeta }, + func(list *v1beta1.FRRK8sConfigurationList) []*v1beta1.FRRK8sConfiguration { + return gentype.ToPointerSlice(list.Items) + }, + func(list *v1beta1.FRRK8sConfigurationList, items []*v1beta1.FRRK8sConfiguration) { + list.Items = gentype.FromPointerSlice(items) + }, + ), + fake, + } +} diff --git a/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/clientset/versioned/typed/api/v1beta1/frrconfiguration.go b/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/clientset/versioned/typed/api/v1beta1/frrconfiguration.go index 8b098aa1cf..7371d41ac2 100644 --- a/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/clientset/versioned/typed/api/v1beta1/frrconfiguration.go +++ b/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/clientset/versioned/typed/api/v1beta1/frrconfiguration.go @@ -5,9 +5,9 @@ package v1beta1 import ( - "context" + context "context" - v1beta1 "github.com/metallb/frr-k8s/api/v1beta1" + apiv1beta1 "github.com/metallb/frr-k8s/api/v1beta1" scheme "github.com/metallb/frr-k8s/pkg/client/clientset/versioned/scheme" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" @@ -23,33 +23,34 @@ type FRRConfigurationsGetter interface { // FRRConfigurationInterface has methods to work with FRRConfiguration resources. type FRRConfigurationInterface interface { - Create(ctx context.Context, fRRConfiguration *v1beta1.FRRConfiguration, opts v1.CreateOptions) (*v1beta1.FRRConfiguration, error) - Update(ctx context.Context, fRRConfiguration *v1beta1.FRRConfiguration, opts v1.UpdateOptions) (*v1beta1.FRRConfiguration, error) + Create(ctx context.Context, fRRConfiguration *apiv1beta1.FRRConfiguration, opts v1.CreateOptions) (*apiv1beta1.FRRConfiguration, error) + Update(ctx context.Context, fRRConfiguration *apiv1beta1.FRRConfiguration, opts v1.UpdateOptions) (*apiv1beta1.FRRConfiguration, error) // Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). - UpdateStatus(ctx context.Context, fRRConfiguration *v1beta1.FRRConfiguration, opts v1.UpdateOptions) (*v1beta1.FRRConfiguration, error) + UpdateStatus(ctx context.Context, fRRConfiguration *apiv1beta1.FRRConfiguration, opts v1.UpdateOptions) (*apiv1beta1.FRRConfiguration, error) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error - Get(ctx context.Context, name string, opts v1.GetOptions) (*v1beta1.FRRConfiguration, error) - List(ctx context.Context, opts v1.ListOptions) (*v1beta1.FRRConfigurationList, error) + Get(ctx context.Context, name string, opts v1.GetOptions) (*apiv1beta1.FRRConfiguration, error) + List(ctx context.Context, opts v1.ListOptions) (*apiv1beta1.FRRConfigurationList, error) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) - Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1beta1.FRRConfiguration, err error) + Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *apiv1beta1.FRRConfiguration, err error) FRRConfigurationExpansion } // fRRConfigurations implements FRRConfigurationInterface type fRRConfigurations struct { - *gentype.ClientWithList[*v1beta1.FRRConfiguration, *v1beta1.FRRConfigurationList] + *gentype.ClientWithList[*apiv1beta1.FRRConfiguration, *apiv1beta1.FRRConfigurationList] } // newFRRConfigurations returns a FRRConfigurations func newFRRConfigurations(c *ApiV1beta1Client, namespace string) *fRRConfigurations { return &fRRConfigurations{ - gentype.NewClientWithList[*v1beta1.FRRConfiguration, *v1beta1.FRRConfigurationList]( + gentype.NewClientWithList[*apiv1beta1.FRRConfiguration, *apiv1beta1.FRRConfigurationList]( "frrconfigurations", c.RESTClient(), scheme.ParameterCodec, namespace, - func() *v1beta1.FRRConfiguration { return &v1beta1.FRRConfiguration{} }, - func() *v1beta1.FRRConfigurationList { return &v1beta1.FRRConfigurationList{} }), + func() *apiv1beta1.FRRConfiguration { return &apiv1beta1.FRRConfiguration{} }, + func() *apiv1beta1.FRRConfigurationList { return &apiv1beta1.FRRConfigurationList{} }, + ), } } diff --git a/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/clientset/versioned/typed/api/v1beta1/frrk8sconfiguration.go b/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/clientset/versioned/typed/api/v1beta1/frrk8sconfiguration.go new file mode 100644 index 0000000000..efb43d3b1c --- /dev/null +++ b/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/clientset/versioned/typed/api/v1beta1/frrk8sconfiguration.go @@ -0,0 +1,56 @@ +// SPDX-License-Identifier:Apache-2.0 + +// Code generated by client-gen. DO NOT EDIT. + +package v1beta1 + +import ( + context "context" + + apiv1beta1 "github.com/metallb/frr-k8s/api/v1beta1" + scheme "github.com/metallb/frr-k8s/pkg/client/clientset/versioned/scheme" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + gentype "k8s.io/client-go/gentype" +) + +// FRRK8sConfigurationsGetter has a method to return a FRRK8sConfigurationInterface. +// A group's client should implement this interface. +type FRRK8sConfigurationsGetter interface { + FRRK8sConfigurations(namespace string) FRRK8sConfigurationInterface +} + +// FRRK8sConfigurationInterface has methods to work with FRRK8sConfiguration resources. +type FRRK8sConfigurationInterface interface { + Create(ctx context.Context, fRRK8sConfiguration *apiv1beta1.FRRK8sConfiguration, opts v1.CreateOptions) (*apiv1beta1.FRRK8sConfiguration, error) + Update(ctx context.Context, fRRK8sConfiguration *apiv1beta1.FRRK8sConfiguration, opts v1.UpdateOptions) (*apiv1beta1.FRRK8sConfiguration, error) + // Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). + UpdateStatus(ctx context.Context, fRRK8sConfiguration *apiv1beta1.FRRK8sConfiguration, opts v1.UpdateOptions) (*apiv1beta1.FRRK8sConfiguration, error) + Delete(ctx context.Context, name string, opts v1.DeleteOptions) error + DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error + Get(ctx context.Context, name string, opts v1.GetOptions) (*apiv1beta1.FRRK8sConfiguration, error) + List(ctx context.Context, opts v1.ListOptions) (*apiv1beta1.FRRK8sConfigurationList, error) + Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) + Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *apiv1beta1.FRRK8sConfiguration, err error) + FRRK8sConfigurationExpansion +} + +// fRRK8sConfigurations implements FRRK8sConfigurationInterface +type fRRK8sConfigurations struct { + *gentype.ClientWithList[*apiv1beta1.FRRK8sConfiguration, *apiv1beta1.FRRK8sConfigurationList] +} + +// newFRRK8sConfigurations returns a FRRK8sConfigurations +func newFRRK8sConfigurations(c *ApiV1beta1Client, namespace string) *fRRK8sConfigurations { + return &fRRK8sConfigurations{ + gentype.NewClientWithList[*apiv1beta1.FRRK8sConfiguration, *apiv1beta1.FRRK8sConfigurationList]( + "frrk8sconfigurations", + c.RESTClient(), + scheme.ParameterCodec, + namespace, + func() *apiv1beta1.FRRK8sConfiguration { return &apiv1beta1.FRRK8sConfiguration{} }, + func() *apiv1beta1.FRRK8sConfigurationList { return &apiv1beta1.FRRK8sConfigurationList{} }, + ), + } +} diff --git a/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/clientset/versioned/typed/api/v1beta1/generated_expansion.go b/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/clientset/versioned/typed/api/v1beta1/generated_expansion.go index 864abd79f8..5f0bee86ff 100644 --- a/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/clientset/versioned/typed/api/v1beta1/generated_expansion.go +++ b/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/clientset/versioned/typed/api/v1beta1/generated_expansion.go @@ -5,3 +5,5 @@ package v1beta1 type FRRConfigurationExpansion interface{} + +type FRRK8sConfigurationExpansion interface{} diff --git a/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/informers/externalversions/api/v1beta1/frrconfiguration.go b/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/informers/externalversions/api/v1beta1/frrconfiguration.go index 93414be862..c41ce49422 100644 --- a/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/informers/externalversions/api/v1beta1/frrconfiguration.go +++ b/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/informers/externalversions/api/v1beta1/frrconfiguration.go @@ -5,13 +5,13 @@ package v1beta1 import ( - "context" + context "context" time "time" - apiv1beta1 "github.com/metallb/frr-k8s/api/v1beta1" + frrk8sapiv1beta1 "github.com/metallb/frr-k8s/api/v1beta1" versioned "github.com/metallb/frr-k8s/pkg/client/clientset/versioned" internalinterfaces "github.com/metallb/frr-k8s/pkg/client/informers/externalversions/internalinterfaces" - v1beta1 "github.com/metallb/frr-k8s/pkg/client/listers/api/v1beta1" + apiv1beta1 "github.com/metallb/frr-k8s/pkg/client/listers/api/v1beta1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" watch "k8s.io/apimachinery/pkg/watch" @@ -22,7 +22,7 @@ import ( // FRRConfigurations. type FRRConfigurationInformer interface { Informer() cache.SharedIndexInformer - Lister() v1beta1.FRRConfigurationLister + Lister() apiv1beta1.FRRConfigurationLister } type fRRConfigurationInformer struct { @@ -48,16 +48,28 @@ func NewFilteredFRRConfigurationInformer(client versioned.Interface, namespace s if tweakListOptions != nil { tweakListOptions(&options) } - return client.ApiV1beta1().FRRConfigurations(namespace).List(context.TODO(), options) + return client.ApiV1beta1().FRRConfigurations(namespace).List(context.Background(), options) }, WatchFunc: func(options v1.ListOptions) (watch.Interface, error) { if tweakListOptions != nil { tweakListOptions(&options) } - return client.ApiV1beta1().FRRConfigurations(namespace).Watch(context.TODO(), options) + return client.ApiV1beta1().FRRConfigurations(namespace).Watch(context.Background(), options) + }, + ListWithContextFunc: func(ctx context.Context, options v1.ListOptions) (runtime.Object, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.ApiV1beta1().FRRConfigurations(namespace).List(ctx, options) + }, + WatchFuncWithContext: func(ctx context.Context, options v1.ListOptions) (watch.Interface, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.ApiV1beta1().FRRConfigurations(namespace).Watch(ctx, options) }, }, - &apiv1beta1.FRRConfiguration{}, + &frrk8sapiv1beta1.FRRConfiguration{}, resyncPeriod, indexers, ) @@ -68,9 +80,9 @@ func (f *fRRConfigurationInformer) defaultInformer(client versioned.Interface, r } func (f *fRRConfigurationInformer) Informer() cache.SharedIndexInformer { - return f.factory.InformerFor(&apiv1beta1.FRRConfiguration{}, f.defaultInformer) + return f.factory.InformerFor(&frrk8sapiv1beta1.FRRConfiguration{}, f.defaultInformer) } -func (f *fRRConfigurationInformer) Lister() v1beta1.FRRConfigurationLister { - return v1beta1.NewFRRConfigurationLister(f.Informer().GetIndexer()) +func (f *fRRConfigurationInformer) Lister() apiv1beta1.FRRConfigurationLister { + return apiv1beta1.NewFRRConfigurationLister(f.Informer().GetIndexer()) } diff --git a/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/informers/externalversions/api/v1beta1/frrk8sconfiguration.go b/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/informers/externalversions/api/v1beta1/frrk8sconfiguration.go new file mode 100644 index 0000000000..97743fbf75 --- /dev/null +++ b/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/informers/externalversions/api/v1beta1/frrk8sconfiguration.go @@ -0,0 +1,88 @@ +// SPDX-License-Identifier:Apache-2.0 + +// Code generated by informer-gen. DO NOT EDIT. + +package v1beta1 + +import ( + context "context" + time "time" + + frrk8sapiv1beta1 "github.com/metallb/frr-k8s/api/v1beta1" + versioned "github.com/metallb/frr-k8s/pkg/client/clientset/versioned" + internalinterfaces "github.com/metallb/frr-k8s/pkg/client/informers/externalversions/internalinterfaces" + apiv1beta1 "github.com/metallb/frr-k8s/pkg/client/listers/api/v1beta1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + watch "k8s.io/apimachinery/pkg/watch" + cache "k8s.io/client-go/tools/cache" +) + +// FRRK8sConfigurationInformer provides access to a shared informer and lister for +// FRRK8sConfigurations. +type FRRK8sConfigurationInformer interface { + Informer() cache.SharedIndexInformer + Lister() apiv1beta1.FRRK8sConfigurationLister +} + +type fRRK8sConfigurationInformer struct { + factory internalinterfaces.SharedInformerFactory + tweakListOptions internalinterfaces.TweakListOptionsFunc + namespace string +} + +// NewFRRK8sConfigurationInformer constructs a new informer for FRRK8sConfiguration type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewFRRK8sConfigurationInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers) cache.SharedIndexInformer { + return NewFilteredFRRK8sConfigurationInformer(client, namespace, resyncPeriod, indexers, nil) +} + +// NewFilteredFRRK8sConfigurationInformer constructs a new informer for FRRK8sConfiguration type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewFilteredFRRK8sConfigurationInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer { + return cache.NewSharedIndexInformer( + &cache.ListWatch{ + ListFunc: func(options v1.ListOptions) (runtime.Object, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.ApiV1beta1().FRRK8sConfigurations(namespace).List(context.Background(), options) + }, + WatchFunc: func(options v1.ListOptions) (watch.Interface, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.ApiV1beta1().FRRK8sConfigurations(namespace).Watch(context.Background(), options) + }, + ListWithContextFunc: func(ctx context.Context, options v1.ListOptions) (runtime.Object, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.ApiV1beta1().FRRK8sConfigurations(namespace).List(ctx, options) + }, + WatchFuncWithContext: func(ctx context.Context, options v1.ListOptions) (watch.Interface, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.ApiV1beta1().FRRK8sConfigurations(namespace).Watch(ctx, options) + }, + }, + &frrk8sapiv1beta1.FRRK8sConfiguration{}, + resyncPeriod, + indexers, + ) +} + +func (f *fRRK8sConfigurationInformer) defaultInformer(client versioned.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer { + return NewFilteredFRRK8sConfigurationInformer(client, f.namespace, resyncPeriod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, f.tweakListOptions) +} + +func (f *fRRK8sConfigurationInformer) Informer() cache.SharedIndexInformer { + return f.factory.InformerFor(&frrk8sapiv1beta1.FRRK8sConfiguration{}, f.defaultInformer) +} + +func (f *fRRK8sConfigurationInformer) Lister() apiv1beta1.FRRK8sConfigurationLister { + return apiv1beta1.NewFRRK8sConfigurationLister(f.Informer().GetIndexer()) +} diff --git a/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/informers/externalversions/api/v1beta1/interface.go b/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/informers/externalversions/api/v1beta1/interface.go index f87efaa202..bb691a410e 100644 --- a/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/informers/externalversions/api/v1beta1/interface.go +++ b/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/informers/externalversions/api/v1beta1/interface.go @@ -12,6 +12,8 @@ import ( type Interface interface { // FRRConfigurations returns a FRRConfigurationInformer. FRRConfigurations() FRRConfigurationInformer + // FRRK8sConfigurations returns a FRRK8sConfigurationInformer. + FRRK8sConfigurations() FRRK8sConfigurationInformer } type version struct { @@ -29,3 +31,8 @@ func New(f internalinterfaces.SharedInformerFactory, namespace string, tweakList func (v *version) FRRConfigurations() FRRConfigurationInformer { return &fRRConfigurationInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions} } + +// FRRK8sConfigurations returns a FRRK8sConfigurationInformer. +func (v *version) FRRK8sConfigurations() FRRK8sConfigurationInformer { + return &fRRK8sConfigurationInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions} +} diff --git a/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/informers/externalversions/generic.go b/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/informers/externalversions/generic.go index 9c85f3dd1d..aefb72f9a1 100644 --- a/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/informers/externalversions/generic.go +++ b/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/informers/externalversions/generic.go @@ -5,7 +5,7 @@ package externalversions import ( - "fmt" + fmt "fmt" v1beta1 "github.com/metallb/frr-k8s/api/v1beta1" schema "k8s.io/apimachinery/pkg/runtime/schema" @@ -41,6 +41,8 @@ func (f *sharedInformerFactory) ForResource(resource schema.GroupVersionResource // Group=api, Version=v1beta1 case v1beta1.SchemeGroupVersion.WithResource("frrconfigurations"): return &genericInformer{resource: resource.GroupResource(), informer: f.Api().V1beta1().FRRConfigurations().Informer()}, nil + case v1beta1.SchemeGroupVersion.WithResource("frrk8sconfigurations"): + return &genericInformer{resource: resource.GroupResource(), informer: f.Api().V1beta1().FRRK8sConfigurations().Informer()}, nil } diff --git a/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/listers/api/v1beta1/expansion_generated.go b/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/listers/api/v1beta1/expansion_generated.go index 4b69446dc4..20536ff40b 100644 --- a/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/listers/api/v1beta1/expansion_generated.go +++ b/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/listers/api/v1beta1/expansion_generated.go @@ -11,3 +11,11 @@ type FRRConfigurationListerExpansion interface{} // FRRConfigurationNamespaceListerExpansion allows custom methods to be added to // FRRConfigurationNamespaceLister. type FRRConfigurationNamespaceListerExpansion interface{} + +// FRRK8sConfigurationListerExpansion allows custom methods to be added to +// FRRK8sConfigurationLister. +type FRRK8sConfigurationListerExpansion interface{} + +// FRRK8sConfigurationNamespaceListerExpansion allows custom methods to be added to +// FRRK8sConfigurationNamespaceLister. +type FRRK8sConfigurationNamespaceListerExpansion interface{} diff --git a/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/listers/api/v1beta1/frrconfiguration.go b/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/listers/api/v1beta1/frrconfiguration.go index f0799a932d..d918af62b1 100644 --- a/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/listers/api/v1beta1/frrconfiguration.go +++ b/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/listers/api/v1beta1/frrconfiguration.go @@ -5,10 +5,10 @@ package v1beta1 import ( - v1beta1 "github.com/metallb/frr-k8s/api/v1beta1" - "k8s.io/apimachinery/pkg/labels" - "k8s.io/client-go/listers" - "k8s.io/client-go/tools/cache" + apiv1beta1 "github.com/metallb/frr-k8s/api/v1beta1" + labels "k8s.io/apimachinery/pkg/labels" + listers "k8s.io/client-go/listers" + cache "k8s.io/client-go/tools/cache" ) // FRRConfigurationLister helps list FRRConfigurations. @@ -16,7 +16,7 @@ import ( type FRRConfigurationLister interface { // List lists all FRRConfigurations in the indexer. // Objects returned here must be treated as read-only. - List(selector labels.Selector) (ret []*v1beta1.FRRConfiguration, err error) + List(selector labels.Selector) (ret []*apiv1beta1.FRRConfiguration, err error) // FRRConfigurations returns an object that can list and get FRRConfigurations. FRRConfigurations(namespace string) FRRConfigurationNamespaceLister FRRConfigurationListerExpansion @@ -24,17 +24,17 @@ type FRRConfigurationLister interface { // fRRConfigurationLister implements the FRRConfigurationLister interface. type fRRConfigurationLister struct { - listers.ResourceIndexer[*v1beta1.FRRConfiguration] + listers.ResourceIndexer[*apiv1beta1.FRRConfiguration] } // NewFRRConfigurationLister returns a new FRRConfigurationLister. func NewFRRConfigurationLister(indexer cache.Indexer) FRRConfigurationLister { - return &fRRConfigurationLister{listers.New[*v1beta1.FRRConfiguration](indexer, v1beta1.Resource("frrconfiguration"))} + return &fRRConfigurationLister{listers.New[*apiv1beta1.FRRConfiguration](indexer, apiv1beta1.Resource("frrconfiguration"))} } // FRRConfigurations returns an object that can list and get FRRConfigurations. func (s *fRRConfigurationLister) FRRConfigurations(namespace string) FRRConfigurationNamespaceLister { - return fRRConfigurationNamespaceLister{listers.NewNamespaced[*v1beta1.FRRConfiguration](s.ResourceIndexer, namespace)} + return fRRConfigurationNamespaceLister{listers.NewNamespaced[*apiv1beta1.FRRConfiguration](s.ResourceIndexer, namespace)} } // FRRConfigurationNamespaceLister helps list and get FRRConfigurations. @@ -42,15 +42,15 @@ func (s *fRRConfigurationLister) FRRConfigurations(namespace string) FRRConfigur type FRRConfigurationNamespaceLister interface { // List lists all FRRConfigurations in the indexer for a given namespace. // Objects returned here must be treated as read-only. - List(selector labels.Selector) (ret []*v1beta1.FRRConfiguration, err error) + List(selector labels.Selector) (ret []*apiv1beta1.FRRConfiguration, err error) // Get retrieves the FRRConfiguration from the indexer for a given namespace and name. // Objects returned here must be treated as read-only. - Get(name string) (*v1beta1.FRRConfiguration, error) + Get(name string) (*apiv1beta1.FRRConfiguration, error) FRRConfigurationNamespaceListerExpansion } // fRRConfigurationNamespaceLister implements the FRRConfigurationNamespaceLister // interface. type fRRConfigurationNamespaceLister struct { - listers.ResourceIndexer[*v1beta1.FRRConfiguration] + listers.ResourceIndexer[*apiv1beta1.FRRConfiguration] } diff --git a/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/listers/api/v1beta1/frrk8sconfiguration.go b/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/listers/api/v1beta1/frrk8sconfiguration.go new file mode 100644 index 0000000000..393404c3cc --- /dev/null +++ b/go-controller/vendor/github.com/metallb/frr-k8s/pkg/client/listers/api/v1beta1/frrk8sconfiguration.go @@ -0,0 +1,56 @@ +// SPDX-License-Identifier:Apache-2.0 + +// Code generated by lister-gen. DO NOT EDIT. + +package v1beta1 + +import ( + apiv1beta1 "github.com/metallb/frr-k8s/api/v1beta1" + labels "k8s.io/apimachinery/pkg/labels" + listers "k8s.io/client-go/listers" + cache "k8s.io/client-go/tools/cache" +) + +// FRRK8sConfigurationLister helps list FRRK8sConfigurations. +// All objects returned here must be treated as read-only. +type FRRK8sConfigurationLister interface { + // List lists all FRRK8sConfigurations in the indexer. + // Objects returned here must be treated as read-only. + List(selector labels.Selector) (ret []*apiv1beta1.FRRK8sConfiguration, err error) + // FRRK8sConfigurations returns an object that can list and get FRRK8sConfigurations. + FRRK8sConfigurations(namespace string) FRRK8sConfigurationNamespaceLister + FRRK8sConfigurationListerExpansion +} + +// fRRK8sConfigurationLister implements the FRRK8sConfigurationLister interface. +type fRRK8sConfigurationLister struct { + listers.ResourceIndexer[*apiv1beta1.FRRK8sConfiguration] +} + +// NewFRRK8sConfigurationLister returns a new FRRK8sConfigurationLister. +func NewFRRK8sConfigurationLister(indexer cache.Indexer) FRRK8sConfigurationLister { + return &fRRK8sConfigurationLister{listers.New[*apiv1beta1.FRRK8sConfiguration](indexer, apiv1beta1.Resource("frrk8sconfiguration"))} +} + +// FRRK8sConfigurations returns an object that can list and get FRRK8sConfigurations. +func (s *fRRK8sConfigurationLister) FRRK8sConfigurations(namespace string) FRRK8sConfigurationNamespaceLister { + return fRRK8sConfigurationNamespaceLister{listers.NewNamespaced[*apiv1beta1.FRRK8sConfiguration](s.ResourceIndexer, namespace)} +} + +// FRRK8sConfigurationNamespaceLister helps list and get FRRK8sConfigurations. +// All objects returned here must be treated as read-only. +type FRRK8sConfigurationNamespaceLister interface { + // List lists all FRRK8sConfigurations in the indexer for a given namespace. + // Objects returned here must be treated as read-only. + List(selector labels.Selector) (ret []*apiv1beta1.FRRK8sConfiguration, err error) + // Get retrieves the FRRK8sConfiguration from the indexer for a given namespace and name. + // Objects returned here must be treated as read-only. + Get(name string) (*apiv1beta1.FRRK8sConfiguration, error) + FRRK8sConfigurationNamespaceListerExpansion +} + +// fRRK8sConfigurationNamespaceLister implements the FRRK8sConfigurationNamespaceLister +// interface. +type fRRK8sConfigurationNamespaceLister struct { + listers.ResourceIndexer[*apiv1beta1.FRRK8sConfiguration] +} diff --git a/go-controller/vendor/go.opentelemetry.io/otel/attribute/internal/attribute.go b/go-controller/vendor/go.opentelemetry.io/otel/attribute/internal/attribute.go index 0875504302..7f5eae877d 100644 --- a/go-controller/vendor/go.opentelemetry.io/otel/attribute/internal/attribute.go +++ b/go-controller/vendor/go.opentelemetry.io/otel/attribute/internal/attribute.go @@ -13,32 +13,28 @@ import ( // BoolSliceValue converts a bool slice into an array with same elements as slice. func BoolSliceValue(v []bool) any { - var zero bool - cp := reflect.New(reflect.ArrayOf(len(v), reflect.TypeOf(zero))).Elem() + cp := reflect.New(reflect.ArrayOf(len(v), reflect.TypeFor[bool]())).Elem() reflect.Copy(cp, reflect.ValueOf(v)) return cp.Interface() } // Int64SliceValue converts an int64 slice into an array with same elements as slice. func Int64SliceValue(v []int64) any { - var zero int64 - cp := reflect.New(reflect.ArrayOf(len(v), reflect.TypeOf(zero))).Elem() + cp := reflect.New(reflect.ArrayOf(len(v), reflect.TypeFor[int64]())).Elem() reflect.Copy(cp, reflect.ValueOf(v)) return cp.Interface() } // Float64SliceValue converts a float64 slice into an array with same elements as slice. func Float64SliceValue(v []float64) any { - var zero float64 - cp := reflect.New(reflect.ArrayOf(len(v), reflect.TypeOf(zero))).Elem() + cp := reflect.New(reflect.ArrayOf(len(v), reflect.TypeFor[float64]())).Elem() reflect.Copy(cp, reflect.ValueOf(v)) return cp.Interface() } // StringSliceValue converts a string slice into an array with same elements as slice. func StringSliceValue(v []string) any { - var zero string - cp := reflect.New(reflect.ArrayOf(len(v), reflect.TypeOf(zero))).Elem() + cp := reflect.New(reflect.ArrayOf(len(v), reflect.TypeFor[string]())).Elem() reflect.Copy(cp, reflect.ValueOf(v)) return cp.Interface() } diff --git a/go-controller/vendor/go.opentelemetry.io/otel/attribute/set.go b/go-controller/vendor/go.opentelemetry.io/otel/attribute/set.go index 911d557ee5..6572c98b12 100644 --- a/go-controller/vendor/go.opentelemetry.io/otel/attribute/set.go +++ b/go-controller/vendor/go.opentelemetry.io/otel/attribute/set.go @@ -58,7 +58,7 @@ func isComparable[T comparable](t T) T { return t } var ( // keyValueType is used in computeDistinctReflect. - keyValueType = reflect.TypeOf(KeyValue{}) + keyValueType = reflect.TypeFor[KeyValue]() // emptyHash is the hash of an empty set. emptyHash = xxhash.New().Sum64() diff --git a/go-controller/vendor/go.opentelemetry.io/otel/attribute/value.go b/go-controller/vendor/go.opentelemetry.io/otel/attribute/value.go index 653c33a861..5931e71291 100644 --- a/go-controller/vendor/go.opentelemetry.io/otel/attribute/value.go +++ b/go-controller/vendor/go.opentelemetry.io/otel/attribute/value.go @@ -66,8 +66,7 @@ func IntValue(v int) Value { // IntSliceValue creates an INTSLICE Value. func IntSliceValue(v []int) Value { - var int64Val int64 - cp := reflect.New(reflect.ArrayOf(len(v), reflect.TypeOf(int64Val))) + cp := reflect.New(reflect.ArrayOf(len(v), reflect.TypeFor[int64]())) for i, val := range v { cp.Elem().Index(i).SetInt(int64(val)) } diff --git a/go-controller/vendor/go.opentelemetry.io/otel/semconv/v1.37.0/MIGRATION.md b/go-controller/vendor/go.opentelemetry.io/otel/semconv/v1.37.0/MIGRATION.md deleted file mode 100644 index 2480547895..0000000000 --- a/go-controller/vendor/go.opentelemetry.io/otel/semconv/v1.37.0/MIGRATION.md +++ /dev/null @@ -1,41 +0,0 @@ - -# Migration from v1.36.0 to v1.37.0 - -The `go.opentelemetry.io/otel/semconv/v1.37.0` package should be a drop-in replacement for `go.opentelemetry.io/otel/semconv/v1.36.0` with the following exceptions. - -## Removed - -The following declarations have been removed. -Refer to the [OpenTelemetry Semantic Conventions documentation] for deprecation instructions. - -If the type is not listed in the documentation as deprecated, it has been removed in this version due to lack of applicability or use. -If you use any of these non-deprecated declarations in your Go application, please [open an issue] describing your use-case. - -- `ContainerRuntime` -- `ContainerRuntimeKey` -- `GenAIOpenAIRequestServiceTierAuto` -- `GenAIOpenAIRequestServiceTierDefault` -- `GenAIOpenAIRequestServiceTierKey` -- `GenAIOpenAIResponseServiceTier` -- `GenAIOpenAIResponseServiceTierKey` -- `GenAIOpenAIResponseSystemFingerprint` -- `GenAIOpenAIResponseSystemFingerprintKey` -- `GenAISystemAWSBedrock` -- `GenAISystemAnthropic` -- `GenAISystemAzureAIInference` -- `GenAISystemAzureAIOpenAI` -- `GenAISystemCohere` -- `GenAISystemDeepseek` -- `GenAISystemGCPGemini` -- `GenAISystemGCPGenAI` -- `GenAISystemGCPVertexAI` -- `GenAISystemGroq` -- `GenAISystemIBMWatsonxAI` -- `GenAISystemKey` -- `GenAISystemMistralAI` -- `GenAISystemOpenAI` -- `GenAISystemPerplexity` -- `GenAISystemXai` - -[OpenTelemetry Semantic Conventions documentation]: https://github.com/open-telemetry/semantic-conventions -[open an issue]: https://github.com/open-telemetry/opentelemetry-go/issues/new?template=Blank+issue diff --git a/go-controller/vendor/go.opentelemetry.io/otel/semconv/v1.37.0/README.md b/go-controller/vendor/go.opentelemetry.io/otel/semconv/v1.37.0/README.md deleted file mode 100644 index d795247f32..0000000000 --- a/go-controller/vendor/go.opentelemetry.io/otel/semconv/v1.37.0/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# Semconv v1.37.0 - -[![PkgGoDev](https://pkg.go.dev/badge/go.opentelemetry.io/otel/semconv/v1.37.0)](https://pkg.go.dev/go.opentelemetry.io/otel/semconv/v1.37.0) diff --git a/go-controller/vendor/go.opentelemetry.io/otel/semconv/v1.39.0/MIGRATION.md b/go-controller/vendor/go.opentelemetry.io/otel/semconv/v1.39.0/MIGRATION.md new file mode 100644 index 0000000000..fed7013e6f --- /dev/null +++ b/go-controller/vendor/go.opentelemetry.io/otel/semconv/v1.39.0/MIGRATION.md @@ -0,0 +1,78 @@ + +# Migration from v1.38.0 to v1.39.0 + +The `go.opentelemetry.io/otel/semconv/v1.39.0` package should be a drop-in replacement for `go.opentelemetry.io/otel/semconv/v1.38.0` with the following exceptions. + +## Removed + +The following declarations have been removed. +Refer to the [OpenTelemetry Semantic Conventions documentation] for deprecation instructions. + +If the type is not listed in the documentation as deprecated, it has been removed in this version due to lack of applicability or use. +If you use any of these non-deprecated declarations in your Go application, please [open an issue] describing your use-case. + +- `LinuxMemorySlabStateKey` +- `LinuxMemorySlabStateReclaimable` +- `LinuxMemorySlabStateUnreclaimable` +- `PeerService` +- `PeerServiceKey` +- `RPCConnectRPCErrorCodeAborted` +- `RPCConnectRPCErrorCodeAlreadyExists` +- `RPCConnectRPCErrorCodeCancelled` +- `RPCConnectRPCErrorCodeDataLoss` +- `RPCConnectRPCErrorCodeDeadlineExceeded` +- `RPCConnectRPCErrorCodeFailedPrecondition` +- `RPCConnectRPCErrorCodeInternal` +- `RPCConnectRPCErrorCodeInvalidArgument` +- `RPCConnectRPCErrorCodeKey` +- `RPCConnectRPCErrorCodeNotFound` +- `RPCConnectRPCErrorCodeOutOfRange` +- `RPCConnectRPCErrorCodePermissionDenied` +- `RPCConnectRPCErrorCodeResourceExhausted` +- `RPCConnectRPCErrorCodeUnauthenticated` +- `RPCConnectRPCErrorCodeUnavailable` +- `RPCConnectRPCErrorCodeUnimplemented` +- `RPCConnectRPCErrorCodeUnknown` +- `RPCConnectRPCRequestMetadata` +- `RPCConnectRPCResponseMetadata` +- `RPCGRPCRequestMetadata` +- `RPCGRPCResponseMetadata` +- `RPCGRPCStatusCodeAborted` +- `RPCGRPCStatusCodeAlreadyExists` +- `RPCGRPCStatusCodeCancelled` +- `RPCGRPCStatusCodeDataLoss` +- `RPCGRPCStatusCodeDeadlineExceeded` +- `RPCGRPCStatusCodeFailedPrecondition` +- `RPCGRPCStatusCodeInternal` +- `RPCGRPCStatusCodeInvalidArgument` +- `RPCGRPCStatusCodeKey` +- `RPCGRPCStatusCodeNotFound` +- `RPCGRPCStatusCodeOk` +- `RPCGRPCStatusCodeOutOfRange` +- `RPCGRPCStatusCodePermissionDenied` +- `RPCGRPCStatusCodeResourceExhausted` +- `RPCGRPCStatusCodeUnauthenticated` +- `RPCGRPCStatusCodeUnavailable` +- `RPCGRPCStatusCodeUnimplemented` +- `RPCGRPCStatusCodeUnknown` +- `RPCJSONRPCErrorCode` +- `RPCJSONRPCErrorCodeKey` +- `RPCJSONRPCErrorMessage` +- `RPCJSONRPCErrorMessageKey` +- `RPCJSONRPCRequestID` +- `RPCJSONRPCRequestIDKey` +- `RPCJSONRPCVersion` +- `RPCJSONRPCVersionKey` +- `RPCService` +- `RPCServiceKey` +- `RPCSystemApacheDubbo` +- `RPCSystemConnectRPC` +- `RPCSystemDotnetWcf` +- `RPCSystemGRPC` +- `RPCSystemJSONRPC` +- `RPCSystemJavaRmi` +- `RPCSystemKey` +- `RPCSystemOncRPC` + +[OpenTelemetry Semantic Conventions documentation]: https://github.com/open-telemetry/semantic-conventions +[open an issue]: https://github.com/open-telemetry/opentelemetry-go/issues/new?template=Blank+issue diff --git a/go-controller/vendor/go.opentelemetry.io/otel/semconv/v1.39.0/README.md b/go-controller/vendor/go.opentelemetry.io/otel/semconv/v1.39.0/README.md new file mode 100644 index 0000000000..4b0e6f7f3e --- /dev/null +++ b/go-controller/vendor/go.opentelemetry.io/otel/semconv/v1.39.0/README.md @@ -0,0 +1,3 @@ +# Semconv v1.39.0 + +[![PkgGoDev](https://pkg.go.dev/badge/go.opentelemetry.io/otel/semconv/v1.39.0)](https://pkg.go.dev/go.opentelemetry.io/otel/semconv/v1.39.0) diff --git a/go-controller/vendor/go.opentelemetry.io/otel/semconv/v1.37.0/attribute_group.go b/go-controller/vendor/go.opentelemetry.io/otel/semconv/v1.39.0/attribute_group.go similarity index 90% rename from go-controller/vendor/go.opentelemetry.io/otel/semconv/v1.37.0/attribute_group.go rename to go-controller/vendor/go.opentelemetry.io/otel/semconv/v1.39.0/attribute_group.go index b6b27498f2..080365fc19 100644 --- a/go-controller/vendor/go.opentelemetry.io/otel/semconv/v1.37.0/attribute_group.go +++ b/go-controller/vendor/go.opentelemetry.io/otel/semconv/v1.39.0/attribute_group.go @@ -3,7 +3,7 @@ // Code generated from semantic convention specification. DO NOT EDIT. -package semconv // import "go.opentelemetry.io/otel/semconv/v1.37.0" +package semconv // import "go.opentelemetry.io/otel/semconv/v1.39.0" import "go.opentelemetry.io/otel/attribute" @@ -187,6 +187,38 @@ const ( // Examples: 12, 99 AppScreenCoordinateYKey = attribute.Key("app.screen.coordinate.y") + // AppScreenIDKey is the attribute Key conforming to the "app.screen.id" + // semantic conventions. It represents an identifier that uniquely + // differentiates this screen from other screens in the same application. + // + // Type: string + // RequirementLevel: Recommended + // Stability: Development + // + // Examples: "f9bc787d-ff05-48ad-90e1-fca1d46130b3", + // "com.example.app.MainActivity", "com.example.shop.ProductDetailFragment", + // "MyApp.ProfileView", "MyApp.ProfileViewController" + // Note: A screen represents only the part of the device display drawn by the + // app. It typically contains multiple widgets or UI components and is larger in + // scope than individual widgets. Multiple screens can coexist on the same + // display simultaneously (e.g., split view on tablets). + AppScreenIDKey = attribute.Key("app.screen.id") + + // AppScreenNameKey is the attribute Key conforming to the "app.screen.name" + // semantic conventions. It represents the name of an application screen. + // + // Type: string + // RequirementLevel: Recommended + // Stability: Development + // + // Examples: "MainActivity", "ProductDetailFragment", "ProfileView", + // "ProfileViewController" + // Note: A screen represents only the part of the device display drawn by the + // app. It typically contains multiple widgets or UI components and is larger in + // scope than individual widgets. Multiple screens can coexist on the same + // display simultaneously (e.g., split view on tablets). + AppScreenNameKey = attribute.Key("app.screen.name") + // AppWidgetIDKey is the attribute Key conforming to the "app.widget.id" // semantic conventions. It represents an identifier that uniquely // differentiates this widget from other widgets in the same application. @@ -262,6 +294,20 @@ func AppScreenCoordinateY(val int) attribute.KeyValue { return AppScreenCoordinateYKey.Int(val) } +// AppScreenID returns an attribute KeyValue conforming to the "app.screen.id" +// semantic conventions. It represents an identifier that uniquely differentiates +// this screen from other screens in the same application. +func AppScreenID(val string) attribute.KeyValue { + return AppScreenIDKey.String(val) +} + +// AppScreenName returns an attribute KeyValue conforming to the +// "app.screen.name" semantic conventions. It represents the name of an +// application screen. +func AppScreenName(val string) attribute.KeyValue { + return AppScreenNameKey.String(val) +} + // AppWidgetID returns an attribute KeyValue conforming to the "app.widget.id" // semantic conventions. It represents an identifier that uniquely differentiates // this widget from other widgets in the same application. @@ -1662,7 +1708,7 @@ const ( // Examples: "North Central US", "Australia East", "Australia Southeast" // Note: Region name matches the format of `displayName` in [Azure Location API] // - // [Azure Location API]: https://learn.microsoft.com/rest/api/subscription/subscriptions/list-locations?view=rest-subscription-2021-10-01&tabs=HTTP#location + // [Azure Location API]: https://learn.microsoft.com/rest/api/resources/subscriptions/list-locations AzureCosmosDBOperationContactedRegionsKey = attribute.Key("azure.cosmosdb.operation.contacted_regions") // AzureCosmosDBOperationRequestChargeKey is the attribute Key conforming to the @@ -2646,6 +2692,9 @@ func CloudResourceID(val string) attribute.KeyValue { // Enum values for cloud.platform var ( + // Akamai Cloud Compute + // Stability: development + CloudPlatformAkamaiCloudCompute = CloudPlatformKey.String("akamai_cloud.compute") // Alibaba Cloud Elastic Compute Service // Stability: development CloudPlatformAlibabaCloudECS = CloudPlatformKey.String("alibaba_cloud_ecs") @@ -2697,6 +2746,9 @@ var ( // Azure Red Hat OpenShift // Stability: development CloudPlatformAzureOpenShift = CloudPlatformKey.String("azure.openshift") + // Google Vertex AI Agent Engine + // Stability: development + CloudPlatformGCPAgentEngine = CloudPlatformKey.String("gcp.agent_engine") // Google Bare Metal Solution (BMS) // Stability: development CloudPlatformGCPBareMetalSolution = CloudPlatformKey.String("gcp_bare_metal_solution") @@ -2718,6 +2770,9 @@ var ( // Red Hat OpenShift on Google Cloud // Stability: development CloudPlatformGCPOpenShift = CloudPlatformKey.String("gcp_openshift") + // Server on Hetzner Cloud + // Stability: development + CloudPlatformHetznerCloudServer = CloudPlatformKey.String("hetzner.cloud_server") // Red Hat OpenShift on IBM Cloud // Stability: development CloudPlatformIBMCloudOpenShift = CloudPlatformKey.String("ibm_cloud_openshift") @@ -2736,10 +2791,16 @@ var ( // Tencent Cloud Serverless Cloud Function (SCF) // Stability: development CloudPlatformTencentCloudSCF = CloudPlatformKey.String("tencent_cloud_scf") + // Vultr Cloud Compute + // Stability: development + CloudPlatformVultrCloudCompute = CloudPlatformKey.String("vultr.cloud_compute") ) // Enum values for cloud.provider var ( + // Akamai Cloud + // Stability: development + CloudProviderAkamaiCloud = CloudProviderKey.String("akamai_cloud") // Alibaba Cloud // Stability: development CloudProviderAlibabaCloud = CloudProviderKey.String("alibaba_cloud") @@ -2755,6 +2816,9 @@ var ( // Heroku Platform as a Service // Stability: development CloudProviderHeroku = CloudProviderKey.String("heroku") + // Hetzner + // Stability: development + CloudProviderHetzner = CloudProviderKey.String("hetzner") // IBM Cloud // Stability: development CloudProviderIBMCloud = CloudProviderKey.String("ibm_cloud") @@ -2764,6 +2828,9 @@ var ( // Tencent Cloud // Stability: development CloudProviderTencentCloud = CloudProviderKey.String("tencent_cloud") + // Vultr + // Stability: development + CloudProviderVultr = CloudProviderKey.String("vultr") ) // Namespace: cloudevents @@ -3364,7 +3431,7 @@ const ( // // Type: string // RequirementLevel: Recommended - // Stability: Development + // Stability: Alpha // // Examples: "a3bf90e006b2" // @@ -3391,7 +3458,7 @@ const ( // environments. Consider using `oci.manifest.digest` if it is important to // identify the same image in different environments/runtimes. // - // [API]: https://docs.docker.com/engine/api/v1.43/#tag/Container/operation/ContainerInspect + // [API]: https://docs.docker.com/reference/api/engine/version/v1.52/#tag/Container/operation/ContainerInspect ContainerImageIDKey = attribute.Key("container.image.id") // ContainerImageNameKey is the attribute Key conforming to the @@ -3400,7 +3467,7 @@ const ( // // Type: string // RequirementLevel: Recommended - // Stability: Development + // Stability: Alpha // // Examples: "gcr.io/opentelemetry/operator" ContainerImageNameKey = attribute.Key("container.image.name") @@ -3411,14 +3478,14 @@ const ( // // Type: string[] // RequirementLevel: Recommended - // Stability: Development + // Stability: Alpha // // Examples: // "example@sha256:afcc7f1ac1b49db317a7196c902e61c6c3c4607d63599ee1a82d702d249a0ccb", // "internal.registry.example.com:5000/example@sha256:b69959407d21e8a062e0416bf13405bb2b71ed7a84dde4158ebafacfa06f5578" // Note: [Docker] and [CRI] report those under the `RepoDigests` field. // - // [Docker]: https://docs.docker.com/engine/api/v1.43/#tag/Image/operation/ImageInspect + // [Docker]: https://docs.docker.com/reference/api/engine/version/v1.52/#tag/Image/operation/ImageInspect // [CRI]: https://github.com/kubernetes/cri-api/blob/c75ef5b473bbe2d0a4fc92f82235efd665ea8e9f/pkg/apis/runtime/v1/api.proto#L1237-L1238 ContainerImageRepoDigestsKey = attribute.Key("container.image.repo_digests") @@ -3430,11 +3497,11 @@ const ( // // Type: string[] // RequirementLevel: Recommended - // Stability: Development + // Stability: Alpha // // Examples: "v1.27.1", "3.5.7-0" // - // [Docker Image Inspect]: https://docs.docker.com/engine/api/v1.43/#tag/Image/operation/ImageInspect + // [Docker Image Inspect]: https://docs.docker.com/reference/api/engine/version/v1.52/#tag/Image/operation/ImageInspect ContainerImageTagsKey = attribute.Key("container.image.tags") // ContainerNameKey is the attribute Key conforming to the "container.name" @@ -3560,7 +3627,7 @@ func ContainerImageRepoDigests(val ...string) attribute.KeyValue { // `` section of the full name for example from // `registry.example.com/my-org/my-image:`. // -// [Docker Image Inspect]: https://docs.docker.com/engine/api/v1.43/#tag/Image/operation/ImageInspect +// [Docker Image Inspect]: https://docs.docker.com/reference/api/engine/version/v1.52/#tag/Image/operation/ImageInspect func ContainerImageTags(val ...string) attribute.KeyValue { return ContainerImageTagsKey.StringSlice(val) } @@ -3789,7 +3856,7 @@ const ( // [Generating query summary] // section. // - // [Generating query summary]: /docs/database/database-spans.md#generating-a-summary-of-the-query + // [Generating query summary]: /docs/db/database-spans.md#generating-a-summary-of-the-query DBQuerySummaryKey = attribute.Key("db.query.summary") // DBQueryTextKey is the attribute Key conforming to the "db.query.text" @@ -3811,7 +3878,7 @@ const ( // passed as parameter values, and the benefit to observability of capturing the // static part of the query text by default outweighs the risk. // - // [Sanitization of `db.query.text`]: /docs/database/database-spans.md#sanitization-of-dbquerytext + // [Sanitization of `db.query.text`]: /docs/db/database-spans.md#sanitization-of-dbquerytext DBQueryTextKey = attribute.Key("db.query.text") // DBResponseReturnedRowsKey is the attribute Key conforming to the @@ -4463,10 +4530,8 @@ const ( // Stability: Development // // Examples: "www.example.com", "opentelemetry.io" - // Note: If the name field contains non-printable characters (below 32 or above - // 126), those characters should be represented as escaped base 10 integers - // (\DDD). Back slashes and quotes should be escaped. Tabs, carriage returns, - // and line feeds should be converted to \t, \r, and \n respectively. + // Note: The name represents the queried domain name as it appears in the DNS + // query without any additional normalization. DNSQuestionNameKey = attribute.Key("dns.question.name") ) @@ -4609,7 +4674,7 @@ const ( // `error.type`. // // If a specific domain defines its own set of error identifiers (such as HTTP - // or gRPC status codes), + // or RPC status codes), // it's RECOMMENDED to: // // - Use a domain-specific attribute @@ -4927,7 +4992,7 @@ const ( // // [function version]: https://docs.aws.amazon.com/lambda/latest/dg/configuration-versions.html // [revision]: https://cloud.google.com/run/docs/managing/revisions - // [`K_REVISION` environment variable]: https://cloud.google.com/functions/docs/env-var#runtime_environment_variables_set_automatically + // [`K_REVISION` environment variable]: https://cloud.google.com/run/docs/container-contract#services-env-vars FaaSVersionKey = attribute.Key("faas.version") ) @@ -5729,6 +5794,119 @@ const ( // Examples: "my-workload" GCPAppHubWorkloadIDKey = attribute.Key("gcp.apphub.workload.id") + // GCPAppHubDestinationApplicationContainerKey is the attribute Key conforming + // to the "gcp.apphub_destination.application.container" semantic conventions. + // It represents the container within GCP where the AppHub destination + // application is defined. + // + // Type: string + // RequirementLevel: Recommended + // Stability: Development + // + // Examples: "projects/my-container-project" + GCPAppHubDestinationApplicationContainerKey = attribute.Key("gcp.apphub_destination.application.container") + + // GCPAppHubDestinationApplicationIDKey is the attribute Key conforming to the + // "gcp.apphub_destination.application.id" semantic conventions. It represents + // the name of the destination application as configured in AppHub. + // + // Type: string + // RequirementLevel: Recommended + // Stability: Development + // + // Examples: "my-application" + GCPAppHubDestinationApplicationIDKey = attribute.Key("gcp.apphub_destination.application.id") + + // GCPAppHubDestinationApplicationLocationKey is the attribute Key conforming to + // the "gcp.apphub_destination.application.location" semantic conventions. It + // represents the GCP zone or region where the destination application is + // defined. + // + // Type: string + // RequirementLevel: Recommended + // Stability: Development + // + // Examples: "us-central1" + GCPAppHubDestinationApplicationLocationKey = attribute.Key("gcp.apphub_destination.application.location") + + // GCPAppHubDestinationServiceCriticalityTypeKey is the attribute Key conforming + // to the "gcp.apphub_destination.service.criticality_type" semantic + // conventions. It represents the criticality of a destination workload + // indicates its importance to the business as specified in [AppHub type enum]. + // + // Type: Enum + // RequirementLevel: Recommended + // Stability: Development + // + // Examples: + // + // [AppHub type enum]: https://cloud.google.com/app-hub/docs/reference/rest/v1/Attributes#type + GCPAppHubDestinationServiceCriticalityTypeKey = attribute.Key("gcp.apphub_destination.service.criticality_type") + + // GCPAppHubDestinationServiceEnvironmentTypeKey is the attribute Key conforming + // to the "gcp.apphub_destination.service.environment_type" semantic + // conventions. It represents the software lifecycle stage of a destination + // service as defined [AppHub environment type]. + // + // Type: Enum + // RequirementLevel: Recommended + // Stability: Development + // + // Examples: + // + // [AppHub environment type]: https://cloud.google.com/app-hub/docs/reference/rest/v1/Attributes#type_1 + GCPAppHubDestinationServiceEnvironmentTypeKey = attribute.Key("gcp.apphub_destination.service.environment_type") + + // GCPAppHubDestinationServiceIDKey is the attribute Key conforming to the + // "gcp.apphub_destination.service.id" semantic conventions. It represents the + // name of the destination service as configured in AppHub. + // + // Type: string + // RequirementLevel: Recommended + // Stability: Development + // + // Examples: "my-service" + GCPAppHubDestinationServiceIDKey = attribute.Key("gcp.apphub_destination.service.id") + + // GCPAppHubDestinationWorkloadCriticalityTypeKey is the attribute Key + // conforming to the "gcp.apphub_destination.workload.criticality_type" semantic + // conventions. It represents the criticality of a destination workload + // indicates its importance to the business as specified in [AppHub type enum]. + // + // Type: Enum + // RequirementLevel: Recommended + // Stability: Development + // + // Examples: + // + // [AppHub type enum]: https://cloud.google.com/app-hub/docs/reference/rest/v1/Attributes#type + GCPAppHubDestinationWorkloadCriticalityTypeKey = attribute.Key("gcp.apphub_destination.workload.criticality_type") + + // GCPAppHubDestinationWorkloadEnvironmentTypeKey is the attribute Key + // conforming to the "gcp.apphub_destination.workload.environment_type" semantic + // conventions. It represents the environment of a destination workload is the + // stage of a software lifecycle as provided in the [AppHub environment type]. + // + // Type: Enum + // RequirementLevel: Recommended + // Stability: Development + // + // Examples: + // + // [AppHub environment type]: https://cloud.google.com/app-hub/docs/reference/rest/v1/Attributes#type_1 + GCPAppHubDestinationWorkloadEnvironmentTypeKey = attribute.Key("gcp.apphub_destination.workload.environment_type") + + // GCPAppHubDestinationWorkloadIDKey is the attribute Key conforming to the + // "gcp.apphub_destination.workload.id" semantic conventions. It represents the + // name of the destination workload as configured in AppHub. + // + // Type: string + // RequirementLevel: Recommended + // Stability: Development + // + // Examples: "my-workload" + GCPAppHubDestinationWorkloadIDKey = attribute.Key("gcp.apphub_destination.workload.id") + // GCPClientServiceKey is the attribute Key conforming to the // "gcp.client.service" semantic conventions. It represents the identifies the // Google Cloud service for which the official client library is intended. @@ -5839,6 +6017,43 @@ func GCPAppHubWorkloadID(val string) attribute.KeyValue { return GCPAppHubWorkloadIDKey.String(val) } +// GCPAppHubDestinationApplicationContainer returns an attribute KeyValue +// conforming to the "gcp.apphub_destination.application.container" semantic +// conventions. It represents the container within GCP where the AppHub +// destination application is defined. +func GCPAppHubDestinationApplicationContainer(val string) attribute.KeyValue { + return GCPAppHubDestinationApplicationContainerKey.String(val) +} + +// GCPAppHubDestinationApplicationID returns an attribute KeyValue conforming to +// the "gcp.apphub_destination.application.id" semantic conventions. It +// represents the name of the destination application as configured in AppHub. +func GCPAppHubDestinationApplicationID(val string) attribute.KeyValue { + return GCPAppHubDestinationApplicationIDKey.String(val) +} + +// GCPAppHubDestinationApplicationLocation returns an attribute KeyValue +// conforming to the "gcp.apphub_destination.application.location" semantic +// conventions. It represents the GCP zone or region where the destination +// application is defined. +func GCPAppHubDestinationApplicationLocation(val string) attribute.KeyValue { + return GCPAppHubDestinationApplicationLocationKey.String(val) +} + +// GCPAppHubDestinationServiceID returns an attribute KeyValue conforming to the +// "gcp.apphub_destination.service.id" semantic conventions. It represents the +// name of the destination service as configured in AppHub. +func GCPAppHubDestinationServiceID(val string) attribute.KeyValue { + return GCPAppHubDestinationServiceIDKey.String(val) +} + +// GCPAppHubDestinationWorkloadID returns an attribute KeyValue conforming to the +// "gcp.apphub_destination.workload.id" semantic conventions. It represents the +// name of the destination workload as configured in AppHub. +func GCPAppHubDestinationWorkloadID(val string) attribute.KeyValue { + return GCPAppHubDestinationWorkloadIDKey.String(val) +} + // GCPClientService returns an attribute KeyValue conforming to the // "gcp.client.service" semantic conventions. It represents the identifies the // Google Cloud service for which the official client library is intended. @@ -5952,6 +6167,70 @@ var ( GCPAppHubWorkloadEnvironmentTypeDevelopment = GCPAppHubWorkloadEnvironmentTypeKey.String("DEVELOPMENT") ) +// Enum values for gcp.apphub_destination.service.criticality_type +var ( + // Mission critical service. + // Stability: development + GCPAppHubDestinationServiceCriticalityTypeMissionCritical = GCPAppHubDestinationServiceCriticalityTypeKey.String("MISSION_CRITICAL") + // High impact. + // Stability: development + GCPAppHubDestinationServiceCriticalityTypeHigh = GCPAppHubDestinationServiceCriticalityTypeKey.String("HIGH") + // Medium impact. + // Stability: development + GCPAppHubDestinationServiceCriticalityTypeMedium = GCPAppHubDestinationServiceCriticalityTypeKey.String("MEDIUM") + // Low impact. + // Stability: development + GCPAppHubDestinationServiceCriticalityTypeLow = GCPAppHubDestinationServiceCriticalityTypeKey.String("LOW") +) + +// Enum values for gcp.apphub_destination.service.environment_type +var ( + // Production environment. + // Stability: development + GCPAppHubDestinationServiceEnvironmentTypeProduction = GCPAppHubDestinationServiceEnvironmentTypeKey.String("PRODUCTION") + // Staging environment. + // Stability: development + GCPAppHubDestinationServiceEnvironmentTypeStaging = GCPAppHubDestinationServiceEnvironmentTypeKey.String("STAGING") + // Test environment. + // Stability: development + GCPAppHubDestinationServiceEnvironmentTypeTest = GCPAppHubDestinationServiceEnvironmentTypeKey.String("TEST") + // Development environment. + // Stability: development + GCPAppHubDestinationServiceEnvironmentTypeDevelopment = GCPAppHubDestinationServiceEnvironmentTypeKey.String("DEVELOPMENT") +) + +// Enum values for gcp.apphub_destination.workload.criticality_type +var ( + // Mission critical service. + // Stability: development + GCPAppHubDestinationWorkloadCriticalityTypeMissionCritical = GCPAppHubDestinationWorkloadCriticalityTypeKey.String("MISSION_CRITICAL") + // High impact. + // Stability: development + GCPAppHubDestinationWorkloadCriticalityTypeHigh = GCPAppHubDestinationWorkloadCriticalityTypeKey.String("HIGH") + // Medium impact. + // Stability: development + GCPAppHubDestinationWorkloadCriticalityTypeMedium = GCPAppHubDestinationWorkloadCriticalityTypeKey.String("MEDIUM") + // Low impact. + // Stability: development + GCPAppHubDestinationWorkloadCriticalityTypeLow = GCPAppHubDestinationWorkloadCriticalityTypeKey.String("LOW") +) + +// Enum values for gcp.apphub_destination.workload.environment_type +var ( + // Production environment. + // Stability: development + GCPAppHubDestinationWorkloadEnvironmentTypeProduction = GCPAppHubDestinationWorkloadEnvironmentTypeKey.String("PRODUCTION") + // Staging environment. + // Stability: development + GCPAppHubDestinationWorkloadEnvironmentTypeStaging = GCPAppHubDestinationWorkloadEnvironmentTypeKey.String("STAGING") + // Test environment. + // Stability: development + GCPAppHubDestinationWorkloadEnvironmentTypeTest = GCPAppHubDestinationWorkloadEnvironmentTypeKey.String("TEST") + // Development environment. + // Stability: development + GCPAppHubDestinationWorkloadEnvironmentTypeDevelopment = GCPAppHubDestinationWorkloadEnvironmentTypeKey.String("DEVELOPMENT") +) + // Namespace: gen_ai const ( // GenAIAgentDescriptionKey is the attribute Key conforming to the @@ -6017,6 +6296,68 @@ const ( // `db.*`, to further identify and describe the data source. GenAIDataSourceIDKey = attribute.Key("gen_ai.data_source.id") + // GenAIEmbeddingsDimensionCountKey is the attribute Key conforming to the + // "gen_ai.embeddings.dimension.count" semantic conventions. It represents the + // number of dimensions the resulting output embeddings should have. + // + // Type: int + // RequirementLevel: Recommended + // Stability: Development + // + // Examples: 512, 1024 + GenAIEmbeddingsDimensionCountKey = attribute.Key("gen_ai.embeddings.dimension.count") + + // GenAIEvaluationExplanationKey is the attribute Key conforming to the + // "gen_ai.evaluation.explanation" semantic conventions. It represents a + // free-form explanation for the assigned score provided by the evaluator. + // + // Type: string + // RequirementLevel: Recommended + // Stability: Development + // + // Examples: "The response is factually accurate but lacks sufficient detail to + // fully address the question." + GenAIEvaluationExplanationKey = attribute.Key("gen_ai.evaluation.explanation") + + // GenAIEvaluationNameKey is the attribute Key conforming to the + // "gen_ai.evaluation.name" semantic conventions. It represents the name of the + // evaluation metric used for the GenAI response. + // + // Type: string + // RequirementLevel: Recommended + // Stability: Development + // + // Examples: "Relevance", "IntentResolution" + GenAIEvaluationNameKey = attribute.Key("gen_ai.evaluation.name") + + // GenAIEvaluationScoreLabelKey is the attribute Key conforming to the + // "gen_ai.evaluation.score.label" semantic conventions. It represents the human + // readable label for evaluation. + // + // Type: string + // RequirementLevel: Recommended + // Stability: Development + // + // Examples: "relevant", "not_relevant", "correct", "incorrect", "pass", "fail" + // Note: This attribute provides a human-readable interpretation of the + // evaluation score produced by an evaluator. For example, a score value of 1 + // could mean "relevant" in one evaluation system and "not relevant" in another, + // depending on the scoring range and evaluator. The label SHOULD have low + // cardinality. Possible values depend on the evaluation metric and evaluator + // used; implementations SHOULD document the possible values. + GenAIEvaluationScoreLabelKey = attribute.Key("gen_ai.evaluation.score.label") + + // GenAIEvaluationScoreValueKey is the attribute Key conforming to the + // "gen_ai.evaluation.score.value" semantic conventions. It represents the + // evaluation score returned by the evaluator. + // + // Type: double + // RequirementLevel: Recommended + // Stability: Development + // + // Examples: 4.0 + GenAIEvaluationScoreValueKey = attribute.Key("gen_ai.evaluation.score.value") + // GenAIInputMessagesKey is the attribute Key conforming to the // "gen_ai.input.messages" semantic conventions. It represents the chat history // provided to the model as an input. @@ -6125,6 +6466,17 @@ const ( // `gen_ai.output.{type}.*` attributes. GenAIOutputTypeKey = attribute.Key("gen_ai.output.type") + // GenAIPromptNameKey is the attribute Key conforming to the + // "gen_ai.prompt.name" semantic conventions. It represents the name of the + // prompt that uniquely identifies it. + // + // Type: string + // RequirementLevel: Recommended + // Stability: Development + // + // Examples: "analyze-code" + GenAIPromptNameKey = attribute.Key("gen_ai.prompt.name") + // GenAIProviderNameKey is the attribute Key conforming to the // "gen_ai.provider.name" semantic conventions. It represents the Generative AI // provider as identified by the client or server instrumentation. @@ -6360,6 +6712,26 @@ const ( // Examples: "input", "output" GenAITokenTypeKey = attribute.Key("gen_ai.token.type") + // GenAIToolCallArgumentsKey is the attribute Key conforming to the + // "gen_ai.tool.call.arguments" semantic conventions. It represents the + // parameters passed to the tool call. + // + // Type: any + // RequirementLevel: Recommended + // Stability: Development + // + // Examples: "{\n "location": "San Francisco?",\n "date": "2025-10-01"\n}\n" + // Note: > [!WARNING] + // + // > This attribute may contain sensitive information. + // + // It's expected to be an object - in case a serialized string is available + // to the instrumentation, the instrumentation SHOULD do the best effort to + // deserialize it to an object. When recorded on spans, it MAY be recorded as a + // JSON string if structured format is not supported and SHOULD be recorded in + // structured form otherwise. + GenAIToolCallArgumentsKey = attribute.Key("gen_ai.tool.call.arguments") + // GenAIToolCallIDKey is the attribute Key conforming to the // "gen_ai.tool.call.id" semantic conventions. It represents the tool call // identifier. @@ -6371,6 +6743,56 @@ const ( // Examples: "call_mszuSIzqtI65i1wAUOE8w5H4" GenAIToolCallIDKey = attribute.Key("gen_ai.tool.call.id") + // GenAIToolCallResultKey is the attribute Key conforming to the + // "gen_ai.tool.call.result" semantic conventions. It represents the result + // returned by the tool call (if any and if execution was successful). + // + // Type: any + // RequirementLevel: Recommended + // Stability: Development + // + // Examples: "{\n "temperature_range": {\n "high": 75,\n "low": 60\n },\n + // "conditions": "sunny"\n}\n" + // Note: > [!WARNING] + // + // > This attribute may contain sensitive information. + // + // It's expected to be an object - in case a serialized string is available + // to the instrumentation, the instrumentation SHOULD do the best effort to + // deserialize it to an object. When recorded on spans, it MAY be recorded as a + // JSON string if structured format is not supported and SHOULD be recorded in + // structured form otherwise. + GenAIToolCallResultKey = attribute.Key("gen_ai.tool.call.result") + + // GenAIToolDefinitionsKey is the attribute Key conforming to the + // "gen_ai.tool.definitions" semantic conventions. It represents the list of + // source system tool definitions available to the GenAI agent or model. + // + // Type: any + // RequirementLevel: Recommended + // Stability: Development + // + // Examples: "[\n {\n "type": "function",\n "name": "get_current_weather",\n + // "description": "Get the current weather in a given location",\n "parameters": + // {\n "type": "object",\n "properties": {\n "location": {\n "type": "string",\n + // "description": "The city and state, e.g. San Francisco, CA"\n },\n "unit": + // {\n "type": "string",\n "enum": [\n "celsius",\n "fahrenheit"\n ]\n }\n },\n + // "required": [\n "location",\n "unit"\n ]\n }\n }\n]\n" + // Note: The value of this attribute matches source system tool definition + // format. + // + // It's expected to be an array of objects where each object represents a tool + // definition. In case a serialized string is available + // to the instrumentation, the instrumentation SHOULD do the best effort to + // deserialize it to an array. When recorded on spans, it MAY be recorded as a + // JSON string if structured format is not supported and SHOULD be recorded in + // structured form otherwise. + // + // Since this attribute could be large, it's NOT RECOMMENDED to populate + // it by default. Instrumentations MAY provide a way to enable + // populating this attribute. + GenAIToolDefinitionsKey = attribute.Key("gen_ai.tool.definitions") + // GenAIToolDescriptionKey is the attribute Key conforming to the // "gen_ai.tool.description" semantic conventions. It represents the tool // description. @@ -6473,6 +6895,48 @@ func GenAIDataSourceID(val string) attribute.KeyValue { return GenAIDataSourceIDKey.String(val) } +// GenAIEmbeddingsDimensionCount returns an attribute KeyValue conforming to the +// "gen_ai.embeddings.dimension.count" semantic conventions. It represents the +// number of dimensions the resulting output embeddings should have. +func GenAIEmbeddingsDimensionCount(val int) attribute.KeyValue { + return GenAIEmbeddingsDimensionCountKey.Int(val) +} + +// GenAIEvaluationExplanation returns an attribute KeyValue conforming to the +// "gen_ai.evaluation.explanation" semantic conventions. It represents a +// free-form explanation for the assigned score provided by the evaluator. +func GenAIEvaluationExplanation(val string) attribute.KeyValue { + return GenAIEvaluationExplanationKey.String(val) +} + +// GenAIEvaluationName returns an attribute KeyValue conforming to the +// "gen_ai.evaluation.name" semantic conventions. It represents the name of the +// evaluation metric used for the GenAI response. +func GenAIEvaluationName(val string) attribute.KeyValue { + return GenAIEvaluationNameKey.String(val) +} + +// GenAIEvaluationScoreLabel returns an attribute KeyValue conforming to the +// "gen_ai.evaluation.score.label" semantic conventions. It represents the human +// readable label for evaluation. +func GenAIEvaluationScoreLabel(val string) attribute.KeyValue { + return GenAIEvaluationScoreLabelKey.String(val) +} + +// GenAIEvaluationScoreValue returns an attribute KeyValue conforming to the +// "gen_ai.evaluation.score.value" semantic conventions. It represents the +// evaluation score returned by the evaluator. +func GenAIEvaluationScoreValue(val float64) attribute.KeyValue { + return GenAIEvaluationScoreValueKey.Float64(val) +} + +// GenAIPromptName returns an attribute KeyValue conforming to the +// "gen_ai.prompt.name" semantic conventions. It represents the name of the +// prompt that uniquely identifies it. +func GenAIPromptName(val string) attribute.KeyValue { + return GenAIPromptNameKey.String(val) +} + // GenAIRequestChoiceCount returns an attribute KeyValue conforming to the // "gen_ai.request.choice.count" semantic conventions. It represents the target // number of candidate completions to return. @@ -7413,8 +7877,9 @@ const ( // Examples: "GET", "POST", "HEAD" // Note: HTTP request method value SHOULD be "known" to the instrumentation. // By default, this convention defines "known" methods as the ones listed in - // [RFC9110] - // and the PATCH method defined in [RFC5789]. + // [RFC9110], + // the PATCH method defined in [RFC5789] + // and the QUERY method defined in [httpbis-safe-method-w-body]. // // If the HTTP request method is not known to instrumentation, it MUST set the // `http.request.method` attribute to `_OTHER`. @@ -7437,6 +7902,7 @@ const ( // // [RFC9110]: https://www.rfc-editor.org/rfc/rfc9110.html#name-methods // [RFC5789]: https://www.rfc-editor.org/rfc/rfc5789.html + // [httpbis-safe-method-w-body]: https://datatracker.ietf.org/doc/draft-ietf-httpbis-safe-method-w-body/?include_text=1 HTTPRequestMethodKey = attribute.Key("http.request.method") // HTTPRequestMethodOriginalKey is the attribute Key conforming to the @@ -7514,19 +7980,36 @@ const ( HTTPResponseStatusCodeKey = attribute.Key("http.response.status_code") // HTTPRouteKey is the attribute Key conforming to the "http.route" semantic - // conventions. It represents the matched route, that is, the path template in - // the format used by the respective server framework. + // conventions. It represents the matched route template for the request. This + // MUST be low-cardinality and include all static path segments, with dynamic + // path segments represented with placeholders. // // Type: string // RequirementLevel: Recommended // Stability: Stable // - // Examples: "/users/:userID?", "{controller}/{action}/{id?}" + // Examples: "/users/:userID?", "my-controller/my-action/{id?}" // Note: MUST NOT be populated when this is not supported by the HTTP server // framework as the route attribute should have low-cardinality and the URI path // can NOT substitute it. // SHOULD include the [application root] if there is one. // + // A static path segment is a part of the route template with a fixed, + // low-cardinality value. This includes literal strings like `/users/` and + // placeholders that + // are constrained to a finite, predefined set of values, e.g. `{controller}` or + // `{action}`. + // + // A dynamic path segment is a placeholder for a value that can have high + // cardinality and is not constrained to a predefined list like static path + // segments. + // + // Instrumentations SHOULD use routing information provided by the corresponding + // web framework. They SHOULD pick the most precise source of routing + // information and MAY + // support custom route formatting. Instrumentations SHOULD document the format + // and the API used to obtain the route string. + // // [application root]: /docs/http/http-spans.md#http-server-definitions HTTPRouteKey = attribute.Key("http.route") ) @@ -7613,8 +8096,9 @@ func HTTPResponseStatusCode(val int) attribute.KeyValue { } // HTTPRoute returns an attribute KeyValue conforming to the "http.route" -// semantic conventions. It represents the matched route, that is, the path -// template in the format used by the respective server framework. +// semantic conventions. It represents the matched route template for the +// request. This MUST be low-cardinality and include all static path segments, +// with dynamic path segments represented with placeholders. func HTTPRoute(val string) attribute.KeyValue { return HTTPRouteKey.String(val) } @@ -7658,6 +8142,9 @@ var ( // TRACE method. // Stability: stable HTTPRequestMethodTrace = HTTPRequestMethodKey.String("TRACE") + // QUERY method. + // Stability: development + HTTPRequestMethodQuery = HTTPRequestMethodKey.String("QUERY") // Any HTTP method that the instrumentation has no prior knowledge of. // Stability: stable HTTPRequestMethodOther = HTTPRequestMethodKey.String("_OTHER") @@ -8300,6 +8787,57 @@ var ( IOSAppStateTerminate = IOSAppStateKey.String("terminate") ) +// Namespace: jsonrpc +const ( + // JSONRPCProtocolVersionKey is the attribute Key conforming to the + // "jsonrpc.protocol.version" semantic conventions. It represents the protocol + // version, as specified in the `jsonrpc` property of the request and its + // corresponding response. + // + // Type: string + // RequirementLevel: Recommended + // Stability: Development + // + // Examples: "2.0", "1.0" + JSONRPCProtocolVersionKey = attribute.Key("jsonrpc.protocol.version") + + // JSONRPCRequestIDKey is the attribute Key conforming to the + // "jsonrpc.request.id" semantic conventions. It represents a string + // representation of the `id` property of the request and its corresponding + // response. + // + // Type: string + // RequirementLevel: Recommended + // Stability: Development + // + // Examples: "10", "request-7" + // Note: Under the [JSON-RPC specification], the `id` property may be a string, + // number, null, or omitted entirely. When omitted, the request is treated as a + // notification. Using `null` is not equivalent to omitting the `id`, but it is + // discouraged. + // Instrumentations SHOULD NOT capture this attribute when the `id` is `null` or + // omitted. + // + // [JSON-RPC specification]: https://www.jsonrpc.org/specification + JSONRPCRequestIDKey = attribute.Key("jsonrpc.request.id") +) + +// JSONRPCProtocolVersion returns an attribute KeyValue conforming to the +// "jsonrpc.protocol.version" semantic conventions. It represents the protocol +// version, as specified in the `jsonrpc` property of the request and its +// corresponding response. +func JSONRPCProtocolVersion(val string) attribute.KeyValue { + return JSONRPCProtocolVersionKey.String(val) +} + +// JSONRPCRequestID returns an attribute KeyValue conforming to the +// "jsonrpc.request.id" semantic conventions. It represents a string +// representation of the `id` property of the request and its corresponding +// response. +func JSONRPCRequestID(val string) attribute.KeyValue { + return JSONRPCRequestIDKey.String(val) +} + // Namespace: k8s const ( // K8SClusterNameKey is the attribute Key conforming to the "k8s.cluster.name" @@ -8307,7 +8845,7 @@ const ( // // Type: string // RequirementLevel: Recommended - // Stability: Development + // Stability: Alpha // // Examples: "opentelemetry-cluster" K8SClusterNameKey = attribute.Key("k8s.cluster.name") @@ -8318,7 +8856,7 @@ const ( // // Type: string // RequirementLevel: Recommended - // Stability: Development + // Stability: Alpha // // Examples: "218fc5a9-a5f1-4b54-aa05-46717d0ab26d" // Note: K8s doesn't have support for obtaining a cluster ID. If this is ever @@ -8354,7 +8892,7 @@ const ( // // Type: string // RequirementLevel: Recommended - // Stability: Development + // Stability: Alpha // // Examples: "redis" K8SContainerNameKey = attribute.Key("k8s.container.name") @@ -8366,7 +8904,7 @@ const ( // // Type: int // RequirementLevel: Recommended - // Stability: Development + // Stability: Alpha // // Examples: K8SContainerRestartCountKey = attribute.Key("k8s.container.restart_count") @@ -8417,7 +8955,7 @@ const ( // // Type: string // RequirementLevel: Recommended - // Stability: Development + // Stability: Alpha // // Examples: "opentelemetry" K8SCronJobNameKey = attribute.Key("k8s.cronjob.name") @@ -8427,7 +8965,7 @@ const ( // // Type: string // RequirementLevel: Recommended - // Stability: Development + // Stability: Alpha // // Examples: "275ecb36-5aa8-4c2a-9c47-d8bb681b9aff" K8SCronJobUIDKey = attribute.Key("k8s.cronjob.uid") @@ -8438,7 +8976,7 @@ const ( // // Type: string // RequirementLevel: Recommended - // Stability: Development + // Stability: Alpha // // Examples: "opentelemetry" K8SDaemonSetNameKey = attribute.Key("k8s.daemonset.name") @@ -8448,7 +8986,7 @@ const ( // // Type: string // RequirementLevel: Recommended - // Stability: Development + // Stability: Alpha // // Examples: "275ecb36-5aa8-4c2a-9c47-d8bb681b9aff" K8SDaemonSetUIDKey = attribute.Key("k8s.daemonset.uid") @@ -8459,7 +8997,7 @@ const ( // // Type: string // RequirementLevel: Recommended - // Stability: Development + // Stability: Alpha // // Examples: "opentelemetry" K8SDeploymentNameKey = attribute.Key("k8s.deployment.name") @@ -8470,7 +9008,7 @@ const ( // // Type: string // RequirementLevel: Recommended - // Stability: Development + // Stability: Alpha // // Examples: "275ecb36-5aa8-4c2a-9c47-d8bb681b9aff" K8SDeploymentUIDKey = attribute.Key("k8s.deployment.uid") @@ -8560,7 +9098,7 @@ const ( // // Type: string // RequirementLevel: Recommended - // Stability: Development + // Stability: Alpha // // Examples: "opentelemetry" K8SJobNameKey = attribute.Key("k8s.job.name") @@ -8570,7 +9108,7 @@ const ( // // Type: string // RequirementLevel: Recommended - // Stability: Development + // Stability: Alpha // // Examples: "275ecb36-5aa8-4c2a-9c47-d8bb681b9aff" K8SJobUIDKey = attribute.Key("k8s.job.uid") @@ -8581,7 +9119,7 @@ const ( // // Type: string // RequirementLevel: Recommended - // Stability: Development + // Stability: Alpha // // Examples: "default" K8SNamespaceNameKey = attribute.Key("k8s.namespace.name") @@ -8646,7 +9184,7 @@ const ( // // Type: string // RequirementLevel: Recommended - // Stability: Development + // Stability: Alpha // // Examples: "node-1" K8SNodeNameKey = attribute.Key("k8s.node.name") @@ -8656,27 +9194,106 @@ const ( // // Type: string // RequirementLevel: Recommended - // Stability: Development + // Stability: Alpha // // Examples: "1eb3a0c6-0477-4080-a9cb-0cb7db65c6a2" K8SNodeUIDKey = attribute.Key("k8s.node.uid") + // K8SPodHostnameKey is the attribute Key conforming to the "k8s.pod.hostname" + // semantic conventions. It represents the specifies the hostname of the Pod. + // + // Type: string + // RequirementLevel: Recommended + // Stability: Alpha + // + // Examples: "collector-gateway" + // Note: The K8s Pod spec has an optional hostname field, which can be used to + // specify a hostname. + // Refer to [K8s docs] + // for more information about this field. + // + // This attribute aligns with the `hostname` field of the + // [K8s PodSpec]. + // + // [K8s docs]: https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/#pod-hostname-and-subdomain-field + // [K8s PodSpec]: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.34/#podspec-v1-core + K8SPodHostnameKey = attribute.Key("k8s.pod.hostname") + + // K8SPodIPKey is the attribute Key conforming to the "k8s.pod.ip" semantic + // conventions. It represents the IP address allocated to the Pod. + // + // Type: string + // RequirementLevel: Recommended + // Stability: Alpha + // + // Examples: "172.18.0.2" + // Note: This attribute aligns with the `podIP` field of the + // [K8s PodStatus]. + // + // [K8s PodStatus]: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.34/#podstatus-v1-core + K8SPodIPKey = attribute.Key("k8s.pod.ip") + // K8SPodNameKey is the attribute Key conforming to the "k8s.pod.name" semantic // conventions. It represents the name of the Pod. // // Type: string // RequirementLevel: Recommended - // Stability: Development + // Stability: Alpha // // Examples: "opentelemetry-pod-autoconf" K8SPodNameKey = attribute.Key("k8s.pod.name") + // K8SPodStartTimeKey is the attribute Key conforming to the + // "k8s.pod.start_time" semantic conventions. It represents the start timestamp + // of the Pod. + // + // Type: string + // RequirementLevel: Recommended + // Stability: Alpha + // + // Examples: "2025-12-04T08:41:03Z" + // Note: Date and time at which the object was acknowledged by the Kubelet. + // This is before the Kubelet pulled the container image(s) for the pod. + // + // This attribute aligns with the `startTime` field of the + // [K8s PodStatus], + // in ISO 8601 (RFC 3339 compatible) format. + // + // [K8s PodStatus]: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.34/#podstatus-v1-core + K8SPodStartTimeKey = attribute.Key("k8s.pod.start_time") + + // K8SPodStatusPhaseKey is the attribute Key conforming to the + // "k8s.pod.status.phase" semantic conventions. It represents the phase for the + // pod. Corresponds to the `phase` field of the: [K8s PodStatus]. + // + // Type: Enum + // RequirementLevel: Recommended + // Stability: Development + // + // Examples: "Pending", "Running" + // + // [K8s PodStatus]: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.33/#podstatus-v1-core + K8SPodStatusPhaseKey = attribute.Key("k8s.pod.status.phase") + + // K8SPodStatusReasonKey is the attribute Key conforming to the + // "k8s.pod.status.reason" semantic conventions. It represents the reason for + // the pod state. Corresponds to the `reason` field of the: [K8s PodStatus]. + // + // Type: Enum + // RequirementLevel: Recommended + // Stability: Development + // + // Examples: "Evicted", "NodeAffinity" + // + // [K8s PodStatus]: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.33/#podstatus-v1-core + K8SPodStatusReasonKey = attribute.Key("k8s.pod.status.reason") + // K8SPodUIDKey is the attribute Key conforming to the "k8s.pod.uid" semantic // conventions. It represents the UID of the Pod. // // Type: string // RequirementLevel: Recommended - // Stability: Development + // Stability: Alpha // // Examples: "275ecb36-5aa8-4c2a-9c47-d8bb681b9aff" K8SPodUIDKey = attribute.Key("k8s.pod.uid") @@ -8687,7 +9304,7 @@ const ( // // Type: string // RequirementLevel: Recommended - // Stability: Development + // Stability: Alpha // // Examples: "opentelemetry" K8SReplicaSetNameKey = attribute.Key("k8s.replicaset.name") @@ -8698,7 +9315,7 @@ const ( // // Type: string // RequirementLevel: Recommended - // Stability: Development + // Stability: Alpha // // Examples: "275ecb36-5aa8-4c2a-9c47-d8bb681b9aff" K8SReplicaSetUIDKey = attribute.Key("k8s.replicaset.uid") @@ -8752,7 +9369,7 @@ const ( // Kubernetes for object count quotas. See // [Kubernetes Resource Quotas documentation] for more details. // - // [Kubernetes Resource Quotas documentation]: https://kubernetes.io/docs/concepts/policy/resource-quotas/#object-count-quota + // [Kubernetes Resource Quotas documentation]: https://kubernetes.io/docs/concepts/policy/resource-quotas/#quota-on-object-count K8SResourceQuotaResourceNameKey = attribute.Key("k8s.resourcequota.resource_name") // K8SResourceQuotaUIDKey is the attribute Key conforming to the @@ -8772,7 +9389,7 @@ const ( // // Type: string // RequirementLevel: Recommended - // Stability: Development + // Stability: Alpha // // Examples: "opentelemetry" K8SStatefulSetNameKey = attribute.Key("k8s.statefulset.name") @@ -8783,7 +9400,7 @@ const ( // // Type: string // RequirementLevel: Recommended - // Stability: Development + // Stability: Alpha // // Examples: "275ecb36-5aa8-4c2a-9c47-d8bb681b9aff" K8SStatefulSetUIDKey = attribute.Key("k8s.statefulset.uid") @@ -9082,6 +9699,19 @@ func K8SPodAnnotation(key string, val string) attribute.KeyValue { return attribute.String("k8s.pod.annotation."+key, val) } +// K8SPodHostname returns an attribute KeyValue conforming to the +// "k8s.pod.hostname" semantic conventions. It represents the specifies the +// hostname of the Pod. +func K8SPodHostname(val string) attribute.KeyValue { + return K8SPodHostnameKey.String(val) +} + +// K8SPodIP returns an attribute KeyValue conforming to the "k8s.pod.ip" semantic +// conventions. It represents the IP address allocated to the Pod. +func K8SPodIP(val string) attribute.KeyValue { + return K8SPodIPKey.String(val) +} + // K8SPodLabel returns an attribute KeyValue conforming to the "k8s.pod.label" // semantic conventions. It represents the label placed on the Pod, the `` // being the label name, the value being the label value. @@ -9095,6 +9725,13 @@ func K8SPodName(val string) attribute.KeyValue { return K8SPodNameKey.String(val) } +// K8SPodStartTime returns an attribute KeyValue conforming to the +// "k8s.pod.start_time" semantic conventions. It represents the start timestamp +// of the Pod. +func K8SPodStartTime(val string) attribute.KeyValue { + return K8SPodStartTimeKey.String(val) +} + // K8SPodUID returns an attribute KeyValue conforming to the "k8s.pod.uid" // semantic conventions. It represents the UID of the Pod. func K8SPodUID(val string) attribute.KeyValue { @@ -9303,6 +9940,61 @@ var ( K8SNodeConditionTypeNetworkUnavailable = K8SNodeConditionTypeKey.String("NetworkUnavailable") ) +// Enum values for k8s.pod.status.phase +var ( + // The pod has been accepted by the system, but one or more of the containers + // has not been started. This includes time before being bound to a node, as + // well as time spent pulling images onto the host. + // + // Stability: development + K8SPodStatusPhasePending = K8SPodStatusPhaseKey.String("Pending") + // The pod has been bound to a node and all of the containers have been started. + // At least one container is still running or is in the process of being + // restarted. + // + // Stability: development + K8SPodStatusPhaseRunning = K8SPodStatusPhaseKey.String("Running") + // All containers in the pod have voluntarily terminated with a container exit + // code of 0, and the system is not going to restart any of these containers. + // + // Stability: development + K8SPodStatusPhaseSucceeded = K8SPodStatusPhaseKey.String("Succeeded") + // All containers in the pod have terminated, and at least one container has + // terminated in a failure (exited with a non-zero exit code or was stopped by + // the system). + // + // Stability: development + K8SPodStatusPhaseFailed = K8SPodStatusPhaseKey.String("Failed") + // For some reason the state of the pod could not be obtained, typically due to + // an error in communicating with the host of the pod. + // + // Stability: development + K8SPodStatusPhaseUnknown = K8SPodStatusPhaseKey.String("Unknown") +) + +// Enum values for k8s.pod.status.reason +var ( + // The pod is evicted. + // Stability: development + K8SPodStatusReasonEvicted = K8SPodStatusReasonKey.String("Evicted") + // The pod is in a status because of its node affinity + // Stability: development + K8SPodStatusReasonNodeAffinity = K8SPodStatusReasonKey.String("NodeAffinity") + // The reason on a pod when its state cannot be confirmed as kubelet is + // unresponsive on the node it is (was) running. + // + // Stability: development + K8SPodStatusReasonNodeLost = K8SPodStatusReasonKey.String("NodeLost") + // The node is shutdown + // Stability: development + K8SPodStatusReasonShutdown = K8SPodStatusReasonKey.String("Shutdown") + // The pod was rejected admission to the node because of an error during + // admission that could not be categorized. + // + // Stability: development + K8SPodStatusReasonUnexpectedAdmissionError = K8SPodStatusReasonKey.String("UnexpectedAdmissionError") +) + // Enum values for k8s.volume.type var ( // A [persistentVolumeClaim] volume @@ -9337,30 +10029,6 @@ var ( K8SVolumeTypeLocal = K8SVolumeTypeKey.String("local") ) -// Namespace: linux -const ( - // LinuxMemorySlabStateKey is the attribute Key conforming to the - // "linux.memory.slab.state" semantic conventions. It represents the Linux Slab - // memory state. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "reclaimable", "unreclaimable" - LinuxMemorySlabStateKey = attribute.Key("linux.memory.slab.state") -) - -// Enum values for linux.memory.slab.state -var ( - // reclaimable - // Stability: development - LinuxMemorySlabStateReclaimable = LinuxMemorySlabStateKey.String("reclaimable") - // unreclaimable - // Stability: development - LinuxMemorySlabStateUnreclaimable = LinuxMemorySlabStateKey.String("unreclaimable") -) - // Namespace: log const ( // LogFileNameKey is the attribute Key conforming to the "log.file.name" @@ -9521,6 +10189,188 @@ func MainframeLparName(val string) attribute.KeyValue { return MainframeLparNameKey.String(val) } +// Namespace: mcp +const ( + // McpMethodNameKey is the attribute Key conforming to the "mcp.method.name" + // semantic conventions. It represents the name of the request or notification + // method. + // + // Type: Enum + // RequirementLevel: Recommended + // Stability: Development + // + // Examples: + McpMethodNameKey = attribute.Key("mcp.method.name") + + // McpProtocolVersionKey is the attribute Key conforming to the + // "mcp.protocol.version" semantic conventions. It represents the [version] of + // the Model Context Protocol used. + // + // Type: string + // RequirementLevel: Recommended + // Stability: Development + // + // Examples: "2025-06-18" + // + // [version]: https://modelcontextprotocol.io/specification/versioning + McpProtocolVersionKey = attribute.Key("mcp.protocol.version") + + // McpResourceURIKey is the attribute Key conforming to the "mcp.resource.uri" + // semantic conventions. It represents the value of the resource uri. + // + // Type: string + // RequirementLevel: Recommended + // Stability: Development + // + // Examples: "postgres://database/customers/schema", + // "file:///home/user/documents/report.pdf" + // Note: This is a URI of the resource provided in the following requests or + // notifications: `resources/read`, `resources/subscribe`, + // `resources/unsubscribe`, or `notifications/resources/updated`. + McpResourceURIKey = attribute.Key("mcp.resource.uri") + + // McpSessionIDKey is the attribute Key conforming to the "mcp.session.id" + // semantic conventions. It represents the identifies [MCP session]. + // + // Type: string + // RequirementLevel: Recommended + // Stability: Development + // + // Examples: "191c4850af6c49e08843a3f6c80e5046" + // + // [MCP session]: https://modelcontextprotocol.io/specification/2025-06-18/basic/transports#session-management + McpSessionIDKey = attribute.Key("mcp.session.id") +) + +// McpProtocolVersion returns an attribute KeyValue conforming to the +// "mcp.protocol.version" semantic conventions. It represents the [version] of +// the Model Context Protocol used. +// +// [version]: https://modelcontextprotocol.io/specification/versioning +func McpProtocolVersion(val string) attribute.KeyValue { + return McpProtocolVersionKey.String(val) +} + +// McpResourceURI returns an attribute KeyValue conforming to the +// "mcp.resource.uri" semantic conventions. It represents the value of the +// resource uri. +func McpResourceURI(val string) attribute.KeyValue { + return McpResourceURIKey.String(val) +} + +// McpSessionID returns an attribute KeyValue conforming to the "mcp.session.id" +// semantic conventions. It represents the identifies [MCP session]. +// +// [MCP session]: https://modelcontextprotocol.io/specification/2025-06-18/basic/transports#session-management +func McpSessionID(val string) attribute.KeyValue { + return McpSessionIDKey.String(val) +} + +// Enum values for mcp.method.name +var ( + // Notification cancelling a previously-issued request. + // + // Stability: development + McpMethodNameNotificationsCancelled = McpMethodNameKey.String("notifications/cancelled") + // Request to initialize the MCP client. + // + // Stability: development + McpMethodNameInitialize = McpMethodNameKey.String("initialize") + // Notification indicating that the MCP client has been initialized. + // + // Stability: development + McpMethodNameNotificationsInitialized = McpMethodNameKey.String("notifications/initialized") + // Notification indicating the progress for a long-running operation. + // + // Stability: development + McpMethodNameNotificationsProgress = McpMethodNameKey.String("notifications/progress") + // Request to check that the other party is still alive. + // + // Stability: development + McpMethodNamePing = McpMethodNameKey.String("ping") + // Request to list resources available on server. + // + // Stability: development + McpMethodNameResourcesList = McpMethodNameKey.String("resources/list") + // Request to list resource templates available on server. + // + // Stability: development + McpMethodNameResourcesTemplatesList = McpMethodNameKey.String("resources/templates/list") + // Request to read a resource. + // + // Stability: development + McpMethodNameResourcesRead = McpMethodNameKey.String("resources/read") + // Notification indicating that the list of resources has changed. + // + // Stability: development + McpMethodNameNotificationsResourcesListChanged = McpMethodNameKey.String("notifications/resources/list_changed") + // Request to subscribe to a resource. + // + // Stability: development + McpMethodNameResourcesSubscribe = McpMethodNameKey.String("resources/subscribe") + // Request to unsubscribe from resource updates. + // + // Stability: development + McpMethodNameResourcesUnsubscribe = McpMethodNameKey.String("resources/unsubscribe") + // Notification indicating that a resource has been updated. + // + // Stability: development + McpMethodNameNotificationsResourcesUpdated = McpMethodNameKey.String("notifications/resources/updated") + // Request to list prompts available on server. + // + // Stability: development + McpMethodNamePromptsList = McpMethodNameKey.String("prompts/list") + // Request to get a prompt. + // + // Stability: development + McpMethodNamePromptsGet = McpMethodNameKey.String("prompts/get") + // Notification indicating that the list of prompts has changed. + // + // Stability: development + McpMethodNameNotificationsPromptsListChanged = McpMethodNameKey.String("notifications/prompts/list_changed") + // Request to list tools available on server. + // + // Stability: development + McpMethodNameToolsList = McpMethodNameKey.String("tools/list") + // Request to call a tool. + // + // Stability: development + McpMethodNameToolsCall = McpMethodNameKey.String("tools/call") + // Notification indicating that the list of tools has changed. + // + // Stability: development + McpMethodNameNotificationsToolsListChanged = McpMethodNameKey.String("notifications/tools/list_changed") + // Request to set the logging level. + // + // Stability: development + McpMethodNameLoggingSetLevel = McpMethodNameKey.String("logging/setLevel") + // Notification indicating that a message has been received. + // + // Stability: development + McpMethodNameNotificationsMessage = McpMethodNameKey.String("notifications/message") + // Request to create a sampling message. + // + // Stability: development + McpMethodNameSamplingCreateMessage = McpMethodNameKey.String("sampling/createMessage") + // Request to complete a prompt. + // + // Stability: development + McpMethodNameCompletionComplete = McpMethodNameKey.String("completion/complete") + // Request to list roots available on server. + // + // Stability: development + McpMethodNameRootsList = McpMethodNameKey.String("roots/list") + // Notification indicating that the list of roots has changed. + // + // Stability: development + McpMethodNameNotificationsRootsListChanged = McpMethodNameKey.String("notifications/roots/list_changed") + // Request from the server to elicit additional information from the user via + // the client + // + // Stability: development + McpMethodNameElicitationCreate = McpMethodNameKey.String("elicitation/create") +) + // Namespace: messaging const ( // MessagingBatchMessageCountKey is the attribute Key conforming to the @@ -10772,6 +11622,47 @@ var ( NetworkTypeIPv6 = NetworkTypeKey.String("ipv6") ) +// Namespace: nfs +const ( + // NfsOperationNameKey is the attribute Key conforming to the + // "nfs.operation.name" semantic conventions. It represents the NFSv4+ operation + // name. + // + // Type: string + // RequirementLevel: Recommended + // Stability: Development + // + // Examples: "OPEN", "READ", "GETATTR" + NfsOperationNameKey = attribute.Key("nfs.operation.name") + + // NfsServerRepcacheStatusKey is the attribute Key conforming to the + // "nfs.server.repcache.status" semantic conventions. It represents the linux: + // one of "hit" (NFSD_STATS_RC_HITS), "miss" (NFSD_STATS_RC_MISSES), or + // "nocache" (NFSD_STATS_RC_NOCACHE -- uncacheable). + // + // Type: string + // RequirementLevel: Recommended + // Stability: Development + // + // Examples: hit + NfsServerRepcacheStatusKey = attribute.Key("nfs.server.repcache.status") +) + +// NfsOperationName returns an attribute KeyValue conforming to the +// "nfs.operation.name" semantic conventions. It represents the NFSv4+ operation +// name. +func NfsOperationName(val string) attribute.KeyValue { + return NfsOperationNameKey.String(val) +} + +// NfsServerRepcacheStatus returns an attribute KeyValue conforming to the +// "nfs.server.repcache.status" semantic conventions. It represents the linux: +// one of "hit" (NFSD_STATS_RC_HITS), "miss" (NFSD_STATS_RC_MISSES), or "nocache" +// (NFSD_STATS_RC_NOCACHE -- uncacheable). +func NfsServerRepcacheStatus(val string) attribute.KeyValue { + return NfsServerRepcacheStatusKey.String(val) +} + // Namespace: oci const ( // OCIManifestDigestKey is the attribute Key conforming to the @@ -10803,6 +11694,80 @@ func OCIManifestDigest(val string) attribute.KeyValue { return OCIManifestDigestKey.String(val) } +// Namespace: onc_rpc +const ( + // OncRPCProcedureNameKey is the attribute Key conforming to the + // "onc_rpc.procedure.name" semantic conventions. It represents the ONC/Sun RPC + // procedure name. + // + // Type: string + // RequirementLevel: Recommended + // Stability: Development + // + // Examples: "OPEN", "READ", "GETATTR" + OncRPCProcedureNameKey = attribute.Key("onc_rpc.procedure.name") + + // OncRPCProcedureNumberKey is the attribute Key conforming to the + // "onc_rpc.procedure.number" semantic conventions. It represents the ONC/Sun + // RPC procedure number. + // + // Type: int + // RequirementLevel: Recommended + // Stability: Development + // + // Examples: + OncRPCProcedureNumberKey = attribute.Key("onc_rpc.procedure.number") + + // OncRPCProgramNameKey is the attribute Key conforming to the + // "onc_rpc.program.name" semantic conventions. It represents the ONC/Sun RPC + // program name. + // + // Type: string + // RequirementLevel: Recommended + // Stability: Development + // + // Examples: "portmapper", "nfs" + OncRPCProgramNameKey = attribute.Key("onc_rpc.program.name") + + // OncRPCVersionKey is the attribute Key conforming to the "onc_rpc.version" + // semantic conventions. It represents the ONC/Sun RPC program version. + // + // Type: int + // RequirementLevel: Recommended + // Stability: Development + // + // Examples: + OncRPCVersionKey = attribute.Key("onc_rpc.version") +) + +// OncRPCProcedureName returns an attribute KeyValue conforming to the +// "onc_rpc.procedure.name" semantic conventions. It represents the ONC/Sun RPC +// procedure name. +func OncRPCProcedureName(val string) attribute.KeyValue { + return OncRPCProcedureNameKey.String(val) +} + +// OncRPCProcedureNumber returns an attribute KeyValue conforming to the +// "onc_rpc.procedure.number" semantic conventions. It represents the ONC/Sun RPC +// procedure number. +func OncRPCProcedureNumber(val int) attribute.KeyValue { + return OncRPCProcedureNumberKey.Int(val) +} + +// OncRPCProgramName returns an attribute KeyValue conforming to the +// "onc_rpc.program.name" semantic conventions. It represents the ONC/Sun RPC +// program name. +func OncRPCProgramName(val string) attribute.KeyValue { + return OncRPCProgramNameKey.String(val) +} + +// OncRPCVersion returns an attribute KeyValue conforming to the +// "onc_rpc.version" semantic conventions. It represents the ONC/Sun RPC program +// version. +func OncRPCVersion(val int) attribute.KeyValue { + return OncRPCVersionKey.Int(val) +} + // Namespace: openai const ( // OpenAIRequestServiceTierKey is the attribute Key conforming to the @@ -10863,26 +11828,65 @@ var ( OpenAIRequestServiceTierDefault = OpenAIRequestServiceTierKey.String("default") ) -// Namespace: opentracing +// Namespace: openshift const ( - // OpenTracingRefTypeKey is the attribute Key conforming to the - // "opentracing.ref_type" semantic conventions. It represents the parent-child - // Reference type. + // OpenShiftClusterquotaNameKey is the attribute Key conforming to the + // "openshift.clusterquota.name" semantic conventions. It represents the name of + // the cluster quota. // - // Type: Enum + // Type: string // RequirementLevel: Recommended // Stability: Development // - // Examples: - // Note: The causal relationship between a child Span and a parent Span. - OpenTracingRefTypeKey = attribute.Key("opentracing.ref_type") -) + // Examples: "opentelemetry" + OpenShiftClusterquotaNameKey = attribute.Key("openshift.clusterquota.name") -// Enum values for opentracing.ref_type -var ( - // The parent Span depends on the child Span in some capacity - // Stability: development - OpenTracingRefTypeChildOf = OpenTracingRefTypeKey.String("child_of") + // OpenShiftClusterquotaUIDKey is the attribute Key conforming to the + // "openshift.clusterquota.uid" semantic conventions. It represents the UID of + // the cluster quota. + // + // Type: string + // RequirementLevel: Recommended + // Stability: Development + // + // Examples: "275ecb36-5aa8-4c2a-9c47-d8bb681b9aff" + OpenShiftClusterquotaUIDKey = attribute.Key("openshift.clusterquota.uid") +) + +// OpenShiftClusterquotaName returns an attribute KeyValue conforming to the +// "openshift.clusterquota.name" semantic conventions. It represents the name of +// the cluster quota. +func OpenShiftClusterquotaName(val string) attribute.KeyValue { + return OpenShiftClusterquotaNameKey.String(val) +} + +// OpenShiftClusterquotaUID returns an attribute KeyValue conforming to the +// "openshift.clusterquota.uid" semantic conventions. It represents the UID of +// the cluster quota. +func OpenShiftClusterquotaUID(val string) attribute.KeyValue { + return OpenShiftClusterquotaUIDKey.String(val) +} + +// Namespace: opentracing +const ( + // OpenTracingRefTypeKey is the attribute Key conforming to the + // "opentracing.ref_type" semantic conventions. It represents the parent-child + // Reference type. + // + // Type: Enum + // RequirementLevel: Recommended + // Stability: Development + // + // Examples: + // Note: The causal relationship between a child Span and a parent Span. + OpenTracingRefTypeKey = attribute.Key("opentracing.ref_type") +) + +// Enum values for opentracing.ref_type +var ( + // The parent Span depends on the child Span in some capacity + // Stability: development + OpenTracingRefTypeChildOf = OpenTracingRefTypeKey.String("child_of") // The parent Span doesn't depend in any way on the result of the child Span // Stability: development OpenTracingRefTypeFollowsFrom = OpenTracingRefTypeKey.String("follows_from") @@ -11064,6 +12068,20 @@ const ( // E.g. for Java the fully qualified classname SHOULD be used in this case. OTelComponentTypeKey = attribute.Key("otel.component.type") + // OTelEventNameKey is the attribute Key conforming to the "otel.event.name" + // semantic conventions. It represents the identifies the class / type of event. + // + // Type: string + // RequirementLevel: Recommended + // Stability: Development + // + // Examples: "browser.mouse.click", "device.app.lifecycle" + // Note: This attribute SHOULD be used by non-OTLP exporters when destination + // does not support `EventName` or equivalent field. This attribute MAY be used + // by applications using existing logging libraries so that it can be used to + // set the `EventName` field by Collector or SDK components. + OTelEventNameKey = attribute.Key("otel.event.name") + // OTelScopeNameKey is the attribute Key conforming to the "otel.scope.name" // semantic conventions. It represents the name of the instrumentation scope - ( // `InstrumentationScope.Name` in OTLP). @@ -11153,6 +12171,13 @@ func OTelComponentName(val string) attribute.KeyValue { return OTelComponentNameKey.String(val) } +// OTelEventName returns an attribute KeyValue conforming to the +// "otel.event.name" semantic conventions. It represents the identifies the class +// / type of event. +func OTelEventName(val string) attribute.KeyValue { + return OTelEventNameKey.String(val) +} + // OTelScopeName returns an attribute KeyValue conforming to the // "otel.scope.name" semantic conventions. It represents the name of the // instrumentation scope - (`InstrumentationScope.Name` in OTLP). @@ -11290,31 +12315,183 @@ var ( OTelStatusCodeError = OTelStatusCodeKey.String("ERROR") ) -// Namespace: peer +// Namespace: pprof const ( - // PeerServiceKey is the attribute Key conforming to the "peer.service" semantic - // conventions. It represents the [`service.name`] of the remote service. SHOULD - // be equal to the actual `service.name` resource attribute of the remote - // service if any. + // PprofLocationIsFoldedKey is the attribute Key conforming to the + // "pprof.location.is_folded" semantic conventions. It represents the provides + // an indication that multiple symbols map to this location's address, for + // example due to identical code folding by the linker. In that case the line + // information represents one of the multiple symbols. This field must be + // recomputed when the symbolization state of the profile changes. + // + // Type: boolean + // RequirementLevel: Recommended + // Stability: Development + // + // Examples: + PprofLocationIsFoldedKey = attribute.Key("pprof.location.is_folded") + + // PprofMappingHasFilenamesKey is the attribute Key conforming to the + // "pprof.mapping.has_filenames" semantic conventions. It represents the + // indicates that there are filenames related to this mapping. + // + // Type: boolean + // RequirementLevel: Recommended + // Stability: Development + // + // Examples: + PprofMappingHasFilenamesKey = attribute.Key("pprof.mapping.has_filenames") + + // PprofMappingHasFunctionsKey is the attribute Key conforming to the + // "pprof.mapping.has_functions" semantic conventions. It represents the + // indicates that there are functions related to this mapping. + // + // Type: boolean + // RequirementLevel: Recommended + // Stability: Development + // + // Examples: + PprofMappingHasFunctionsKey = attribute.Key("pprof.mapping.has_functions") + + // PprofMappingHasInlineFramesKey is the attribute Key conforming to the + // "pprof.mapping.has_inline_frames" semantic conventions. It represents the + // indicates that there are inline frames related to this mapping. + // + // Type: boolean + // RequirementLevel: Recommended + // Stability: Development + // + // Examples: + PprofMappingHasInlineFramesKey = attribute.Key("pprof.mapping.has_inline_frames") + + // PprofMappingHasLineNumbersKey is the attribute Key conforming to the + // "pprof.mapping.has_line_numbers" semantic conventions. It represents the + // indicates that there are line numbers related to this mapping. + // + // Type: boolean + // RequirementLevel: Recommended + // Stability: Development + // + // Examples: + PprofMappingHasLineNumbersKey = attribute.Key("pprof.mapping.has_line_numbers") + + // PprofProfileCommentKey is the attribute Key conforming to the + // "pprof.profile.comment" semantic conventions. It represents the free-form + // text associated with the profile. This field should not be used to store any + // machine-readable information, it is only for human-friendly content. + // + // Type: string[] + // RequirementLevel: Recommended + // Stability: Development + // + // Examples: "hello world", "bazinga" + PprofProfileCommentKey = attribute.Key("pprof.profile.comment") + + // PprofProfileDocURLKey is the attribute Key conforming to the + // "pprof.profile.doc_url" semantic conventions. It represents the documentation + // link for this profile type. // // Type: string // RequirementLevel: Recommended // Stability: Development // - // Examples: AuthTokenCache + // Examples: "http://pprof.example.com/cpu-profile.html" + // Note: The URL must be absolute and may be missing if the profile was + // generated by code that did not supply a link + PprofProfileDocURLKey = attribute.Key("pprof.profile.doc_url") + + // PprofProfileDropFramesKey is the attribute Key conforming to the + // "pprof.profile.drop_frames" semantic conventions. It represents the frames + // with Function.function_name fully matching the regexp will be dropped from + // the samples, along with their successors. // - // [`service.name`]: /docs/resource/README.md#service - PeerServiceKey = attribute.Key("peer.service") + // Type: string + // RequirementLevel: Recommended + // Stability: Development + // + // Examples: "/foobar/" + PprofProfileDropFramesKey = attribute.Key("pprof.profile.drop_frames") + + // PprofProfileKeepFramesKey is the attribute Key conforming to the + // "pprof.profile.keep_frames" semantic conventions. It represents the frames + // with Function.function_name fully matching the regexp will be kept, even if + // it matches drop_frames. + // + // Type: string + // RequirementLevel: Recommended + // Stability: Development + // + // Examples: "/bazinga/" + PprofProfileKeepFramesKey = attribute.Key("pprof.profile.keep_frames") ) -// PeerService returns an attribute KeyValue conforming to the "peer.service" -// semantic conventions. It represents the [`service.name`] of the remote -// service. SHOULD be equal to the actual `service.name` resource attribute of -// the remote service if any. -// -// [`service.name`]: /docs/resource/README.md#service -func PeerService(val string) attribute.KeyValue { - return PeerServiceKey.String(val) +// PprofLocationIsFolded returns an attribute KeyValue conforming to the +// "pprof.location.is_folded" semantic conventions. It represents the provides an +// indication that multiple symbols map to this location's address, for example +// due to identical code folding by the linker. In that case the line information +// represents one of the multiple symbols. This field must be recomputed when the +// symbolization state of the profile changes. +func PprofLocationIsFolded(val bool) attribute.KeyValue { + return PprofLocationIsFoldedKey.Bool(val) +} + +// PprofMappingHasFilenames returns an attribute KeyValue conforming to the +// "pprof.mapping.has_filenames" semantic conventions. It represents the +// indicates that there are filenames related to this mapping. +func PprofMappingHasFilenames(val bool) attribute.KeyValue { + return PprofMappingHasFilenamesKey.Bool(val) +} + +// PprofMappingHasFunctions returns an attribute KeyValue conforming to the +// "pprof.mapping.has_functions" semantic conventions. It represents the +// indicates that there are functions related to this mapping. +func PprofMappingHasFunctions(val bool) attribute.KeyValue { + return PprofMappingHasFunctionsKey.Bool(val) +} + +// PprofMappingHasInlineFrames returns an attribute KeyValue conforming to the +// "pprof.mapping.has_inline_frames" semantic conventions. It represents the +// indicates that there are inline frames related to this mapping. +func PprofMappingHasInlineFrames(val bool) attribute.KeyValue { + return PprofMappingHasInlineFramesKey.Bool(val) +} + +// PprofMappingHasLineNumbers returns an attribute KeyValue conforming to the +// "pprof.mapping.has_line_numbers" semantic conventions. It represents the +// indicates that there are line numbers related to this mapping. +func PprofMappingHasLineNumbers(val bool) attribute.KeyValue { + return PprofMappingHasLineNumbersKey.Bool(val) +} + +// PprofProfileComment returns an attribute KeyValue conforming to the +// "pprof.profile.comment" semantic conventions. It represents the free-form text +// associated with the profile. This field should not be used to store any +// machine-readable information, it is only for human-friendly content. +func PprofProfileComment(val ...string) attribute.KeyValue { + return PprofProfileCommentKey.StringSlice(val) +} + +// PprofProfileDocURL returns an attribute KeyValue conforming to the +// "pprof.profile.doc_url" semantic conventions. It represents the documentation +// link for this profile type. +func PprofProfileDocURL(val string) attribute.KeyValue { + return PprofProfileDocURLKey.String(val) +} + +// PprofProfileDropFrames returns an attribute KeyValue conforming to the +// "pprof.profile.drop_frames" semantic conventions. It represents the frames +// with Function.function_name fully matching the regexp will be dropped from the +// samples, along with their successors. +func PprofProfileDropFrames(val string) attribute.KeyValue { + return PprofProfileDropFramesKey.String(val) +} + +// PprofProfileKeepFrames returns an attribute KeyValue conforming to the +// "pprof.profile.keep_frames" semantic conventions. It represents the frames +// with Function.function_name fully matching the regexp will be kept, even if it +// matches drop_frames. +func PprofProfileKeepFrames(val string) attribute.KeyValue { + return PprofProfileKeepFramesKey.String(val) } // Namespace: process @@ -11378,7 +12555,7 @@ const ( ProcessCommandLineKey = attribute.Key("process.command_line") // ProcessContextSwitchTypeKey is the attribute Key conforming to the - // "process.context_switch_type" semantic conventions. It represents the + // "process.context_switch.type" semantic conventions. It represents the // specifies whether the context switches for this data point were voluntary or // involuntary. // @@ -11387,7 +12564,7 @@ const ( // Stability: Development // // Examples: - ProcessContextSwitchTypeKey = attribute.Key("process.context_switch_type") + ProcessContextSwitchTypeKey = attribute.Key("process.context_switch.type") // ProcessCreationTimeKey is the attribute Key conforming to the // "process.creation.time" semantic conventions. It represents the date and time @@ -11534,18 +12711,6 @@ const ( // Examples: "root" ProcessOwnerKey = attribute.Key("process.owner") - // ProcessPagingFaultTypeKey is the attribute Key conforming to the - // "process.paging.fault_type" semantic conventions. It represents the type of - // page fault for this data point. Type `major` is for major/hard page faults, - // and `minor` is for minor/soft page faults. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: - ProcessPagingFaultTypeKey = attribute.Key("process.paging.fault_type") - // ProcessParentPIDKey is the attribute Key conforming to the // "process.parent_pid" semantic conventions. It represents the parent Process // identifier (PPID). @@ -11657,6 +12822,19 @@ const ( // Examples: 14 ProcessSessionLeaderPIDKey = attribute.Key("process.session_leader.pid") + // ProcessStateKey is the attribute Key conforming to the "process.state" + // semantic conventions. It represents the process state, e.g., + // [Linux Process State Codes]. + // + // Type: Enum + // RequirementLevel: Recommended + // Stability: Development + // + // Examples: "running" + // + // [Linux Process State Codes]: https://man7.org/linux/man-pages/man1/ps.1.html#PROCESS_STATE_CODES + ProcessStateKey = attribute.Key("process.state") + // ProcessTitleKey is the attribute Key conforming to the "process.title" // semantic conventions. It represents the process title (proctitle). // @@ -11958,7 +13136,7 @@ func ProcessWorkingDirectory(val string) attribute.KeyValue { return ProcessWorkingDirectoryKey.String(val) } -// Enum values for process.context_switch_type +// Enum values for process.context_switch.type var ( // voluntary // Stability: development @@ -11968,14 +13146,20 @@ var ( ProcessContextSwitchTypeInvoluntary = ProcessContextSwitchTypeKey.String("involuntary") ) -// Enum values for process.paging.fault_type +// Enum values for process.state var ( - // major + // running // Stability: development - ProcessPagingFaultTypeMajor = ProcessPagingFaultTypeKey.String("major") - // minor + ProcessStateRunning = ProcessStateKey.String("running") + // sleeping + // Stability: development + ProcessStateSleeping = ProcessStateKey.String("sleeping") + // stopped + // Stability: development + ProcessStateStopped = ProcessStateKey.String("stopped") + // defunct // Stability: development - ProcessPagingFaultTypeMinor = ProcessPagingFaultTypeKey.String("minor") + ProcessStateDefunct = ProcessStateKey.String("defunct") ) // Namespace: profile @@ -12074,80 +13258,6 @@ var ( // Namespace: rpc const ( - // RPCConnectRPCErrorCodeKey is the attribute Key conforming to the - // "rpc.connect_rpc.error_code" semantic conventions. It represents the - // [error codes] of the Connect request. Error codes are always string values. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: - // - // [error codes]: https://connectrpc.com//docs/protocol/#error-codes - RPCConnectRPCErrorCodeKey = attribute.Key("rpc.connect_rpc.error_code") - - // RPCGRPCStatusCodeKey is the attribute Key conforming to the - // "rpc.grpc.status_code" semantic conventions. It represents the - // [numeric status code] of the gRPC request. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: - // - // [numeric status code]: https://github.com/grpc/grpc/blob/v1.33.2/doc/statuscodes.md - RPCGRPCStatusCodeKey = attribute.Key("rpc.grpc.status_code") - - // RPCJSONRPCErrorCodeKey is the attribute Key conforming to the - // "rpc.jsonrpc.error_code" semantic conventions. It represents the `error.code` - // property of response if it is an error response. - // - // Type: int - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: -32700, 100 - RPCJSONRPCErrorCodeKey = attribute.Key("rpc.jsonrpc.error_code") - - // RPCJSONRPCErrorMessageKey is the attribute Key conforming to the - // "rpc.jsonrpc.error_message" semantic conventions. It represents the - // `error.message` property of response if it is an error response. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "Parse error", "User already exists" - RPCJSONRPCErrorMessageKey = attribute.Key("rpc.jsonrpc.error_message") - - // RPCJSONRPCRequestIDKey is the attribute Key conforming to the - // "rpc.jsonrpc.request_id" semantic conventions. It represents the `id` - // property of request or response. Since protocol allows id to be int, string, - // `null` or missing (for notifications), value is expected to be cast to string - // for simplicity. Use empty string in case of `null` value. Omit entirely if - // this is a notification. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "10", "request-7", "" - RPCJSONRPCRequestIDKey = attribute.Key("rpc.jsonrpc.request_id") - - // RPCJSONRPCVersionKey is the attribute Key conforming to the - // "rpc.jsonrpc.version" semantic conventions. It represents the protocol - // version as in `jsonrpc` property of request/response. Since JSON-RPC 1.0 - // doesn't specify this, the value can be omitted. - // - // Type: string - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "2.0", "1.0" - RPCJSONRPCVersionKey = attribute.Key("rpc.jsonrpc.version") - // RPCMessageCompressedSizeKey is the attribute Key conforming to the // "rpc.message.compressed_size" semantic conventions. It represents the // compressed size of the message in bytes. @@ -12195,114 +13305,84 @@ const ( RPCMessageUncompressedSizeKey = attribute.Key("rpc.message.uncompressed_size") // RPCMethodKey is the attribute Key conforming to the "rpc.method" semantic - // conventions. It represents the name of the (logical) method being called, - // must be equal to the $method part in the span name. + // conventions. It represents the fully-qualified logical name of the method + // from the RPC interface perspective. // // Type: string // RequirementLevel: Recommended // Stability: Development // - // Examples: exampleMethod - // Note: This is the logical name of the method from the RPC interface - // perspective, which can be different from the name of any implementing - // method/function. The `code.function.name` attribute may be used to store the - // latter (e.g., method actually executing the call on the server side, RPC - // client stub method on the client side). + // Examples: "com.example.ExampleService/exampleMethod", "EchoService/Echo", + // "_OTHER" + // Note: The method name MAY have unbounded cardinality in edge or error cases. + // + // Some RPC frameworks or libraries provide a fixed set of recognized methods + // for client stubs and server implementations. Instrumentations for such + // frameworks MUST set this attribute to the original method name only + // when the method is recognized by the framework or library. + // + // When the method is not recognized, for example, when the server receives + // a request for a method that is not predefined on the server, or when + // instrumentation is not able to reliably detect if the method is predefined, + // the attribute MUST be set to `_OTHER`. In such cases, tracing + // instrumentations MUST also set `rpc.method_original` attribute to + // the original method value. + // + // If the RPC instrumentation could end up converting valid RPC methods to + // `_OTHER`, then it SHOULD provide a way to configure the list of recognized + // RPC methods. + // + // The `rpc.method` can be different from the name of any implementing + // method/function. + // The `code.function.name` attribute may be used to record the fully-qualified + // method actually executing the call on the server side, or the + // RPC client stub method on the client side. RPCMethodKey = attribute.Key("rpc.method") - // RPCServiceKey is the attribute Key conforming to the "rpc.service" semantic - // conventions. It represents the full (logical) name of the service being - // called, including its package name, if applicable. + // RPCMethodOriginalKey is the attribute Key conforming to the + // "rpc.method_original" semantic conventions. It represents the original name + // of the method used by the client. + // + // Type: string + // RequirementLevel: Recommended + // Stability: Development + // + // Examples: "com.myservice.EchoService/catchAll", + // "com.myservice.EchoService/unknownMethod", "InvalidMethod" + RPCMethodOriginalKey = attribute.Key("rpc.method_original") + + // RPCResponseStatusCodeKey is the attribute Key conforming to the + // "rpc.response.status_code" semantic conventions. It represents the status + // code of the RPC returned by the RPC server or generated by the client. // // Type: string // RequirementLevel: Recommended // Stability: Development // - // Examples: myservice.EchoService - // Note: This is the logical name of the service from the RPC interface - // perspective, which can be different from the name of any implementing class. - // The `code.namespace` attribute may be used to store the latter (despite the - // attribute name, it may include a class name; e.g., class with method actually - // executing the call on the server side, RPC client stub class on the client - // side). - RPCServiceKey = attribute.Key("rpc.service") + // Examples: "OK", "DEADLINE_EXCEEDED", "-32602" + // Note: Usually it represents an error code, but may also represent partial + // success, warning, or differentiate between various types of successful + // outcomes. + // Semantic conventions for individual RPC frameworks SHOULD document what + // `rpc.response.status_code` means in the context of that system and which + // values are considered to represent errors. + RPCResponseStatusCodeKey = attribute.Key("rpc.response.status_code") - // RPCSystemKey is the attribute Key conforming to the "rpc.system" semantic - // conventions. It represents a string identifying the remoting system. See - // below for a list of well-known identifiers. + // RPCSystemNameKey is the attribute Key conforming to the "rpc.system.name" + // semantic conventions. It represents the Remote Procedure Call (RPC) system. // // Type: Enum // RequirementLevel: Recommended // Stability: Development // // Examples: - RPCSystemKey = attribute.Key("rpc.system") + // Note: The client and server RPC systems may differ for the same RPC + // interaction. For example, a client may use Apache Dubbo or Connect RPC to + // communicate with a server that uses gRPC since both protocols provide + // compatibility with gRPC. + RPCSystemNameKey = attribute.Key("rpc.system.name") ) -// RPCConnectRPCRequestMetadata returns an attribute KeyValue conforming to the -// "rpc.connect_rpc.request.metadata" semantic conventions. It represents the -// connect request metadata, `` being the normalized Connect Metadata key -// (lowercase), the value being the metadata values. -func RPCConnectRPCRequestMetadata(key string, val ...string) attribute.KeyValue { - return attribute.StringSlice("rpc.connect_rpc.request.metadata."+key, val) -} - -// RPCConnectRPCResponseMetadata returns an attribute KeyValue conforming to the -// "rpc.connect_rpc.response.metadata" semantic conventions. It represents the -// connect response metadata, `` being the normalized Connect Metadata key -// (lowercase), the value being the metadata values. -func RPCConnectRPCResponseMetadata(key string, val ...string) attribute.KeyValue { - return attribute.StringSlice("rpc.connect_rpc.response.metadata."+key, val) -} - -// RPCGRPCRequestMetadata returns an attribute KeyValue conforming to the -// "rpc.grpc.request.metadata" semantic conventions. It represents the gRPC -// request metadata, `` being the normalized gRPC Metadata key (lowercase), -// the value being the metadata values. -func RPCGRPCRequestMetadata(key string, val ...string) attribute.KeyValue { - return attribute.StringSlice("rpc.grpc.request.metadata."+key, val) -} - -// RPCGRPCResponseMetadata returns an attribute KeyValue conforming to the -// "rpc.grpc.response.metadata" semantic conventions. It represents the gRPC -// response metadata, `` being the normalized gRPC Metadata key (lowercase), -// the value being the metadata values. -func RPCGRPCResponseMetadata(key string, val ...string) attribute.KeyValue { - return attribute.StringSlice("rpc.grpc.response.metadata."+key, val) -} - -// RPCJSONRPCErrorCode returns an attribute KeyValue conforming to the -// "rpc.jsonrpc.error_code" semantic conventions. It represents the `error.code` -// property of response if it is an error response. -func RPCJSONRPCErrorCode(val int) attribute.KeyValue { - return RPCJSONRPCErrorCodeKey.Int(val) -} - -// RPCJSONRPCErrorMessage returns an attribute KeyValue conforming to the -// "rpc.jsonrpc.error_message" semantic conventions. It represents the -// `error.message` property of response if it is an error response. -func RPCJSONRPCErrorMessage(val string) attribute.KeyValue { - return RPCJSONRPCErrorMessageKey.String(val) -} - -// RPCJSONRPCRequestID returns an attribute KeyValue conforming to the -// "rpc.jsonrpc.request_id" semantic conventions. It represents the `id` property -// of request or response. Since protocol allows id to be int, string, `null` or -// missing (for notifications), value is expected to be cast to string for -// simplicity. Use empty string in case of `null` value. Omit entirely if this is -// a notification. -func RPCJSONRPCRequestID(val string) attribute.KeyValue { - return RPCJSONRPCRequestIDKey.String(val) -} - -// RPCJSONRPCVersion returns an attribute KeyValue conforming to the -// "rpc.jsonrpc.version" semantic conventions. It represents the protocol version -// as in `jsonrpc` property of request/response. Since JSON-RPC 1.0 doesn't -// specify this, the value can be omitted. -func RPCJSONRPCVersion(val string) attribute.KeyValue { - return RPCJSONRPCVersionKey.String(val) -} - // RPCMessageCompressedSize returns an attribute KeyValue conforming to the // "rpc.message.compressed_size" semantic conventions. It represents the // compressed size of the message in bytes. @@ -12325,125 +13405,41 @@ func RPCMessageUncompressedSize(val int) attribute.KeyValue { } // RPCMethod returns an attribute KeyValue conforming to the "rpc.method" -// semantic conventions. It represents the name of the (logical) method being -// called, must be equal to the $method part in the span name. +// semantic conventions. It represents the fully-qualified logical name of the +// method from the RPC interface perspective. func RPCMethod(val string) attribute.KeyValue { return RPCMethodKey.String(val) } -// RPCService returns an attribute KeyValue conforming to the "rpc.service" -// semantic conventions. It represents the full (logical) name of the service -// being called, including its package name, if applicable. -func RPCService(val string) attribute.KeyValue { - return RPCServiceKey.String(val) +// RPCMethodOriginal returns an attribute KeyValue conforming to the +// "rpc.method_original" semantic conventions. It represents the original name of +// the method used by the client. +func RPCMethodOriginal(val string) attribute.KeyValue { + return RPCMethodOriginalKey.String(val) } -// Enum values for rpc.connect_rpc.error_code -var ( - // cancelled - // Stability: development - RPCConnectRPCErrorCodeCancelled = RPCConnectRPCErrorCodeKey.String("cancelled") - // unknown - // Stability: development - RPCConnectRPCErrorCodeUnknown = RPCConnectRPCErrorCodeKey.String("unknown") - // invalid_argument - // Stability: development - RPCConnectRPCErrorCodeInvalidArgument = RPCConnectRPCErrorCodeKey.String("invalid_argument") - // deadline_exceeded - // Stability: development - RPCConnectRPCErrorCodeDeadlineExceeded = RPCConnectRPCErrorCodeKey.String("deadline_exceeded") - // not_found - // Stability: development - RPCConnectRPCErrorCodeNotFound = RPCConnectRPCErrorCodeKey.String("not_found") - // already_exists - // Stability: development - RPCConnectRPCErrorCodeAlreadyExists = RPCConnectRPCErrorCodeKey.String("already_exists") - // permission_denied - // Stability: development - RPCConnectRPCErrorCodePermissionDenied = RPCConnectRPCErrorCodeKey.String("permission_denied") - // resource_exhausted - // Stability: development - RPCConnectRPCErrorCodeResourceExhausted = RPCConnectRPCErrorCodeKey.String("resource_exhausted") - // failed_precondition - // Stability: development - RPCConnectRPCErrorCodeFailedPrecondition = RPCConnectRPCErrorCodeKey.String("failed_precondition") - // aborted - // Stability: development - RPCConnectRPCErrorCodeAborted = RPCConnectRPCErrorCodeKey.String("aborted") - // out_of_range - // Stability: development - RPCConnectRPCErrorCodeOutOfRange = RPCConnectRPCErrorCodeKey.String("out_of_range") - // unimplemented - // Stability: development - RPCConnectRPCErrorCodeUnimplemented = RPCConnectRPCErrorCodeKey.String("unimplemented") - // internal - // Stability: development - RPCConnectRPCErrorCodeInternal = RPCConnectRPCErrorCodeKey.String("internal") - // unavailable - // Stability: development - RPCConnectRPCErrorCodeUnavailable = RPCConnectRPCErrorCodeKey.String("unavailable") - // data_loss - // Stability: development - RPCConnectRPCErrorCodeDataLoss = RPCConnectRPCErrorCodeKey.String("data_loss") - // unauthenticated - // Stability: development - RPCConnectRPCErrorCodeUnauthenticated = RPCConnectRPCErrorCodeKey.String("unauthenticated") -) +// RPCRequestMetadata returns an attribute KeyValue conforming to the +// "rpc.request.metadata" semantic conventions. It represents the RPC request +// metadata, `` being the normalized RPC metadata key (lowercase), the value +// being the metadata values. +func RPCRequestMetadata(key string, val ...string) attribute.KeyValue { + return attribute.StringSlice("rpc.request.metadata."+key, val) +} -// Enum values for rpc.grpc.status_code -var ( - // OK - // Stability: development - RPCGRPCStatusCodeOk = RPCGRPCStatusCodeKey.Int(0) - // CANCELLED - // Stability: development - RPCGRPCStatusCodeCancelled = RPCGRPCStatusCodeKey.Int(1) - // UNKNOWN - // Stability: development - RPCGRPCStatusCodeUnknown = RPCGRPCStatusCodeKey.Int(2) - // INVALID_ARGUMENT - // Stability: development - RPCGRPCStatusCodeInvalidArgument = RPCGRPCStatusCodeKey.Int(3) - // DEADLINE_EXCEEDED - // Stability: development - RPCGRPCStatusCodeDeadlineExceeded = RPCGRPCStatusCodeKey.Int(4) - // NOT_FOUND - // Stability: development - RPCGRPCStatusCodeNotFound = RPCGRPCStatusCodeKey.Int(5) - // ALREADY_EXISTS - // Stability: development - RPCGRPCStatusCodeAlreadyExists = RPCGRPCStatusCodeKey.Int(6) - // PERMISSION_DENIED - // Stability: development - RPCGRPCStatusCodePermissionDenied = RPCGRPCStatusCodeKey.Int(7) - // RESOURCE_EXHAUSTED - // Stability: development - RPCGRPCStatusCodeResourceExhausted = RPCGRPCStatusCodeKey.Int(8) - // FAILED_PRECONDITION - // Stability: development - RPCGRPCStatusCodeFailedPrecondition = RPCGRPCStatusCodeKey.Int(9) - // ABORTED - // Stability: development - RPCGRPCStatusCodeAborted = RPCGRPCStatusCodeKey.Int(10) - // OUT_OF_RANGE - // Stability: development - RPCGRPCStatusCodeOutOfRange = RPCGRPCStatusCodeKey.Int(11) - // UNIMPLEMENTED - // Stability: development - RPCGRPCStatusCodeUnimplemented = RPCGRPCStatusCodeKey.Int(12) - // INTERNAL - // Stability: development - RPCGRPCStatusCodeInternal = RPCGRPCStatusCodeKey.Int(13) - // UNAVAILABLE - // Stability: development - RPCGRPCStatusCodeUnavailable = RPCGRPCStatusCodeKey.Int(14) - // DATA_LOSS - // Stability: development - RPCGRPCStatusCodeDataLoss = RPCGRPCStatusCodeKey.Int(15) - // UNAUTHENTICATED - // Stability: development - RPCGRPCStatusCodeUnauthenticated = RPCGRPCStatusCodeKey.Int(16) -) +// RPCResponseMetadata returns an attribute KeyValue conforming to the +// "rpc.response.metadata" semantic conventions. It represents the RPC response +// metadata, `` being the normalized RPC metadata key (lowercase), the value +// being the metadata values. +func RPCResponseMetadata(key string, val ...string) attribute.KeyValue { + return attribute.StringSlice("rpc.response.metadata."+key, val) +} + +// RPCResponseStatusCode returns an attribute KeyValue conforming to the +// "rpc.response.status_code" semantic conventions. It represents the status code +// of the RPC returned by the RPC server or generated by the client. +func RPCResponseStatusCode(val string) attribute.KeyValue { + return RPCResponseStatusCodeKey.String(val) +} // Enum values for rpc.message.type var ( @@ -12455,23 +13451,28 @@ var ( RPCMessageTypeReceived = RPCMessageTypeKey.String("RECEIVED") ) -// Enum values for rpc.system +// Enum values for rpc.system.name var ( - // gRPC + // [gRPC] // Stability: development - RPCSystemGRPC = RPCSystemKey.String("grpc") - // Java RMI - // Stability: development - RPCSystemJavaRmi = RPCSystemKey.String("java_rmi") - // .NET WCF + // + // [gRPC]: https://grpc.io/ + RPCSystemNameGRPC = RPCSystemNameKey.String("grpc") + // [Apache Dubbo] // Stability: development - RPCSystemDotnetWcf = RPCSystemKey.String("dotnet_wcf") - // Apache Dubbo + // + // [Apache Dubbo]: https://dubbo.apache.org/ + RPCSystemNameDubbo = RPCSystemNameKey.String("dubbo") + // [Connect RPC] // Stability: development - RPCSystemApacheDubbo = RPCSystemKey.String("apache_dubbo") - // Connect RPC + // + // [Connect RPC]: https://connectrpc.com/ + RPCSystemNameConnectrpc = RPCSystemNameKey.String("connectrpc") + // [JSON-RPC] // Stability: development - RPCSystemConnectRPC = RPCSystemKey.String("connect_rpc") + // + // [JSON-RPC]: https://www.jsonrpc.org/ + RPCSystemNameJSONRPC = RPCSystemNameKey.String("jsonrpc") ) // Namespace: security_rule @@ -12766,9 +13767,38 @@ const ( // namespace. ServiceNamespaceKey = attribute.Key("service.namespace") + // ServicePeerNameKey is the attribute Key conforming to the "service.peer.name" + // semantic conventions. It represents the logical name of the service on the + // other side of the connection. SHOULD be equal to the actual [`service.name`] + // resource attribute of the remote service if any. + // + // Type: string + // RequirementLevel: Recommended + // Stability: Development + // + // Examples: "shoppingcart" + // + // [`service.name`]: /docs/resource/README.md#service + ServicePeerNameKey = attribute.Key("service.peer.name") + + // ServicePeerNamespaceKey is the attribute Key conforming to the + // "service.peer.namespace" semantic conventions. It represents the logical + // namespace of the service on the other side of the connection. SHOULD be equal + // to the actual [`service.namespace`] resource attribute of the remote service + // if any. + // + // Type: string + // RequirementLevel: Recommended + // Stability: Development + // + // Examples: "Shop" + // + // [`service.namespace`]: /docs/resource/README.md#service + ServicePeerNamespaceKey = attribute.Key("service.peer.namespace") + // ServiceVersionKey is the attribute Key conforming to the "service.version" - // semantic conventions. It represents the version string of the service API or - // implementation. The format is not defined by these conventions. + // semantic conventions. It represents the version string of the service + // component. The format is not defined by these conventions. // // Type: string // RequirementLevel: Recommended @@ -12798,10 +13828,30 @@ func ServiceNamespace(val string) attribute.KeyValue { return ServiceNamespaceKey.String(val) } +// ServicePeerName returns an attribute KeyValue conforming to the +// "service.peer.name" semantic conventions. It represents the logical name of +// the service on the other side of the connection. SHOULD be equal to the actual +// [`service.name`] resource attribute of the remote service if any. +// +// [`service.name`]: /docs/resource/README.md#service +func ServicePeerName(val string) attribute.KeyValue { + return ServicePeerNameKey.String(val) +} + +// ServicePeerNamespace returns an attribute KeyValue conforming to the +// "service.peer.namespace" semantic conventions. It represents the logical +// namespace of the service on the other side of the connection. SHOULD be equal +// to the actual [`service.namespace`] resource attribute of the remote service +// if any. +// +// [`service.namespace`]: /docs/resource/README.md#service +func ServicePeerNamespace(val string) attribute.KeyValue { + return ServicePeerNamespaceKey.String(val) +} + // ServiceVersion returns an attribute KeyValue conforming to the // "service.version" semantic conventions. It represents the version string of -// the service API or implementation. The format is not defined by these -// conventions. +// the service component. The format is not defined by these conventions. func ServiceVersion(val string) attribute.KeyValue { return ServiceVersionKey.String(val) } @@ -12940,17 +13990,6 @@ func SourcePort(val int) attribute.KeyValue { // Namespace: system const ( - // SystemCPULogicalNumberKey is the attribute Key conforming to the - // "system.cpu.logical_number" semantic conventions. It represents the - // deprecated, use `cpu.logical_number` instead. - // - // Type: int - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: 1 - SystemCPULogicalNumberKey = attribute.Key("system.cpu.logical_number") - // SystemDeviceKey is the attribute Key conforming to the "system.device" // semantic conventions. It represents the device identifier. // @@ -13005,6 +14044,17 @@ const ( // Examples: "ext4" SystemFilesystemTypeKey = attribute.Key("system.filesystem.type") + // SystemMemoryLinuxSlabStateKey is the attribute Key conforming to the + // "system.memory.linux.slab.state" semantic conventions. It represents the + // Linux Slab memory state. + // + // Type: Enum + // RequirementLevel: Recommended + // Stability: Development + // + // Examples: "reclaimable", "unreclaimable" + SystemMemoryLinuxSlabStateKey = attribute.Key("system.memory.linux.slab.state") + // SystemMemoryStateKey is the attribute Key conforming to the // "system.memory.state" semantic conventions. It represents the memory state. // @@ -13026,49 +14076,29 @@ const ( // Examples: "in" SystemPagingDirectionKey = attribute.Key("system.paging.direction") - // SystemPagingStateKey is the attribute Key conforming to the - // "system.paging.state" semantic conventions. It represents the memory paging - // state. - // - // Type: Enum - // RequirementLevel: Recommended - // Stability: Development - // - // Examples: "free" - SystemPagingStateKey = attribute.Key("system.paging.state") - - // SystemPagingTypeKey is the attribute Key conforming to the - // "system.paging.type" semantic conventions. It represents the memory paging - // type. + // SystemPagingFaultTypeKey is the attribute Key conforming to the + // "system.paging.fault.type" semantic conventions. It represents the paging + // fault type. // // Type: Enum // RequirementLevel: Recommended // Stability: Development // // Examples: "minor" - SystemPagingTypeKey = attribute.Key("system.paging.type") + SystemPagingFaultTypeKey = attribute.Key("system.paging.fault.type") - // SystemProcessStatusKey is the attribute Key conforming to the - // "system.process.status" semantic conventions. It represents the process - // state, e.g., [Linux Process State Codes]. + // SystemPagingStateKey is the attribute Key conforming to the + // "system.paging.state" semantic conventions. It represents the memory paging + // state. // // Type: Enum // RequirementLevel: Recommended // Stability: Development // - // Examples: "running" - // - // [Linux Process State Codes]: https://man7.org/linux/man-pages/man1/ps.1.html#PROCESS_STATE_CODES - SystemProcessStatusKey = attribute.Key("system.process.status") + // Examples: "free" + SystemPagingStateKey = attribute.Key("system.paging.state") ) -// SystemCPULogicalNumber returns an attribute KeyValue conforming to the -// "system.cpu.logical_number" semantic conventions. It represents the -// deprecated, use `cpu.logical_number` instead. -func SystemCPULogicalNumber(val int) attribute.KeyValue { - return SystemCPULogicalNumberKey.Int(val) -} - // SystemDevice returns an attribute KeyValue conforming to the "system.device" // semantic conventions. It represents the device identifier. func SystemDevice(val string) attribute.KeyValue { @@ -13124,6 +14154,16 @@ var ( SystemFilesystemTypeExt4 = SystemFilesystemTypeKey.String("ext4") ) +// Enum values for system.memory.linux.slab.state +var ( + // reclaimable + // Stability: development + SystemMemoryLinuxSlabStateReclaimable = SystemMemoryLinuxSlabStateKey.String("reclaimable") + // unreclaimable + // Stability: development + SystemMemoryLinuxSlabStateUnreclaimable = SystemMemoryLinuxSlabStateKey.String("unreclaimable") +) + // Enum values for system.memory.state var ( // Actual used virtual memory in bytes. @@ -13150,40 +14190,24 @@ var ( SystemPagingDirectionOut = SystemPagingDirectionKey.String("out") ) -// Enum values for system.paging.state -var ( - // used - // Stability: development - SystemPagingStateUsed = SystemPagingStateKey.String("used") - // free - // Stability: development - SystemPagingStateFree = SystemPagingStateKey.String("free") -) - -// Enum values for system.paging.type +// Enum values for system.paging.fault.type var ( // major // Stability: development - SystemPagingTypeMajor = SystemPagingTypeKey.String("major") + SystemPagingFaultTypeMajor = SystemPagingFaultTypeKey.String("major") // minor // Stability: development - SystemPagingTypeMinor = SystemPagingTypeKey.String("minor") + SystemPagingFaultTypeMinor = SystemPagingFaultTypeKey.String("minor") ) -// Enum values for system.process.status +// Enum values for system.paging.state var ( - // running - // Stability: development - SystemProcessStatusRunning = SystemProcessStatusKey.String("running") - // sleeping - // Stability: development - SystemProcessStatusSleeping = SystemProcessStatusKey.String("sleeping") - // stopped + // used // Stability: development - SystemProcessStatusStopped = SystemProcessStatusKey.String("stopped") - // defunct + SystemPagingStateUsed = SystemPagingStateKey.String("used") + // free // Stability: development - SystemProcessStatusDefunct = SystemProcessStatusKey.String("defunct") + SystemPagingStateFree = SystemPagingStateKey.String("free") ) // Namespace: telemetry @@ -13438,6 +14462,18 @@ const ( // Type: int // RequirementLevel: Recommended // Stability: Development + // + // Note: + // Examples of where the value can be extracted from: + // + // | Language or platform | Source | + // | --- | --- | + // | JVM | `Thread.currentThread().threadId()` | + // | .NET | `Thread.CurrentThread.ManagedThreadId` | + // | Python | `threading.current_thread().ident` | + // | Ruby | `Thread.current.object_id` | + // | C++ | `std::this_thread::get_id()` | + // | Erlang | `erlang:self()` | ThreadIDKey = attribute.Key("thread.id") // ThreadNameKey is the attribute Key conforming to the "thread.name" semantic @@ -13448,6 +14484,16 @@ const ( // Stability: Development // // Examples: main + // Note: + // Examples of where the value can be extracted from: + // + // | Language or platform | Source | + // | --- | --- | + // | JVM | `Thread.currentThread().getName()` | + // | .NET | `Thread.CurrentThread.Name` | + // | Python | `threading.current_thread().name` | + // | Ruby | `Thread.current.name` | + // | Erlang | `erlang:process_info(self(), registered_name)` | ThreadNameKey = attribute.Key("thread.name") ) @@ -14515,7 +15561,7 @@ const ( // significant name SHOULD be selected. In such a scenario it should align with // `user_agent.version` // - // [Example]: https://www.whatsmyua.info + // [Example]: https://uaparser.dev/#demo UserAgentNameKey = attribute.Key("user_agent.name") // UserAgentOriginalKey is the attribute Key conforming to the @@ -14598,7 +15644,7 @@ const ( // significant version SHOULD be selected. In such a scenario it should align // with `user_agent.name` // - // [Example]: https://www.whatsmyua.info + // [Example]: https://uaparser.dev/#demo UserAgentVersionKey = attribute.Key("user_agent.version") ) @@ -14890,7 +15936,7 @@ const ( // Note: In Git Version Control Systems, the canonical URL SHOULD NOT include // the `.git` extension. // - // [canonical URL]: https://support.google.com/webmasters/answer/10347851?hl=en#:~:text=A%20canonical%20URL%20is%20the,Google%20chooses%20one%20as%20canonical. + // [canonical URL]: https://support.google.com/webmasters/answer/10347851 VCSRepositoryURLFullKey = attribute.Key("vcs.repository.url.full") // VCSRevisionDeltaDirectionKey is the attribute Key conforming to the @@ -14980,7 +16026,7 @@ func VCSRepositoryName(val string) attribute.KeyValue { // [canonical URL] of the repository providing the complete HTTP(S) address in // order to locate and identify the repository through a browser. // -// [canonical URL]: https://support.google.com/webmasters/answer/10347851?hl=en#:~:text=A%20canonical%20URL%20is%20the,Google%20chooses%20one%20as%20canonical. +// [canonical URL]: https://support.google.com/webmasters/answer/10347851 func VCSRepositoryURLFull(val string) attribute.KeyValue { return VCSRepositoryURLFullKey.String(val) } diff --git a/go-controller/vendor/go.opentelemetry.io/otel/semconv/v1.37.0/doc.go b/go-controller/vendor/go.opentelemetry.io/otel/semconv/v1.39.0/doc.go similarity index 96% rename from go-controller/vendor/go.opentelemetry.io/otel/semconv/v1.37.0/doc.go rename to go-controller/vendor/go.opentelemetry.io/otel/semconv/v1.39.0/doc.go index 1110103210..852362ef77 100644 --- a/go-controller/vendor/go.opentelemetry.io/otel/semconv/v1.37.0/doc.go +++ b/go-controller/vendor/go.opentelemetry.io/otel/semconv/v1.39.0/doc.go @@ -4,6 +4,6 @@ // Package semconv implements OpenTelemetry semantic conventions. // // OpenTelemetry semantic conventions are agreed standardized naming -// patterns for OpenTelemetry things. This package represents the v1.37.0 +// patterns for OpenTelemetry things. This package represents the v1.39.0 // version of the OpenTelemetry semantic conventions. -package semconv // import "go.opentelemetry.io/otel/semconv/v1.37.0" +package semconv // import "go.opentelemetry.io/otel/semconv/v1.39.0" diff --git a/go-controller/vendor/go.opentelemetry.io/otel/semconv/v1.37.0/error_type.go b/go-controller/vendor/go.opentelemetry.io/otel/semconv/v1.39.0/error_type.go similarity index 99% rename from go-controller/vendor/go.opentelemetry.io/otel/semconv/v1.37.0/error_type.go rename to go-controller/vendor/go.opentelemetry.io/otel/semconv/v1.39.0/error_type.go index 267979c051..84cf636a72 100644 --- a/go-controller/vendor/go.opentelemetry.io/otel/semconv/v1.37.0/error_type.go +++ b/go-controller/vendor/go.opentelemetry.io/otel/semconv/v1.39.0/error_type.go @@ -1,7 +1,7 @@ // Copyright The OpenTelemetry Authors // SPDX-License-Identifier: Apache-2.0 -package semconv // import "go.opentelemetry.io/otel/semconv/v1.37.0" +package semconv // import "go.opentelemetry.io/otel/semconv/v1.39.0" import ( "reflect" diff --git a/go-controller/vendor/go.opentelemetry.io/otel/semconv/v1.37.0/exception.go b/go-controller/vendor/go.opentelemetry.io/otel/semconv/v1.39.0/exception.go similarity index 98% rename from go-controller/vendor/go.opentelemetry.io/otel/semconv/v1.37.0/exception.go rename to go-controller/vendor/go.opentelemetry.io/otel/semconv/v1.39.0/exception.go index e67469a4f6..7b688ecc33 100644 --- a/go-controller/vendor/go.opentelemetry.io/otel/semconv/v1.37.0/exception.go +++ b/go-controller/vendor/go.opentelemetry.io/otel/semconv/v1.39.0/exception.go @@ -1,7 +1,7 @@ // Copyright The OpenTelemetry Authors // SPDX-License-Identifier: Apache-2.0 -package semconv // import "go.opentelemetry.io/otel/semconv/v1.37.0" +package semconv // import "go.opentelemetry.io/otel/semconv/v1.39.0" const ( // ExceptionEventName is the name of the Span event representing an exception. diff --git a/go-controller/vendor/go.opentelemetry.io/otel/semconv/v1.37.0/schema.go b/go-controller/vendor/go.opentelemetry.io/otel/semconv/v1.39.0/schema.go similarity index 85% rename from go-controller/vendor/go.opentelemetry.io/otel/semconv/v1.37.0/schema.go rename to go-controller/vendor/go.opentelemetry.io/otel/semconv/v1.39.0/schema.go index f8a0b70441..e1a199d89b 100644 --- a/go-controller/vendor/go.opentelemetry.io/otel/semconv/v1.37.0/schema.go +++ b/go-controller/vendor/go.opentelemetry.io/otel/semconv/v1.39.0/schema.go @@ -1,9 +1,9 @@ // Copyright The OpenTelemetry Authors // SPDX-License-Identifier: Apache-2.0 -package semconv // import "go.opentelemetry.io/otel/semconv/v1.37.0" +package semconv // import "go.opentelemetry.io/otel/semconv/v1.39.0" // SchemaURL is the schema URL that matches the version of the semantic conventions // that this package defines. Semconv packages starting from v1.4.0 must declare // non-empty schema URL in the form https://opentelemetry.io/schemas/ -const SchemaURL = "https://opentelemetry.io/schemas/1.37.0" +const SchemaURL = "https://opentelemetry.io/schemas/1.39.0" diff --git a/go-controller/vendor/go.opentelemetry.io/otel/trace/auto.go b/go-controller/vendor/go.opentelemetry.io/otel/trace/auto.go index 8763936a84..604fdab446 100644 --- a/go-controller/vendor/go.opentelemetry.io/otel/trace/auto.go +++ b/go-controller/vendor/go.opentelemetry.io/otel/trace/auto.go @@ -20,7 +20,7 @@ import ( "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/codes" - semconv "go.opentelemetry.io/otel/semconv/v1.37.0" + semconv "go.opentelemetry.io/otel/semconv/v1.39.0" "go.opentelemetry.io/otel/trace/embedded" "go.opentelemetry.io/otel/trace/internal/telemetry" ) diff --git a/go-controller/vendor/golang.org/x/sys/cpu/cpu_x86.go b/go-controller/vendor/golang.org/x/sys/cpu/cpu_x86.go index 1e642f3304..f5723d4f7e 100644 --- a/go-controller/vendor/golang.org/x/sys/cpu/cpu_x86.go +++ b/go-controller/vendor/golang.org/x/sys/cpu/cpu_x86.go @@ -64,6 +64,80 @@ func initOptions() { func archInit() { + // From internal/cpu + const ( + // eax bits + cpuid_AVXVNNI = 1 << 4 + + // ecx bits + cpuid_SSE3 = 1 << 0 + cpuid_PCLMULQDQ = 1 << 1 + cpuid_AVX512VBMI = 1 << 1 + cpuid_AVX512VBMI2 = 1 << 6 + cpuid_SSSE3 = 1 << 9 + cpuid_AVX512GFNI = 1 << 8 + cpuid_AVX512VAES = 1 << 9 + cpuid_AVX512VNNI = 1 << 11 + cpuid_AVX512BITALG = 1 << 12 + cpuid_FMA = 1 << 12 + cpuid_AVX512VPOPCNTDQ = 1 << 14 + cpuid_SSE41 = 1 << 19 + cpuid_SSE42 = 1 << 20 + cpuid_POPCNT = 1 << 23 + cpuid_AES = 1 << 25 + cpuid_OSXSAVE = 1 << 27 + cpuid_AVX = 1 << 28 + + // "Extended Feature Flag" bits returned in EBX for CPUID EAX=0x7 ECX=0x0 + cpuid_BMI1 = 1 << 3 + cpuid_AVX2 = 1 << 5 + cpuid_BMI2 = 1 << 8 + cpuid_ERMS = 1 << 9 + cpuid_AVX512F = 1 << 16 + cpuid_AVX512DQ = 1 << 17 + cpuid_ADX = 1 << 19 + cpuid_AVX512CD = 1 << 28 + cpuid_SHA = 1 << 29 + cpuid_AVX512BW = 1 << 30 + cpuid_AVX512VL = 1 << 31 + + // "Extended Feature Flag" bits returned in ECX for CPUID EAX=0x7 ECX=0x0 + cpuid_AVX512_VBMI = 1 << 1 + cpuid_AVX512_VBMI2 = 1 << 6 + cpuid_GFNI = 1 << 8 + cpuid_AVX512VPCLMULQDQ = 1 << 10 + cpuid_AVX512_BITALG = 1 << 12 + + // edx bits + cpuid_FSRM = 1 << 4 + // edx bits for CPUID 0x80000001 + cpuid_RDTSCP = 1 << 27 + ) + // Additional constants not in internal/cpu + const ( + // eax=1: edx + cpuid_SSE2 = 1 << 26 + // eax=1: ecx + cpuid_CX16 = 1 << 13 + cpuid_RDRAND = 1 << 30 + // eax=7,ecx=0: ebx + cpuid_RDSEED = 1 << 18 + cpuid_AVX512IFMA = 1 << 21 + cpuid_AVX512PF = 1 << 26 + cpuid_AVX512ER = 1 << 27 + // eax=7,ecx=0: edx + cpuid_AVX5124VNNIW = 1 << 2 + cpuid_AVX5124FMAPS = 1 << 3 + cpuid_AMXBF16 = 1 << 22 + cpuid_AMXTile = 1 << 24 + cpuid_AMXInt8 = 1 << 25 + // eax=7,ecx=1: eax + cpuid_AVX512BF16 = 1 << 5 + cpuid_AVXIFMA = 1 << 23 + // eax=7,ecx=1: edx + cpuid_AVXVNNIInt8 = 1 << 4 + ) + Initialized = true maxID, _, _, _ := cpuid(0, 0) @@ -73,90 +147,90 @@ func archInit() { } _, _, ecx1, edx1 := cpuid(1, 0) - X86.HasSSE2 = isSet(26, edx1) - - X86.HasSSE3 = isSet(0, ecx1) - X86.HasPCLMULQDQ = isSet(1, ecx1) - X86.HasSSSE3 = isSet(9, ecx1) - X86.HasFMA = isSet(12, ecx1) - X86.HasCX16 = isSet(13, ecx1) - X86.HasSSE41 = isSet(19, ecx1) - X86.HasSSE42 = isSet(20, ecx1) - X86.HasPOPCNT = isSet(23, ecx1) - X86.HasAES = isSet(25, ecx1) - X86.HasOSXSAVE = isSet(27, ecx1) - X86.HasRDRAND = isSet(30, ecx1) + X86.HasSSE2 = isSet(edx1, cpuid_SSE2) + + X86.HasSSE3 = isSet(ecx1, cpuid_SSE3) + X86.HasPCLMULQDQ = isSet(ecx1, cpuid_PCLMULQDQ) + X86.HasSSSE3 = isSet(ecx1, cpuid_SSSE3) + X86.HasFMA = isSet(ecx1, cpuid_FMA) + X86.HasCX16 = isSet(ecx1, cpuid_CX16) + X86.HasSSE41 = isSet(ecx1, cpuid_SSE41) + X86.HasSSE42 = isSet(ecx1, cpuid_SSE42) + X86.HasPOPCNT = isSet(ecx1, cpuid_POPCNT) + X86.HasAES = isSet(ecx1, cpuid_AES) + X86.HasOSXSAVE = isSet(ecx1, cpuid_OSXSAVE) + X86.HasRDRAND = isSet(ecx1, cpuid_RDRAND) var osSupportsAVX, osSupportsAVX512 bool // For XGETBV, OSXSAVE bit is required and sufficient. if X86.HasOSXSAVE { eax, _ := xgetbv() // Check if XMM and YMM registers have OS support. - osSupportsAVX = isSet(1, eax) && isSet(2, eax) + osSupportsAVX = isSet(eax, 1<<1) && isSet(eax, 1<<2) if runtime.GOOS == "darwin" { // Darwin requires special AVX512 checks, see cpu_darwin_x86.go osSupportsAVX512 = osSupportsAVX && darwinSupportsAVX512() } else { // Check if OPMASK and ZMM registers have OS support. - osSupportsAVX512 = osSupportsAVX && isSet(5, eax) && isSet(6, eax) && isSet(7, eax) + osSupportsAVX512 = osSupportsAVX && isSet(eax, 1<<5) && isSet(eax, 1<<6) && isSet(eax, 1<<7) } } - X86.HasAVX = isSet(28, ecx1) && osSupportsAVX + X86.HasAVX = isSet(ecx1, cpuid_AVX) && osSupportsAVX if maxID < 7 { return } eax7, ebx7, ecx7, edx7 := cpuid(7, 0) - X86.HasBMI1 = isSet(3, ebx7) - X86.HasAVX2 = isSet(5, ebx7) && osSupportsAVX - X86.HasBMI2 = isSet(8, ebx7) - X86.HasERMS = isSet(9, ebx7) - X86.HasRDSEED = isSet(18, ebx7) - X86.HasADX = isSet(19, ebx7) - - X86.HasAVX512 = isSet(16, ebx7) && osSupportsAVX512 // Because avx-512 foundation is the core required extension + X86.HasBMI1 = isSet(ebx7, cpuid_BMI1) + X86.HasAVX2 = isSet(ebx7, cpuid_AVX2) && osSupportsAVX + X86.HasBMI2 = isSet(ebx7, cpuid_BMI2) + X86.HasERMS = isSet(ebx7, cpuid_ERMS) + X86.HasRDSEED = isSet(ebx7, cpuid_RDSEED) + X86.HasADX = isSet(ebx7, cpuid_ADX) + + X86.HasAVX512 = isSet(ebx7, cpuid_AVX512F) && osSupportsAVX512 // Because avx-512 foundation is the core required extension if X86.HasAVX512 { X86.HasAVX512F = true - X86.HasAVX512CD = isSet(28, ebx7) - X86.HasAVX512ER = isSet(27, ebx7) - X86.HasAVX512PF = isSet(26, ebx7) - X86.HasAVX512VL = isSet(31, ebx7) - X86.HasAVX512BW = isSet(30, ebx7) - X86.HasAVX512DQ = isSet(17, ebx7) - X86.HasAVX512IFMA = isSet(21, ebx7) - X86.HasAVX512VBMI = isSet(1, ecx7) - X86.HasAVX5124VNNIW = isSet(2, edx7) - X86.HasAVX5124FMAPS = isSet(3, edx7) - X86.HasAVX512VPOPCNTDQ = isSet(14, ecx7) - X86.HasAVX512VPCLMULQDQ = isSet(10, ecx7) - X86.HasAVX512VNNI = isSet(11, ecx7) - X86.HasAVX512GFNI = isSet(8, ecx7) - X86.HasAVX512VAES = isSet(9, ecx7) - X86.HasAVX512VBMI2 = isSet(6, ecx7) - X86.HasAVX512BITALG = isSet(12, ecx7) + X86.HasAVX512CD = isSet(ebx7, cpuid_AVX512CD) + X86.HasAVX512ER = isSet(ebx7, cpuid_AVX512ER) + X86.HasAVX512PF = isSet(ebx7, cpuid_AVX512PF) + X86.HasAVX512VL = isSet(ebx7, cpuid_AVX512VL) + X86.HasAVX512BW = isSet(ebx7, cpuid_AVX512BW) + X86.HasAVX512DQ = isSet(ebx7, cpuid_AVX512DQ) + X86.HasAVX512IFMA = isSet(ebx7, cpuid_AVX512IFMA) + X86.HasAVX512VBMI = isSet(ecx7, cpuid_AVX512_VBMI) + X86.HasAVX5124VNNIW = isSet(edx7, cpuid_AVX5124VNNIW) + X86.HasAVX5124FMAPS = isSet(edx7, cpuid_AVX5124FMAPS) + X86.HasAVX512VPOPCNTDQ = isSet(ecx7, cpuid_AVX512VPOPCNTDQ) + X86.HasAVX512VPCLMULQDQ = isSet(ecx7, cpuid_AVX512VPCLMULQDQ) + X86.HasAVX512VNNI = isSet(ecx7, cpuid_AVX512VNNI) + X86.HasAVX512GFNI = isSet(ecx7, cpuid_AVX512GFNI) + X86.HasAVX512VAES = isSet(ecx7, cpuid_AVX512VAES) + X86.HasAVX512VBMI2 = isSet(ecx7, cpuid_AVX512VBMI2) + X86.HasAVX512BITALG = isSet(ecx7, cpuid_AVX512BITALG) } - X86.HasAMXTile = isSet(24, edx7) - X86.HasAMXInt8 = isSet(25, edx7) - X86.HasAMXBF16 = isSet(22, edx7) + X86.HasAMXTile = isSet(edx7, cpuid_AMXTile) + X86.HasAMXInt8 = isSet(edx7, cpuid_AMXInt8) + X86.HasAMXBF16 = isSet(edx7, cpuid_AMXBF16) // These features depend on the second level of extended features. if eax7 >= 1 { eax71, _, _, edx71 := cpuid(7, 1) if X86.HasAVX512 { - X86.HasAVX512BF16 = isSet(5, eax71) + X86.HasAVX512BF16 = isSet(eax71, cpuid_AVX512BF16) } if X86.HasAVX { - X86.HasAVXIFMA = isSet(23, eax71) - X86.HasAVXVNNI = isSet(4, eax71) - X86.HasAVXVNNIInt8 = isSet(4, edx71) + X86.HasAVXIFMA = isSet(eax71, cpuid_AVXIFMA) + X86.HasAVXVNNI = isSet(eax71, cpuid_AVXVNNI) + X86.HasAVXVNNIInt8 = isSet(edx71, cpuid_AVXVNNIInt8) } } } -func isSet(bitpos uint, value uint32) bool { - return value&(1< Date: Thu, 4 Jun 2026 09:45:22 -0700 Subject: [PATCH 07/51] ovn: remove legacy external-gateway annotation handling Drop the annotation-driven external-gateway feature (routing-external-gws, routing-namespaces, routing-network, bfd-enabled) from DefaultNetworkController; AdminPolicyBasedExternalRoute is now the only path. Removes the pod/namespace-GW handlers, the now-dead route add primitives, and the nsInfo external-gateway fields. The shared delete path stays (APB delegates pod/namespace-deletion cleanup to it). Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Mykola Yurchenko --- .../ovn/base_network_controller_namespace.go | 11 - go-controller/pkg/ovn/egressgw.go | 489 ------------------ go-controller/pkg/ovn/namespace.go | 187 +------ go-controller/pkg/ovn/ovn.go | 74 +-- go-controller/pkg/ovn/pods.go | 42 +- 5 files changed, 13 insertions(+), 790 deletions(-) diff --git a/go-controller/pkg/ovn/base_network_controller_namespace.go b/go-controller/pkg/ovn/base_network_controller_namespace.go index 75e0733f32..f84bfa5984 100644 --- a/go-controller/pkg/ovn/base_network_controller_namespace.go +++ b/go-controller/pkg/ovn/base_network_controller_namespace.go @@ -46,15 +46,6 @@ type namespaceInfo struct { // Namespace can take oc.networkPolicies key Lock while holding nsInfo lock, the opposite should never happen. relatedNetworkPolicies map[string]bool - // routingExternalGWs is a slice of net.IP containing the values parsed from - // annotation k8s.ovn.org/routing-external-gws - routingExternalGWs gatewayInfo - - // routingExternalPodGWs contains a map of all pods serving as exgws as well as their - // exgw IPs - // key is _ - routingExternalPodGWs map[string]gatewayInfo - multicastEnabled bool // If not empty, then it has to be set to a logging a severity level, e.g. "notice", "alert", etc @@ -239,8 +230,6 @@ func (bnc *BaseNetworkController) ensureNamespaceLockedCommon(ns string, readOnl nsInfo = &namespaceInfo{ relatedNetworkPolicies: map[string]bool{}, multicastEnabled: false, - routingExternalPodGWs: make(map[string]gatewayInfo), - routingExternalGWs: gatewayInfo{gws: sets.New[string](), bfdEnabled: false}, } // we are creating nsInfo and going to set it in namespaces map // so safe to hold the lock while we create and add it diff --git a/go-controller/pkg/ovn/egressgw.go b/go-controller/pkg/ovn/egressgw.go index 475a484b19..bd4cb22e29 100644 --- a/go-controller/pkg/ovn/egressgw.go +++ b/go-controller/pkg/ovn/egressgw.go @@ -4,21 +4,16 @@ package ovn import ( - "encoding/json" "errors" "fmt" "net" "regexp" - "strings" - - nettypes "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" ktypes "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/klog/v2" - v1pod "k8s.io/kubernetes/pkg/api/v1/pod" utilnet "k8s.io/utils/net" libovsdbclient "github.com/ovn-kubernetes/libovsdb/client" @@ -27,116 +22,12 @@ import ( "github.com/ovn-kubernetes/ovn-kubernetes/go-controller/pkg/config" "github.com/ovn-kubernetes/ovn-kubernetes/go-controller/pkg/factory" libovsdbops "github.com/ovn-kubernetes/ovn-kubernetes/go-controller/pkg/libovsdb/ops" - libovsdbutil "github.com/ovn-kubernetes/ovn-kubernetes/go-controller/pkg/libovsdb/util" "github.com/ovn-kubernetes/ovn-kubernetes/go-controller/pkg/nbdb" apbroutecontroller "github.com/ovn-kubernetes/ovn-kubernetes/go-controller/pkg/ovn/controller/apbroute" "github.com/ovn-kubernetes/ovn-kubernetes/go-controller/pkg/types" "github.com/ovn-kubernetes/ovn-kubernetes/go-controller/pkg/util" ) -type gatewayInfo struct { - gws sets.Set[string] - bfdEnabled bool -} - -// addPodExternalGW handles detecting if a pod is serving as an external gateway for namespace(s) and adding routes -// to all pods in that namespace -func (oc *DefaultNetworkController) addPodExternalGW(pod *corev1.Pod) error { - podRoutingNamespaceAnno := pod.Annotations[util.RoutingNamespaceAnnotation] - if podRoutingNamespaceAnno == "" { - return nil - } - enableBFD := false - if _, ok := pod.Annotations[util.BfdAnnotation]; ok { - enableBFD = true - } - - klog.Infof("External gateway pod: %s, detected for namespace(s) %s", pod.Name, podRoutingNamespaceAnno) - - // If an external gateway pod is in terminating or not ready state then don't add the - // routes for the external gateway pod - if util.PodTerminating(pod) || !v1pod.IsPodReadyConditionTrue(pod.Status) { - klog.Warningf("External gateway pod cannot serve traffic; it's in terminating or not ready state: %s/%s", pod.Namespace, pod.Name) - return nil - } - - foundGws, err := getExGwPodIPs(pod) - if err != nil { - klog.Errorf("Error getting exgw IPs for pod: %s, error: %v", pod.Name, err) - oc.recordPodEvent("ErrorAddingLogicalPort", err, pod) - return nil - } - - // if we found any gateways then we need to update current pods routing in the relevant namespace - if len(foundGws) == 0 { - klog.Warningf("No valid gateway IPs found for requested external gateway pod: %s", pod.Name) - return nil - } - - for _, namespace := range strings.Split(podRoutingNamespaceAnno, ",") { - err := oc.addPodExternalGWForNamespace(namespace, pod, gatewayInfo{gws: foundGws, bfdEnabled: enableBFD}) - if err != nil { - return err - } - } - return nil -} - -// addPodExternalGWForNamespace handles adding routes to all pods in that namespace for a pod GW -func (oc *DefaultNetworkController) addPodExternalGWForNamespace(namespace string, pod *corev1.Pod, egress gatewayInfo) error { - nsInfo, nsUnlock, err := oc.ensureNamespaceLocked(namespace, false, nil) - if err != nil { - return fmt.Errorf("failed to ensure namespace locked: %v", err) - } - tmpPodGWs := oc.getRoutingPodGWs(nsInfo) - tmpPodGWs[makePodGWKey(pod)] = egress - if err = validateRoutingPodGWs(tmpPodGWs); err != nil { - nsUnlock() - return fmt.Errorf("unable to add pod: %s/%s as external gateway for namespace: %s, error: %v", - pod.Namespace, pod.Name, namespace, err) - } - nsInfo.routingExternalPodGWs[makePodGWKey(pod)] = egress - existingGWs := sets.NewString() - for _, gwInfo := range nsInfo.routingExternalPodGWs { - existingGWs.Insert(gwInfo.gws.UnsortedList()...) - } - if oc.zone != types.OvnDefaultZone { - existingGWs.Insert(nsInfo.routingExternalGWs.gws.UnsortedList()...) - } - nsUnlock() - - klog.Infof("Adding routes for external gateway pod: %s, next hops: %q, namespace: %s, bfd-enabled: %t", - pod.Name, strings.Join(egress.gws.UnsortedList(), ","), namespace, egress.bfdEnabled) - err = oc.addGWRoutesForNamespace(namespace, egress) - if err != nil { - return err - } - // add the exgw podIP to the namespace's k8s.ovn.org/external-gw-pod-ips list - if oc.zone == types.OvnDefaultZone { - // In single-zone deployments (default zone), ovnkube-controller patches - // the "k8s.ovn.org/external-gw-pod-ips" namespace annotation; ovnkube-node - // watches it and flushes conntrack on every node. In multi-zone - // interconnect, ovnkube-controller flushes conntrack directly and skips - // the annotation. - if err := util.UpdateExternalGatewayPodIPsAnnotation(oc.kube, namespace, existingGWs.List()); err != nil { - klog.Errorf("Unable to update %s/%v annotation for namespace %s: %v", util.ExternalGatewayPodIPsAnnotation, existingGWs, namespace, err) - } - } else { - // flush here since we know we have added an egressgw pod and we also know the full list of existing gatewayIPs - gatewayIPs, err := oc.apbExternalRouteController.GetAdminPolicyBasedExternalRouteIPsForTargetNamespace(namespace) - if err != nil { - return fmt.Errorf("unable to retrieve gateway IPs for Admin Policy Based External Route objects: %w", err) - } - gatewayIPs = gatewayIPs.Insert(existingGWs.List()...) - err = oc.syncConntrackForExternalGateways(namespace, gatewayIPs) // best effort - if err != nil { - klog.Errorf("Syncing conntrack entries for egressGW pod %v serving the namespace %s failed: %v", - egress, namespace, err) - } - } - return nil -} - func (oc *DefaultNetworkController) syncConntrackForExternalGateways(namespace string, gwIPsToKeep sets.Set[string]) error { return util.SyncConntrackForExternalGateways(gwIPsToKeep, oc.isPodInLocalZone, func() ([]*corev1.Pod, error) { return oc.watchFactory.GetPods(namespace) @@ -156,19 +47,6 @@ func (oc *DefaultNetworkController) checkAndDeleteStaleConntrackEntries() { klog.Errorf("Unable to retrieve gateway IPs for Admin Policy Based External Route objects for ns %s: %v", namespace.Name, err) return } - // by now the nsInfo cache must be repaired for this feature fully; - // however this introduces cache lock scale concern by doing this every minute - // versus previously this was done purely using annotations - nsInfo, nsUnlock, err := oc.ensureNamespaceLocked(namespace.Name, false, nil) - if err != nil { - klog.Errorf("Failed to ensure namespace %s locked: %v", namespace, err) - return - } - for _, gwInfo := range nsInfo.routingExternalPodGWs { - existingGWs.Insert(gwInfo.gws.UnsortedList()...) - } - existingGWs.Insert(nsInfo.routingExternalGWs.gws.UnsortedList()...) - nsUnlock() if len(existingGWs) > 0 { pods, err := oc.watchFactory.GetPods(namespace.Name) if err != nil { @@ -187,16 +65,6 @@ func (oc *DefaultNetworkController) checkAndDeleteStaleConntrackEntries() { } } -// addExternalGWsForNamespace handles adding annotated gw routes to all pods in namespace -// This should only be called with a lock on nsInfo -func (oc *DefaultNetworkController) addExternalGWsForNamespace(egress gatewayInfo, nsInfo *namespaceInfo, namespace string) error { - if egress.gws == nil { - return fmt.Errorf("unable to add gateways routes for namespace: %s, gateways are nil", namespace) - } - nsInfo.routingExternalGWs = egress - return oc.addGWRoutesForNamespace(namespace, egress) -} - func (oc *DefaultNetworkController) isPodInLocalZone(pod *corev1.Pod) (bool, error) { node, err := oc.watchFactory.GetNode(pod.Spec.NodeName) if err != nil { @@ -205,87 +73,6 @@ func (oc *DefaultNetworkController) isPodInLocalZone(pod *corev1.Pod) (bool, err return oc.isLocalZoneNode(node), nil } -// addGWRoutesForNamespace handles adding routes for all existing pods in namespace -func (oc *DefaultNetworkController) addGWRoutesForNamespace(namespace string, egress gatewayInfo) error { - existingPods, err := oc.watchFactory.GetPods(namespace) - if err != nil { - return fmt.Errorf("failed to get all the pods (%v)", err) - } - for _, pod := range existingPods { - if util.PodCompleted(pod) || util.PodWantsHostNetwork(pod) { - continue - } - podIPs := make([]*net.IPNet, 0) - for _, podIP := range pod.Status.PodIPs { - podIP := &net.IPNet{IP: utilnet.ParseIPSloppy(podIP.IP)} - podIP.Mask = util.GetIPFullMask(podIP.IP) - podIPs = append(podIPs, podIP) - } - if len(podIPs) == 0 { - klog.Warningf("Will not add gateway routes pod %s/%s. IPs not found!", pod.Namespace, pod.Name) - continue - } - if config.Gateway.DisableSNATMultipleGWs { - // delete all perPodSNATs (if this pod was controlled by egressIP controller, it will stop working since - // a pod cannot be used for multiple-external-gateways and egressIPs at the same time) - if err = oc.deletePodSNAT(pod.Spec.NodeName, []*net.IPNet{}, podIPs); err != nil { - klog.Error(err.Error()) - } - } - podNsName := ktypes.NamespacedName{Namespace: pod.Namespace, Name: pod.Name} - if err := oc.addGWRoutesForPod([]*gatewayInfo{&egress}, podIPs, podNsName, pod.Spec.NodeName); err != nil { - return err - } - } - return nil -} - -func (oc *DefaultNetworkController) createBFDStaticRoute(bfdEnabled bool, gw string, podIP, gr, port, mask string) error { - lrsr := nbdb.LogicalRouterStaticRoute{ - Policy: &nbdb.LogicalRouterStaticRoutePolicySrcIP, - Options: map[string]string{ - "ecmp_symmetric_reply": "true", - }, - Nexthop: gw, - IPPrefix: podIP + mask, - OutputPort: &port, - } - - ops := []ovsdb.Operation{} - var err error - if bfdEnabled { - bfd := nbdb.BFD{ - DstIP: gw, - LogicalPort: port, - } - ops, err = libovsdbops.CreateOrUpdateBFDOps(oc.nbClient, ops, &bfd) - if err != nil { - return fmt.Errorf("error creating or updating BFD %+v: %v", bfd, err) - } - lrsr.BFD = &bfd.UUID - } - - p := func(item *nbdb.LogicalRouterStaticRoute) bool { - return item.IPPrefix == lrsr.IPPrefix && - item.Nexthop == lrsr.Nexthop && - item.OutputPort != nil && - *item.OutputPort == *lrsr.OutputPort && - item.Policy == lrsr.Policy - } - ops, err = libovsdbops.CreateOrUpdateLogicalRouterStaticRoutesWithPredicateOps(oc.nbClient, ops, gr, &lrsr, p, - &lrsr.Options) - if err != nil { - return fmt.Errorf("error creating or updating static route %+v on router %s: %v", lrsr, gr, err) - } - - _, err = libovsdbops.TransactAndCheck(oc.nbClient, ops) - if err != nil { - return fmt.Errorf("error transacting static route: %v", err) - } - - return nil -} - func (oc *DefaultNetworkController) deleteLogicalRouterStaticRoute(podIP, mask, gw, gr string) error { p := func(item *nbdb.LogicalRouterStaticRoute) bool { return item.Policy != nil && @@ -345,86 +132,6 @@ func (oc *DefaultNetworkController) deletePodGWRoute(routeInfo *apbroutecontroll return oc.cleanUpBFDEntry(gw, gr, portPrefix) } -// deletePodExternalGW detects if a given pod is acting as an external GW and removes all routes in all namespaces -// associated with that pod -func (oc *DefaultNetworkController) deletePodExternalGW(pod *corev1.Pod) (err error) { - podRoutingNamespaceAnno := pod.Annotations[util.RoutingNamespaceAnnotation] - if podRoutingNamespaceAnno == "" { - return nil - } - klog.Infof("Deleting routes for external gateway pod: %s, for namespace(s) %s", pod.Name, - podRoutingNamespaceAnno) - for _, namespace := range strings.Split(podRoutingNamespaceAnno, ",") { - if err := oc.deletePodGWRoutesForNamespace(pod, namespace); err != nil { - // if we encounter error while deleting things in one namespace we return and don't try subsequent namespaces - return fmt.Errorf("failed to delete ecmp routes for pod %s in namespace %s", pod.Name, namespace) - } - } - return nil -} - -// deletePodGwRoutesForNamespace handles deleting all routes in a namespace for a specific pod GW -func (oc *DefaultNetworkController) deletePodGWRoutesForNamespace(pod *corev1.Pod, namespace string) (err error) { - nsInfo, nsUnlock := oc.getNamespaceLocked(namespace, false) - if nsInfo == nil { - return nil - } - podGWKey := makePodGWKey(pod) - // check if any gateways were stored for this pod - foundGws, ok := nsInfo.routingExternalPodGWs[podGWKey] - delete(nsInfo.routingExternalPodGWs, podGWKey) - existingGWs := sets.New[string]() - for _, gwInfo := range nsInfo.routingExternalPodGWs { - existingGWs.Insert(gwInfo.gws.UnsortedList()...) - } - if oc.zone != types.OvnDefaultZone { - existingGWs.Insert(nsInfo.routingExternalGWs.gws.UnsortedList()...) - } - nsUnlock() - - if !ok || len(foundGws.gws) == 0 { - klog.Infof("No gateways found to remove for annotated gateway pod: %s on namespace: %s", - pod, namespace) - return nil - } - - if err := oc.deleteGWRoutesForNamespace(namespace, foundGws.gws); err != nil { - // add the entry back to nsInfo for retrying later - nsInfo, nsUnlock := oc.getNamespaceLocked(namespace, false) - if nsInfo == nil { - return fmt.Errorf("failed to get nsInfo %s to add back all the gw routes: %w", namespace, err) - } - // we add back all the gw routes as we don't know which specific route for which pod error-ed out - nsInfo.routingExternalPodGWs[podGWKey] = foundGws - nsUnlock() - return fmt.Errorf("failed to delete GW routes for pod %s: %w", pod.Name, err) - } - // remove the exgw podIP from the namespace's k8s.ovn.org/external-gw-pod-ips list - if oc.zone == types.OvnDefaultZone { - // In single-zone deployments (default zone), ovnkube-controller patches - // the "k8s.ovn.org/external-gw-pod-ips" namespace annotation; ovnkube-node - // watches it and flushes conntrack on every node. In multi-zone - // interconnect, ovnkube-controller flushes conntrack directly and skips - // the annotation. - if err := util.UpdateExternalGatewayPodIPsAnnotation(oc.kube, namespace, sets.List(existingGWs)); err != nil { - klog.Errorf("Unable to update %s/%v annotation for namespace %s: %v", util.ExternalGatewayPodIPsAnnotation, existingGWs, namespace, err) - } - } else { - // flush here since we know we have deleted an egressgw pod and we also know the full list of existing gatewayIPs - gatewayIPs, err := oc.apbExternalRouteController.GetAdminPolicyBasedExternalRouteIPsForTargetNamespace(namespace) - if err != nil { - return fmt.Errorf("unable to retrieve gateway IPs for Admin Policy Based External Route objects: %w", err) - } - gatewayIPs = gatewayIPs.Insert(sets.List(existingGWs)...) - err = oc.syncConntrackForExternalGateways(namespace, gatewayIPs) // best effort - if err != nil { - klog.Errorf("Syncing conntrack entries for egressGWs %+v serving the namespace %s failed: %v", - gatewayIPs, namespace, err) - } - } - return nil -} - // deleteGwRoutesForNamespace handles deleting routes to gateways for a pod on a specific GR. // If a set of gateways is given, only routes for that gateway are deleted. If no gateways // are given, all routes for the namespace are deleted. @@ -493,88 +200,6 @@ func (oc *DefaultNetworkController) deleteGWRoutesForPod(name ktypes.NamespacedN }) } -// addEgressGwRoutesForPod handles adding all routes to gateways for a pod on a specific GR -func (oc *DefaultNetworkController) addGWRoutesForPod(gateways []*gatewayInfo, podIfAddrs []*net.IPNet, podNsName ktypes.NamespacedName, node string) error { - pod, err := oc.watchFactory.PodCoreInformer().Lister().Pods(podNsName.Namespace).Get(podNsName.Name) - if err != nil { - return err - } - - local, err := oc.isPodInLocalZone(pod) - if err != nil { - return err - } - if !local { - klog.V(4).Infof("Not adding exgw routes for pod %s not in the local zone %s", podNsName, oc.zone) - return nil - } - - gr := oc.GetNetworkScopedGWRouterName(node) - - routesAdded := 0 - portPrefix, err := oc.extSwitchPrefix(node) - if err != nil { - klog.Infof("Failed to find ext switch prefix for %s %v", node, err) - return err - } - - port := portPrefix + types.GWRouterToExtSwitchPrefix + gr - - return oc.externalGatewayRouteInfo.CreateOrLoad(podNsName, func(routeInfo *apbroutecontroller.RouteInfo) error { - policyGWIPs, err := oc.apbExternalRouteController.GetDynamicGatewayIPsForTargetNamespace(podNsName.Namespace) - if err != nil { - return err - } - policyStaticGWIPs, err := oc.apbExternalRouteController.GetStaticGatewayIPsForTargetNamespace(podNsName.Namespace) - if err != nil { - return err - } - policyGWIPs = policyGWIPs.Union(policyStaticGWIPs) - - for _, podIPNet := range podIfAddrs { - for _, gateway := range gateways { - // TODO (trozet): use the go bindings here and batch commands - // validate the ip and gateway belong to the same address family - gws, err := util.MatchAllIPStringFamily(utilnet.IsIPv6(podIPNet.IP), gateway.gws.UnsortedList()) - if err == nil { - podIP := podIPNet.IP.String() - for _, gw := range gws { - // if route was already programmed, skip it - foundGR, ok := routeInfo.PodExternalRoutes[podIP][gw] - if (ok && foundGR == gr) || policyGWIPs.Has(gw) { - routesAdded++ - continue - } - mask := util.GetIPFullMaskString(podIP) - - if err := oc.createBFDStaticRoute(gateway.bfdEnabled, gw, podIP, gr, port, mask); err != nil { - return err - } - if routeInfo.PodExternalRoutes[podIP] == nil { - routeInfo.PodExternalRoutes[podIP] = make(map[string]string) - } - routeInfo.PodExternalRoutes[podIP][gw] = gr - routesAdded++ - if len(routeInfo.PodExternalRoutes[podIP]) == 1 { - if err := oc.addHybridRoutePolicyForPod(podIPNet.IP, node); err != nil { - return err - } - } - } - } else { - klog.Warningf("Address families for the pod address %s and gateway %s did not match", podIPNet.IP.String(), gateway.gws) - } - } - } - // if no routes are added return an error - if routesAdded < 1 { - return fmt.Errorf("gateway specified for namespace %s with gateway addresses %v but no valid routes exist for pod: %s", - podNsName.Namespace, podIfAddrs, podNsName.Name) - } - return nil - }) -} - // deletePodSNAT removes per pod SNAT rules towards the nodeIP that are applied to the GR where the pod resides // used when disableSNATMultipleGWs=true func (oc *DefaultNetworkController) deletePodSNAT(nodeName string, extIPs, podIPNets []*net.IPNet) error { @@ -689,82 +314,6 @@ func addOrUpdatePodSNATOps(nbClient libovsdbclient.Client, gwRouterName string, return ops, nil } -// addHybridRoutePolicyForPod handles adding a higher priority allow policy to allow traffic to be routed normally -// by ecmp routes. -// WARNING: updates same db entries as apbroutecontroller. Make sure to call only when route is not managed by -// apbroute controller. -func (oc *DefaultNetworkController) addHybridRoutePolicyForPod(podIP net.IP, node string) error { - if config.Gateway.Mode == config.GatewayModeLocal { - // Add podIP to the node's address_set. - asIndex := apbroutecontroller.GetHybridRouteAddrSetDbIDs(node, oc.controllerName) - as, err := oc.addressSetFactory.EnsureAddressSet(asIndex) - if err != nil { - return fmt.Errorf("cannot ensure that addressSet for node %s exists %v", node, err) - } - err = as.AddAddresses([]string{podIP.String()}) - if err != nil { - return fmt.Errorf("unable to add PodIP %s: to the address set %s, err: %v", podIP.String(), node, err) - } - - // add allow policy to bypass lr-policy in GR - ipv4HashedAS, ipv6HashedAS := as.GetASHashNames() - var l3Prefix string - var matchSrcAS string - isIPv6 := utilnet.IsIPv6(podIP) - if isIPv6 { - l3Prefix = "ip6" - matchSrcAS = ipv6HashedAS - } else { - l3Prefix = "ip4" - matchSrcAS = ipv4HashedAS - } - - // get the GR to join switch ip address - grJoinIfAddrs, err := libovsdbutil.GetLRPAddrs(oc.nbClient, types.GWRouterToJoinSwitchPrefix+oc.GetNetworkScopedGWRouterName(node)) - if err != nil { - return fmt.Errorf("unable to find IP address for node: %s, %s port, err: %v", node, types.GWRouterToJoinSwitchPrefix, err) - } - grJoinIfAddr, err := util.MatchFirstIPNetFamily(utilnet.IsIPv6(podIP), grJoinIfAddrs) - if err != nil { - return fmt.Errorf("failed to match gateway router join interface IPs: %v, err: %v", grJoinIfAddr, err) - } - - var matchDst string - var clusterL3Prefix string - for _, clusterSubnet := range config.Default.ClusterSubnets { - if utilnet.IsIPv6CIDR(clusterSubnet.CIDR) { - clusterL3Prefix = "ip6" - } else { - clusterL3Prefix = "ip4" - } - if l3Prefix != clusterL3Prefix { - continue - } - matchDst += fmt.Sprintf(" && %s.dst != %s", clusterL3Prefix, clusterSubnet.CIDR) - } - - // traffic destined outside of cluster subnet go to GR - matchStr := fmt.Sprintf(`inport == "%s%s" && %s.src == $%s`, types.RouterToSwitchPrefix, node, l3Prefix, matchSrcAS) - matchStr += matchDst - - logicalRouterPolicy := nbdb.LogicalRouterPolicy{ - Priority: types.HybridOverlayReroutePriority, - Action: nbdb.LogicalRouterPolicyActionReroute, - Nexthops: []string{grJoinIfAddr.IP.String()}, - Match: matchStr, - } - p := func(item *nbdb.LogicalRouterPolicy) bool { - return item.Priority == logicalRouterPolicy.Priority && strings.Contains(item.Match, matchSrcAS) - } - err = libovsdbops.CreateOrUpdateLogicalRouterPolicyWithPredicate(oc.nbClient, oc.GetNetworkScopedClusterRouterName(), - &logicalRouterPolicy, p, &logicalRouterPolicy.Nexthops, &logicalRouterPolicy.Match, &logicalRouterPolicy.Action) - if err != nil { - return fmt.Errorf("failed to add policy route %+v to %s: %v", logicalRouterPolicy, oc.GetNetworkScopedClusterRouterName(), err) - } - } - return nil -} - // delHybridRoutePolicyForPod handles deleting a logical route policy that // forces pod egress traffic to be rerouted to a gateway router for local gateway mode. // WARNING: updates same db entries as apbroutecontroller. Make sure to call only when route is not managed by @@ -933,41 +482,3 @@ func (oc *DefaultNetworkController) extSwitchPrefix(nodeName string) (string, er } return "", nil } - -func getExGwPodIPs(gatewayPod *corev1.Pod) (sets.Set[string], error) { - foundGws := sets.New[string]() - if gatewayPod.Annotations[util.RoutingNetworkAnnotation] != "" { - var multusNetworks []nettypes.NetworkStatus - err := json.Unmarshal([]byte(gatewayPod.ObjectMeta.Annotations[nettypes.NetworkStatusAnnot]), &multusNetworks) - if err != nil { - return nil, fmt.Errorf("unable to unmarshall annotation k8s.v1.cni.cncf.io/network-status on pod %s: %v", - gatewayPod.Name, err) - } - for _, multusNetwork := range multusNetworks { - if multusNetwork.Name == gatewayPod.Annotations[util.RoutingNetworkAnnotation] { - for _, gwIP := range multusNetwork.IPs { - ip := net.ParseIP(gwIP) - if ip != nil { - foundGws.Insert(ip.String()) - } - } - } - } - } else if gatewayPod.Spec.HostNetwork { - for _, podIP := range gatewayPod.Status.PodIPs { - ip := utilnet.ParseIPSloppy(podIP.IP) - if ip != nil { - foundGws.Insert(ip.String()) - } - } - } else { - return nil, fmt.Errorf("ignoring pod %s as an external gateway candidate. Invalid combination "+ - "of host network: %t and routing-network annotation: %s", gatewayPod.Name, gatewayPod.Spec.HostNetwork, - gatewayPod.Annotations[util.RoutingNetworkAnnotation]) - } - return foundGws, nil -} - -func makePodGWKey(pod *corev1.Pod) string { - return fmt.Sprintf("%s_%s", pod.Namespace, pod.Name) -} diff --git a/go-controller/pkg/ovn/namespace.go b/go-controller/pkg/ovn/namespace.go index 9ae52f80a4..93ad1b35b4 100644 --- a/go-controller/pkg/ovn/namespace.go +++ b/go-controller/pkg/ovn/namespace.go @@ -5,79 +5,34 @@ package ovn import ( "fmt" - "net" "time" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/labels" - "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/tools/cache" "k8s.io/klog/v2" "github.com/ovn-kubernetes/libovsdb/ovsdb" - "github.com/ovn-kubernetes/ovn-kubernetes/go-controller/pkg/config" - libovsdbops "github.com/ovn-kubernetes/ovn-kubernetes/go-controller/pkg/libovsdb/ops" - "github.com/ovn-kubernetes/ovn-kubernetes/go-controller/pkg/types" "github.com/ovn-kubernetes/ovn-kubernetes/go-controller/pkg/util" utilerrors "github.com/ovn-kubernetes/ovn-kubernetes/go-controller/pkg/util/errors" ) -func (oc *DefaultNetworkController) getRoutingExternalGWs(nsInfo *namespaceInfo) *gatewayInfo { - res := gatewayInfo{} - // return a copy of the object so it can be handled without the - // namespace locked - res.bfdEnabled = nsInfo.routingExternalGWs.bfdEnabled - res.gws = sets.New(nsInfo.routingExternalGWs.gws.UnsortedList()...) - return &res -} - -// wrapper function to log if there are duplicate gateway IPs present in the cache -func validateRoutingPodGWs(podGWs map[string]gatewayInfo) error { - // map to hold IP/podName - ipTracker := make(map[string]string) - for podName, gwInfo := range podGWs { - for _, gwIP := range gwInfo.gws.UnsortedList() { - if foundPod, ok := ipTracker[gwIP]; ok { - return fmt.Errorf("duplicate IP found in ECMP Pod route cache! IP: %q, first pod: %q, second "+ - "pod: %q", gwIP, podName, foundPod) - } - ipTracker[gwIP] = podName - } - } - return nil -} - -func (oc *DefaultNetworkController) getRoutingPodGWs(nsInfo *namespaceInfo) map[string]gatewayInfo { - // return a copy of the object so it can be handled without the - // namespace locked - res := make(map[string]gatewayInfo) - for k, v := range nsInfo.routingExternalPodGWs { - item := gatewayInfo{ - bfdEnabled: v.bfdEnabled, - gws: sets.New(v.gws.UnsortedList()...), - } - res[k] = item - } - return res -} - -// addLocalPodToNamespace returns pod's routing gateway info and the ops needed -// to add pod's IP to the namespace's address set and port group. -func (oc *DefaultNetworkController) addLocalPodToNamespace(ns string, portUUID string) (*gatewayInfo, map[string]gatewayInfo, []ovsdb.Operation, error) { - var err error +// addLocalPodToNamespace returns the ops needed to add pod's IP to the +// namespace's address set and port group. +func (oc *DefaultNetworkController) addLocalPodToNamespace(ns string, portUUID string) ([]ovsdb.Operation, error) { nsInfo, nsUnlock, err := oc.ensureNamespaceLocked(ns, true, nil) if err != nil { - return nil, nil, nil, fmt.Errorf("failed to ensure namespace locked: %v", err) + return nil, fmt.Errorf("failed to ensure namespace locked: %v", err) } defer nsUnlock() ops, err := oc.addLocalPodToNamespaceLocked(nsInfo, portUUID) if err != nil { - return nil, nil, nil, err + return nil, err } - return oc.getRoutingExternalGWs(nsInfo), oc.getRoutingPodGWs(nsInfo), ops, nil + return ops, nil } func isNamespaceMulticastEnabled(annotations map[string]string) bool { @@ -106,22 +61,6 @@ func (oc *DefaultNetworkController) AddNamespace(ns *corev1.Namespace) error { func (oc *DefaultNetworkController) configureNamespace(nsInfo *namespaceInfo, ns *corev1.Namespace) error { var errors []error - if annotation, ok := ns.Annotations[util.RoutingExternalGWsAnnotation]; ok { - exGateways, err := util.ParseRoutingExternalGWAnnotation(annotation) - if err != nil { - errors = append(errors, fmt.Errorf("failed to parse external gateway annotation (%v)", err)) - } else { - _, bfdEnabled := ns.Annotations[util.BfdAnnotation] - err = oc.addExternalGWsForNamespace(gatewayInfo{gws: exGateways, bfdEnabled: bfdEnabled}, nsInfo, ns.Name) - if err != nil { - errors = append(errors, fmt.Errorf("failed to add external gateway for namespace %s (%v)", ns.Name, err)) - } - } - if _, ok := ns.Annotations[util.BfdAnnotation]; ok { - nsInfo.routingExternalGWs.bfdEnabled = true - } - } - if err := oc.configureNamespaceCommon(nsInfo, ns); err != nil { errors = append(errors, err) } @@ -139,120 +78,6 @@ func (oc *DefaultNetworkController) updateNamespace(old, newer *corev1.Namespace } defer nsUnlock() - gwAnnotation := newer.Annotations[util.RoutingExternalGWsAnnotation] - oldGWAnnotation := old.Annotations[util.RoutingExternalGWsAnnotation] - _, newBFDEnabled := newer.Annotations[util.BfdAnnotation] - _, oldBFDEnabled := old.Annotations[util.BfdAnnotation] - - if gwAnnotation != oldGWAnnotation || newBFDEnabled != oldBFDEnabled { - // if old gw annotation was empty, new one must not be empty, so we should remove any per pod SNAT towards nodeIP - if oldGWAnnotation == "" { - if config.Gateway.DisableSNATMultipleGWs { - existingPods, err := oc.watchFactory.GetPods(old.Name) - if err != nil { - errors = append(errors, fmt.Errorf("failed to get all the pods (%v)", err)) - } - for _, pod := range existingPods { - if !oc.isPodScheduledinLocalZone(pod) { - continue - } - - logicalPort := util.GetLogicalPortName(pod.Namespace, pod.Name) - if util.PodWantsHostNetwork(pod) { - continue - } - podIPs, err := util.GetPodIPsOfNetwork(pod, oc.GetNetInfo(), nil) - if err != nil { - errors = append(errors, fmt.Errorf("unable to get pod %q IPs for SNAT rule removal err (%v)", logicalPort, err)) - } - ips := make([]*net.IPNet, 0, len(podIPs)) - for _, podIP := range podIPs { - ips = append(ips, &net.IPNet{IP: podIP}) - } - if len(ips) > 0 { - if extIPs, err := getExternalIPsGR(oc.watchFactory, pod.Spec.NodeName); err != nil { - errors = append(errors, err) - } else if err = oc.deletePodSNAT(pod.Spec.NodeName, extIPs, ips); err != nil { - errors = append(errors, err) - } - } - } - } - } else { - if err := oc.deleteGWRoutesForNamespace(old.Name, nil); err != nil { - errors = append(errors, err) - } - nsInfo.routingExternalGWs = gatewayInfo{} - } - exGateways, err := util.ParseRoutingExternalGWAnnotation(gwAnnotation) - if err != nil { - errors = append(errors, err) - } else { - if exGateways.Len() != 0 { - err = oc.addExternalGWsForNamespace(gatewayInfo{gws: exGateways, bfdEnabled: newBFDEnabled}, nsInfo, old.Name) - if err != nil { - errors = append(errors, err) - } - } - } - if oc.zone != types.OvnDefaultZone { - // In multi-zone interconnect, ovnkube-controller flushes conntrack - // directly here rather than using the "k8s.ovn.org/external-gw-pod-ips" - // namespace annotation that ovnkube-node watches in single-zone - // deployments. - gatewayIPs, err := oc.apbExternalRouteController.GetAdminPolicyBasedExternalRouteIPsForTargetNamespace(old.Name) - if err != nil { - return fmt.Errorf("unable to retrieve gateway IPs for Admin Policy Based External Route objects for namespace %s: %w", old.Name, err) - } - for _, gwInfo := range nsInfo.routingExternalPodGWs { - gatewayIPs.Insert(gwInfo.gws.UnsortedList()...) - } - gatewayIPs.Insert(nsInfo.routingExternalGWs.gws.UnsortedList()...) - err = oc.syncConntrackForExternalGateways(old.Name, gatewayIPs) // best effort - if err != nil { - klog.Errorf("Syncing conntrack entries for egressGWs %+v serving the namespace %s failed: %v", - gatewayIPs, old.Name, err) - } - } - // if new annotation is empty, exgws were removed, may need to add SNAT per pod - // check if there are any pod gateways serving this namespace as well - if gwAnnotation == "" && len(nsInfo.routingExternalPodGWs) == 0 && config.Gateway.DisableSNATMultipleGWs { - existingPods, err := oc.watchFactory.GetPods(old.Name) - if err != nil { - errors = append(errors, fmt.Errorf("failed to get all the pods (%v)", err)) - } - for _, pod := range existingPods { - if !oc.isPodScheduledinLocalZone(pod) && !util.PodNeedsSNAT(pod) { - continue - } - podAnnotation, err := util.UnmarshalPodAnnotation(pod.Annotations, types.DefaultNetworkName) - if err != nil { - errors = append(errors, err) - } else { - // Helper function to handle the complex SNAT operations - handleSNATOps := func() error { - ops, err := oc.AddPodSNATOps(pod.Spec.NodeName, podAnnotation.IPs) - if err != nil { - return err - } - - // Execute all operations in a single transaction - if len(ops) > 0 { - _, err = libovsdbops.TransactAndCheck(oc.nbClient, ops) - if err != nil { - return fmt.Errorf("failed to update SNAT for pod %s on router %s: %v", pod.Name, oc.GetNetworkScopedGWRouterName(pod.Spec.NodeName), err) - } - } - return nil - } - - if err := handleSNATOps(); err != nil { - errors = append(errors, err) - } - } - } - } - } aclAnnotation := newer.Annotations[util.AclLoggingAnnotation] oldACLAnnotation := old.Annotations[util.AclLoggingAnnotation] // support for ACL logging update, if new annotation is empty, make sure we propagate new setting diff --git a/go-controller/pkg/ovn/ovn.go b/go-controller/pkg/ovn/ovn.go index b44aada806..61fd788015 100644 --- a/go-controller/pkg/ovn/ovn.go +++ b/go-controller/pkg/ovn/ovn.go @@ -10,14 +10,11 @@ import ( "sync" "time" - nettypes "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" - corev1 "k8s.io/api/core/v1" "k8s.io/client-go/kubernetes/scheme" listers "k8s.io/client-go/listers/core/v1" ref "k8s.io/client-go/tools/reference" "k8s.io/klog/v2" - v1pod "k8s.io/kubernetes/pkg/api/v1/pod" libovsdbclient "github.com/ovn-kubernetes/libovsdb/client" @@ -102,20 +99,6 @@ func (oc *DefaultNetworkController) recordPodEvent(reason string, addErr error, } } -func exGatewayAnnotationsChanged(oldPod, newPod *corev1.Pod) bool { - return oldPod.Annotations[util.RoutingNamespaceAnnotation] != newPod.Annotations[util.RoutingNamespaceAnnotation] || - oldPod.Annotations[util.RoutingNetworkAnnotation] != newPod.Annotations[util.RoutingNetworkAnnotation] || - oldPod.Annotations[util.BfdAnnotation] != newPod.Annotations[util.BfdAnnotation] -} - -func networkStatusAnnotationsChanged(oldPod, newPod *corev1.Pod) bool { - return oldPod.Annotations[nettypes.NetworkStatusAnnot] != newPod.Annotations[nettypes.NetworkStatusAnnot] -} - -func podBecameReady(oldPod, newPod *corev1.Pod) bool { - return !v1pod.IsPodReadyConditionTrue(oldPod.Status) && v1pod.IsPodReadyConditionTrue(newPod.Status) -} - // ensurePod tries to set up a pod. It returns nil on success and error on failure; failure // indicates the pod set up should be retried later. func (oc *DefaultNetworkController) ensurePod(oldPod, pod *corev1.Pod, addPort bool) error { @@ -124,14 +107,6 @@ func (oc *DefaultNetworkController) ensurePod(oldPod, pod *corev1.Pod, addPort b return nil } - // If an external gateway pod is in terminating or not ready state then remove the - // routes for the external gateway pod - if util.PodTerminating(pod) || !v1pod.IsPodReadyConditionTrue(pod.Status) { - if err := oc.deletePodExternalGW(pod); err != nil { - return fmt.Errorf("ensurePod failed %s/%s: %w", pod.Namespace, pod.Name, err) - } - } - if oc.isPodScheduledinLocalZone(pod) { klog.V(5).Infof("Ensuring zone local for Pod %s/%s in node %s", pod.Namespace, pod.Name, pod.Spec.NodeName) return oc.ensureLocalZonePod(oldPod, pod, addPort) @@ -156,26 +131,10 @@ func (oc *DefaultNetworkController) ensureLocalZonePod(oldPod, pod *corev1.Pod, }() } - if oldPod != nil && (exGatewayAnnotationsChanged(oldPod, pod) || networkStatusAnnotationsChanged(oldPod, pod)) { - // No matter if a pod is ovn networked, or host networked, we still need to check for exgw - // annotations. If the pod is ovn networked and is in update reschedule, addLogicalPort will take - // care of updating the exgw updates - if err := oc.deletePodExternalGW(oldPod); err != nil { - return fmt.Errorf("ensurePod failed %s/%s: %w", pod.Namespace, pod.Name, err) - } - } - if !util.PodWantsHostNetwork(pod) && addPort { if err := oc.addLogicalPort(pod); err != nil { return fmt.Errorf("addLogicalPort failed for %s/%s: %w", pod.Namespace, pod.Name, err) } - } else { - // either pod is host-networked or its an update for a normal pod (addPort=false case) - if oldPod == nil || exGatewayAnnotationsChanged(oldPod, pod) || networkStatusAnnotationsChanged(oldPod, pod) || podBecameReady(oldPod, pod) { - if err := oc.addPodExternalGW(pod); err != nil { - return fmt.Errorf("addPodExternalGW failed for %s/%s: %w", pod.Namespace, pod.Name, err) - } - } } // update open ports for UDN pods on pod update. @@ -205,28 +164,10 @@ func (oc *DefaultNetworkController) ensureLocalZonePod(oldPod, pod *corev1.Pod, } // ensureRemoteZonePod tries to set up remote zone pod bits required to interconnect it. -// - Reconciles external-gateway annotations on the remote pod // - For live-migratable VMs, ensures remote-zone pod-to-node routes // // It returns nil on success and error on failure; failure indicates the pod set up should be retried later. -func (oc *DefaultNetworkController) ensureRemoteZonePod(oldPod, pod *corev1.Pod) error { - //FIXME: Update comments & reduce code duplication. - // check if this remote pod is serving as an external GW. - if oldPod != nil && (exGatewayAnnotationsChanged(oldPod, pod) || networkStatusAnnotationsChanged(oldPod, pod)) { - // Delete the routes in the namespace associated with this remote oldPod if its acting as an external GW - if err := oc.deletePodExternalGW(oldPod); err != nil { - return fmt.Errorf("deletePodExternalGW failed for remote pod %s/%s: %w", oldPod.Namespace, oldPod.Name, err) - } - } - - // either pod is host-networked or its an update for a normal pod (addPort=false case) - if oldPod == nil || exGatewayAnnotationsChanged(oldPod, pod) || networkStatusAnnotationsChanged(oldPod, pod) || podBecameReady(oldPod, pod) { - // check if this remote pod is serving as an external GW. If so add the routes in the namespace - // associated with this remote pod - if err := oc.addPodExternalGW(pod); err != nil { - return fmt.Errorf("addPodExternalGW failed for remote pod %s/%s: %v", pod.Namespace, pod.Name, err) - } - } +func (oc *DefaultNetworkController) ensureRemoteZonePod(_, pod *corev1.Pod) error { if kubevirt.IsPodLiveMigratable(pod) { return kubevirt.EnsureRemoteZonePodAddressesToNodeRoute(oc.watchFactory, oc.nbClient, pod) } @@ -268,10 +209,6 @@ func (oc *DefaultNetworkController) removeLocalZonePod(pod *corev1.Pod, portInfo }() } if util.PodWantsHostNetwork(pod) { - if err := oc.deletePodExternalGW(pod); err != nil { - return fmt.Errorf("unable to delete external gateway routes for pod %s: %w", - getPodNamespacedName(pod), err) - } return nil } if err := oc.deleteLogicalPort(pod, portInfo); err != nil { @@ -284,15 +221,8 @@ func (oc *DefaultNetworkController) removeLocalZonePod(pod *corev1.Pod, portInfo // removeRemoteZonePod tries to tear down a remote zone pod bits. It returns nil on success and error on failure; // failure indicates the pod tear down should be retried later. -// It removes the remote pod ips from the namespace address set and if its an external gw pod, removes -// its routes. +// It removes the remote pod ips from the namespace address set. func (oc *DefaultNetworkController) removeRemoteZonePod(pod *corev1.Pod) error { - // Delete the routes in the namespace associated with this remote pod if it was acting as an external GW - if err := oc.deletePodExternalGW(pod); err != nil { - return fmt.Errorf("unable to delete external gateway routes for remote pod %s: %w", - getPodNamespacedName(pod), err) - } - // while this check is only intended for local pods, we also need it for // remote live migrated pods that might have been allocated from this zone if oc.wasPodReleasedBeforeStartup(string(pod.UID), ovntypes.DefaultNetworkName) { diff --git a/go-controller/pkg/ovn/pods.go b/go-controller/pkg/ovn/pods.go index dc5477131e..ee7f3458a7 100644 --- a/go-controller/pkg/ovn/pods.go +++ b/go-controller/pkg/ovn/pods.go @@ -177,9 +177,6 @@ func (oc *DefaultNetworkController) deleteLogicalPort(pod *corev1.Pod, portInfo podDesc := pod.Namespace + "/" + pod.Name klog.Infof("Deleting pod: %s", podDesc) - if err = oc.deletePodExternalGW(pod); err != nil { - return fmt.Errorf("unable to delete external gateway routes for pod %s: %w", podDesc, err) - } if pod.Spec.HostNetwork { return nil } @@ -300,39 +297,16 @@ func (oc *DefaultNetworkController) addLogicalPort(pod *corev1.Pod) (err error) } // Ensure the namespace/nsInfo exists - routingExternalGWs, routingPodGWs, addOps, err := oc.addLocalPodToNamespace(pod.Namespace, lsp.UUID) + addOps, err := oc.addLocalPodToNamespace(pod.Namespace, lsp.UUID) if err != nil { return err } ops = append(ops, addOps...) - // if we have any external or pod Gateways, add routes - gateways := make([]*gatewayInfo, 0, len(routingExternalGWs.gws)+len(routingPodGWs)) - - if len(routingExternalGWs.gws) > 0 { - gateways = append(gateways, routingExternalGWs) - } - for key := range routingPodGWs { - gw := routingPodGWs[key] - if len(gw.gws) > 0 { - if err = validateRoutingPodGWs(routingPodGWs); err != nil { - klog.Error(err) - } - gateways = append(gateways, &gw) - } else { - klog.Warningf("Found routingPodGW with no gateways ip set for namespace %s", pod.Namespace) - } - } - - if len(gateways) > 0 { - podNsName := ktypes.NamespacedName{Namespace: pod.Namespace, Name: pod.Name} - err = oc.addGWRoutesForPod(gateways, podAnnotation.IPs, podNsName, pod.Spec.NodeName) - if err != nil { - return err - } - } else if config.Gateway.DisableSNATMultipleGWs { - // Add NAT rules to pods if disable SNAT is set and does not have - // namespace annotations to go through external egress router + if config.Gateway.DisableSNATMultipleGWs { + // Add NAT rules to pods if disable SNAT is set. External gateway routes + // (and the associated per-pod SNAT removal) are programmed by the + // AdminPolicyBasedExternalRoute controller. snatOps, err := oc.AddPodSNATOps(pod.Spec.NodeName, podAnnotation.IPs) if err != nil { return err @@ -355,12 +329,6 @@ func (oc *DefaultNetworkController) addLogicalPort(pod *corev1.Pod) (err error) txOkCallBack() oc.podRecorder.AddLSP(pod.UID, oc.GetNetInfo()) - // check if this pod is serving as an external GW - err = oc.addPodExternalGW(pod) - if err != nil { - return fmt.Errorf("failed to handle external GW check: %v", err) - } - // if somehow lspUUID is empty, there is a bug here with interpreting OVSDB results if len(lsp.UUID) == 0 { return fmt.Errorf("UUID is empty from LSP: %+v", *lsp) From 2d0beded68d898724439f2417957951bb7f97c21 Mon Sep 17 00:00:00 2001 From: Mykola Yurchenko Date: Thu, 4 Jun 2026 09:45:22 -0700 Subject: [PATCH 08/51] apbroute: stop honoring legacy external-gateway annotations Remove the annotation readers and the logic that shielded annotation-derived gateway IPs from deletion; the CRD path is unchanged. Also fixes a stale error string and drops inert annotations from a test fixture. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Mykola Yurchenko --- .../external_controller_namespace_test.go | 5 +- .../apbroute/external_controller_pod.go | 2 +- .../apbroute/external_controller_policy.go | 82 +------------ .../pkg/ovn/controller/apbroute/repair.go | 108 +----------------- 4 files changed, 7 insertions(+), 190 deletions(-) diff --git a/go-controller/pkg/ovn/controller/apbroute/external_controller_namespace_test.go b/go-controller/pkg/ovn/controller/apbroute/external_controller_namespace_test.go index 155e1a368f..6cc090a6a4 100644 --- a/go-controller/pkg/ovn/controller/apbroute/external_controller_namespace_test.go +++ b/go-controller/pkg/ovn/controller/apbroute/external_controller_namespace_test.go @@ -200,9 +200,8 @@ var _ = Describe("OVN External Gateway namespace", func() { annotatedPodGW = &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{Name: "annotatedPod", Namespace: namespaceGW.Name, Labels: map[string]string{"name": "annotatedPod"}, - Annotations: map[string]string{"k8s.ovn.org/routing-namespaces": "test", - "k8s.ovn.org/routing-network": "", - nettypes.NetworkStatusAnnot: fmt.Sprintf(network_status, annotatedPodIP)}, + Annotations: map[string]string{ + nettypes.NetworkStatusAnnot: fmt.Sprintf(network_status, annotatedPodIP)}, }, Status: corev1.PodStatus{ PodIPs: []corev1.PodIP{{IP: annotatedPodIP}}, diff --git a/go-controller/pkg/ovn/controller/apbroute/external_controller_pod.go b/go-controller/pkg/ovn/controller/apbroute/external_controller_pod.go index ebbdf03c3b..d10fa08883 100644 --- a/go-controller/pkg/ovn/controller/apbroute/external_controller_pod.go +++ b/go-controller/pkg/ovn/controller/apbroute/external_controller_pod.go @@ -48,7 +48,7 @@ func getExGwPodIPs(gatewayPod *corev1.Pod, networkName string) (sets.Set[string] return getPodIPs(gatewayPod), nil } return nil, fmt.Errorf("ignoring pod %s as an external gateway candidate. Invalid combination "+ - "of host network: %t and routing-network annotation: %s", gatewayPod.Name, gatewayPod.Spec.HostNetwork, + "of host network: %t and network attachment name: %s", gatewayPod.Name, gatewayPod.Spec.HostNetwork, networkName) } diff --git a/go-controller/pkg/ovn/controller/apbroute/external_controller_policy.go b/go-controller/pkg/ovn/controller/apbroute/external_controller_policy.go index cc07df2113..065f0d3dc0 100644 --- a/go-controller/pkg/ovn/controller/apbroute/external_controller_policy.go +++ b/go-controller/pkg/ovn/controller/apbroute/external_controller_policy.go @@ -6,7 +6,6 @@ package apbroute import ( "fmt" "net" - "strings" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -102,20 +101,6 @@ func (m *externalPolicyManager) updateRoutePolicy(existingPolicy *routePolicySta allGWIPsToDelete := sets.New[string]() allGWIPsToKeep := sets.New[string]() - // Static Hops - annotatedNamespaceGWIPs, err := m.calculateAnnotatedNamespaceGatewayIPsForNamespace(targetNamespace) - if err != nil { - return err - } - - // Dynamic Hops - annotatedPodGWIPs, err := m.calculateAnnotatedPodGatewayIPsForNamespace(targetNamespace) - if err != nil { - return err - } - - allGWIPsToKeep = allGWIPsToKeep.Union(annotatedNamespaceGWIPs).Union(annotatedPodGWIPs) - // track which pods should be removed from targetPods podsToDelete := []ktypes.NamespacedName{} for podNamespacedName, existingPodConfig := range targetPods { @@ -139,10 +124,8 @@ func (m *externalPolicyManager) updateRoutePolicy(existingPolicy *routePolicySta insertSet(gwIPsToLeave, existingGW.Gateways) } } - // don't delete ips from annotations - ipsToDelete := gwIPsToDelete.Difference(annotatedNamespaceGWIPs) - insertSet(allGWIPsToDelete, ipsToDelete) + insertSet(allGWIPsToDelete, gwIPsToDelete) insertSet(allGWIPsToKeep, gwIPsToLeave) // Dynamic Hops @@ -161,10 +144,8 @@ func (m *externalPolicyManager) updateRoutePolicy(existingPolicy *routePolicySta insertSet(gwIPsToLeave, existingGW.Gateways) } } - // don't delete ips from annotations - ipsToDelete = gwIPsToDelete.Difference(annotatedPodGWIPs) - insertSet(allGWIPsToDelete, ipsToDelete) + insertSet(allGWIPsToDelete, gwIPsToDelete) insertSet(allGWIPsToKeep, gwIPsToLeave) if staticGWsToDelete.Len() > 0 || dynamicGWsToDelete.Len() > 0 { @@ -265,65 +246,6 @@ func (m *externalPolicyManager) applyPodConfig(pod *corev1.Pod, existingPodConfi return nil } -// calculateAnnotatedNamespaceGatewayIPsForNamespace retrieves the list of IPs defined by the legacy annotation gateway logic for namespaces. -// this function is used when deleting gateway IPs to ensure that IPs that overlap with the annotation logic are not deleted from the network resource -// (north bound or conntrack) when the given IP is deleted when removing the policy that references them. -func (m *externalPolicyManager) calculateAnnotatedNamespaceGatewayIPsForNamespace(targetNamespace string) (sets.Set[string], error) { - namespace, err := m.namespaceLister.Get(targetNamespace) - if err != nil { - if apierrors.IsNotFound(err) { - return sets.New[string](), nil - } - return nil, err - } - - if annotation, ok := namespace.Annotations[util.RoutingExternalGWsAnnotation]; ok { - exGateways, err := util.ParseRoutingExternalGWAnnotation(annotation) - if err != nil { - return nil, err - } - return exGateways, nil - } - return sets.New[string](), nil - -} - -// calculateAnnotatedPodGatewayIPsForNamespace retrieves the list of IPs defined by the legacy annotation gateway logic for pods. -// this function is used when deleting gateway IPs to ensure that IPs that overlap with the annotation logic are not deleted from the network resource -// (north bound or conntrack) when the given IP is deleted when removing the policy that references them. -func (m *externalPolicyManager) calculateAnnotatedPodGatewayIPsForNamespace(targetNamespace string) (sets.Set[string], error) { - gwIPs := sets.New[string]() - podList, err := m.podLister.Pods(targetNamespace).List(labels.Everything()) - if err != nil { - return nil, err - } - - for _, pod := range podList { - networkName, ok := pod.Annotations[util.RoutingNetworkAnnotation] - if !ok { - continue - } - targetNamespaces, ok := pod.Annotations[util.RoutingNamespaceAnnotation] - if !ok { - continue - } - foundGws, err := getExGwPodIPs(pod, networkName) - if err != nil { - klog.Errorf("Error getting exgw IPs for pod: %s, error: %v", pod.Name, err) - return nil, err - } - if foundGws.Len() == 0 { - klog.Errorf("No pod IPs found for pod %s/%s", pod.Namespace, pod.Name) - continue - } - tmpNs := sets.New(strings.Split(targetNamespaces, ",")...) - if tmpNs.Has(targetNamespaces) { - insertSet(gwIPs, foundGws) - } - } - return gwIPs, nil -} - func (m *externalPolicyManager) processStaticHopsGatewayInformation(hops []*adminpolicybasedrouteapi.StaticHop) (*gateway_info.GatewayInfoList, error) { gwList := gateway_info.NewGatewayInfoList() diff --git a/go-controller/pkg/ovn/controller/apbroute/repair.go b/go-controller/pkg/ovn/controller/apbroute/repair.go index 160b6e62c0..bc264376af 100644 --- a/go-controller/pkg/ovn/controller/apbroute/repair.go +++ b/go-controller/pkg/ovn/controller/apbroute/repair.go @@ -9,7 +9,6 @@ import ( "strings" "time" - corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/labels" ktypes "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" @@ -71,18 +70,12 @@ func (c *ExternalGatewayMasterController) Repair() error { return nil } - annotatedGWIPsMap, err := c.buildExternalIPGatewaysFromAnnotations() - if err != nil { - return fmt.Errorf("cannot retrieve the annotated gateway IPs:%w", err) - } - // compare caches and see if OVN routes are stale for podIP, ovnRoutes := range ovnRouteCache { // pod IP does not exist in the cluster // remove route and any hybrid policy expectedNextHopsPolicy, okPolicy := policyGWIPsMap[podIP] - expectedNextHopsAnnotation, okAnnotation := annotatedGWIPsMap[podIP] - if !okPolicy && !okAnnotation { + if !okPolicy { // No external gateways found for this Pod IP continue } @@ -112,9 +105,6 @@ func (c *ExternalGatewayMasterController) Repair() error { continue } } - if expectedNextHopsAnnotation != nil { - ovnRoute.shouldExist = c.processOVNRoute(ovnRoute, expectedNextHopsAnnotation.gwList, podIP, expectedNextHopsAnnotation, false) - } } } @@ -177,8 +167,7 @@ func (c *ExternalGatewayMasterController) Repair() error { for ip, node := range ovnHybridCache { // check if this pod IP has a corresponding policy, if not, remove it _, okPolicy := policyGWIPsMap[ip] - _, okAnnotation := annotatedGWIPsMap[ip] - if !okPolicy && !okAnnotation { + if !okPolicy { klog.Infof("CleanHybridPRoutes: Removing IP: %s from hybrid route policy", ip) if err := c.nbClient.delHybridRoutePolicyForPod(net.ParseIP(ip), node); err != nil { return fmt.Errorf("CleanHybridPRoutes: error while removing hybrid policy for pod IP: %s, on node: %s, error: %v", @@ -282,99 +271,6 @@ func (c *ExternalGatewayMasterController) processOVNRoute(ovnRoute *ovnRoute, gw return false } -func (c *ExternalGatewayMasterController) buildExternalIPGatewaysFromAnnotations() (map[string]*managedGWIPs, error) { - clusterRouteCache := make(map[string]*managedGWIPs, 0) - - nsList, err := c.mgr.namespaceLister.List(labels.Everything()) - if err != nil { - return nil, err - } - for _, ns := range nsList { - if nsGWIPs, ok := ns.Annotations[util.RoutingExternalGWsAnnotation]; ok && nsGWIPs != "" { - var bfdEnabled bool - ips := sets.Set[string]{} - for _, ip := range strings.Split(nsGWIPs, ",") { - podIPStr := utilnet.ParseIPSloppy(ip).String() - ips.Insert(podIPStr) - } - - if _, ok := ns.Annotations[util.BfdAnnotation]; ok { - bfdEnabled = true - } - gwInfo := gateway_info.NewGatewayInfo(ips, bfdEnabled) - nsPodList, err := c.mgr.podLister.Pods(ns.Name).List(labels.Everything()) - if err != nil { - return nil, err - } - // set static gateway ips for all pods in the namespace - populateManagedGWIPsCacheForPods(gwInfo, clusterRouteCache, nsPodList) - } - } - - podList, err := c.mgr.podLister.List(labels.Everything()) - if err != nil { - return nil, err - } - for _, pod := range podList { - networkName, ok := pod.Annotations[util.RoutingNetworkAnnotation] - if !ok { - continue - } - targetNamespaces, ok := pod.Annotations[util.RoutingNamespaceAnnotation] - if !ok { - continue - } - foundGws, err := getExGwPodIPs(pod, networkName) - if err != nil { - klog.Errorf("Error getting exgw IPs for pod: %s, error: %v", pod.Name, err) - return nil, err - } - if foundGws.Len() == 0 { - klog.Errorf("No pod IPs found for pod %s/%s", pod.Namespace, pod.Name) - continue - } - var bfdEnabled bool - if _, ok := pod.Annotations[util.BfdAnnotation]; ok { - bfdEnabled = true - } - gwInfo := gateway_info.NewGatewayInfo(foundGws, bfdEnabled) - for _, targetNs := range strings.Split(targetNamespaces, ",") { - nsPodList, err := c.mgr.podLister.Pods(targetNs).List(labels.Everything()) - if err != nil { - return nil, err - } - // set dynamic gateway ips for all pods in the targetNamespaces - populateManagedGWIPsCacheForPods(gwInfo, clusterRouteCache, nsPodList) - } - } - return clusterRouteCache, nil -} - -func populateManagedGWIPsCacheForPods(gwInfo *gateway_info.GatewayInfo, cache map[string]*managedGWIPs, podList []*corev1.Pod) { - for gwIP := range gwInfo.Gateways { - for _, pod := range podList { - // ignore completed pods, host networked pods, pods not scheduled - if util.PodWantsHostNetwork(pod) || util.PodCompleted(pod) || !util.PodScheduled(pod) { - continue - } - for _, podIP := range pod.Status.PodIPs { - podIPStr := utilnet.ParseIPSloppy(podIP.IP).String() - if utilnet.IsIPv6String(gwIP) != utilnet.IsIPv6String(podIPStr) { - continue - } - if _, ok := cache[podIPStr]; !ok { - cache[podIPStr] = &managedGWIPs{ - namespacedName: ktypes.NamespacedName{Namespace: pod.Namespace, Name: pod.Name}, - nodeName: pod.Spec.NodeName, - gwList: gateway_info.NewGatewayInfoList(), - } - } - cache[podIPStr].gwList.InsertOverwrite(gateway_info.NewGatewayInfo(sets.New(gwIP), gwInfo.BFDEnabled)) - } - } - } -} - // Build cache of routes in OVN // map[podIP][]ovnRoute type ovnRoute struct { From 52634fa1a4e9242214a04df2ba5063c18b542f05 Mon Sep 17 00:00:00 2001 From: Mykola Yurchenko Date: Thu, 4 Jun 2026 09:45:22 -0700 Subject: [PATCH 09/51] util,node,config: drop legacy external-gateway annotations Remove the four annotation constants and ParseRoutingExternalGWAnnotation, the routing-external-gws read in the ovnkube-node conntrack path, and update the disable-snat-multiple-gws docs (config + Helm) to reference APB. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Mykola Yurchenko --- go-controller/pkg/config/config.go | 2 +- .../node/default_node_network_controller.go | 7 ++--- .../pkg/util/namespace_annotation.go | 30 ++----------------- helm/ovn-kubernetes/README.md | 2 +- .../values-single-node-zone-dpu.yaml | 2 +- .../values-single-node-zone.yaml | 2 +- 6 files changed, 8 insertions(+), 37 deletions(-) diff --git a/go-controller/pkg/config/config.go b/go-controller/pkg/config/config.go index 2627af0a1e..35b7866df2 100644 --- a/go-controller/pkg/config/config.go +++ b/go-controller/pkg/config/config.go @@ -553,7 +553,7 @@ type GatewayConfig struct { VLANID uint `gcfg:"vlan-id"` // NodeportEnable sets whether to provide Kubernetes NodePort service or not NodeportEnable bool `gcfg:"nodeport"` - // DisableSNATMultipleGws sets whether to disable SNAT of egress traffic in namespaces annotated with routing-external-gws + // DisableSNATMultipleGws sets whether to disable SNAT of egress traffic in namespaces with external gateways configured via AdminPolicyBasedExternalRoute CRs // only applicable to the default network not for UDNs DisableSNATMultipleGWs bool `gcfg:"disable-snat-multiple-gws"` // V4JoinSubnet to be used in the cluster diff --git a/go-controller/pkg/node/default_node_network_controller.go b/go-controller/pkg/node/default_node_network_controller.go index d36607d417..8ce2263f91 100644 --- a/go-controller/pkg/node/default_node_network_controller.go +++ b/go-controller/pkg/node/default_node_network_controller.go @@ -1312,8 +1312,7 @@ func exGatewayPodsAnnotationsChanged(oldNs, newNs *corev1.Namespace) bool { // In reality we only care about exgw pod deletions, however since the list of IPs is not expected to change // that often, let's check for *any* changes to these annotations compared to their previous state and trigger // the logic for checking if we need to delete any conntrack entries - return (oldNs.Annotations[util.ExternalGatewayPodIPsAnnotation] != newNs.Annotations[util.ExternalGatewayPodIPsAnnotation]) || - (oldNs.Annotations[util.RoutingExternalGWsAnnotation] != newNs.Annotations[util.RoutingExternalGWsAnnotation]) + return oldNs.Annotations[util.ExternalGatewayPodIPsAnnotation] != newNs.Annotations[util.ExternalGatewayPodIPsAnnotation] } func (nc *DefaultNodeNetworkController) checkAndDeleteStaleConntrackEntries() { @@ -1322,9 +1321,8 @@ func (nc *DefaultNodeNetworkController) checkAndDeleteStaleConntrackEntries() { klog.Errorf("Unable to get pods from informer: %v", err) } for _, namespace := range namespaces { - _, foundRoutingExternalGWsAnnotation := namespace.Annotations[util.RoutingExternalGWsAnnotation] _, foundExternalGatewayPodIPsAnnotation := namespace.Annotations[util.ExternalGatewayPodIPsAnnotation] - if foundRoutingExternalGWsAnnotation || foundExternalGatewayPodIPsAnnotation { + if foundExternalGatewayPodIPsAnnotation { pods, err := nc.watchFactory.GetPods(namespace.Name) if err != nil { klog.Warningf("Unable to get pods from informer for namespace %s: %v", namespace.Name, err) @@ -1345,7 +1343,6 @@ func (nc *DefaultNodeNetworkController) syncConntrackForExternalGateways(newNs * } // loop through all the IPs on the annotations; ARP for their MACs and form an allowlist gatewayIPs = gatewayIPs.Insert(strings.Split(newNs.Annotations[util.ExternalGatewayPodIPsAnnotation], ",")...) - gatewayIPs = gatewayIPs.Insert(strings.Split(newNs.Annotations[util.RoutingExternalGWsAnnotation], ",")...) return util.SyncConntrackForExternalGateways(gatewayIPs, nil, func() ([]*corev1.Pod, error) { return nc.watchFactory.GetPods(newNs.Name) diff --git a/go-controller/pkg/util/namespace_annotation.go b/go-controller/pkg/util/namespace_annotation.go index 32c898de85..882d9e34f9 100644 --- a/go-controller/pkg/util/namespace_annotation.go +++ b/go-controller/pkg/util/namespace_annotation.go @@ -5,23 +5,16 @@ package util import ( "fmt" - "net" "strings" - "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/klog/v2" - "github.com/ovn-kubernetes/ovn-kubernetes/go-controller/pkg/kube" ) const ( // Annotation used to enable/disable multicast in the namespace NsMulticastAnnotation = "k8s.ovn.org/multicast-enabled" - // Annotations used by multiple external gateways feature - RoutingExternalGWsAnnotation = "k8s.ovn.org/routing-external-gws" - RoutingNamespaceAnnotation = "k8s.ovn.org/routing-namespaces" - RoutingNetworkAnnotation = "k8s.ovn.org/routing-network" - BfdAnnotation = "k8s.ovn.org/bfd-enabled" + // ExternalGatewayPodIPsAnnotation is an internal annotation used to signal + // ovnkube-node to flush conntrack for external gateway pod IPs. ExternalGatewayPodIPsAnnotation = "k8s.ovn.org/external-gw-pod-ips" // Annotation for enabling ACL logging to controller's log file AclLoggingAnnotation = "k8s.ovn.org/acl-logging" @@ -35,22 +28,3 @@ func UpdateExternalGatewayPodIPsAnnotation(k kube.Interface, namespace string, e } return nil } - -func ParseRoutingExternalGWAnnotation(annotation string) (sets.Set[string], error) { - ipTracker := sets.New[string]() - if annotation == "" { - return ipTracker, nil - } - for _, v := range strings.Split(annotation, ",") { - parsedAnnotation := net.ParseIP(v) - if parsedAnnotation == nil { - return nil, fmt.Errorf("could not parse routing external gw annotation value %s", v) - } - if ipTracker.Has(parsedAnnotation.String()) { - klog.Warningf("Duplicate IP detected in routing external gw annotation: %s", annotation) - continue - } - ipTracker.Insert(parsedAnnotation.String()) - } - return ipTracker, nil -} diff --git a/helm/ovn-kubernetes/README.md b/helm/ovn-kubernetes/README.md index 84f82123c0..294513d559 100644 --- a/helm/ovn-kubernetes/README.md +++ b/helm/ovn-kubernetes/README.md @@ -64,7 +64,7 @@ false "" - Whether to disable SNAT of egress traffic in namespaces annotated with routing-external-gws + Whether to disable SNAT of egress traffic in namespaces with external gateways configured via AdminPolicyBasedExternalRoute CRs global.dockerConfigSecret diff --git a/helm/ovn-kubernetes/values-single-node-zone-dpu.yaml b/helm/ovn-kubernetes/values-single-node-zone-dpu.yaml index 0848788563..f93eb6b0d7 100644 --- a/helm/ovn-kubernetes/values-single-node-zone-dpu.yaml +++ b/helm/ovn-kubernetes/values-single-node-zone-dpu.yaml @@ -110,7 +110,7 @@ global: enablePersistentIPs: false # -- Configure to use DNSNameResolver feature with ovn-kubernetes enableDNSNameResolver: false - # -- Whether to disable SNAT of egress traffic in namespaces annotated with routing-external-gws + # -- Whether to disable SNAT of egress traffic in namespaces with external gateways configured via AdminPolicyBasedExternalRoute CRs disableSnatMultipleGws: "" # -- Controls if forwarding is allowed on OVNK controlled interfaces # @default -- false diff --git a/helm/ovn-kubernetes/values-single-node-zone.yaml b/helm/ovn-kubernetes/values-single-node-zone.yaml index a96bc54f73..bb1440ecbf 100644 --- a/helm/ovn-kubernetes/values-single-node-zone.yaml +++ b/helm/ovn-kubernetes/values-single-node-zone.yaml @@ -127,7 +127,7 @@ global: enableDNSNameResolver: false # -- Configure to allow ICMP and ICMPv6 traffic to bypass NetworkPolicy deny rules allowICMPNetworkPolicy: false - # -- Whether to disable SNAT of egress traffic in namespaces annotated with routing-external-gws + # -- Whether to disable SNAT of egress traffic in namespaces with external gateways configured via AdminPolicyBasedExternalRoute CRs disableSnatMultipleGws: "" # -- Controls if forwarding is allowed on OVNK controlled interfaces # @default -- false From be576f25c81a3f18a83004146a953eea39377c71 Mon Sep 17 00:00:00 2001 From: Mykola Yurchenko Date: Thu, 4 Jun 2026 09:45:22 -0700 Subject: [PATCH 10/51] tests,e2e: remove legacy external-gateway annotation specs Trim egressgw_test.go to the surviving delete-hybrid/SNAT cases and drop the annotation-based Contexts and now-dead helpers from the external_gateways e2e suite, keeping the APB CRD specs. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Mykola Yurchenko --- go-controller/pkg/ovn/egressgw_test.go | 3196 +----------------------- test/e2e/external_gateways.go | 1848 +------------- 2 files changed, 66 insertions(+), 4978 deletions(-) diff --git a/go-controller/pkg/ovn/egressgw_test.go b/go-controller/pkg/ovn/egressgw_test.go index 4f76c9452d..4184b614c0 100644 --- a/go-controller/pkg/ovn/egressgw_test.go +++ b/go-controller/pkg/ovn/egressgw_test.go @@ -4,26 +4,17 @@ package ovn import ( - "context" - "encoding/json" - "fmt" "net" - "sync" - "time" - nettypes "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" "github.com/onsi/ginkgo/v2" "github.com/onsi/gomega" "github.com/urfave/cli/v2" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/sets" "github.com/ovn-kubernetes/ovn-kubernetes/go-controller/pkg/config" - adminpolicybasedrouteapi "github.com/ovn-kubernetes/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1" "github.com/ovn-kubernetes/ovn-kubernetes/go-controller/pkg/kube" - libovsdbops "github.com/ovn-kubernetes/ovn-kubernetes/go-controller/pkg/libovsdb/ops" "github.com/ovn-kubernetes/ovn-kubernetes/go-controller/pkg/nbdb" addressset "github.com/ovn-kubernetes/ovn-kubernetes/go-controller/pkg/ovn/address_set" "github.com/ovn-kubernetes/ovn-kubernetes/go-controller/pkg/ovn/controller/apbroute" @@ -41,8 +32,6 @@ var _ = ginkgo.Describe("OVN Egress Gateway Operations", func() { app *cli.App fakeOvn *FakeOVN - bfd1NamedUUID = "bfd-1-UUID" - bfd2NamedUUID = "bfd-2-UUID" logicalRouterPort = "rtoe-GR_node1" ) @@ -62,3190 +51,7 @@ var _ = ginkgo.Describe("OVN Egress Gateway Operations", func() { fakeOvn.shutdown() }) - ginkgo.Context("on setting namespace gateway annotations", func() { - - ginkgo.DescribeTable("reconciles an new pod with namespace single exgw annotation already set", func(bfd bool, finalNB []libovsdbtest.TestData) { - app.Action = func(*cli.Context) error { - - namespaceT := *ovntest.NewNamespace(namespaceName) - namespaceT.Annotations = map[string]string{"k8s.ovn.org/routing-external-gws": "9.0.0.1"} - if bfd { - namespaceT.Annotations["k8s.ovn.org/bfd-enabled"] = "" - } - t := newTPod( - "node1", - "10.128.1.0/24", - "10.128.1.2", - "10.128.1.1", - "myPod", - "10.128.1.3", - "0a:58:0a:80:01:03", - namespaceT.Name, - ) - - fakeOvn.startWithDBSetup( - libovsdbtest.TestSetup{ - NBData: []libovsdbtest.TestData{ - &nbdb.LogicalSwitch{ - UUID: "node1", - Name: "node1", - }, - &nbdb.LogicalRouter{ - UUID: "GR_node1-UUID", - Name: "GR_node1", - }, - }, - }, - &corev1.NamespaceList{ - Items: []corev1.Namespace{ - namespaceT, - }, - }, - &corev1.NodeList{ - Items: []corev1.Node{ - *newNode("node1", "192.168.126.202/24"), - }, - }, - &corev1.PodList{ - Items: []corev1.Pod{ - *ovntest.NewPod(t.namespace, t.podName, t.nodeName, t.podIP), - }, - }, - ) - - t.populateLogicalSwitchCache(fakeOvn) - - injectNode(fakeOvn) - err := fakeOvn.controller.WatchNamespaces() - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fakeOvn.controller.WatchPods() - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - gomega.Eventually(func() string { return getPodAnnotations(fakeOvn.fakeClient.KubeClient, t.namespace, t.podName) }, 2).Should(gomega.MatchJSON(t.getAnnotationsJson())) - gomega.Eventually(fakeOvn.nbClient).Should(libovsdbtest.HaveData(finalNB)) - return nil - } - - err := app.Run([]string{app.Name}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }, ginkgo.Entry("No BFD", false, []libovsdbtest.TestData{ - &nbdb.LogicalSwitchPort{ - UUID: "lsp1", - Addresses: []string{"0a:58:0a:80:01:03 10.128.1.3"}, - ExternalIDs: map[string]string{ - "pod": "true", - "namespace": namespaceName, - }, - Name: "namespace1_myPod", - Options: map[string]string{ - "iface-id-ver": "myPod", - libovsdbops.RequestedChassis: chassisIDForNode("node1"), - }, - PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, - }, - &nbdb.LogicalSwitch{ - UUID: "node1", - Name: "node1", - Ports: []string{"lsp1"}, - }, - &nbdb.LogicalRouterStaticRoute{ - UUID: "static-route-1-UUID", - IPPrefix: "10.128.1.3/32", - Nexthop: "9.0.0.1", - Policy: &nbdb.LogicalRouterStaticRoutePolicySrcIP, - OutputPort: &logicalRouterPort, - Options: map[string]string{ - "ecmp_symmetric_reply": "true", - }, - }, - &nbdb.LogicalRouter{ - UUID: "GR_node1-UUID", - Name: "GR_node1", - StaticRoutes: []string{"static-route-1-UUID"}, - }, - }), - ginkgo.Entry("BFD Enabled", true, []libovsdbtest.TestData{ - &nbdb.LogicalSwitchPort{ - UUID: "lsp1", - Addresses: []string{"0a:58:0a:80:01:03 10.128.1.3"}, - ExternalIDs: map[string]string{ - "pod": "true", - "namespace": namespaceName, - }, - Name: "namespace1_myPod", - Options: map[string]string{ - "iface-id-ver": "myPod", - libovsdbops.RequestedChassis: chassisIDForNode("node1"), - }, - PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, - }, - &nbdb.LogicalSwitch{ - UUID: "node1", - Name: "node1", - Ports: []string{"lsp1"}, - }, - &nbdb.BFD{ - UUID: bfd1NamedUUID, - DstIP: "9.0.0.1", - LogicalPort: "rtoe-GR_node1", - }, - &nbdb.LogicalRouterStaticRoute{ - UUID: "static-route-1-UUID", - IPPrefix: "10.128.1.3/32", - Nexthop: "9.0.0.1", - BFD: &bfd1NamedUUID, - Policy: &nbdb.LogicalRouterStaticRoutePolicySrcIP, - OutputPort: &logicalRouterPort, - Options: map[string]string{ - "ecmp_symmetric_reply": "true", - }, - }, - &nbdb.LogicalRouter{ - UUID: "GR_node1-UUID", - Name: "GR_node1", - StaticRoutes: []string{"static-route-1-UUID"}, - }, - })) - - ginkgo.DescribeTable("reconciles an new pod with namespace single exgw annotation already set with pod event first", func(bfd bool, finalNB []libovsdbtest.TestData) { - app.Action = func(*cli.Context) error { - - namespaceT := *ovntest.NewNamespace(namespaceName) - namespaceT.Annotations = map[string]string{"k8s.ovn.org/routing-external-gws": "9.0.0.1"} - if bfd { - namespaceT.Annotations["k8s.ovn.org/bfd-enabled"] = "" - } - t := newTPod( - "node1", - "10.128.1.0/24", - "10.128.1.2", - "10.128.1.1", - "myPod", - "10.128.1.3", - "0a:58:0a:80:01:03", - namespaceT.Name, - ) - - fakeOvn.startWithDBSetup( - libovsdbtest.TestSetup{ - NBData: []libovsdbtest.TestData{ - &nbdb.LogicalSwitch{ - UUID: "node1", - Name: "node1", - }, - &nbdb.LogicalRouter{ - UUID: "GR_node1-UUID", - Name: "GR_node1", - }, - }, - }, - &corev1.NodeList{ - Items: []corev1.Node{ - *newNode("node1", "192.168.126.202/24"), - }, - }, - &corev1.PodList{ - Items: []corev1.Pod{ - *ovntest.NewPod(t.namespace, t.podName, t.nodeName, t.podIP), - }, - }, - ) - t.populateLogicalSwitchCache(fakeOvn) - - injectNode(fakeOvn) - err := fakeOvn.controller.WatchNamespaces() - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fakeOvn.controller.WatchPods() - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - _, err = fakeOvn.fakeClient.KubeClient.CoreV1().Namespaces().Create(context.TODO(), &namespaceT, metav1.CreateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - gomega.Eventually(func() string { return getPodAnnotations(fakeOvn.fakeClient.KubeClient, t.namespace, t.podName) }, 2).Should(gomega.MatchJSON(t.getAnnotationsJson())) - gomega.Eventually(fakeOvn.nbClient).Should(libovsdbtest.HaveData(finalNB)) - return nil - } - - err := app.Run([]string{app.Name}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }, ginkgo.Entry("No BFD", false, []libovsdbtest.TestData{ - &nbdb.LogicalSwitchPort{ - UUID: "lsp1", - Addresses: []string{"0a:58:0a:80:01:03 10.128.1.3"}, - ExternalIDs: map[string]string{ - "pod": "true", - "namespace": namespaceName, - }, - Name: "namespace1_myPod", - Options: map[string]string{ - "iface-id-ver": "myPod", - libovsdbops.RequestedChassis: chassisIDForNode("node1"), - }, - PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, - }, - &nbdb.LogicalSwitch{ - UUID: "node1", - Name: "node1", - Ports: []string{"lsp1"}, - }, - &nbdb.LogicalRouterStaticRoute{ - UUID: "static-route-1-UUID", - IPPrefix: "10.128.1.3/32", - Nexthop: "9.0.0.1", - Policy: &nbdb.LogicalRouterStaticRoutePolicySrcIP, - OutputPort: &logicalRouterPort, - Options: map[string]string{ - "ecmp_symmetric_reply": "true", - }, - }, - &nbdb.LogicalRouter{ - UUID: "GR_node1-UUID", - Name: "GR_node1", - StaticRoutes: []string{"static-route-1-UUID"}, - }, - }), - ginkgo.Entry("BFD Enabled", true, []libovsdbtest.TestData{ - &nbdb.LogicalSwitchPort{ - UUID: "lsp1", - Addresses: []string{"0a:58:0a:80:01:03 10.128.1.3"}, - ExternalIDs: map[string]string{ - "pod": "true", - "namespace": namespaceName, - }, - Name: "namespace1_myPod", - Options: map[string]string{ - "iface-id-ver": "myPod", - libovsdbops.RequestedChassis: chassisIDForNode("node1"), - }, - PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, - }, - &nbdb.LogicalSwitch{ - UUID: "node1", - Name: "node1", - Ports: []string{"lsp1"}, - }, - &nbdb.BFD{ - UUID: bfd1NamedUUID, - DstIP: "9.0.0.1", - LogicalPort: "rtoe-GR_node1", - }, - &nbdb.LogicalRouterStaticRoute{ - UUID: "static-route-1-UUID", - IPPrefix: "10.128.1.3/32", - Nexthop: "9.0.0.1", - BFD: &bfd1NamedUUID, - Policy: &nbdb.LogicalRouterStaticRoutePolicySrcIP, - OutputPort: &logicalRouterPort, - Options: map[string]string{ - "ecmp_symmetric_reply": "true", - }, - }, - &nbdb.LogicalRouter{ - UUID: "GR_node1-UUID", - Name: "GR_node1", - StaticRoutes: []string{"static-route-1-UUID"}, - }, - })) - - ginkgo.DescribeTable("reconciles an new pod with namespace double exgw annotation already set", func(bfd bool, finalNB []libovsdbtest.TestData) { - - app.Action = func(*cli.Context) error { - - namespaceT := *ovntest.NewNamespace(namespaceName) - namespaceT.Annotations = map[string]string{"k8s.ovn.org/routing-external-gws": "9.0.0.1,9.0.0.2"} - if bfd { - namespaceT.Annotations["k8s.ovn.org/bfd-enabled"] = "" - } - t := newTPod( - "node1", - "10.128.1.0/24", - "10.128.1.2", - "10.128.1.1", - "myPod", - "10.128.1.3", - "0a:58:0a:80:01:03", - namespaceT.Name, - ) - - fakeOvn.startWithDBSetup( - libovsdbtest.TestSetup{ - NBData: []libovsdbtest.TestData{ - &nbdb.LogicalSwitch{ - UUID: "node1", - Name: "node1", - }, - &nbdb.LogicalRouter{ - UUID: "GR_node1-UUID", - Name: "GR_node1", - }, - }, - }, - &corev1.NamespaceList{ - Items: []corev1.Namespace{ - namespaceT, - }, - }, - &corev1.NodeList{ - Items: []corev1.Node{ - *newNode("node1", "192.168.126.202/24"), - }, - }, - &corev1.PodList{ - Items: []corev1.Pod{ - *ovntest.NewPod(t.namespace, t.podName, t.nodeName, t.podIP), - }, - }, - ) - t.populateLogicalSwitchCache(fakeOvn) - - injectNode(fakeOvn) - err := fakeOvn.controller.WatchNamespaces() - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fakeOvn.controller.WatchPods() - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - gomega.Eventually(func() string { return getPodAnnotations(fakeOvn.fakeClient.KubeClient, t.namespace, t.podName) }, 2).Should(gomega.MatchJSON(t.getAnnotationsJson())) - gomega.Eventually(fakeOvn.nbClient).Should(libovsdbtest.HaveData(finalNB)) - return nil - } - - err := app.Run([]string{app.Name}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }, - ginkgo.Entry("No BFD", false, []libovsdbtest.TestData{ - &nbdb.LogicalSwitchPort{ - UUID: "lsp1", - Addresses: []string{"0a:58:0a:80:01:03 10.128.1.3"}, - ExternalIDs: map[string]string{ - "pod": "true", - "namespace": namespaceName, - }, - Name: "namespace1_myPod", - Options: map[string]string{ - "iface-id-ver": "myPod", - libovsdbops.RequestedChassis: chassisIDForNode("node1"), - }, - PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, - }, - &nbdb.LogicalSwitch{ - UUID: "node1", - Name: "node1", - Ports: []string{"lsp1"}, - }, - &nbdb.LogicalRouterStaticRoute{ - UUID: "static-route-1-UUID", - IPPrefix: "10.128.1.3/32", - Nexthop: "9.0.0.1", - Policy: &nbdb.LogicalRouterStaticRoutePolicySrcIP, - OutputPort: &logicalRouterPort, - Options: map[string]string{ - "ecmp_symmetric_reply": "true", - }, - }, - &nbdb.LogicalRouterStaticRoute{ - UUID: "static-route-2-UUID", - IPPrefix: "10.128.1.3/32", - Nexthop: "9.0.0.2", - Policy: &nbdb.LogicalRouterStaticRoutePolicySrcIP, - OutputPort: &logicalRouterPort, - Options: map[string]string{ - "ecmp_symmetric_reply": "true", - }, - }, - &nbdb.LogicalRouter{ - UUID: "GR_node1-UUID", - Name: "GR_node1", - StaticRoutes: []string{"static-route-1-UUID", "static-route-2-UUID"}, - }, - }), - ginkgo.Entry("BFD Enabled", true, []libovsdbtest.TestData{ - &nbdb.LogicalSwitchPort{ - UUID: "lsp1", - Addresses: []string{"0a:58:0a:80:01:03 10.128.1.3"}, - ExternalIDs: map[string]string{ - "pod": "true", - "namespace": namespaceName, - }, - Name: "namespace1_myPod", - Options: map[string]string{ - "iface-id-ver": "myPod", - libovsdbops.RequestedChassis: chassisIDForNode("node1"), - }, - PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, - }, - &nbdb.LogicalSwitch{ - UUID: "node1", - Name: "node1", - Ports: []string{"lsp1"}, - }, - &nbdb.BFD{ - UUID: bfd1NamedUUID, - DstIP: "9.0.0.1", - LogicalPort: "rtoe-GR_node1", - }, - &nbdb.BFD{ - UUID: bfd2NamedUUID, - DstIP: "9.0.0.2", - LogicalPort: "rtoe-GR_node1", - }, - &nbdb.LogicalRouterStaticRoute{ - UUID: "static-route-1-UUID", - IPPrefix: "10.128.1.3/32", - Nexthop: "9.0.0.1", - BFD: &bfd1NamedUUID, - Policy: &nbdb.LogicalRouterStaticRoutePolicySrcIP, - OutputPort: &logicalRouterPort, - Options: map[string]string{ - "ecmp_symmetric_reply": "true", - }, - }, - &nbdb.LogicalRouterStaticRoute{ - UUID: "static-route-2-UUID", - IPPrefix: "10.128.1.3/32", - Nexthop: "9.0.0.2", - Policy: &nbdb.LogicalRouterStaticRoutePolicySrcIP, - BFD: &bfd2NamedUUID, - OutputPort: &logicalRouterPort, - Options: map[string]string{ - "ecmp_symmetric_reply": "true", - }, - }, - &nbdb.LogicalRouter{ - UUID: "GR_node1-UUID", - Name: "GR_node1", - StaticRoutes: []string{"static-route-1-UUID", "static-route-2-UUID"}, - }, - }), - ) - - ginkgo.DescribeTable("reconciles deleting a pod with namespace double exgw annotation already set", - func(bfd bool, - initNB []libovsdbtest.TestData, - finalNB []libovsdbtest.TestData, - ) { - app.Action = func(*cli.Context) error { - - namespaceT := *ovntest.NewNamespace(namespaceName) - namespaceT.Annotations = map[string]string{"k8s.ovn.org/routing-external-gws": "9.0.0.1,9.0.0.2"} - if bfd { - namespaceT.Annotations["k8s.ovn.org/bfd-enabled"] = "" - } - t := newTPod( - "node1", - "10.128.1.0/24", - "10.128.1.2", - "10.128.1.1", - "myPod", - "10.128.1.3", - "0a:58:0a:80:01:03", - namespaceT.Name, - ) - - fakeOvn.startWithDBSetup( - libovsdbtest.TestSetup{ - NBData: initNB, - }, - &corev1.NamespaceList{ - Items: []corev1.Namespace{ - namespaceT, - }, - }, - &corev1.NodeList{ - Items: []corev1.Node{ - *newNode("node1", "192.168.126.202/24"), - }, - }, - &corev1.PodList{ - Items: []corev1.Pod{ - *ovntest.NewPod(t.namespace, t.podName, t.nodeName, t.podIP), - }, - }, - ) - t.populateLogicalSwitchCache(fakeOvn) - - injectNode(fakeOvn) - err := fakeOvn.controller.WatchNamespaces() - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fakeOvn.controller.WatchPods() - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - gomega.Eventually(func() string { return getPodAnnotations(fakeOvn.fakeClient.KubeClient, t.namespace, t.podName) }, 2).Should(gomega.MatchJSON(t.getAnnotationsJson())) - - err = fakeOvn.fakeClient.KubeClient.CoreV1().Pods(t.namespace).Delete(context.TODO(), t.podName, *metav1.NewDeleteOptions(0)) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Eventually(fakeOvn.nbClient).Should(libovsdbtest.HaveData(finalNB)) - return nil - } - err := app.Run([]string{app.Name}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }, - ginkgo.Entry("No BFD", false, - []libovsdbtest.TestData{ - &nbdb.LogicalSwitch{ - UUID: "node1", - Name: "node1", - }, - &nbdb.LogicalRouterStaticRoute{ - UUID: "static-route-1-UUID", - IPPrefix: "10.128.1.3/32", - Nexthop: "9.0.0.1", - Policy: &nbdb.LogicalRouterStaticRoutePolicySrcIP, - OutputPort: &logicalRouterPort, - Options: map[string]string{ - "ecmp_symmetric_reply": "true", - }, - }, - &nbdb.LogicalRouterStaticRoute{ - UUID: "static-route-2-UUID", - IPPrefix: "10.128.1.3/32", - Nexthop: "9.0.0.2", - Policy: &nbdb.LogicalRouterStaticRoutePolicySrcIP, - OutputPort: &logicalRouterPort, - Options: map[string]string{ - "ecmp_symmetric_reply": "true", - }, - }, - &nbdb.LogicalRouter{ - UUID: "GR_node1-UUID", - Name: "GR_node1", - StaticRoutes: []string{"static-route-1-UUID", "static-route-2-UUID"}, - }, - }, - []libovsdbtest.TestData{ - &nbdb.LogicalSwitch{ - UUID: "node1", - Name: "node1", - }, - &nbdb.LogicalRouter{ - UUID: "GR_node1-UUID", - Name: "GR_node1", - StaticRoutes: []string{}, - }, - }, - ), - ginkgo.Entry("BFD", true, - []libovsdbtest.TestData{ - &nbdb.LogicalSwitch{ - UUID: "node1", - Name: "node1", - }, - &nbdb.BFD{ - UUID: bfd1NamedUUID, - DstIP: "9.0.0.1", - LogicalPort: "rtoe-GR_node1", - }, - &nbdb.BFD{ - UUID: bfd2NamedUUID, - DstIP: "9.0.0.2", - LogicalPort: "rtoe-GR_node1", - }, - &nbdb.LogicalRouterStaticRoute{ - UUID: "static-route-1-UUID", - IPPrefix: "10.128.1.3/32", - Nexthop: "9.0.0.1", - Policy: &nbdb.LogicalRouterStaticRoutePolicySrcIP, - BFD: &bfd1NamedUUID, - OutputPort: &logicalRouterPort, - Options: map[string]string{ - "ecmp_symmetric_reply": "true", - }, - }, - &nbdb.LogicalRouterStaticRoute{ - UUID: "static-route-2-UUID", - IPPrefix: "10.128.1.3/32", - Nexthop: "9.0.0.2", - Policy: &nbdb.LogicalRouterStaticRoutePolicySrcIP, - BFD: &bfd2NamedUUID, - OutputPort: &logicalRouterPort, - Options: map[string]string{ - "ecmp_symmetric_reply": "true", - }, - }, - &nbdb.LogicalRouter{ - UUID: "GR_node1-UUID", - Name: "GR_node1", - StaticRoutes: []string{"static-route-1-UUID", "static-route-2-UUID"}, - }, - }, - []libovsdbtest.TestData{ - &nbdb.LogicalSwitch{ - UUID: "node1", - Name: "node1", - }, - &nbdb.LogicalRouter{ - UUID: "GR_node1-UUID", - Name: "GR_node1", - StaticRoutes: []string{}, - }, - }, - ), - ) - - ginkgo.DescribeTable("reconciles deleting a pod with namespace double exgw annotation already set IPV6", - func(bfd bool, - initNB []libovsdbtest.TestData, - finalNB []libovsdbtest.TestData) { - app.Action = func(*cli.Context) error { - namespaceT := *ovntest.NewNamespace(namespaceName) - namespaceT.Annotations = map[string]string{"k8s.ovn.org/routing-external-gws": "fd2e:6f44:5dd8::89,fd2e:6f44:5dd8::76"} - if bfd { - namespaceT.Annotations["k8s.ovn.org/bfd-enabled"] = "" - } - t := newTPod( - "node1", - "fd00:10:244:2::0/64", - "fd00:10:244:2::2", - "fd00:10:244:2::1", - "myPod", - "fd00:10:244:2::3", - "0a:58:49:a1:93:cb", - namespaceT.Name, - ) - - fakeOvn.startWithDBSetup( - libovsdbtest.TestSetup{ - NBData: initNB, - }, - &corev1.NamespaceList{ - Items: []corev1.Namespace{ - namespaceT, - }, - }, - &corev1.NodeList{ - Items: []corev1.Node{ - *newNode("node1", "192.168.126.202/24"), - }, - }, - &corev1.PodList{ - Items: []corev1.Pod{ - *ovntest.NewPod(t.namespace, t.podName, t.nodeName, t.podIP), - }, - }, - ) - config.IPv6Mode = true - t.populateLogicalSwitchCache(fakeOvn) - injectNode(fakeOvn) - err := fakeOvn.controller.WatchNamespaces() - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fakeOvn.controller.WatchPods() - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - gomega.Eventually(func() string { return getPodAnnotations(fakeOvn.fakeClient.KubeClient, t.namespace, t.podName) }, 2).Should(gomega.MatchJSON(t.getAnnotationsJson())) - - err = fakeOvn.fakeClient.KubeClient.CoreV1().Pods(t.namespace).Delete(context.TODO(), t.podName, *metav1.NewDeleteOptions(0)) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Eventually(fakeOvn.nbClient).Should(libovsdbtest.HaveData(finalNB...)) - return nil - } - err := app.Run([]string{app.Name}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }, - ginkgo.Entry("BFD IPV6", true, []libovsdbtest.TestData{ - &nbdb.LogicalSwitch{ - UUID: "node1", - Name: "node1", - }, - &nbdb.LogicalRouterStaticRoute{ - UUID: "static-route-1-UUID", - IPPrefix: "fd00:10:244:2::3/128", - BFD: &bfd1NamedUUID, - OutputPort: &logicalRouterPort, - Nexthop: "fd2e:6f44:5dd8::89", - Policy: &nbdb.LogicalRouterStaticRoutePolicySrcIP, - Options: map[string]string{ - "ecmp_symmetric_reply": "true", - }, - }, - &nbdb.LogicalRouterStaticRoute{ - UUID: "static-route-2-UUID", - IPPrefix: "fd00:10:244:2::3/128", - BFD: &bfd1NamedUUID, - OutputPort: &logicalRouterPort, - Nexthop: "fd2e:6f44:5dd8::76", - Policy: &nbdb.LogicalRouterStaticRoutePolicySrcIP, - Options: map[string]string{ - "ecmp_symmetric_reply": "true", - }, - }, - &nbdb.BFD{ - UUID: bfd2NamedUUID, - DstIP: "fd2e:6f44:5dd8::76", - LogicalPort: "rtoe-GR_node1", - }, - &nbdb.BFD{ - UUID: bfd1NamedUUID, - DstIP: "fd2e:6f44:5dd8::89", - LogicalPort: "rtoe-GR_node1", - }, - &nbdb.LogicalRouter{ - UUID: "GR_node1-UUID", - Name: "GR_node1", - StaticRoutes: []string{"static-route-1-UUID", "static-route-2-UUID"}, - }, - }, - []libovsdbtest.TestData{ - &nbdb.LogicalSwitch{ - UUID: "node1", - Name: "node1", - }, - &nbdb.LogicalRouter{ - UUID: "GR_node1-UUID", - Name: "GR_node1", - }, - }, - ), - ) - - ginkgo.DescribeTable("reconciles deleting a exgw namespace with active pod", - func(bfd bool, - initNB []libovsdbtest.TestData, - finalNB []libovsdbtest.TestData, - ) { - app.Action = func(*cli.Context) error { - - namespaceT := *ovntest.NewNamespace(namespaceName) - namespaceT.Annotations = map[string]string{"k8s.ovn.org/routing-external-gws": "9.0.0.1,9.0.0.2"} - if bfd { - namespaceT.Annotations["k8s.ovn.org/bfd-enabled"] = "" - } - t := newTPod( - "node1", - "10.128.1.0/24", - "10.128.1.2", - "10.128.1.1", - "myPod", - "10.128.1.3", - "0a:58:0a:80:01:03", - namespaceT.Name, - ) - - fakeOvn.startWithDBSetup( - libovsdbtest.TestSetup{ - NBData: initNB, - }, - &corev1.NamespaceList{ - Items: []corev1.Namespace{ - namespaceT, - }, - }, - &corev1.NodeList{ - Items: []corev1.Node{ - *newNode("node1", "192.168.126.202/24"), - }, - }, - &corev1.PodList{ - Items: []corev1.Pod{ - *ovntest.NewPod(t.namespace, t.podName, t.nodeName, t.podIP), - }, - }, - ) - t.populateLogicalSwitchCache(fakeOvn) - - injectNode(fakeOvn) - err := fakeOvn.controller.WatchNamespaces() - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fakeOvn.controller.WatchPods() - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - gomega.Eventually(func() string { return getPodAnnotations(fakeOvn.fakeClient.KubeClient, t.namespace, t.podName) }, 2).Should(gomega.MatchJSON(t.getAnnotationsJson())) - - err = fakeOvn.fakeClient.KubeClient.CoreV1().Namespaces().Delete(context.TODO(), t.namespace, *metav1.NewDeleteOptions(0)) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Eventually(fakeOvn.nbClient).Should(libovsdbtest.HaveData(finalNB)) - return nil - } - - err := app.Run([]string{app.Name}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }, - ginkgo.Entry("No BFD", false, - []libovsdbtest.TestData{ - &nbdb.LogicalSwitch{ - UUID: "node1", - Name: "node1", - }, - &nbdb.LogicalRouterStaticRoute{ - UUID: "static-route-1-UUID", - IPPrefix: "10.128.1.3/32", - Nexthop: "9.0.0.1", - Policy: &nbdb.LogicalRouterStaticRoutePolicySrcIP, - OutputPort: &logicalRouterPort, - Options: map[string]string{ - "ecmp_symmetric_reply": "true", - }, - }, - &nbdb.LogicalRouterStaticRoute{ - UUID: "static-route-2-UUID", - IPPrefix: "10.128.1.3/32", - Nexthop: "9.0.0.2", - Policy: &nbdb.LogicalRouterStaticRoutePolicySrcIP, - OutputPort: &logicalRouterPort, - Options: map[string]string{ - "ecmp_symmetric_reply": "true", - }, - }, - &nbdb.LogicalRouter{ - UUID: "GR_node1-UUID", - Name: "GR_node1", - StaticRoutes: []string{"static-route-1-UUID", "static-route-2-UUID"}, - }, - }, - []libovsdbtest.TestData{ - &nbdb.LogicalSwitchPort{ - UUID: "lsp1", - Addresses: []string{"0a:58:0a:80:01:03 10.128.1.3"}, - ExternalIDs: map[string]string{ - "pod": "true", - "namespace": namespaceName, - }, - Name: "namespace1_myPod", - Options: map[string]string{ - "iface-id-ver": "myPod", - libovsdbops.RequestedChassis: chassisIDForNode("node1"), - }, - PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, - }, - &nbdb.LogicalSwitch{ - UUID: "node1", - Name: "node1", - Ports: []string{"lsp1"}, - }, - &nbdb.LogicalRouter{ - UUID: "GR_node1-UUID", - Name: "GR_node1", - StaticRoutes: []string{}, - }, - }, - ), - ginkgo.Entry("BFD", true, - []libovsdbtest.TestData{ - &nbdb.LogicalSwitch{ - UUID: "node1", - Name: "node1", - }, - &nbdb.BFD{ - UUID: "bfd1-UUID", - DstIP: "9.0.0.1", - LogicalPort: "rtoe-GR_node1", - }, - &nbdb.BFD{ - UUID: "bfd2-UUID", - DstIP: "9.0.0.2", - LogicalPort: "rtoe-GR_node1", - }, - &nbdb.LogicalRouterStaticRoute{ - UUID: "static-route-1-UUID", - IPPrefix: "10.128.1.3/32", - Nexthop: "9.0.0.1", - BFD: &bfd1NamedUUID, - Policy: &nbdb.LogicalRouterStaticRoutePolicySrcIP, - OutputPort: &logicalRouterPort, - Options: map[string]string{ - "ecmp_symmetric_reply": "true", - }, - }, - &nbdb.LogicalRouterStaticRoute{ - UUID: "static-route-2-UUID", - IPPrefix: "10.128.1.3/32", - BFD: &bfd2NamedUUID, - Nexthop: "9.0.0.2", - Policy: &nbdb.LogicalRouterStaticRoutePolicySrcIP, - OutputPort: &logicalRouterPort, - Options: map[string]string{ - "ecmp_symmetric_reply": "true", - }, - }, - &nbdb.LogicalRouter{ - UUID: "GR_node1-UUID", - Name: "GR_node1", - StaticRoutes: []string{"static-route-1-UUID", "static-route-2-UUID"}, - }, - }, - []libovsdbtest.TestData{ - &nbdb.LogicalSwitchPort{ - UUID: "lsp1", - Addresses: []string{"0a:58:0a:80:01:03 10.128.1.3"}, - ExternalIDs: map[string]string{ - "pod": "true", - "namespace": namespaceName, - }, - Name: "namespace1_myPod", - Options: map[string]string{ - "iface-id-ver": "myPod", - libovsdbops.RequestedChassis: chassisIDForNode("node1"), - }, - PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, - }, - &nbdb.LogicalSwitch{ - UUID: "node1", - Name: "node1", - Ports: []string{"lsp1"}, - }, - &nbdb.LogicalRouter{ - UUID: "GR_node1-UUID", - Name: "GR_node1", - StaticRoutes: []string{}, - }, - }, - )) - }) - - ginkgo.Context("on setting pod gateway annotations", func() { - var ( - namespace2Name = "namespace2" - gwPodName = "gwPod" - ) - ginkgo.DescribeTable("reconciles a host networked pod acting as a exgw for another namespace for new pod", func(bfd bool, finalNB []libovsdbtest.TestData) { - app.Action = func(*cli.Context) error { - - namespaceT := *ovntest.NewNamespace(namespaceName) - namespaceX := *ovntest.NewNamespace(namespace2Name) - t := newTPod( - "node1", - "10.128.1.0/24", - "10.128.1.2", - "10.128.1.1", - "myPod", - "10.128.1.3", - "0a:58:0a:80:01:03", - namespaceT.Name, - ) - gwPod := *ovntest.NewPod(namespaceX.Name, gwPodName, "node2", "9.0.0.1") - gwPod.Annotations = map[string]string{"k8s.ovn.org/routing-namespaces": namespaceT.Name} - if bfd { - gwPod.Annotations["k8s.ovn.org/bfd-enabled"] = "" - } - gwPod.Spec.HostNetwork = true - fakeOvn.startWithDBSetup( - libovsdbtest.TestSetup{ - NBData: []libovsdbtest.TestData{ - &nbdb.LogicalSwitch{ - UUID: "node1", - Name: "node1", - }, - &nbdb.LogicalSwitch{ - UUID: "node2", - Name: "node2", - }, - &nbdb.LogicalRouter{ - UUID: "GR_node1-UUID", - Name: "GR_node1", - }, - }, - }, - &corev1.NamespaceList{ - Items: []corev1.Namespace{ - namespaceT, namespaceX, - }, - }, - &corev1.NodeList{ - Items: []corev1.Node{ - *newNode("node2", "192.168.126.51/24"), - }, - }, - &corev1.PodList{ - Items: []corev1.Pod{ - gwPod, - }, - }, - ) - t.populateLogicalSwitchCache(fakeOvn) - err := fakeOvn.controller.lsManager.AddOrUpdateSwitch("node2", []*net.IPNet{ovntest.MustParseIPNet("10.128.2.0/24")}, nil) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - injectNode(fakeOvn) - err = fakeOvn.controller.WatchNamespaces() - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fakeOvn.controller.WatchPods() - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - _, err = fakeOvn.fakeClient.KubeClient.CoreV1().Pods(t.namespace).Create(context.TODO(), ovntest.NewPod(t.namespace, t.podName, t.nodeName, t.podIP), metav1.CreateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - gomega.Eventually(func() string { return getPodAnnotations(fakeOvn.fakeClient.KubeClient, t.namespace, t.podName) }, 2).Should(gomega.MatchJSON(t.getAnnotationsJson())) - gomega.Eventually(func() string { - return getNamespaceAnnotations(fakeOvn.fakeClient.KubeClient, namespaceT.Name)[util.ExternalGatewayPodIPsAnnotation] - }).Should(gomega.Equal("9.0.0.1")) - gomega.Eventually(fakeOvn.nbClient).Should(libovsdbtest.HaveData(finalNB)) - return nil - } - - err := app.Run([]string{app.Name}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }, ginkgo.Entry("No BFD", false, []libovsdbtest.TestData{ - &nbdb.LogicalSwitchPort{ - UUID: "lsp1", - Addresses: []string{"0a:58:0a:80:01:03 10.128.1.3"}, - ExternalIDs: map[string]string{ - "pod": "true", - "namespace": namespaceName, - }, - Name: "namespace1_myPod", - Options: map[string]string{ - "iface-id-ver": "myPod", - libovsdbops.RequestedChassis: chassisIDForNode("node1"), - }, - PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, - }, - &nbdb.LogicalSwitch{ - UUID: "node1", - Name: "node1", - Ports: []string{"lsp1"}, - }, - &nbdb.LogicalSwitch{ - UUID: "node2", - Name: "node2", - }, - &nbdb.LogicalRouterStaticRoute{ - UUID: "static-route-1-UUID", - IPPrefix: "10.128.1.3/32", - Nexthop: "9.0.0.1", - Policy: &nbdb.LogicalRouterStaticRoutePolicySrcIP, - OutputPort: &logicalRouterPort, - Options: map[string]string{ - "ecmp_symmetric_reply": "true", - }, - }, - &nbdb.LogicalRouter{ - UUID: "GR_node1-UUID", - Name: "GR_node1", - StaticRoutes: []string{"static-route-1-UUID"}, - }, - }), - ginkgo.Entry("BFD Enabled", true, []libovsdbtest.TestData{ - &nbdb.LogicalSwitchPort{ - UUID: "lsp1", - Addresses: []string{"0a:58:0a:80:01:03 10.128.1.3"}, - ExternalIDs: map[string]string{ - "pod": "true", - "namespace": namespaceName, - }, - Name: "namespace1_myPod", - Options: map[string]string{ - "iface-id-ver": "myPod", - libovsdbops.RequestedChassis: chassisIDForNode("node1"), - }, - PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, - }, - &nbdb.LogicalSwitch{ - UUID: "node1", - Name: "node1", - Ports: []string{"lsp1"}, - }, - &nbdb.LogicalSwitch{ - UUID: "node2", - Name: "node2", - }, - &nbdb.BFD{ - UUID: bfd1NamedUUID, - DstIP: "9.0.0.1", - LogicalPort: "rtoe-GR_node1", - }, - &nbdb.LogicalRouterStaticRoute{ - UUID: "static-route-1-UUID", - IPPrefix: "10.128.1.3/32", - Nexthop: "9.0.0.1", - BFD: &bfd1NamedUUID, - Policy: &nbdb.LogicalRouterStaticRoutePolicySrcIP, - OutputPort: &logicalRouterPort, - Options: map[string]string{ - "ecmp_symmetric_reply": "true", - }, - }, - &nbdb.LogicalRouter{ - UUID: "GR_node1-UUID", - Name: "GR_node1", - StaticRoutes: []string{"static-route-1-UUID"}, - }, - })) - - ginkgo.DescribeTable("reconciles a host networked pod acting as a exgw for another namespace for existing pod", func(bfd bool, finalNB []libovsdbtest.TestData) { - app.Action = func(*cli.Context) error { - - namespaceT := *ovntest.NewNamespace(namespaceName) - namespaceX := *ovntest.NewNamespace(namespace2Name) - t := newTPod( - "node1", - "10.128.1.0/24", - "10.128.1.2", - "10.128.1.1", - "myPod", - "10.128.1.3", - "0a:58:0a:80:01:03", - namespaceT.Name, - ) - gwPod := *ovntest.NewPod(namespaceX.Name, gwPodName, "node2", "9.0.0.1") - gwPod.Annotations = map[string]string{"k8s.ovn.org/routing-namespaces": namespaceT.Name} - if bfd { - gwPod.Annotations["k8s.ovn.org/bfd-enabled"] = "" - } - gwPod.Spec.HostNetwork = true - fakeOvn.startWithDBSetup( - libovsdbtest.TestSetup{ - NBData: []libovsdbtest.TestData{ - &nbdb.LogicalSwitch{ - UUID: "node1", - Name: "node1", - }, - &nbdb.LogicalSwitch{ - UUID: "node2", - Name: "node2", - }, - &nbdb.LogicalRouter{ - UUID: "GR_node1-UUID", - Name: "GR_node1", - }, - }, - }, - &corev1.NamespaceList{ - Items: []corev1.Namespace{ - namespaceT, namespaceX, - }, - }, - &corev1.NodeList{ - Items: []corev1.Node{ - *newNode("node2", "192.168.126.51/24"), - }, - }, - &corev1.PodList{ - Items: []corev1.Pod{ - *ovntest.NewPod(t.namespace, t.podName, t.nodeName, t.podIP), - }, - }, - ) - t.populateLogicalSwitchCache(fakeOvn) - err := fakeOvn.controller.lsManager.AddOrUpdateSwitch("node2", []*net.IPNet{ovntest.MustParseIPNet("10.128.2.0/24")}, nil) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - injectNode(fakeOvn) - err = fakeOvn.controller.WatchNamespaces() - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fakeOvn.controller.WatchPods() - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - _, err = fakeOvn.fakeClient.KubeClient.CoreV1().Pods(namespaceX.Name).Create(context.TODO(), &gwPod, metav1.CreateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Eventually(fakeOvn.nbClient).Should(libovsdbtest.HaveData(finalNB)) - gomega.Eventually(func() string { - return getNamespaceAnnotations(fakeOvn.fakeClient.KubeClient, namespaceT.Name)[util.ExternalGatewayPodIPsAnnotation] - }).Should(gomega.Equal("9.0.0.1")) - return nil - } - - err := app.Run([]string{app.Name}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }, ginkgo.Entry("No BFD", false, []libovsdbtest.TestData{ - &nbdb.LogicalSwitchPort{ - UUID: "lsp1", - Addresses: []string{"0a:58:0a:80:01:03 10.128.1.3"}, - ExternalIDs: map[string]string{ - "pod": "true", - "namespace": namespaceName, - }, - Name: "namespace1_myPod", - Options: map[string]string{ - "iface-id-ver": "myPod", - libovsdbops.RequestedChassis: chassisIDForNode("node1"), - }, - PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, - }, - &nbdb.LogicalSwitch{ - UUID: "node1", - Name: "node1", - Ports: []string{"lsp1"}, - }, - &nbdb.LogicalSwitch{ - UUID: "node2", - Name: "node2", - }, - &nbdb.LogicalRouterStaticRoute{ - UUID: "static-route-1-UUID", - IPPrefix: "10.128.1.3/32", - Nexthop: "9.0.0.1", - Policy: &nbdb.LogicalRouterStaticRoutePolicySrcIP, - OutputPort: &logicalRouterPort, - Options: map[string]string{ - "ecmp_symmetric_reply": "true", - }, - }, - &nbdb.LogicalRouter{ - UUID: "GR_node1-UUID", - Name: "GR_node1", - StaticRoutes: []string{"static-route-1-UUID"}, - }, - }), - ginkgo.Entry("BFD Enabled", true, []libovsdbtest.TestData{ - &nbdb.LogicalSwitchPort{ - UUID: "lsp1", - Addresses: []string{"0a:58:0a:80:01:03 10.128.1.3"}, - ExternalIDs: map[string]string{ - "pod": "true", - "namespace": namespaceName, - }, - Name: "namespace1_myPod", - Options: map[string]string{ - "iface-id-ver": "myPod", - libovsdbops.RequestedChassis: chassisIDForNode("node1"), - }, - PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, - }, - &nbdb.LogicalSwitch{ - UUID: "node1", - Name: "node1", - Ports: []string{"lsp1"}, - }, - &nbdb.LogicalSwitch{ - UUID: "node2", - Name: "node2", - }, - &nbdb.BFD{ - UUID: bfd1NamedUUID, - DstIP: "9.0.0.1", - LogicalPort: "rtoe-GR_node1", - }, - &nbdb.LogicalRouterStaticRoute{ - UUID: "static-route-1-UUID", - IPPrefix: "10.128.1.3/32", - Nexthop: "9.0.0.1", - BFD: &bfd1NamedUUID, - Policy: &nbdb.LogicalRouterStaticRoutePolicySrcIP, - OutputPort: &logicalRouterPort, - Options: map[string]string{ - "ecmp_symmetric_reply": "true", - }, - }, - &nbdb.LogicalRouter{ - UUID: "GR_node1-UUID", - Name: "GR_node1", - StaticRoutes: []string{"static-route-1-UUID"}, - }, - })) - - ginkgo.DescribeTable("reconciles a multus networked pod acting as a exgw for another namespace for new pod", func(bfd bool, finalNB []libovsdbtest.TestData) { - app.Action = func(*cli.Context) error { - ns := nettypes.NetworkStatus{Name: "dummy", IPs: []string{"11.0.0.1"}} - networkStatuses := []nettypes.NetworkStatus{ns} - nsEncoded, err := json.Marshal(networkStatuses) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - namespaceT := *ovntest.NewNamespace(namespaceName) - namespaceX := *ovntest.NewNamespace(namespace2Name) - t := newTPod( - "node1", - "10.128.1.0/24", - "10.128.1.2", - "10.128.1.1", - "myPod", - "10.128.1.3", - "0a:58:0a:80:01:03", - namespaceT.Name, - ) - gwPod := *ovntest.NewPod(namespaceX.Name, gwPodName, "node2", "9.0.0.1") - gwPod.Annotations = map[string]string{ - "k8s.ovn.org/routing-namespaces": namespaceT.Name, - "k8s.ovn.org/routing-network": "dummy", - "k8s.v1.cni.cncf.io/network-status": string(nsEncoded), - } - if bfd { - gwPod.Annotations["k8s.ovn.org/bfd-enabled"] = "" - } - gwPod.Spec.HostNetwork = true - fakeOvn.startWithDBSetup( - libovsdbtest.TestSetup{ - NBData: []libovsdbtest.TestData{ - &nbdb.LogicalSwitch{ - UUID: "node1", - Name: "node1", - }, - &nbdb.LogicalSwitch{ - UUID: "node2", - Name: "node2", - }, - &nbdb.LogicalRouter{ - UUID: "GR_node1-UUID", - Name: "GR_node1", - }, - }, - }, - &corev1.NamespaceList{ - Items: []corev1.Namespace{ - namespaceT, namespaceX, - }, - }, - &corev1.NodeList{ - Items: []corev1.Node{ - *newNode("node2", "192.168.126.51/24"), - }, - }, - &corev1.PodList{ - Items: []corev1.Pod{ - gwPod, - }, - }, - ) - t.populateLogicalSwitchCache(fakeOvn) - err = fakeOvn.controller.lsManager.AddOrUpdateSwitch("node2", []*net.IPNet{ovntest.MustParseIPNet("10.128.2.0/24")}, nil) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - injectNode(fakeOvn) - err = fakeOvn.controller.WatchNamespaces() - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fakeOvn.controller.WatchPods() - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - _, err = fakeOvn.fakeClient.KubeClient.CoreV1().Pods(t.namespace).Create(context.TODO(), ovntest.NewPod(t.namespace, t.podName, t.nodeName, t.podIP), metav1.CreateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - gomega.Eventually(func() string { return getPodAnnotations(fakeOvn.fakeClient.KubeClient, t.namespace, t.podName) }, 2).Should(gomega.MatchJSON(t.getAnnotationsJson())) - gomega.Eventually(func() string { - return getNamespaceAnnotations(fakeOvn.fakeClient.KubeClient, namespaceT.Name)[util.ExternalGatewayPodIPsAnnotation] - }).Should(gomega.Equal("11.0.0.1")) - gomega.Eventually(fakeOvn.nbClient).Should(libovsdbtest.HaveData(finalNB)) - return nil - } - - err := app.Run([]string{app.Name}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }, ginkgo.Entry("No BFD", false, []libovsdbtest.TestData{ - &nbdb.LogicalSwitchPort{ - UUID: "lsp1", - Addresses: []string{"0a:58:0a:80:01:03 10.128.1.3"}, - ExternalIDs: map[string]string{ - "pod": "true", - "namespace": namespaceName, - }, - Name: "namespace1_myPod", - Options: map[string]string{ - "iface-id-ver": "myPod", - libovsdbops.RequestedChassis: chassisIDForNode("node1"), - }, - PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, - }, - &nbdb.LogicalSwitch{ - UUID: "node1", - Name: "node1", - Ports: []string{"lsp1"}, - }, - &nbdb.LogicalSwitch{ - UUID: "node2", - Name: "node2", - }, - &nbdb.LogicalRouterStaticRoute{ - UUID: "static-route-1-UUID", - IPPrefix: "10.128.1.3/32", - Nexthop: "11.0.0.1", - Policy: &nbdb.LogicalRouterStaticRoutePolicySrcIP, - OutputPort: &logicalRouterPort, - Options: map[string]string{ - "ecmp_symmetric_reply": "true", - }, - }, - &nbdb.LogicalRouter{ - UUID: "GR_node1-UUID", - Name: "GR_node1", - StaticRoutes: []string{"static-route-1-UUID"}, - }, - }), - ginkgo.Entry("BFD Enabled", true, []libovsdbtest.TestData{ - &nbdb.LogicalSwitchPort{ - UUID: "lsp1", - Addresses: []string{"0a:58:0a:80:01:03 10.128.1.3"}, - ExternalIDs: map[string]string{ - "pod": "true", - "namespace": namespaceName, - }, - Name: "namespace1_myPod", - Options: map[string]string{ - "iface-id-ver": "myPod", - libovsdbops.RequestedChassis: chassisIDForNode("node1"), - }, - PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, - }, - &nbdb.LogicalSwitch{ - UUID: "node1", - Name: "node1", - Ports: []string{"lsp1"}, - }, - &nbdb.LogicalSwitch{ - UUID: "node2", - Name: "node2", - }, - &nbdb.BFD{ - UUID: bfd1NamedUUID, - DstIP: "11.0.0.1", - LogicalPort: "rtoe-GR_node1", - }, - &nbdb.LogicalRouterStaticRoute{ - UUID: "static-route-1-UUID", - IPPrefix: "10.128.1.3/32", - Nexthop: "11.0.0.1", - BFD: &bfd1NamedUUID, - Policy: &nbdb.LogicalRouterStaticRoutePolicySrcIP, - OutputPort: &logicalRouterPort, - Options: map[string]string{ - "ecmp_symmetric_reply": "true", - }, - }, - &nbdb.LogicalRouter{ - UUID: "GR_node1-UUID", - Name: "GR_node1", - StaticRoutes: []string{"static-route-1-UUID"}, - }, - })) - - ginkgo.DescribeTable("reconciles deleting a host networked pod acting as a exgw for another namespace for existing pod", - func(bfd bool, - beforeDeleteNB []libovsdbtest.TestData, - afterDeleteNB []libovsdbtest.TestData, - expectedNamespaceAnnotation string, - apbExternalRouteCRList *adminpolicybasedrouteapi.AdminPolicyBasedExternalRouteList) { - app.Action = func(*cli.Context) error { - - namespaceT := *ovntest.NewNamespace(namespaceName) - namespaceX := *ovntest.NewNamespace(namespace2Name) - t := newTPod( - "node1", - "10.128.1.0/24", - "10.128.1.2", - "10.128.1.1", - "myPod", - "10.128.1.3", - "0a:58:0a:80:01:03", - namespaceT.Name, - ) - gwPod := *ovntest.NewPod(namespaceX.Name, gwPodName, "node2", "9.0.0.1") - gwPod.Annotations = map[string]string{"k8s.ovn.org/routing-namespaces": namespaceT.Name} - if bfd { - gwPod.Annotations["k8s.ovn.org/bfd-enabled"] = "" - } - gwPod.Spec.HostNetwork = true - fakeOvn.startWithDBSetup( - libovsdbtest.TestSetup{ - NBData: []libovsdbtest.TestData{ - &nbdb.LogicalSwitch{ - UUID: "node1", - Name: "node1", - }, - &nbdb.LogicalSwitch{ - UUID: "node2", - Name: "node2", - }, - &nbdb.LogicalRouter{ - UUID: "GR_node1-UUID", - Name: "GR_node1", - }, - }, - }, - &corev1.NamespaceList{ - Items: []corev1.Namespace{ - namespaceT, namespaceX, - }, - }, - &corev1.NodeList{ - Items: []corev1.Node{ - *newNode("node1", "192.168.126.202/24"), - *newNode("node2", "192.168.126.50/24"), - }, - }, - &corev1.PodList{ - Items: []corev1.Pod{ - *ovntest.NewPod(t.namespace, t.podName, t.nodeName, t.podIP), - }, - }, - apbExternalRouteCRList, - ) - t.populateLogicalSwitchCache(fakeOvn) - err := fakeOvn.controller.lsManager.AddOrUpdateSwitch("node2", []*net.IPNet{ovntest.MustParseIPNet("10.128.2.0/24")}, nil) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - injectNode(fakeOvn) - err = fakeOvn.controller.WatchNamespaces() - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fakeOvn.controller.WatchPods() - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - fakeOvn.RunAPBExternalPolicyController() - - _, err = fakeOvn.fakeClient.KubeClient.CoreV1().Pods(namespaceX.Name).Create(context.TODO(), &gwPod, metav1.CreateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Eventually(fakeOvn.nbClient).Should(libovsdbtest.HaveData(beforeDeleteNB)) - gomega.Eventually(func() string { - return getNamespaceAnnotations(fakeOvn.fakeClient.KubeClient, namespaceT.Name)[util.ExternalGatewayPodIPsAnnotation] - }).Should(gomega.Equal("9.0.0.1")) - - err = fakeOvn.fakeClient.KubeClient.CoreV1().Pods(namespaceX.Name).Delete(context.TODO(), gwPod.Name, *metav1.NewDeleteOptions(0)) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Eventually(fakeOvn.nbClient).Should(libovsdbtest.HaveData(afterDeleteNB)) - gomega.Eventually(func() string { - return getNamespaceAnnotations(fakeOvn.fakeClient.KubeClient, namespaceT.Name)[util.ExternalGatewayPodIPsAnnotation] - }).Should(gomega.Equal(expectedNamespaceAnnotation)) - for _, apbRoutePolicy := range apbExternalRouteCRList.Items { - checkAPBRouteStatus(fakeOvn, apbRoutePolicy.Name, false) - } - return nil - } - - err := app.Run([]string{app.Name}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }, - ginkgo.Entry("No BFD", false, - []libovsdbtest.TestData{ - &nbdb.LogicalSwitchPort{ - UUID: "lsp1", - Addresses: []string{"0a:58:0a:80:01:03 10.128.1.3"}, - ExternalIDs: map[string]string{ - "pod": "true", - "namespace": namespaceName, - }, - Name: "namespace1_myPod", - Options: map[string]string{ - "iface-id-ver": "myPod", - libovsdbops.RequestedChassis: chassisIDForNode("node1"), - }, - PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, - }, - &nbdb.LogicalSwitch{ - UUID: "node1", - Name: "node1", - Ports: []string{"lsp1"}, - }, - &nbdb.LogicalSwitch{ - UUID: "node2", - Name: "node2", - }, - &nbdb.LogicalRouterStaticRoute{ - UUID: "static-route-1-UUID", - IPPrefix: "10.128.1.3/32", - Nexthop: "9.0.0.1", - Policy: &nbdb.LogicalRouterStaticRoutePolicySrcIP, - OutputPort: &logicalRouterPort, - Options: map[string]string{ - "ecmp_symmetric_reply": "true", - }, - }, - &nbdb.LogicalRouter{ - UUID: "GR_node1-UUID", - Name: "GR_node1", - StaticRoutes: []string{"static-route-1-UUID"}, - }, - }, - []libovsdbtest.TestData{ - &nbdb.LogicalSwitchPort{ - UUID: "lsp1", - Addresses: []string{"0a:58:0a:80:01:03 10.128.1.3"}, - ExternalIDs: map[string]string{ - "pod": "true", - "namespace": namespaceName, - }, - Name: "namespace1_myPod", - Options: map[string]string{ - "iface-id-ver": "myPod", - libovsdbops.RequestedChassis: chassisIDForNode("node1"), - }, - PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, - }, - &nbdb.LogicalSwitch{ - UUID: "node1", - Name: "node1", - Ports: []string{"lsp1"}, - }, - &nbdb.LogicalSwitch{ - UUID: "node2", - Name: "node2", - }, - &nbdb.LogicalRouter{ - UUID: "GR_node1-UUID", - Name: "GR_node1", - StaticRoutes: []string{}, - }, - }, - "", - &adminpolicybasedrouteapi.AdminPolicyBasedExternalRouteList{}, - ), - ginkgo.Entry("BFD Enabled", true, []libovsdbtest.TestData{ - &nbdb.LogicalSwitchPort{ - UUID: "lsp1", - Addresses: []string{"0a:58:0a:80:01:03 10.128.1.3"}, - ExternalIDs: map[string]string{ - "pod": "true", - "namespace": namespaceName, - }, - Name: "namespace1_myPod", - Options: map[string]string{ - "iface-id-ver": "myPod", - libovsdbops.RequestedChassis: chassisIDForNode("node1"), - }, - PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, - }, - &nbdb.LogicalSwitch{ - UUID: "node1", - Name: "node1", - Ports: []string{"lsp1"}, - }, - &nbdb.LogicalSwitch{ - UUID: "node2", - Name: "node2", - }, - &nbdb.BFD{ - UUID: bfd1NamedUUID, - DstIP: "9.0.0.1", - LogicalPort: "rtoe-GR_node1", - }, - &nbdb.LogicalRouterStaticRoute{ - UUID: "static-route-1-UUID", - IPPrefix: "10.128.1.3/32", - Nexthop: "9.0.0.1", - BFD: &bfd1NamedUUID, - Policy: &nbdb.LogicalRouterStaticRoutePolicySrcIP, - OutputPort: &logicalRouterPort, - Options: map[string]string{ - "ecmp_symmetric_reply": "true", - }, - }, - &nbdb.LogicalRouter{ - UUID: "GR_node1-UUID", - Name: "GR_node1", - StaticRoutes: []string{"static-route-1-UUID"}, - }, - }, - []libovsdbtest.TestData{ - &nbdb.LogicalSwitchPort{ - UUID: "lsp1", - Addresses: []string{"0a:58:0a:80:01:03 10.128.1.3"}, - ExternalIDs: map[string]string{ - "pod": "true", - "namespace": namespaceName, - }, - Name: "namespace1_myPod", - Options: map[string]string{ - "iface-id-ver": "myPod", - libovsdbops.RequestedChassis: chassisIDForNode("node1"), - }, - PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, - }, - &nbdb.LogicalSwitch{ - UUID: "node1", - Name: "node1", - Ports: []string{"lsp1"}, - }, - &nbdb.LogicalSwitch{ - UUID: "node2", - Name: "node2", - }, - &nbdb.LogicalRouter{ - UUID: "GR_node1-UUID", - Name: "GR_node1", - StaticRoutes: []string{}, - }, - }, - "", - &adminpolicybasedrouteapi.AdminPolicyBasedExternalRouteList{}, - ), - ginkgo.Entry("No BFD and with overlapping APB External Route CR and annotation", false, - []libovsdbtest.TestData{ - &nbdb.LogicalSwitchPort{ - UUID: "lsp1", - Addresses: []string{"0a:58:0a:80:01:03 10.128.1.3"}, - ExternalIDs: map[string]string{ - "pod": "true", - "namespace": namespaceName, - }, - Name: "namespace1_myPod", - Options: map[string]string{ - "iface-id-ver": "myPod", - libovsdbops.RequestedChassis: chassisIDForNode("node1"), - }, - PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, - }, - &nbdb.LogicalSwitch{ - UUID: "node1", - Name: "node1", - Ports: []string{"lsp1"}, - }, - &nbdb.LogicalSwitch{ - UUID: "node2", - Name: "node2", - }, - &nbdb.LogicalRouterStaticRoute{ - UUID: "static-route-1-UUID", - IPPrefix: "10.128.1.3/32", - Nexthop: "9.0.0.1", - Policy: &nbdb.LogicalRouterStaticRoutePolicySrcIP, - OutputPort: &logicalRouterPort, - Options: map[string]string{ - "ecmp_symmetric_reply": "true", - }, - }, - &nbdb.LogicalRouter{ - UUID: "GR_node1-UUID", - Name: "GR_node1", - StaticRoutes: []string{"static-route-1-UUID"}, - }, - }, - []libovsdbtest.TestData{ - &nbdb.LogicalSwitchPort{ - UUID: "lsp1", - Addresses: []string{"0a:58:0a:80:01:03 10.128.1.3"}, - ExternalIDs: map[string]string{ - "pod": "true", - "namespace": namespaceName, - }, - Name: "namespace1_myPod", - Options: map[string]string{ - "iface-id-ver": "myPod", - libovsdbops.RequestedChassis: chassisIDForNode("node1"), - }, - PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, - }, - &nbdb.LogicalSwitch{ - UUID: "node1", - Name: "node1", - Ports: []string{"lsp1"}, - }, - &nbdb.LogicalSwitch{ - UUID: "node2", - Name: "node2", - }, - &nbdb.LogicalRouter{ - UUID: "GR_node1-UUID", - Name: "GR_node1", - StaticRoutes: []string{}, - }, - }, - "", - &adminpolicybasedrouteapi.AdminPolicyBasedExternalRouteList{ - Items: []adminpolicybasedrouteapi.AdminPolicyBasedExternalRoute{ - newPolicy("policy", - &metav1.LabelSelector{MatchLabels: map[string]string{"name": namespaceName}}, - nil, - false, - &metav1.LabelSelector{MatchLabels: map[string]string{"name": namespace2Name}}, - &metav1.LabelSelector{MatchLabels: map[string]string{"name": gwPodName}}, - false, - ""), - }, - }, - ), - ) - ginkgo.DescribeTable("reconciles a host networked pod in terminating or not ready state acting as a exgw for another namespace for existing pod", - func(bfd bool, - terminating bool, - beforeUpdateNB []libovsdbtest.TestData, - afterUpdateNB []libovsdbtest.TestData, - expectedNamespaceAnnotation string, - apbExternalRouteCRList *adminpolicybasedrouteapi.AdminPolicyBasedExternalRouteList) { - app.Action = func(*cli.Context) error { - - namespaceT := *ovntest.NewNamespace(namespaceName) - namespaceX := *ovntest.NewNamespace(namespace2Name) - t := newTPod( - "node1", - "10.128.1.0/24", - "10.128.1.2", - "10.128.1.1", - "myPod", - "10.128.1.3", - "0a:58:0a:80:01:03", - namespaceT.Name, - ) - gwPod := *ovntest.NewPod(namespaceX.Name, gwPodName, "node2", "9.0.0.1") - gwPod.Annotations = map[string]string{"k8s.ovn.org/routing-namespaces": namespaceT.Name} - if bfd { - gwPod.Annotations["k8s.ovn.org/bfd-enabled"] = "" - } - gwPod.Spec.HostNetwork = true - fakeOvn.startWithDBSetup( - libovsdbtest.TestSetup{ - NBData: []libovsdbtest.TestData{ - &nbdb.LogicalSwitch{ - UUID: "node1", - Name: "node1", - }, - &nbdb.LogicalSwitch{ - UUID: "node2", - Name: "node2", - }, - &nbdb.LogicalRouter{ - UUID: "GR_node1-UUID", - Name: "GR_node1", - }, - }, - }, - &corev1.NamespaceList{ - Items: []corev1.Namespace{ - namespaceT, namespaceX, - }, - }, - &corev1.NodeList{ - Items: []corev1.Node{ - *newNode("node1", "192.168.126.202/24"), - *newNode("node2", "192.168.126.50/24"), - }, - }, - &corev1.PodList{ - Items: []corev1.Pod{ - *ovntest.NewPod(t.namespace, t.podName, t.nodeName, t.podIP), - }, - }, - apbExternalRouteCRList, - ) - t.populateLogicalSwitchCache(fakeOvn) - err := fakeOvn.controller.lsManager.AddOrUpdateSwitch("node2", []*net.IPNet{ovntest.MustParseIPNet("10.128.2.0/24")}, nil) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - injectNode(fakeOvn) - err = fakeOvn.controller.WatchNamespaces() - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fakeOvn.controller.WatchPods() - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - fakeOvn.RunAPBExternalPolicyController() - - _, err = fakeOvn.fakeClient.KubeClient.CoreV1().Pods(namespaceX.Name).Create(context.TODO(), &gwPod, metav1.CreateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Eventually(fakeOvn.nbClient).Should(libovsdbtest.HaveData(beforeUpdateNB)) - gomega.Eventually(func() string { - return getNamespaceAnnotations(fakeOvn.fakeClient.KubeClient, namespaceT.Name)[util.ExternalGatewayPodIPsAnnotation] - }).Should(gomega.Equal("9.0.0.1")) - - if terminating { - ginkgo.By("Setting deletion timestamp for the ex gw pod") - gwPod.DeletionTimestamp = &metav1.Time{Time: time.Now().Add(1000 * time.Second)} - _, err = fakeOvn.fakeClient.KubeClient.CoreV1().Pods(namespaceX.Name).Update(context.TODO(), &gwPod, metav1.UpdateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } else { - ginkgo.By("Updating the ex gw pod status to mark it as not ready") - notReadyCondition := corev1.PodCondition{ - Type: corev1.PodReady, - Status: corev1.ConditionFalse, - } - gwPod.Status.Conditions = []corev1.PodCondition{notReadyCondition} - _, err = fakeOvn.fakeClient.KubeClient.CoreV1().Pods(namespaceX.Name).UpdateStatus(context.TODO(), &gwPod, metav1.UpdateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - - gomega.Eventually(fakeOvn.nbClient).Should(libovsdbtest.HaveData(afterUpdateNB)) - gomega.Eventually(func() string { - return getNamespaceAnnotations(fakeOvn.fakeClient.KubeClient, namespaceT.Name)[util.ExternalGatewayPodIPsAnnotation] - }).Should(gomega.Equal(expectedNamespaceAnnotation)) - for _, apbRoutePolicy := range apbExternalRouteCRList.Items { - checkAPBRouteStatus(fakeOvn, apbRoutePolicy.Name, false) - } - return nil - } - - err := app.Run([]string{app.Name}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }, - ginkgo.Entry("No BFD with ex gw pod in terminating state", false, true, - []libovsdbtest.TestData{ - &nbdb.LogicalSwitchPort{ - UUID: "lsp1", - Addresses: []string{"0a:58:0a:80:01:03 10.128.1.3"}, - ExternalIDs: map[string]string{ - "pod": "true", - "namespace": namespaceName, - }, - Name: "namespace1_myPod", - Options: map[string]string{ - "iface-id-ver": "myPod", - "requested-chassis": chassisIDForNode("node1"), - }, - PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, - }, - &nbdb.LogicalSwitch{ - UUID: "node1", - Name: "node1", - Ports: []string{"lsp1"}, - }, - &nbdb.LogicalSwitch{ - UUID: "node2", - Name: "node2", - }, - &nbdb.LogicalRouterStaticRoute{ - UUID: "static-route-1-UUID", - IPPrefix: "10.128.1.3/32", - Nexthop: "9.0.0.1", - Policy: &nbdb.LogicalRouterStaticRoutePolicySrcIP, - OutputPort: &logicalRouterPort, - Options: map[string]string{ - "ecmp_symmetric_reply": "true", - }, - }, - &nbdb.LogicalRouter{ - UUID: "GR_node1-UUID", - Name: "GR_node1", - StaticRoutes: []string{"static-route-1-UUID"}, - }, - }, - []libovsdbtest.TestData{ - &nbdb.LogicalSwitchPort{ - UUID: "lsp1", - Addresses: []string{"0a:58:0a:80:01:03 10.128.1.3"}, - ExternalIDs: map[string]string{ - "pod": "true", - "namespace": namespaceName, - }, - Name: "namespace1_myPod", - Options: map[string]string{ - "iface-id-ver": "myPod", - "requested-chassis": chassisIDForNode("node1"), - }, - PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, - }, - &nbdb.LogicalSwitch{ - UUID: "node1", - Name: "node1", - Ports: []string{"lsp1"}, - }, - &nbdb.LogicalSwitch{ - UUID: "node2", - Name: "node2", - }, - &nbdb.LogicalRouter{ - UUID: "GR_node1-UUID", - Name: "GR_node1", - StaticRoutes: []string{}, - }, - }, - "", - &adminpolicybasedrouteapi.AdminPolicyBasedExternalRouteList{}, - ), - ginkgo.Entry("No BFD with ex gw pod in not ready state", false, false, - []libovsdbtest.TestData{ - &nbdb.LogicalSwitchPort{ - UUID: "lsp1", - Addresses: []string{"0a:58:0a:80:01:03 10.128.1.3"}, - ExternalIDs: map[string]string{ - "pod": "true", - "namespace": namespaceName, - }, - Name: "namespace1_myPod", - Options: map[string]string{ - "iface-id-ver": "myPod", - "requested-chassis": chassisIDForNode("node1"), - }, - PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, - }, - &nbdb.LogicalSwitch{ - UUID: "node1", - Name: "node1", - Ports: []string{"lsp1"}, - }, - &nbdb.LogicalSwitch{ - UUID: "node2", - Name: "node2", - }, - &nbdb.LogicalRouterStaticRoute{ - UUID: "static-route-1-UUID", - IPPrefix: "10.128.1.3/32", - Nexthop: "9.0.0.1", - Policy: &nbdb.LogicalRouterStaticRoutePolicySrcIP, - OutputPort: &logicalRouterPort, - Options: map[string]string{ - "ecmp_symmetric_reply": "true", - }, - }, - &nbdb.LogicalRouter{ - UUID: "GR_node1-UUID", - Name: "GR_node1", - StaticRoutes: []string{"static-route-1-UUID"}, - }, - }, - []libovsdbtest.TestData{ - &nbdb.LogicalSwitchPort{ - UUID: "lsp1", - Addresses: []string{"0a:58:0a:80:01:03 10.128.1.3"}, - ExternalIDs: map[string]string{ - "pod": "true", - "namespace": namespaceName, - }, - Name: "namespace1_myPod", - Options: map[string]string{ - "iface-id-ver": "myPod", - "requested-chassis": chassisIDForNode("node1"), - }, - PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, - }, - &nbdb.LogicalSwitch{ - UUID: "node1", - Name: "node1", - Ports: []string{"lsp1"}, - }, - &nbdb.LogicalSwitch{ - UUID: "node2", - Name: "node2", - }, - &nbdb.LogicalRouter{ - UUID: "GR_node1-UUID", - Name: "GR_node1", - StaticRoutes: []string{}, - }, - }, - "", - &adminpolicybasedrouteapi.AdminPolicyBasedExternalRouteList{}, - ), - ginkgo.Entry("BFD Enabled with ex gw pod in terminating state", true, true, []libovsdbtest.TestData{ - &nbdb.LogicalSwitchPort{ - UUID: "lsp1", - Addresses: []string{"0a:58:0a:80:01:03 10.128.1.3"}, - ExternalIDs: map[string]string{ - "pod": "true", - "namespace": namespaceName, - }, - Name: "namespace1_myPod", - Options: map[string]string{ - "iface-id-ver": "myPod", - "requested-chassis": chassisIDForNode("node1"), - }, - PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, - }, - &nbdb.LogicalSwitch{ - UUID: "node1", - Name: "node1", - Ports: []string{"lsp1"}, - }, - &nbdb.LogicalSwitch{ - UUID: "node2", - Name: "node2", - }, - &nbdb.BFD{ - UUID: bfd1NamedUUID, - DstIP: "9.0.0.1", - LogicalPort: "rtoe-GR_node1", - }, - &nbdb.LogicalRouterStaticRoute{ - UUID: "static-route-1-UUID", - IPPrefix: "10.128.1.3/32", - Nexthop: "9.0.0.1", - BFD: &bfd1NamedUUID, - Policy: &nbdb.LogicalRouterStaticRoutePolicySrcIP, - OutputPort: &logicalRouterPort, - Options: map[string]string{ - "ecmp_symmetric_reply": "true", - }, - }, - &nbdb.LogicalRouter{ - UUID: "GR_node1-UUID", - Name: "GR_node1", - StaticRoutes: []string{"static-route-1-UUID"}, - }, - }, - []libovsdbtest.TestData{ - &nbdb.LogicalSwitchPort{ - UUID: "lsp1", - Addresses: []string{"0a:58:0a:80:01:03 10.128.1.3"}, - ExternalIDs: map[string]string{ - "pod": "true", - "namespace": namespaceName, - }, - Name: "namespace1_myPod", - Options: map[string]string{ - "iface-id-ver": "myPod", - "requested-chassis": chassisIDForNode("node1"), - }, - PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, - }, - &nbdb.LogicalSwitch{ - UUID: "node1", - Name: "node1", - Ports: []string{"lsp1"}, - }, - &nbdb.LogicalSwitch{ - UUID: "node2", - Name: "node2", - }, - &nbdb.LogicalRouter{ - UUID: "GR_node1-UUID", - Name: "GR_node1", - StaticRoutes: []string{}, - }, - }, - "", - &adminpolicybasedrouteapi.AdminPolicyBasedExternalRouteList{}, - ), - ginkgo.Entry("BFD Enabled with ex gw pod in not ready state", true, false, []libovsdbtest.TestData{ - &nbdb.LogicalSwitchPort{ - UUID: "lsp1", - Addresses: []string{"0a:58:0a:80:01:03 10.128.1.3"}, - ExternalIDs: map[string]string{ - "pod": "true", - "namespace": namespaceName, - }, - Name: "namespace1_myPod", - Options: map[string]string{ - "iface-id-ver": "myPod", - "requested-chassis": chassisIDForNode("node1"), - }, - PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, - }, - &nbdb.LogicalSwitch{ - UUID: "node1", - Name: "node1", - Ports: []string{"lsp1"}, - }, - &nbdb.LogicalSwitch{ - UUID: "node2", - Name: "node2", - }, - &nbdb.BFD{ - UUID: bfd1NamedUUID, - DstIP: "9.0.0.1", - LogicalPort: "rtoe-GR_node1", - }, - &nbdb.LogicalRouterStaticRoute{ - UUID: "static-route-1-UUID", - IPPrefix: "10.128.1.3/32", - Nexthop: "9.0.0.1", - BFD: &bfd1NamedUUID, - Policy: &nbdb.LogicalRouterStaticRoutePolicySrcIP, - OutputPort: &logicalRouterPort, - Options: map[string]string{ - "ecmp_symmetric_reply": "true", - }, - }, - &nbdb.LogicalRouter{ - UUID: "GR_node1-UUID", - Name: "GR_node1", - StaticRoutes: []string{"static-route-1-UUID"}, - }, - }, - []libovsdbtest.TestData{ - &nbdb.LogicalSwitchPort{ - UUID: "lsp1", - Addresses: []string{"0a:58:0a:80:01:03 10.128.1.3"}, - ExternalIDs: map[string]string{ - "pod": "true", - "namespace": namespaceName, - }, - Name: "namespace1_myPod", - Options: map[string]string{ - "iface-id-ver": "myPod", - "requested-chassis": chassisIDForNode("node1"), - }, - PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, - }, - &nbdb.LogicalSwitch{ - UUID: "node1", - Name: "node1", - Ports: []string{"lsp1"}, - }, - &nbdb.LogicalSwitch{ - UUID: "node2", - Name: "node2", - }, - &nbdb.LogicalRouter{ - UUID: "GR_node1-UUID", - Name: "GR_node1", - StaticRoutes: []string{}, - }, - }, - "", - &adminpolicybasedrouteapi.AdminPolicyBasedExternalRouteList{}, - ), - ginkgo.Entry("No BFD with ex gw pod in terminating state and with overlapping APB External Route CR and annotation", false, true, - []libovsdbtest.TestData{ - &nbdb.LogicalSwitchPort{ - UUID: "lsp1", - Addresses: []string{"0a:58:0a:80:01:03 10.128.1.3"}, - ExternalIDs: map[string]string{ - "pod": "true", - "namespace": namespaceName, - }, - Name: "namespace1_myPod", - Options: map[string]string{ - "iface-id-ver": "myPod", - "requested-chassis": chassisIDForNode("node1"), - }, - PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, - }, - &nbdb.LogicalSwitch{ - UUID: "node1", - Name: "node1", - Ports: []string{"lsp1"}, - }, - &nbdb.LogicalSwitch{ - UUID: "node2", - Name: "node2", - }, - &nbdb.LogicalRouterStaticRoute{ - UUID: "static-route-1-UUID", - IPPrefix: "10.128.1.3/32", - Nexthop: "9.0.0.1", - Policy: &nbdb.LogicalRouterStaticRoutePolicySrcIP, - OutputPort: &logicalRouterPort, - Options: map[string]string{ - "ecmp_symmetric_reply": "true", - }, - }, - &nbdb.LogicalRouter{ - UUID: "GR_node1-UUID", - Name: "GR_node1", - StaticRoutes: []string{"static-route-1-UUID"}, - }, - }, - []libovsdbtest.TestData{ - &nbdb.LogicalSwitchPort{ - UUID: "lsp1", - Addresses: []string{"0a:58:0a:80:01:03 10.128.1.3"}, - ExternalIDs: map[string]string{ - "pod": "true", - "namespace": namespaceName, - }, - Name: "namespace1_myPod", - Options: map[string]string{ - "iface-id-ver": "myPod", - "requested-chassis": chassisIDForNode("node1"), - }, - PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, - }, - &nbdb.LogicalSwitch{ - UUID: "node1", - Name: "node1", - Ports: []string{"lsp1"}, - }, - &nbdb.LogicalSwitch{ - UUID: "node2", - Name: "node2", - }, - &nbdb.LogicalRouter{ - UUID: "GR_node1-UUID", - Name: "GR_node1", - StaticRoutes: []string{}, - }, - }, - "", - &adminpolicybasedrouteapi.AdminPolicyBasedExternalRouteList{ - Items: []adminpolicybasedrouteapi.AdminPolicyBasedExternalRoute{ - newPolicy("policy", - &metav1.LabelSelector{MatchLabels: map[string]string{"name": namespaceName}}, - nil, - false, - &metav1.LabelSelector{MatchLabels: map[string]string{"name": namespace2Name}}, - &metav1.LabelSelector{MatchLabels: map[string]string{"name": gwPodName}}, - false, - ""), - }, - }, - ), - ginkgo.Entry("No BFD with ex gw pod in not ready state and with overlapping APB External Route CR and annotation", false, false, - []libovsdbtest.TestData{ - &nbdb.LogicalSwitchPort{ - UUID: "lsp1", - Addresses: []string{"0a:58:0a:80:01:03 10.128.1.3"}, - ExternalIDs: map[string]string{ - "pod": "true", - "namespace": namespaceName, - }, - Name: "namespace1_myPod", - Options: map[string]string{ - "iface-id-ver": "myPod", - "requested-chassis": chassisIDForNode("node1"), - }, - PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, - }, - &nbdb.LogicalSwitch{ - UUID: "node1", - Name: "node1", - Ports: []string{"lsp1"}, - }, - &nbdb.LogicalSwitch{ - UUID: "node2", - Name: "node2", - }, - &nbdb.LogicalRouterStaticRoute{ - UUID: "static-route-1-UUID", - IPPrefix: "10.128.1.3/32", - Nexthop: "9.0.0.1", - Policy: &nbdb.LogicalRouterStaticRoutePolicySrcIP, - OutputPort: &logicalRouterPort, - Options: map[string]string{ - "ecmp_symmetric_reply": "true", - }, - }, - &nbdb.LogicalRouter{ - UUID: "GR_node1-UUID", - Name: "GR_node1", - StaticRoutes: []string{"static-route-1-UUID"}, - }, - }, - []libovsdbtest.TestData{ - &nbdb.LogicalSwitchPort{ - UUID: "lsp1", - Addresses: []string{"0a:58:0a:80:01:03 10.128.1.3"}, - ExternalIDs: map[string]string{ - "pod": "true", - "namespace": namespaceName, - }, - Name: "namespace1_myPod", - Options: map[string]string{ - "iface-id-ver": "myPod", - "requested-chassis": chassisIDForNode("node1"), - }, - PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, - }, - &nbdb.LogicalSwitch{ - UUID: "node1", - Name: "node1", - Ports: []string{"lsp1"}, - }, - &nbdb.LogicalSwitch{ - UUID: "node2", - Name: "node2", - }, - &nbdb.LogicalRouter{ - UUID: "GR_node1-UUID", - Name: "GR_node1", - StaticRoutes: []string{}, - }, - }, - "", - &adminpolicybasedrouteapi.AdminPolicyBasedExternalRouteList{ - Items: []adminpolicybasedrouteapi.AdminPolicyBasedExternalRoute{ - newPolicy("policy", - &metav1.LabelSelector{MatchLabels: map[string]string{"name": namespaceName}}, - nil, - false, - &metav1.LabelSelector{MatchLabels: map[string]string{"name": namespace2Name}}, - &metav1.LabelSelector{MatchLabels: map[string]string{"name": gwPodName}}, - false, - ""), - }, - }, - ), - ) - }) - ginkgo.Context("on using bfd", func() { - ginkgo.It("should enable bfd only on the namespace gw when set", func() { - app.Action = func(*cli.Context) error { - - namespaceT := *ovntest.NewNamespace(namespaceName) - namespaceT.Annotations = map[string]string{"k8s.ovn.org/routing-external-gws": "9.0.0.1"} - namespaceT.Annotations["k8s.ovn.org/bfd-enabled"] = "" - namespaceX := *ovntest.NewNamespace("namespace2") - - t := newTPod( - "node1", - "10.128.1.0/24", - "10.128.1.2", - "10.128.1.1", - "myPod", - "10.128.1.3", - "0a:58:0a:80:01:03", - namespaceT.Name, - ) - gwPod := *ovntest.NewPod(namespaceX.Name, "gwPod", "node2", "10.0.0.1") - gwPod.Annotations = map[string]string{"k8s.ovn.org/routing-namespaces": namespaceT.Name} - gwPod.Spec.HostNetwork = true - fakeOvn.startWithDBSetup( - libovsdbtest.TestSetup{ - NBData: []libovsdbtest.TestData{ - &nbdb.LogicalSwitch{ - UUID: "node1", - Name: "node1", - }, - &nbdb.LogicalSwitch{ - UUID: "node2", - Name: "node2", - }, - &nbdb.LogicalRouter{ - UUID: "GR_node1-UUID", - Name: "GR_node1", - }, - }, - }, - &corev1.NamespaceList{ - Items: []corev1.Namespace{ - namespaceT, - }, - }, - &corev1.NodeList{ - Items: []corev1.Node{ - *newNode("node1", "192.168.126.202/24"), - *newNode("node2", "192.168.126.50/24"), - }, - }, - &corev1.PodList{ - Items: []corev1.Pod{ - *ovntest.NewPod(t.namespace, t.podName, t.nodeName, t.podIP), - }, - }, - ) - t.populateLogicalSwitchCache(fakeOvn) - err := fakeOvn.controller.lsManager.AddOrUpdateSwitch("node2", []*net.IPNet{ovntest.MustParseIPNet("10.128.2.0/24")}, nil) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - injectNode(fakeOvn) - err = fakeOvn.controller.WatchNamespaces() - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fakeOvn.controller.WatchPods() - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - _, err = fakeOvn.fakeClient.KubeClient.CoreV1().Pods(namespaceX.Name).Create(context.TODO(), &gwPod, metav1.CreateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - finalNB := []libovsdbtest.TestData{ - &nbdb.LogicalSwitchPort{ - UUID: "lsp1", - Addresses: []string{"0a:58:0a:80:01:03 10.128.1.3"}, - ExternalIDs: map[string]string{ - "pod": "true", - "namespace": namespaceName, - }, - Name: "namespace1_myPod", - Options: map[string]string{ - "iface-id-ver": "myPod", - libovsdbops.RequestedChassis: chassisIDForNode("node1"), - }, - PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, - }, - &nbdb.LogicalSwitch{ - UUID: "node1", - Name: "node1", - Ports: []string{"lsp1"}, - }, - &nbdb.LogicalSwitch{ - UUID: "node2", - Name: "node2", - }, - &nbdb.BFD{ - UUID: bfd1NamedUUID, - DstIP: "9.0.0.1", - LogicalPort: "rtoe-GR_node1", - }, - &nbdb.LogicalRouterStaticRoute{ - UUID: "static-route-1-UUID", - IPPrefix: "10.128.1.3/32", - Nexthop: "9.0.0.1", - BFD: &bfd1NamedUUID, - Policy: &nbdb.LogicalRouterStaticRoutePolicySrcIP, - OutputPort: &logicalRouterPort, - Options: map[string]string{ - "ecmp_symmetric_reply": "true", - }, - }, - &nbdb.LogicalRouterStaticRoute{ - UUID: "static-route-2-UUID", - IPPrefix: "10.128.1.3/32", - Nexthop: "10.0.0.1", - Policy: &nbdb.LogicalRouterStaticRoutePolicySrcIP, - OutputPort: &logicalRouterPort, - Options: map[string]string{ - "ecmp_symmetric_reply": "true", - }, - }, - &nbdb.LogicalRouter{ - UUID: "GR_node1-UUID", - Name: "GR_node1", - StaticRoutes: []string{"static-route-1-UUID", "static-route-2-UUID"}, - }, - } - gomega.Eventually(fakeOvn.nbClient).Should(libovsdbtest.HaveData(finalNB)) - gomega.Eventually(func() string { - return getNamespaceAnnotations(fakeOvn.fakeClient.KubeClient, namespaceT.Name)[util.ExternalGatewayPodIPsAnnotation] - }).Should(gomega.Equal("10.0.0.1")) - return nil - } - - err := app.Run([]string{app.Name}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }) - ginkgo.It("should enable bfd only on the gw pod when set", func() { - app.Action = func(*cli.Context) error { - - namespaceT := *ovntest.NewNamespace(namespaceName) - namespaceT.Annotations = map[string]string{"k8s.ovn.org/routing-external-gws": "9.0.0.1"} - namespaceX := *ovntest.NewNamespace("namespace2") - - t := newTPod( - "node1", - "10.128.1.0/24", - "10.128.1.2", - "10.128.1.1", - "myPod", - "10.128.1.3", - "0a:58:0a:80:01:03", - namespaceT.Name, - ) - gwPod := *ovntest.NewPod(namespaceX.Name, "gwPod", "node2", "10.0.0.1") - gwPod.Annotations = map[string]string{"k8s.ovn.org/routing-namespaces": namespaceT.Name} - gwPod.Annotations["k8s.ovn.org/bfd-enabled"] = "" - - gwPod.Spec.HostNetwork = true - fakeOvn.startWithDBSetup( - libovsdbtest.TestSetup{ - NBData: []libovsdbtest.TestData{ - &nbdb.LogicalSwitch{ - UUID: "node1", - Name: "node1", - }, - &nbdb.LogicalSwitch{ - UUID: "node2", - Name: "node2", - }, - &nbdb.LogicalRouter{ - UUID: "GR_node1-UUID", - Name: "GR_node1", - }, - }, - }, - &corev1.NamespaceList{ - Items: []corev1.Namespace{ - namespaceT, - }, - }, - &corev1.NodeList{ - Items: []corev1.Node{ - *newNode("node1", "192.168.126.202/24"), - *newNode("node2", "192.168.126.50/24"), - }, - }, - &corev1.PodList{ - Items: []corev1.Pod{ - *ovntest.NewPod(t.namespace, t.podName, t.nodeName, t.podIP), - }, - }, - ) - t.populateLogicalSwitchCache(fakeOvn) - err := fakeOvn.controller.lsManager.AddOrUpdateSwitch("node2", []*net.IPNet{ovntest.MustParseIPNet("10.128.2.0/24")}, nil) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - injectNode(fakeOvn) - err = fakeOvn.controller.WatchNamespaces() - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fakeOvn.controller.WatchPods() - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - _, err = fakeOvn.fakeClient.KubeClient.CoreV1().Pods(namespaceX.Name).Create(context.TODO(), &gwPod, metav1.CreateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - finalNB := []libovsdbtest.TestData{ - &nbdb.LogicalSwitchPort{ - UUID: "lsp1", - Addresses: []string{"0a:58:0a:80:01:03 10.128.1.3"}, - ExternalIDs: map[string]string{ - "pod": "true", - "namespace": namespaceName, - }, - Name: "namespace1_myPod", - Options: map[string]string{ - "iface-id-ver": "myPod", - libovsdbops.RequestedChassis: chassisIDForNode("node1"), - }, - PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, - }, - &nbdb.LogicalSwitch{ - UUID: "node1", - Name: "node1", - Ports: []string{"lsp1"}, - }, - &nbdb.LogicalSwitch{ - UUID: "node2", - Name: "node2", - }, - &nbdb.BFD{ - UUID: bfd1NamedUUID, - DstIP: "10.0.0.1", - LogicalPort: "rtoe-GR_node1", - }, - &nbdb.LogicalRouterStaticRoute{ - UUID: "static-route-1-UUID", - IPPrefix: "10.128.1.3/32", - Nexthop: "9.0.0.1", - Policy: &nbdb.LogicalRouterStaticRoutePolicySrcIP, - OutputPort: &logicalRouterPort, - Options: map[string]string{ - "ecmp_symmetric_reply": "true", - }, - }, - &nbdb.LogicalRouterStaticRoute{ - UUID: "static-route-2-UUID", - IPPrefix: "10.128.1.3/32", - Nexthop: "10.0.0.1", - Policy: &nbdb.LogicalRouterStaticRoutePolicySrcIP, - OutputPort: &logicalRouterPort, - BFD: &bfd1NamedUUID, - Options: map[string]string{ - "ecmp_symmetric_reply": "true", - }, - }, - &nbdb.LogicalRouter{ - UUID: "GR_node1-UUID", - Name: "GR_node1", - StaticRoutes: []string{"static-route-1-UUID", "static-route-2-UUID"}, - }, - } - gomega.Eventually(fakeOvn.nbClient).Should(libovsdbtest.HaveData(finalNB)) - gomega.Eventually(func() string { - return getNamespaceAnnotations(fakeOvn.fakeClient.KubeClient, namespaceT.Name)[util.ExternalGatewayPodIPsAnnotation] - }).Should(gomega.Equal("10.0.0.1")) - return nil - } - - err := app.Run([]string{app.Name}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }) - ginkgo.It("should disable bfd when removing the annotation from the namespace", func() { - app.Action = func(*cli.Context) error { - namespaceT := *ovntest.NewNamespace(namespaceName) - namespaceT.Annotations = map[string]string{"k8s.ovn.org/routing-external-gws": "9.0.0.1"} - namespaceT.Annotations["k8s.ovn.org/bfd-enabled"] = "" - - t := newTPod( - "node1", - "10.128.1.0/24", - "10.128.1.2", - "10.128.1.1", - "myPod", - "10.128.1.3", - "0a:58:0a:80:01:03", - namespaceT.Name, - ) - - fakeOvn.startWithDBSetup( - libovsdbtest.TestSetup{ - NBData: []libovsdbtest.TestData{ - &nbdb.LogicalSwitch{ - UUID: "node1", - Name: "node1", - }, - &nbdb.BFD{ - UUID: bfd1NamedUUID, - DstIP: "9.0.0.1", - LogicalPort: "rtoe-GR_node1", - }, - &nbdb.LogicalRouterStaticRoute{ - UUID: "static-route-1-UUID", - IPPrefix: "10.128.1.3/32", - Nexthop: "9.0.0.1", - Policy: &nbdb.LogicalRouterStaticRoutePolicySrcIP, - BFD: &bfd1NamedUUID, - OutputPort: &logicalRouterPort, - Options: map[string]string{ - "ecmp_symmetric_reply": "true", - }, - }, - &nbdb.LogicalRouter{ - UUID: "GR_node1-UUID", - Name: "GR_node1", - StaticRoutes: []string{"static-route-1-UUID"}, - }, - }, - }, - &corev1.NamespaceList{ - Items: []corev1.Namespace{ - namespaceT, - }, - }, - &corev1.NodeList{ - Items: []corev1.Node{ - *newNode("node1", "192.168.126.202/24"), - }, - }, - &corev1.PodList{ - Items: []corev1.Pod{ - *ovntest.NewPod(t.namespace, t.podName, t.nodeName, t.podIP), - }, - }, - ) - t.populateLogicalSwitchCache(fakeOvn) - - injectNode(fakeOvn) - err := fakeOvn.controller.WatchNamespaces() - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fakeOvn.controller.WatchPods() - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - namespaceT.Annotations = map[string]string{"k8s.ovn.org/routing-external-gws": "9.0.0.1"} - _, err = fakeOvn.fakeClient.KubeClient.CoreV1().Namespaces().Update(context.Background(), &namespaceT, metav1.UpdateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - finalNB := []libovsdbtest.TestData{ - &nbdb.LogicalSwitchPort{ - UUID: "lsp1", - Addresses: []string{"0a:58:0a:80:01:03 10.128.1.3"}, - ExternalIDs: map[string]string{ - "pod": "true", - "namespace": namespaceName, - }, - Name: "namespace1_myPod", - Options: map[string]string{ - "iface-id-ver": "myPod", - libovsdbops.RequestedChassis: chassisIDForNode("node1"), - }, - PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, - }, - &nbdb.LogicalSwitch{ - UUID: "node1", - Name: "node1", - Ports: []string{"lsp1"}, - }, - &nbdb.LogicalRouterStaticRoute{ - UUID: "static-route-1-UUID", - IPPrefix: "10.128.1.3/32", - Nexthop: "9.0.0.1", - Policy: &nbdb.LogicalRouterStaticRoutePolicySrcIP, - OutputPort: &logicalRouterPort, - Options: map[string]string{ - "ecmp_symmetric_reply": "true", - }, - }, - &nbdb.LogicalRouter{ - UUID: "GR_node1-UUID", - Name: "GR_node1", - StaticRoutes: []string{"static-route-1-UUID"}, - }, - } - gomega.Eventually(fakeOvn.nbClient).Should(libovsdbtest.HaveData(finalNB)) - return nil - } - - err := app.Run([]string{app.Name}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }) - }) - ginkgo.Context("hybrid route policy operations in lgw mode", func() { - ginkgo.It("add hybrid route policy for pods", func() { - app.Action = func(*cli.Context) error { - config.Gateway.Mode = config.GatewayModeLocal - - fakeOvn.startWithDBSetup( - libovsdbtest.TestSetup{ - NBData: []libovsdbtest.TestData{ - &nbdb.LogicalRouterPort{ - UUID: ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + "node1" + "-UUID", - Name: ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + "node1", - Networks: []string{"100.64.0.4/32"}, - }, - &nbdb.LogicalRouter{ - Name: ovntypes.OVNClusterRouter, - UUID: ovntypes.OVNClusterRouter + "-UUID", - Ports: []string{ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + "node1" + "-UUID"}, - }, - }, - }, - ) - asIndex := apbroute.GetHybridRouteAddrSetDbIDs("node1", ovntypes.DefaultNetworkControllerName) - asv4, _ := addressset.GetHashNamesForAS(asIndex) - finalNB := []libovsdbtest.TestData{ - &nbdb.LogicalRouterPolicy{ - UUID: "2a7a61cb-fb13-4266-a3f0-9ac5c4471123 [u2596996164]", - Priority: ovntypes.HybridOverlayReroutePriority, - Action: nbdb.LogicalRouterPolicyActionReroute, - Nexthops: []string{"100.64.0.4"}, - Match: "inport == \"rtos-node1\" && ip4.src == $" + asv4 + " && ip4.dst != 10.128.0.0/14", - }, - &nbdb.LogicalRouter{ - Name: ovntypes.OVNClusterRouter, - UUID: ovntypes.OVNClusterRouter + "-UUID", - Ports: []string{ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + "node1" + "-UUID"}, - Policies: []string{"2a7a61cb-fb13-4266-a3f0-9ac5c4471123 [u2596996164]"}, - }, - &nbdb.LogicalRouterPort{ - UUID: ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + "node1" + "-UUID", - Name: ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + "node1", - Networks: []string{"100.64.0.4/32"}, - }, - } - - err := fakeOvn.controller.addHybridRoutePolicyForPod(net.ParseIP("10.128.1.3"), "node1") - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Eventually(fakeOvn.nbClient).Should(libovsdbtest.HaveData(finalNB)) - // check if the address-set was created with the podIP - dbIDs := apbroute.GetHybridRouteAddrSetDbIDs("node1", ovntypes.DefaultNetworkControllerName) - fakeOvn.asf.ExpectAddressSetWithAddresses(dbIDs, []string{"10.128.1.3"}) - return nil - } - - err := app.Run([]string{app.Name}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }) - ginkgo.It("should reconcile a pod and create/delete the hybridRoutePolicy accordingly", func() { - app.Action = func(*cli.Context) error { - config.Gateway.Mode = config.GatewayModeLocal - - namespaceT := *ovntest.NewNamespace("namespace1") - namespaceT.Annotations = map[string]string{"k8s.ovn.org/routing-external-gws": "9.0.0.1"} - t := newTPod( - "node1", - "10.128.1.0/24", - "10.128.1.2", - "10.128.1.1", - "myPod", - "10.128.1.3", - "0a:58:0a:80:01:03", - namespaceT.Name, - ) - - fakeOvn.startWithDBSetup( - libovsdbtest.TestSetup{ - NBData: []libovsdbtest.TestData{ - &nbdb.LogicalSwitch{ - UUID: "node1", - Name: "node1", - }, - &nbdb.LogicalRouterPort{ - UUID: ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + "node1" + "-UUID", - Name: ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + "node1", - Networks: []string{"100.64.0.4/32"}, - }, - &nbdb.LogicalRouter{ - UUID: "GR_node1-UUID", - Name: "GR_node1", - Ports: []string{ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + "node1" + "-UUID"}, - }, - &nbdb.LogicalRouter{ - Name: ovntypes.OVNClusterRouter, - UUID: ovntypes.OVNClusterRouter + "-UUID", - }, - }, - }, - &corev1.NamespaceList{ - Items: []corev1.Namespace{ - namespaceT, - }, - }, - &corev1.NodeList{ - Items: []corev1.Node{ - *newNode("node1", "192.168.126.202/24"), - }, - }, - &corev1.PodList{ - Items: []corev1.Pod{ - *ovntest.NewPod(t.namespace, t.podName, t.nodeName, t.podIP), - }, - }, - ) - - t.populateLogicalSwitchCache(fakeOvn) - - injectNode(fakeOvn) - err := fakeOvn.controller.WatchNamespaces() - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fakeOvn.controller.WatchPods() - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - asIndex := apbroute.GetHybridRouteAddrSetDbIDs("node1", ovntypes.DefaultNetworkControllerName) - asv4, _ := addressset.GetHashNamesForAS(asIndex) - nbWithLRP := []libovsdbtest.TestData{ - &nbdb.LogicalRouterPolicy{ - UUID: "lrp1", - Action: "reroute", - Match: "inport == \"rtos-node1\" && ip4.src == $" + asv4 + " && ip4.dst != 10.128.0.0/14", - Nexthops: []string{"100.64.0.4"}, - Priority: ovntypes.HybridOverlayReroutePriority, - }, - &nbdb.LogicalRouterPort{ - UUID: ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + "node1" + "-UUID", - Name: ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + "node1", - Networks: []string{"100.64.0.4/32"}, - }, - &nbdb.LogicalRouterStaticRoute{ - UUID: "static-route-1-UUID", - IPPrefix: "10.128.1.3/32", - Nexthop: "9.0.0.1", - Options: map[string]string{ - "ecmp_symmetric_reply": "true", - }, - OutputPort: &logicalRouterPort, - Policy: &nbdb.LogicalRouterStaticRoutePolicySrcIP, - }, - &nbdb.LogicalSwitch{ - UUID: "493c61b4-2f97-446d-a1f0-1f713b510bbf", - Name: "node1", - Ports: []string{"lsp1"}, - }, - &nbdb.LogicalSwitchPort{ - UUID: "lsp1", - Addresses: []string{"0a:58:0a:80:01:03 10.128.1.3"}, - ExternalIDs: map[string]string{ - "pod": "true", - "namespace": "namespace1", - }, - Name: "namespace1_myPod", - Options: map[string]string{ - libovsdbops.RequestedChassis: chassisIDForNode("node1"), - "iface-id-ver": "myPod", - }, - PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, - }, - &nbdb.LogicalRouter{ - UUID: "e496b76e-18a1-461e-a919-6dcf0b3c35db", - Name: "ovn_cluster_router", - Policies: []string{"lrp1"}, - }, - &nbdb.LogicalRouter{ - UUID: "GR_node1-UUID", - Name: "GR_node1", - Ports: []string{ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + "node1" + "-UUID"}, - StaticRoutes: []string{"static-route-1-UUID"}, - }, - } - - gomega.Eventually(func() string { return getPodAnnotations(fakeOvn.fakeClient.KubeClient, t.namespace, t.podName) }, 2).Should(gomega.MatchJSON(t.getAnnotationsJson())) - gomega.Eventually(fakeOvn.nbClient).Should(libovsdbtest.HaveData(nbWithLRP)) - - err = fakeOvn.fakeClient.KubeClient.CoreV1().Pods(t.namespace).Delete(context.TODO(), t.podName, *metav1.NewDeleteOptions(0)) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - finalNB := []libovsdbtest.TestData{ - &nbdb.LogicalRouterPort{ - UUID: ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + "node1" + "-UUID", - Name: ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + "node1", - Networks: []string{"100.64.0.4/32"}, - }, - &nbdb.LogicalSwitch{ - UUID: "493c61b4-2f97-446d-a1f0-1f713b510bbf", - Name: "node1", - }, - &nbdb.LogicalRouter{ - UUID: "e496b76e-18a1-461e-a919-6dcf0b3c35db", - Name: "ovn_cluster_router", - }, - &nbdb.LogicalRouter{ - UUID: "8945d2c1-bf8a-43ab-aa9f-6130eb525682", - Name: "GR_node1", - Ports: []string{ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + "node1" + "-UUID"}, - }, - } - gomega.Eventually(fakeOvn.nbClient).Should(libovsdbtest.HaveData(finalNB)) - - return nil - } - - err := app.Run([]string{app.Name}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }) - ginkgo.DescribeTable("should keep the hybrid route policy after deleting the namespace gateway annotation when there is an APB External Route CR overlapping the same external gateway IP", func(legacyFirst bool) { - - app.Action = func(*cli.Context) error { - config.Gateway.Mode = config.GatewayModeLocal - - namespaceT := *ovntest.NewNamespace("namespace1") - namespaceT.Annotations = map[string]string{"k8s.ovn.org/routing-external-gws": "9.0.0.1"} - t := newTPod( - "node1", - "10.128.1.0/24", - "10.128.1.2", - "10.128.1.1", - "myPod", - "10.128.1.3", - "0a:58:0a:80:01:03", - namespaceT.Name, - ) - - fakeOvn.startWithDBSetup( - libovsdbtest.TestSetup{ - NBData: []libovsdbtest.TestData{ - &nbdb.LogicalSwitch{ - UUID: "node1", - Name: "node1", - }, - &nbdb.LogicalRouterPort{ - UUID: ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + "node1" + "-UUID", - Name: ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + "node1", - Networks: []string{"100.64.0.4/32"}, - }, - &nbdb.LogicalRouter{ - UUID: "GR_node1-UUID", - Name: "GR_node1", - Ports: []string{ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + "node1" + "-UUID"}, - }, - &nbdb.LogicalRouter{ - Name: ovntypes.OVNClusterRouter, - UUID: ovntypes.OVNClusterRouter + "-UUID", - }, - }, - }, - &corev1.NamespaceList{ - Items: []corev1.Namespace{ - namespaceT, - }, - }, - &corev1.NodeList{ - Items: []corev1.Node{ - *newNode("node1", "192.168.126.202/24"), - }, - }, - &corev1.PodList{ - Items: []corev1.Pod{ - *ovntest.NewPod(t.namespace, t.podName, t.nodeName, t.podIP), - }, - }, - ) - - t.populateLogicalSwitchCache(fakeOvn) - - injectNode(fakeOvn) - - apbRoute := newPolicy("policy", - &metav1.LabelSelector{MatchLabels: map[string]string{"name": namespaceT.Name}}, - sets.New("9.0.0.1"), - false, - nil, - nil, - false, - "", - ) - if !legacyFirst { - // when CR exists, egress_gw code won't do anything - _, err := fakeOvn.fakeClient.AdminPolicyRouteClient.K8sV1().AdminPolicyBasedExternalRoutes().Create( - context.TODO(), &apbRoute, metav1.CreateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - - err := fakeOvn.controller.WatchNamespaces() - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = fakeOvn.controller.WatchPods() - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - fakeOvn.RunAPBExternalPolicyController() - - if legacyFirst { - // create CR after egress_gw has handled namespace annotations - _, err := fakeOvn.fakeClient.AdminPolicyRouteClient.K8sV1().AdminPolicyBasedExternalRoutes().Create( - context.TODO(), &apbRoute, metav1.CreateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - } - - asIndex := apbroute.GetHybridRouteAddrSetDbIDs("node1", ovntypes.DefaultNetworkControllerName) - asv4, _ := addressset.GetHashNamesForAS(asIndex) - nbWithLRP := []libovsdbtest.TestData{ - &nbdb.LogicalRouterPolicy{ - UUID: "lrp1", - Action: "reroute", - Match: "inport == \"rtos-node1\" && ip4.src == $" + asv4 + " && ip4.dst != 10.128.0.0/14", - Nexthops: []string{"100.64.0.4"}, - Priority: ovntypes.HybridOverlayReroutePriority, - }, - &nbdb.LogicalRouterPort{ - UUID: ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + "node1" + "-UUID", - Name: ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + "node1", - Networks: []string{"100.64.0.4/32"}, - }, - &nbdb.LogicalRouterStaticRoute{ - UUID: "static-route-1-UUID", - IPPrefix: "10.128.1.3/32", - Nexthop: "9.0.0.1", - Options: map[string]string{ - "ecmp_symmetric_reply": "true", - }, - OutputPort: &logicalRouterPort, - Policy: &nbdb.LogicalRouterStaticRoutePolicySrcIP, - }, - &nbdb.LogicalSwitch{ - UUID: "493c61b4-2f97-446d-a1f0-1f713b510bbf", - Name: "node1", - Ports: []string{"lsp1"}, - }, - &nbdb.LogicalSwitchPort{ - UUID: "lsp1", - Addresses: []string{"0a:58:0a:80:01:03 10.128.1.3"}, - ExternalIDs: map[string]string{ - "pod": "true", - "namespace": "namespace1", - }, - Name: "namespace1_myPod", - Options: map[string]string{ - libovsdbops.RequestedChassis: chassisIDForNode("node1"), - "iface-id-ver": "myPod", - }, - PortSecurity: []string{"0a:58:0a:80:01:03 10.128.1.3"}, - }, - &nbdb.LogicalRouter{ - UUID: "e496b76e-18a1-461e-a919-6dcf0b3c35db", - Name: "ovn_cluster_router", - Policies: []string{"lrp1"}, - }, - &nbdb.LogicalRouter{ - UUID: "8945d2c1-bf8a-43ab-aa9f-6130eb525682", - Name: "GR_node1", - Ports: []string{ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + "node1" + "-UUID"}, - StaticRoutes: []string{"static-route-1-UUID"}, - }, - } - - gomega.Eventually(func() string { - return getPodAnnotations(fakeOvn.fakeClient.KubeClient, t.namespace, t.podName) - }, 2).Should(gomega.MatchJSON(t.getAnnotationsJson())) - gomega.Eventually(fakeOvn.nbClient).Should(libovsdbtest.HaveData(nbWithLRP)) - - ginkgo.By("Removing the namespace annotation") - namespaceT.Annotations = map[string]string{} - _, err = fakeOvn.fakeClient.KubeClient.CoreV1().Namespaces().Update(context.Background(), &namespaceT, metav1.UpdateOptions{}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - - gomega.Eventually(fakeOvn.nbClient).Should(libovsdbtest.HaveData(nbWithLRP)) - checkAPBRouteStatus(fakeOvn, "policy", false) - return nil - } - - err := app.Run([]string{app.Name}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }, - ginkgo.Entry("when APBRoute handles first", false), - ginkgo.Entry("when external_gw handles first", true)) - - ginkgo.It("should create a single policy for concurrent addHybridRoutePolicy for the same node", func() { - app.Action = func(*cli.Context) error { - config.Gateway.Mode = config.GatewayModeLocal - - fakeOvn.startWithDBSetup( - libovsdbtest.TestSetup{ - NBData: []libovsdbtest.TestData{ - &nbdb.LogicalRouterPort{ - UUID: ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + "node1" + "-UUID", - Name: ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + "node1", - Networks: []string{"100.64.0.4/32"}, - }, - &nbdb.LogicalRouter{ - Name: ovntypes.OVNClusterRouter, - UUID: ovntypes.OVNClusterRouter + "-UUID", - Ports: []string{ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + "node1" + "-UUID"}, - }, - }, - }, - ) - asIndex := apbroute.GetHybridRouteAddrSetDbIDs("node1", ovntypes.DefaultNetworkControllerName) - asv4, _ := addressset.GetHashNamesForAS(asIndex) - finalNB := []libovsdbtest.TestData{ - &nbdb.LogicalRouterPolicy{ - UUID: "lrp1", - Priority: ovntypes.HybridOverlayReroutePriority, - Action: nbdb.LogicalRouterPolicyActionReroute, - Nexthops: []string{"100.64.0.4"}, - Match: "inport == \"rtos-node1\" && ip4.src == $" + asv4 + " && ip4.dst != 10.128.0.0/14", - }, - &nbdb.LogicalRouter{ - Name: ovntypes.OVNClusterRouter, - UUID: ovntypes.OVNClusterRouter + "-UUID", - Ports: []string{ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + "node1" + "-UUID"}, - Policies: []string{"lrp1"}, - }, - &nbdb.LogicalRouterPort{ - UUID: ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + "node1" + "-UUID", - Name: ovntypes.GWRouterToJoinSwitchPrefix + ovntypes.GWRouterPrefix + "node1", - Networks: []string{"100.64.0.4/32"}, - }, - } - - wg := &sync.WaitGroup{} - c := make(chan int) - for i := 1; i <= 5; i++ { - podIndex := i - wg.Add(1) - go func() { - defer ginkgo.GinkgoRecover() - defer wg.Done() - <-c - // errors can happen due to server side concurrency check - _ = fakeOvn.controller.addHybridRoutePolicyForPod(net.ParseIP(fmt.Sprintf("10.128.1.%d", podIndex)), "node1") - }() - } - close(c) - wg.Wait() - gomega.Eventually(fakeOvn.nbClient).Should(libovsdbtest.HaveData(finalNB)) - - err := fakeOvn.controller.addHybridRoutePolicyForPod(net.ParseIP(fmt.Sprintf("10.128.1.%d", 6)), "node1") - // adding another pod after the initial burst should not trigger an error or change db - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - gomega.Eventually(fakeOvn.nbClient).Should(libovsdbtest.HaveData(finalNB)) - - return nil - } - - err := app.Run([]string{app.Name}) - gomega.Expect(err).NotTo(gomega.HaveOccurred()) - }) + ginkgo.Context("hybrid route policy operations in lgw mode", func() { ginkgo.It("delete hybrid route policy for pods", func() { app.Action = func(*cli.Context) error { config.Gateway.Mode = config.GatewayModeLocal diff --git a/test/e2e/external_gateways.go b/test/e2e/external_gateways.go index 498a421ffc..3201c7f305 100644 --- a/test/e2e/external_gateways.go +++ b/test/e2e/external_gateways.go @@ -27,7 +27,6 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/apimachinery/pkg/util/wait" "k8s.io/klog/v2" "k8s.io/kubernetes/test/e2e/framework" e2ekubectl "k8s.io/kubernetes/test/e2e/framework/kubectl" @@ -108,1309 +107,6 @@ var _ = ginkgo.Describe("External Gateway", feature.ExternalGateway, func() { bfdTimeout = 4 * time.Second ) - // Validate pods can reach a network running in a container's loopback address via - // an external gateway running on eth0 of the container without any tunnel encap. - // Next, the test updates the namespace annotation to point to a second container, - // emulating the ext gateway. This test requires shared gateway mode in the job infra. - var _ = ginkgo.Describe("e2e non-vxlan external gateway and update validation", func() { - const ( - svcname string = "multiple-novxlan-externalgw" - gwContainerNameTemplate string = "gw-novxlan-test-container-alt1-%d" - gwContainerNameTemplate2 string = "gw-novxlan-test-container-alt2-%d" - ) - var ( - exGWRemoteIpAlt1 string - exGWRemoteIpAlt2 string - providerCtx infraapi.Context - ) - - f := wrappedTestFramework(svcname) - - // Determine what mode the CI is running in and get relevant endpoint information for the tests - ginkgo.BeforeEach(func() { - providerCtx = infraprovider.Get().NewTestContext() - exGWRemoteIpAlt1 = "10.249.3.1" - exGWRemoteIpAlt2 = "10.249.4.1" - if IsIPv6Cluster(f.ClientSet) { - exGWRemoteIpAlt1 = "fc00:f853:ccd:e793::1" - exGWRemoteIpAlt2 = "fc00:f853:ccd:e794::1" - } - }) - - ginkgo.It("Should validate connectivity without vxlan before and after updating the namespace annotation to a new external gateway", func() { - - var pingSrc string - var validIP net.IP - - isIPv6Cluster := IsIPv6Cluster(f.ClientSet) - srcPingPodName := "e2e-exgw-novxlan-src-ping-pod" - command := []string{"bash", "-c", "sleep 20000"} - testContainer := fmt.Sprintf("%s-container", srcPingPodName) - testContainerFlag := fmt.Sprintf("--container=%s", testContainer) - // start the container that will act as an external gateway - network, err := infraprovider.Get().PrimaryNetwork() - framework.ExpectNoError(err, "failed to get primary network information") - overrideNetworkStr, overrideIPv4, overrideIPv6 := getOverrideNetwork() - if overrideNetworkStr != "" { - overrideNetwork, err := infraprovider.Get().GetNetwork(overrideNetworkStr) - framework.ExpectNoError(err, "over ride network must exist") - network = overrideNetwork - } - externalContainerPort := infraprovider.Get().GetExternalContainerPort() - externalContainer := infraapi.ExternalContainer{Name: getContainerName(gwContainerNameTemplate, externalContainerPort), - Image: images.AgnHost(), Network: network, ExtPort: externalContainerPort, CmdArgs: []string{"pause"}} - externalContainer, err = providerCtx.CreateExternalContainer(externalContainer) - framework.ExpectNoError(err, "failed to start external gateway test container") - if network.Name() == "host" { - // manually cleanup because cleanup doesnt cleanup host network - providerCtx.AddCleanUpFn(func() error { - return providerCtx.DeleteExternalContainer(externalContainer) - }) - } - // non-ha ci mode runs a set of kind nodes prefixed with ovn-worker - nodes, err := e2enode.GetBoundedReadySchedulableNodes(context.TODO(), f.ClientSet, 1) - framework.ExpectNoError(err, "failed to find 3 ready and schedulable nodes") - if len(nodes.Items) < 1 { - framework.Failf("requires at least 1 Nodes") - } - node := &nodes.Items[0] - ni, err := infraprovider.Get().GetK8NodeNetworkInterface(node.Name, network) - framework.ExpectNoError(err, "must get network interface info") - var nodeAddr string - var exGWIpAlt1, exGWRemoteCidrAlt1, exGWRemoteCidrAlt2 string - if isIPv6Cluster { - exGWIpAlt1 = externalContainer.GetIPv6() - if overrideIPv6 != "" { - exGWIpAlt1 = overrideIPv6 - } - exGWRemoteCidrAlt1 = fmt.Sprintf("%s/64", exGWRemoteIpAlt1) - exGWRemoteCidrAlt2 = fmt.Sprintf("%s/64", exGWRemoteIpAlt2) - nodeAddr = ni.IPv6 - } else { - exGWIpAlt1 = externalContainer.GetIPv4() - if overrideIPv4 != "" { - exGWIpAlt1 = overrideIPv4 - } - exGWRemoteCidrAlt1 = fmt.Sprintf("%s/24", exGWRemoteIpAlt1) - exGWRemoteCidrAlt2 = fmt.Sprintf("%s/24", exGWRemoteIpAlt2) - nodeAddr = ni.IPv4 - } - if nodeAddr == "" { - framework.Failf("failed to find node internal IP for node %s", node.Name) - } - // annotate the test namespace - annotateArgs := []string{ - "annotate", - "namespace", - f.Namespace.Name, - fmt.Sprintf("k8s.ovn.org/routing-external-gws=%s", exGWIpAlt1), - } - framework.Logf("Annotating the external gateway test namespace to a container gw: %s ", exGWIpAlt1) - e2ekubectl.RunKubectlOrDie(f.Namespace.Name, annotateArgs...) - - podCIDR, _, err := getNodePodCIDRs(node.Name, "default") - if err != nil { - framework.Failf("Error retrieving the pod cidr from %s %v", node.Name, err) - } - framework.Logf("the pod cidr for node %s is %s", node.Name, podCIDR) - // add loopback interface used to validate all traffic is getting drained through the gateway - _, err = infraprovider.Get().ExecExternalContainerCommand(externalContainer, []string{"ip", "address", "add", exGWRemoteCidrAlt1, "dev", "lo"}) - framework.ExpectNoError(err, "failed to add the loopback ip to dev lo on the test container") - - // Create the pod that will be used as the source for the connectivity test - _, err = createGenericPod(f, srcPingPodName, node.Name, f.Namespace.Name, command) - framework.ExpectNoError(err, "failed to create pod %s/%s", f.Namespace.Name, srcPingPodName) - // wait for pod setup to return a valid address - err = wait.PollImmediate(retryInterval, retryTimeout, func() (bool, error) { - pingSrc = getPodAddress(srcPingPodName, f.Namespace.Name) - validIP = net.ParseIP(pingSrc) - if validIP == nil { - return false, nil - } - return true, nil - }) - // Fail the test if no address is ever retrieved - framework.ExpectNoError(err, "Error trying to get the pod IP address") - // add a host route on the first mock gateway for return traffic to the pod - _, err = infraprovider.Get().ExecExternalContainerCommand(externalContainer, []string{"ip", "route", "add", pingSrc, "via", nodeAddr}) - framework.ExpectNoError(err, "failed to add the pod host route on the test container") - providerCtx.AddCleanUpFn(func() error { - _, err = infraprovider.Get().ExecExternalContainerCommand(externalContainer, []string{"ip", "route", "del", pingSrc, "via", nodeAddr}) - if err != nil { - return fmt.Errorf("failed to add the pod host route on the test container: %v", err) - } - return nil - }) - _, err = infraprovider.Get().ExecExternalContainerCommand(externalContainer, []string{"ping", "-c", "5", pingSrc}) - framework.ExpectNoError(err, "Failed to ping %s from container %s", pingSrc, getContainerName(gwContainerNameTemplate, externalContainerPort)) - - time.Sleep(time.Second * 15) - // Verify the gateway and remote address is reachable from the initial pod - ginkgo.By(fmt.Sprintf("Verifying connectivity without vxlan to the updated annotation and initial external gateway %s and remote address %s", exGWIpAlt1, exGWRemoteIpAlt1)) - _, err = e2ekubectl.RunKubectl(f.Namespace.Name, "exec", srcPingPodName, testContainerFlag, "--", "ping", "-w", "40", exGWRemoteIpAlt1) - framework.ExpectNoError(err, "Failed to ping the first gateway network %s from container %s on node %s: %v", exGWRemoteIpAlt1, testContainer, node.Name, err) - // start the container that will act as a new external gateway that the tests will be updated to use - externalContainer2Port := infraprovider.Get().GetExternalContainerPort() - externalContainer2 := infraapi.ExternalContainer{Name: getContainerName(gwContainerNameTemplate2, externalContainerPort), - Image: images.AgnHost(), Network: network, ExtPort: externalContainer2Port, CmdArgs: []string{"pause"}} - externalContainer2, err = providerCtx.CreateExternalContainer(externalContainer2) - framework.ExpectNoError(err, "failed to start external gateway test container %s", getContainerName(gwContainerNameTemplate2, externalContainerPort)) - if network.Name() == "host" { - // manually cleanup because cleanup doesnt cleanup host network - providerCtx.AddCleanUpFn(func() error { - return providerCtx.DeleteExternalContainer(externalContainer2) - }) - } - var exGWIpAlt2 string - if isIPv6Cluster { - exGWIpAlt2 = externalContainer2.GetIPv6() - } else { - exGWIpAlt2 = externalContainer2.GetIPv4() - } - if exGWIpAlt2 == "" { - framework.Failf("failed to retrieve container %s IP address", getContainerName(gwContainerNameTemplate2, externalContainerPort)) - } - // override the annotation in the test namespace with the new gateway - annotateArgs = []string{ - "annotate", - "namespace", - f.Namespace.Name, - fmt.Sprintf("k8s.ovn.org/routing-external-gws=%s", exGWIpAlt2), - "--overwrite", - } - framework.Logf("Annotating the external gateway test namespace to a new container remote IP:%s gw:%s ", exGWIpAlt2, exGWRemoteIpAlt2) - e2ekubectl.RunKubectlOrDie(f.Namespace.Name, annotateArgs...) - // add loopback interface used to validate all traffic is getting drained through the gateway - _, err = infraprovider.Get().ExecExternalContainerCommand(externalContainer2, []string{"ip", "address", "add", exGWRemoteCidrAlt2, "dev", "lo"}) - framework.ExpectNoError(err, "failed to add the loopback ip to dev lo on the test container %s", getContainerName(gwContainerNameTemplate2, externalContainerPort)) - providerCtx.AddCleanUpFn(func() error { - _, err = infraprovider.Get().ExecExternalContainerCommand(externalContainer2, []string{"ip", "address", "del", exGWRemoteCidrAlt2, "dev", "lo"}) - if err != nil { - return fmt.Errorf("failed to cleanup loopback ip on test container %s: %v", getContainerName(gwContainerNameTemplate2, externalContainerPort), err) - } - return nil - }) - // add a host route on the second mock gateway for return traffic to the pod - _, err = infraprovider.Get().ExecExternalContainerCommand(externalContainer2, []string{"ip", "route", "add", pingSrc, "via", nodeAddr}) - framework.ExpectNoError(err, "failed to add the pod route on the test container %s", getContainerName(gwContainerNameTemplate2, externalContainerPort)) - providerCtx.AddCleanUpFn(func() error { - _, err = infraprovider.Get().ExecExternalContainerCommand(externalContainer2, []string{"ip", "route", "del", pingSrc, "via", nodeAddr}) - if err != nil { - return fmt.Errorf("failed to cleanup route on test container %s: %v", getContainerName(gwContainerNameTemplate2, externalContainerPort), err) - } - return nil - }) - // ping pod from external container - _, err = infraprovider.Get().ExecExternalContainerCommand(externalContainer2, []string{"ping", "-c", "5", pingSrc}) - framework.ExpectNoError(err, "Failed to ping %s from container %s", pingSrc, getContainerName(gwContainerNameTemplate2, externalContainerPort)) - // Verify the updated gateway and remote address is reachable from the initial pod - ginkgo.By(fmt.Sprintf("Verifying connectivity without vxlan to the updated annotation and new external gateway %s and remote IP %s", exGWRemoteIpAlt2, exGWIpAlt2)) - _, err = e2ekubectl.RunKubectl(f.Namespace.Name, "exec", srcPingPodName, testContainerFlag, "--", "ping", "-w", "40", exGWRemoteIpAlt2) - framework.ExpectNoError(err, "Failed to ping the second gateway network %s from container %s on node %s: %v", exGWRemoteIpAlt2, testContainer, node.Name) - }) - }) - - // This test validates ingress traffic sourced from a mock external gateway - // running as a container. Add a namespace annotated with the IP of the - // mock external container's eth0 address. Add a loopback address and a - // route pointing to the pod in the test namespace. Validate connectivity - // sourcing from the mock gateway container loopback to the test ns pod. - var _ = ginkgo.Describe("e2e ingress gateway traffic validation", func() { - const ( - svcname string = "novxlan-externalgw-ingress" - gwContainerTemplate string = "gw-ingress-test-container-%d" - ) - - f := wrappedTestFramework(svcname) - - type nodeInfo struct { - name string - nodeIP string - } - - var ( - workerNodeInfo nodeInfo - isIPv6 bool - providerCtx infraapi.Context - ) - - ginkgo.BeforeEach(func() { - providerCtx = infraprovider.Get().NewTestContext() - // retrieve worker node names - nodes, err := e2enode.GetBoundedReadySchedulableNodes(context.TODO(), f.ClientSet, 3) - framework.ExpectNoError(err) - if len(nodes.Items) < 3 { - framework.Failf( - "Test requires >= 3 Ready nodes, but there are only %v nodes", - len(nodes.Items)) - } - ips := e2enode.CollectAddresses(nodes, corev1.NodeInternalIP) - workerNodeInfo = nodeInfo{ - name: nodes.Items[1].Name, - nodeIP: ips[1], - } - isIPv6 = IsIPv6Cluster(f.ClientSet) - }) - - ginkgo.It("Should validate ingress connectivity from an external gateway", func() { - - var ( - pingDstPod string - dstPingPodName = "e2e-exgw-ingress-ping-pod" - command = []string{"bash", "-c", "sleep 20000"} - exGWLo = "10.30.1.1" - exGWLoCidr = fmt.Sprintf("%s/32", exGWLo) - pingCmd = ipv4PingCommand - pingCount = "3" - ) - if isIPv6 { - exGWLo = "fc00::1" // unique local ipv6 unicast addr as per rfc4193 - exGWLoCidr = fmt.Sprintf("%s/64", exGWLo) - pingCmd = ipv6PingCommand - } - // start the first container that will act as an external gateway - network, err := infraprovider.Get().PrimaryNetwork() - framework.ExpectNoError(err, "failed to get primary network information") - overrideNetworkStr, overrideIPv4, overrideIPv6 := getOverrideNetwork() - if overrideNetworkStr != "" { - overrideNetwork, err := infraprovider.Get().GetNetwork(overrideNetworkStr) - framework.ExpectNoError(err, "over ride network must exist") - network = overrideNetwork - } - externalContainerPort := infraprovider.Get().GetExternalContainerPort() - externalContainer := infraapi.ExternalContainer{Name: getContainerName(gwContainerTemplate, externalContainerPort), Image: images.AgnHost(), Network: network, - CmdArgs: getAgnHostHTTPPortBindCMDArgs(externalContainerPort), ExtPort: externalContainerPort} - externalContainer, err = providerCtx.CreateExternalContainer(externalContainer) - framework.ExpectNoError(err, "failed to start external gateway test container %s", getContainerName(gwContainerTemplate, externalContainerPort)) - if network.Name() == "host" { - // manually cleanup because cleanup doesnt cleanup host network - providerCtx.AddCleanUpFn(func() error { - return providerCtx.DeleteExternalContainer(externalContainer) - }) - } - - exGWIp := externalContainer.GetIPv4() - if overrideIPv4 != "" { - exGWIp = overrideIPv4 - } - if isIPv6 { - exGWIp = externalContainer.GetIPv6() - if overrideIPv6 != "" { - exGWIp = overrideIPv6 - } - } - // annotate the test namespace with the external gateway address - annotateArgs := []string{ - "annotate", - "namespace", - f.Namespace.Name, - fmt.Sprintf("k8s.ovn.org/routing-external-gws=%s", exGWIp), - } - framework.Logf("Annotating the external gateway test namespace to container gateway: %s", exGWIp) - e2ekubectl.RunKubectlOrDie(f.Namespace.Name, annotateArgs...) - primaryNetworkInf, err := infraprovider.Get().GetK8NodeNetworkInterface(workerNodeInfo.name, network) - framework.ExpectNoError(err, "failed to get network interface info for network (%s) on node %s", network, workerNodeInfo.name) - nodeIP := primaryNetworkInf.IPv4 - if isIPv6 { - nodeIP = primaryNetworkInf.IPv6 - } - framework.Logf("the pod side node is %s and the source node ip is %s", workerNodeInfo.name, nodeIP) - podCIDR, _, err := getNodePodCIDRs(workerNodeInfo.name, "default") - if err != nil { - framework.Failf("Error retrieving the pod cidr from %s %v", workerNodeInfo.name, err) - } - framework.Logf("the pod cidr for node %s is %s", workerNodeInfo.name, podCIDR) - - // Create the pod that will be used as the source for the connectivity test - _, err = createGenericPod(f, dstPingPodName, workerNodeInfo.name, f.Namespace.Name, command) - framework.ExpectNoError(err, "failed to create pod %s/%s", f.Namespace.Name, dstPingPodName) - // wait for the pod setup to return a valid address - err = wait.PollImmediate(retryInterval, retryTimeout, func() (bool, error) { - pingDstPod = getPodAddress(dstPingPodName, f.Namespace.Name) - validIP := net.ParseIP(pingDstPod) - if validIP == nil { - return false, nil - } - return true, nil - }) - // fail the test if a pod address is never retrieved - if err != nil { - framework.Failf("Error trying to get the pod IP address") - } - _, err = infraprovider.Get().ExecExternalContainerCommand(externalContainer, []string{"ip", "route", "add", pingDstPod, "via", nodeIP}) - framework.ExpectNoError(err, "failed to add the pod host route on the test container %s", gwContainerTemplate) - providerCtx.AddCleanUpFn(func() error { - _, err = infraprovider.Get().ExecExternalContainerCommand(externalContainer, []string{"ip", "route", "del", pingDstPod, "via", nodeIP}) - if err != nil { - return fmt.Errorf("failed to cleanup route in external container %s: %v", gwContainerTemplate, err) - } - return nil - }) - // add a loopback address to the mock container that will source the ingress test - _, err = infraprovider.Get().ExecExternalContainerCommand(externalContainer, []string{"ip", "address", "add", exGWLoCidr, "dev", "lo"}) - framework.ExpectNoError(err, "failed to add the loopback ip to dev lo on the test container %s", gwContainerTemplate) - providerCtx.AddCleanUpFn(func() error { - _, err = infraprovider.Get().ExecExternalContainerCommand(externalContainer, []string{"ip", "address", "del", exGWLoCidr, "dev", "lo"}) - if err != nil { - return fmt.Errorf("failed to cleanup loopback ip on dev lo within test container %s: %v", gwContainerTemplate, err) - } - return nil - }) - // Validate connectivity from the external gateway loopback to the pod in the test namespace - ginkgo.By(fmt.Sprintf("Validate ingress traffic from the external gateway %s can reach the pod in the exgw annotated namespace", - fmt.Sprintf(gwContainerTemplate, externalContainer.GetPort()))) - // generate traffic that will verify connectivity from the mock external gateway loopback - _, err = infraprovider.Get().ExecExternalContainerCommand(externalContainer, []string{string(pingCmd), "-c", pingCount, - "-I", infraprovider.Get().ExternalContainerPrimaryInterfaceName(), pingDstPod}) - framework.ExpectNoError(err, "failed to ping the pod address %s from mock container %s", pingDstPod, gwContainerTemplate) - }) - }) - - var _ = ginkgo.Context("With annotations", func() { - - // Validate pods can reach a network running in a container's looback address via - // an external gateway running on eth0 of the container without any tunnel encap. - // The traffic will get proxied through an annotated pod in the serving namespace. - var _ = ginkgo.Describe("e2e non-vxlan external gateway through a gateway pod", func() { - const ( - svcname string = "externalgw-pod-novxlan" - gwContainer1Template string = "ex-gw-container1-%d" - gwContainer2Template string = "ex-gw-container2-%d" - srcPingPodName string = "e2e-exgw-src-ping-pod" - gatewayPodName1 string = "e2e-gateway-pod1" - gatewayPodName2 string = "e2e-gateway-pod2" - ecmpRetry int = 20 - testTimeout time.Duration = 20 * time.Second - ) - - var ( - sleepCommand = []string{"bash", "-c", "sleep 20000"} - addressesv4, addressesv6 gatewayTestIPs - servingNamespace string - gwContainers []infraapi.ExternalContainer - providerCtx infraapi.Context - ) - - f := wrappedTestFramework(svcname) - - ginkgo.BeforeEach(func() { - providerCtx = infraprovider.Get().NewTestContext() - // retrieve worker node names - nodes, err := e2enode.GetBoundedReadySchedulableNodes(context.TODO(), f.ClientSet, 3) - framework.ExpectNoError(err) - if len(nodes.Items) < 3 { - framework.Failf( - "Test requires >= 3 Ready nodes, but there are only %v nodes", - len(nodes.Items)) - } - - ns, err := f.CreateNamespace(context.TODO(), "exgw-serving", nil) - framework.ExpectNoError(err) - servingNamespace = ns.Name - network, err := infraprovider.Get().PrimaryNetwork() - framework.ExpectNoError(err, "failed to get primary network information") - overrideNetworkStr, _, _ := getOverrideNetwork() - if overrideNetworkStr != "" { - overrideNetwork, err := infraprovider.Get().GetNetwork(overrideNetworkStr) - framework.ExpectNoError(err, "over ride network must exist") - network = overrideNetwork - } - gwContainers, addressesv4, addressesv6 = setupGatewayContainers(f, providerCtx, nodes, network, gwContainer1Template, gwContainer2Template, - srcPingPodName, gwUDPPort, gwTCPPort, podUDPPort, podTCPPort, ecmpRetry, false) - setupAnnotatedGatewayPods(f, nodes, network, gatewayPodName1, gatewayPodName2, servingNamespace, sleepCommand, addressesv4, addressesv6, false) - }) - - ginkgo.AfterEach(func() { - resetGatewayAnnotations(f) - }) - - ginkgo.DescribeTable("Should validate ICMP connectivity to an external gateway's loopback address via a pod with external gateway CR", - func(addresses *gatewayTestIPs, icmpCommand string) { - if addresses.srcPodIP == "" || addresses.nodeIP == "" { - skipper.Skipf("Skipping as pod ip / node ip are not set pod ip %s node ip %s", addresses.srcPodIP, addresses.nodeIP) - } - - ginkgo.By(fmt.Sprintf("Verifying connectivity to the pod [%s] from external gateways", addresses.srcPodIP)) - for _, gwContainer := range gwContainers { - // Ping from a common IP address that exists on both gateways to ensure test coverage where ingress reply goes back to the same host. - gomega.Eventually(infraprovider.Get().ExecExternalContainerCommand). - WithArguments(gwContainer, []string{"ping", "-B", "-c1", "-W1", "-I", addresses.targetIPs[0], addresses.srcPodIP}). - WithTimeout(testTimeout). - ShouldNot(gomega.BeEmpty(), "Failed to ping %s from container %s", addresses.srcPodIP, gwContainer.Name) - } - - tcpDumpSync := sync.WaitGroup{} - tcpDumpSync.Add(len(gwContainers)) - - for _, gwContainer := range gwContainers { - go checkReceivedPacketsOnExternalContainer(gwContainer, srcPingPodName, anyLink, []string{icmpCommand}, &tcpDumpSync) - } - - pingSync := sync.WaitGroup{} - // Verify the external gateway loopback address running on the external container is reachable and - // that traffic from the source ping pod is proxied through the pod in the serving namespace - ginkgo.By("Verifying connectivity via the gateway namespace to the remote addresses") - for _, t := range addresses.targetIPs { - pingSync.Add(1) - go func(target string) { - defer ginkgo.GinkgoRecover() - defer pingSync.Done() - gomega.Eventually(e2ekubectl.RunKubectl). - WithArguments(f.Namespace.Name, "exec", srcPingPodName, "--", "ping", "-c1", "-W1", target). - WithTimeout(testTimeout). - ShouldNot(gomega.BeEmpty(), "Failed to ping remote gateway %s from pod %s", target, srcPingPodName) - }(t) - } - pingSync.Wait() - tcpDumpSync.Wait() - }, - ginkgo.Entry("ipv4", &addressesv4, "icmp"), - ginkgo.Entry("ipv6", &addressesv6, "icmp6")) - - ginkgo.DescribeTable("Should validate TCP/UDP connectivity to an external gateway's loopback address via a pod with external gateway annotations enabled", - func(protocol string, addresses *gatewayTestIPs, gwPort, podPort int) { - if addresses.srcPodIP == "" || addresses.nodeIP == "" { - skipper.Skipf("Skipping as pod ip / node ip are not set pod ip %s node ip %s", addresses.srcPodIP, addresses.nodeIP) - } - - for _, container := range gwContainers { - reachPodFromGateway(container, addresses.srcPodIP, strconv.Itoa(podPort), srcPingPodName, protocol) - } - - expectedHostNames := make(map[string]struct{}) - for _, c := range gwContainers { - res, err := infraprovider.Get().ExecExternalContainerCommand(c, []string{"hostname"}) - framework.ExpectNoError(err, "failed to run hostname in %s", c) - hostname := strings.TrimSuffix(res, "\n") - framework.Logf("Hostname for %s is %s", c, hostname) - expectedHostNames[hostname] = struct{}{} - } - framework.Logf("Expected hostnames are %v", expectedHostNames) - - ginkgo.By("Checking that external ips are reachable with both gateways") - returnedHostNames := make(map[string]struct{}) - gwIP := addresses.targetIPs[0] - success := false - for i := 0; i < singleTargetRetries; i++ { - args := []string{"exec", srcPingPodName, "--"} - if protocol == "tcp" { - args = append(args, "bash", "-c", fmt.Sprintf("echo | nc -w 1 %s %d", gwIP, gwPort)) - } else { - args = append(args, "bash", "-c", fmt.Sprintf("echo | nc -w 1 -u %s %d", gwIP, gwPort)) - } - res, err := e2ekubectl.RunKubectl(f.Namespace.Name, args...) - framework.ExpectNoError(err, "failed to reach %s (%s)", gwIP, protocol) - hostname := strings.TrimSuffix(res, "\n") - if hostname != "" { - returnedHostNames[hostname] = struct{}{} - } - - if cmp.Equal(returnedHostNames, expectedHostNames) { - success = true - break - } - } - framework.Logf("Received hostnames for protocol %s are %v ", protocol, returnedHostNames) - - if !success { - framework.Failf("Failed to hit all the external gateways via for protocol %s, diff %s", protocol, cmp.Diff(expectedHostNames, returnedHostNames)) - } - - }, - ginkgo.Entry("UDP ipv4", "udp", &addressesv4, gwUDPPort, podUDPPort), - ginkgo.Entry("TCP ipv4", "tcp", &addressesv4, gwTCPPort, podTCPPort), - ginkgo.Entry("UDP ipv6", "udp", &addressesv6, gwUDPPort, podUDPPort), - ginkgo.Entry("TCP ipv6", "tcp", &addressesv6, gwTCPPort, podTCPPort)) - }) - - // Validate pods can reach a network running in multiple container's loopback - // addresses via two external gateways running on eth0 of the container without - // any tunnel encap. This test defines two external gateways and validates ECMP - // functionality to the container loopbacks. To verify traffic reaches the - // gateways, tcpdump is running on the external gateways and will exit successfully - // once an ICMP packet is received from the annotated pod in the k8s cluster. - // Two additional gateways are added to verify the tcp / udp protocols. - // They run the netexec command, and the pod asks to return their hostname. - // The test checks that both hostnames are collected at least once. - var _ = ginkgo.Describe("e2e multiple external gateway validation", func() { - const ( - svcname string = "novxlan-externalgw-ecmp" - gwContainer1Template string = "gw-test-container1-%d" - gwContainer2Template string = "gw-test-container2-%d" - testTimeout time.Duration = 300 * time.Second - ecmpRetry int = 20 - srcPodName = "e2e-exgw-src-pod" - ) - - f := wrappedTestFramework(svcname) - - var gwContainers []infraapi.ExternalContainer - var providerCtx infraapi.Context - var addressesv4, addressesv6 gatewayTestIPs - - ginkgo.BeforeEach(func() { - providerCtx = infraprovider.Get().NewTestContext() - // retrieve worker node names - nodes, err := e2enode.GetBoundedReadySchedulableNodes(context.TODO(), f.ClientSet, 3) - framework.ExpectNoError(err) - if len(nodes.Items) < 3 { - framework.Failf( - "Test requires >= 3 Ready nodes, but there are only %v nodes", - len(nodes.Items)) - } - network, err := infraprovider.Get().PrimaryNetwork() - framework.ExpectNoError(err, "failed to get primary network information") - if overrideNetworkName, _, _ := getOverrideNetwork(); overrideNetworkName == "host" { - skipper.Skipf("Skipping as host network doesn't support multiple external gateways") - } else if overrideNetworkName != "" { - overrideNetwork, err := infraprovider.Get().GetNetwork(overrideNetworkName) - framework.ExpectNoError(err, "over ride network must exist") - network = overrideNetwork - } - gwContainers, addressesv4, addressesv6 = setupGatewayContainers(f, providerCtx, nodes, network, gwContainer1Template, gwContainer2Template, - srcPodName, gwUDPPort, gwTCPPort, podUDPPort, podTCPPort, ecmpRetry, false) - }) - - ginkgo.AfterEach(func() { - resetGatewayAnnotations(f) - }) - - ginkgo.DescribeTable("Should validate ICMP connectivity to multiple external gateways for an ECMP scenario", func(addresses *gatewayTestIPs, icmpToDump string) { - if addresses.srcPodIP == "" || addresses.nodeIP == "" { - skipper.Skipf("Skipping as pod ip / node ip are not set pod ip %s node ip %s", addresses.srcPodIP, addresses.nodeIP) - } - - annotateNamespaceForGateway(f.Namespace.Name, false, addresses.gatewayIPs[:]...) - - ginkgo.By("Verifying connectivity to the pod from external gateways") - for _, gwContainer := range gwContainers { - // Ping from a common IP address that exists on both gateways to ensure test coverage where ingress reply goes back to the same host. - gomega.Eventually(infraprovider.Get().ExecExternalContainerCommand). - WithArguments(gwContainer, []string{"ping", "-B", "-c1", "-W1", "-I", addresses.targetIPs[0], addresses.srcPodIP}). - WithTimeout(testTimeout). - ShouldNot(gomega.BeEmpty(), "Failed to ping %s from container %s", addresses.srcPodIP, gwContainer.Name) - } - - ginkgo.By("Verifying connectivity to the pod from external gateways with large packets > pod MTU") - for _, gwContainer := range gwContainers { - gomega.Eventually(infraprovider.Get().ExecExternalContainerCommand). - WithArguments(gwContainer, []string{"ping", "-s", "1420", "-c1", "-W1", addresses.srcPodIP}). - WithTimeout(testTimeout). - ShouldNot(gomega.BeEmpty(), "Failed to ping %s from container %s", addresses.srcPodIP, gwContainer.Name) - } - - // Verify the gateways and remote loopback addresses are reachable from the pod. - // Iterate checking connectivity to the loopbacks on the gateways until tcpdump see - // the traffic or 20 attempts fail. Odds of a false negative here is ~ (1/2)^20 - ginkgo.By("Verifying ecmp connectivity to the external gateways by iterating through the targets") - - // Check for egress traffic to both gateway loopback addresses using tcpdump, since - // /proc/net/dev counters only record the ingress interface traffic is received on. - // The test will waits until an ICMP packet is matched on the gateways or fail the - // test if a packet to the loopback is not received within the timer interval. - // If an ICMP packet is never detected, return the error via the specified chanel. - - tcpDumpSync := sync.WaitGroup{} - tcpDumpSync.Add(len(gwContainers)) - for _, gwContainer := range gwContainers { - go checkReceivedPacketsOnExternalContainer(gwContainer, srcPodName, anyLink, []string{icmpToDump}, &tcpDumpSync) - } - - pingSync := sync.WaitGroup{} - - // spawn a goroutine to asynchronously (to speed up the test) - // to ping the gateway loopbacks on both containers via ECMP. - for _, address := range addresses.targetIPs { - pingSync.Add(1) - go func(target string) { - defer ginkgo.GinkgoRecover() - defer pingSync.Done() - gomega.Eventually(e2ekubectl.RunKubectl). - WithArguments(f.Namespace.Name, "exec", srcPodName, "--", "ping", "-c1", "-W1", target). - WithTimeout(testTimeout). - ShouldNot(gomega.BeEmpty(), "Failed to ping remote gateway %s from pod %s", target, srcPodName) - }(address) - } - pingSync.Wait() - tcpDumpSync.Wait() - - }, ginkgo.Entry("IPV4", &addressesv4, "icmp"), - ginkgo.Entry("IPV6", &addressesv6, "icmp6")) - - // This test runs a listener on the external container, returning the host name both on tcp and udp. - // The src pod tries to hit the remote address until both the containers are hit. - ginkgo.DescribeTable("Should validate TCP/UDP connectivity to multiple external gateways for a UDP / TCP scenario", func(addresses *gatewayTestIPs, protocol string, gwPort, podPort int) { - if addresses.srcPodIP == "" || addresses.nodeIP == "" { - skipper.Skipf("Skipping as pod ip / node ip are not set pod ip %s node ip %s", addresses.srcPodIP, addresses.nodeIP) - } - - annotateNamespaceForGateway(f.Namespace.Name, false, addresses.gatewayIPs[:]...) - - for _, container := range gwContainers { - reachPodFromGateway(container, addresses.srcPodIP, strconv.Itoa(podPort), srcPodName, protocol) - } - - expectedHostNames := hostNamesForExternalContainers(gwContainers) - framework.Logf("Expected hostnames are %v", expectedHostNames) - - returnedHostNames := make(map[string]struct{}) - success := false - - // Picking only the first address, the one the udp listener is set for - gwIP := addresses.targetIPs[0] - for i := 0; i < singleTargetRetries; i++ { - hostname := pokeHostnameViaNC(srcPodName, f.Namespace.Name, protocol, gwIP, gwPort) - if hostname != "" { - returnedHostNames[hostname] = struct{}{} - } - if cmp.Equal(returnedHostNames, expectedHostNames) { - success = true - break - } - } - - framework.Logf("Received hostnames for protocol %s are %v ", protocol, returnedHostNames) - - if !success { - framework.Failf("Failed to hit all the external gateways via for protocol %s, diff %s", protocol, cmp.Diff(expectedHostNames, returnedHostNames)) - } - - }, ginkgo.Entry("IPV4 udp", &addressesv4, "udp", gwUDPPort, podUDPPort), - ginkgo.Entry("IPV4 tcp", &addressesv4, "tcp", gwTCPPort, podTCPPort), - ginkgo.Entry("IPV6 udp", &addressesv6, "udp", gwUDPPort, podUDPPort), - ginkgo.Entry("IPV6 tcp", &addressesv6, "tcp", gwTCPPort, podTCPPort)) - }) - - var _ = ginkgo.Describe("e2e multiple external gateway stale conntrack entry deletion validation", func() { - const ( - svcname string = "novxlan-externalgw-ecmp" - gwContainer1Template string = "gw-test-container1-%d" - gwContainer2Template string = "gw-test-container2-%d" - srcPodName string = "e2e-exgw-src-pod" - gatewayPodName1 string = "e2e-gateway-pod1" - gatewayPodName2 string = "e2e-gateway-pod2" - ) - - f := wrappedTestFramework(svcname) - - var ( - addressesv4, addressesv6 gatewayTestIPs - externalContainers []infraapi.ExternalContainer - providerCtx infraapi.Context - sleepCommand []string - nodes *corev1.NodeList - err error - servingNamespace string - ) - - ginkgo.BeforeEach(func() { - providerCtx = infraprovider.Get().NewTestContext() - // retrieve worker node names - nodes, err = e2enode.GetBoundedReadySchedulableNodes(context.TODO(), f.ClientSet, 3) - framework.ExpectNoError(err) - if len(nodes.Items) < 3 { - framework.Failf( - "Test requires >= 3 Ready nodes, but there are only %v nodes", - len(nodes.Items)) - } - network, err := infraprovider.Get().PrimaryNetwork() - framework.ExpectNoError(err, "failed to get primary network information") - if overrideNetworkName, _, _ := getOverrideNetwork(); overrideNetworkName != "" { - overrideNetwork, err := infraprovider.Get().GetNetwork(overrideNetworkName) - framework.ExpectNoError(err, "override network must exist") - network = overrideNetwork - } - if network.Name() == "host" { - skipper.Skipf("Skipping as host network doesn't support multiple external gateways") - } - ns, err := f.CreateNamespace(context.TODO(), "exgw-conntrack-serving", nil) - framework.ExpectNoError(err) - servingNamespace = ns.Name - - externalContainers, addressesv4, addressesv6 = setupGatewayContainersForConntrackTest(f, providerCtx, nodes, network, gwContainer1Template, gwContainer2Template, srcPodName) - sleepCommand = []string{"bash", "-c", "trap : TERM INT; sleep infinity & wait"} - _, err = createGenericPod(f, gatewayPodName1, nodes.Items[0].Name, servingNamespace, sleepCommand) - framework.ExpectNoError(err, "Create and annotate the external gw pods to manage the src app pod namespace, failed: %v", err) - _, err = createGenericPod(f, gatewayPodName2, nodes.Items[1].Name, servingNamespace, sleepCommand) - framework.ExpectNoError(err, "Create and annotate the external gw pods to manage the src app pod namespace, failed: %v", err) - }) - - ginkgo.AfterEach(func() { - resetGatewayAnnotations(f) - }) - - ginkgo.DescribeTable("Namespace annotation: Should validate conntrack entry deletion for TCP/UDP traffic via multiple external gateways a.k.a ECMP routes", func(addresses *gatewayTestIPs, protocol string) { - if addresses.srcPodIP == "" || addresses.nodeIP == "" { - skipper.Skipf("Skipping as pod ip / node ip are not set pod ip %s node ip %s", addresses.srcPodIP, addresses.nodeIP) - } - ginkgo.By("Annotate the app namespace to get managed by external gateways") - annotateNamespaceForGateway(f.Namespace.Name, false, addresses.gatewayIPs...) - macAddressGW := make([]string, 2) - network, err := infraprovider.Get().PrimaryNetwork() - framework.ExpectNoError(err, "failed to get primary network information") - if overrideNetworkName, _, _ := getOverrideNetwork(); overrideNetworkName != "" { - overrideNetwork, err := infraprovider.Get().GetNetwork(overrideNetworkName) - framework.ExpectNoError(err, "over ride network must exist") - network = overrideNetwork - } - - for i, externalContainer := range externalContainers { - ginkgo.By("Start iperf3 client from external container to connect to iperf3 server running at the src pod") - _, err = infraprovider.Get().ExecExternalContainerCommand(externalContainer, []string{"iperf3", "-u", "-c", addresses.srcPodIP, - "-p", fmt.Sprintf("%d", 5201+i), "-b", "1M", "-i", "1", "-t", "3", "&"}) - framework.ExpectNoError(err, "failed to execute iperf command from external container") - networkInfo, err := infraprovider.Get().GetExternalContainerNetworkInterface(externalContainer, network) - framework.ExpectNoError(err, "failed to get %s network information for external container %s", network.Name(), externalContainer.Name) - // Trim leading 0s because conntrack dumped labels are just integers - // in hex without leading 0s. - macAddressGW[i] = strings.TrimLeft(strings.Replace(networkInfo.MAC, ":", "", -1), "0") - } - - ginkgo.By("Check if conntrack entries for ECMP routes are created for the 2 external gateways") - nodeName := getPod(f, srcPodName).Spec.NodeName - podConnEntriesWithMACLabelsSet := pokeConntrackEntries(nodeName, addresses.srcPodIP, protocol, macAddressGW) - gomega.Expect(podConnEntriesWithMACLabelsSet).To(gomega.Equal(2)) - totalPodConnEntries := pokeConntrackEntries(nodeName, addresses.srcPodIP, protocol, nil) - gomega.Expect(totalPodConnEntries).To(gomega.Equal(4)) // total conntrack entries for this pod/protocol - - ginkgo.By("Remove second external gateway IP from the app namespace annotation") - annotateNamespaceForGateway(f.Namespace.Name, false, addresses.gatewayIPs[0]) - - ginkgo.By("Check if conntrack entries for ECMP routes are removed for the deleted external gateway if traffic is UDP") - podConnEntriesWithMACLabelsSet = pokeConntrackEntries(nodeName, addresses.srcPodIP, protocol, macAddressGW) - totalPodConnEntries = pokeConntrackEntries(nodeName, addresses.srcPodIP, protocol, nil) - - gomega.Expect(podConnEntriesWithMACLabelsSet).To(gomega.Equal(1)) // we still have the conntrack entry for the remaining gateway - gomega.Expect(totalPodConnEntries).To(gomega.Equal(3)) // 4-1 - - ginkgo.By("Remove first external gateway IP from the app namespace annotation") - annotateNamespaceForGateway(f.Namespace.Name, false, "") - - ginkgo.By("Check if conntrack entries for ECMP routes are removed for the deleted external gateway if traffic is UDP") - podConnEntriesWithMACLabelsSet = pokeConntrackEntries(nodeName, addresses.srcPodIP, protocol, macAddressGW) - totalPodConnEntries = pokeConntrackEntries(nodeName, addresses.srcPodIP, protocol, nil) - - gomega.Expect(podConnEntriesWithMACLabelsSet).To(gomega.Equal(0)) // we don't have any remaining gateways left - gomega.Expect(totalPodConnEntries).To(gomega.Equal(2)) // 4-2 - - }, - ginkgo.Entry("IPV4 udp", &addressesv4, "udp"), - ginkgo.Entry("IPV4 tcp", &addressesv4, "tcp"), - ginkgo.Entry("IPV6 udp", &addressesv6, "udp"), - ginkgo.Entry("IPV6 tcp", &addressesv6, "tcp")) - - ginkgo.DescribeTable("ExternalGWPod annotation: Should validate conntrack entry deletion for TCP/UDP traffic via multiple external gateways a.k.a ECMP routes", func(addresses *gatewayTestIPs, protocol string, removalType GatewayRemovalType) { - if addresses.srcPodIP == "" || addresses.nodeIP == "" { - skipper.Skipf("Skipping as pod ip / node ip are not set pod ip %s node ip %s", addresses.srcPodIP, addresses.nodeIP) - } - - if removalType == GatewayNotReady { - recreatePodWithReadinessProbe(f, gatewayPodName2, nodes.Items[1].Name, servingNamespace, sleepCommand, nil) - } - - ginkgo.By("Annotate the external gw pods to manage the src app pod namespace") - for i, gwPod := range []string{gatewayPodName1, gatewayPodName2} { - networkIPs := fmt.Sprintf("\"%s\"", addresses.gatewayIPs[i]) - if addresses.srcPodIP != "" && addresses.nodeIP != "" { - networkIPs = fmt.Sprintf("\"%s\", \"%s\"", addresses.gatewayIPs[i], addresses.gatewayIPs[i]) - } - annotatePodForGateway(gwPod, servingNamespace, f.Namespace.Name, networkIPs, false) - } - - network, err := infraprovider.Get().PrimaryNetwork() - framework.ExpectNoError(err, "failed to get primary network information") - if overrideNetworkName, _, _ := getOverrideNetwork(); overrideNetworkName != "" { - overrideNetwork, err := infraprovider.Get().GetNetwork(overrideNetworkName) - framework.ExpectNoError(err, "over ride network must exist") - network = overrideNetwork - } - macAddressGW := make([]string, 2) - for i, container := range externalContainers { - ginkgo.By("Start iperf3 client from external container to connect to iperf3 server running at the src pod") - cmd := []string{"iperf3", "-u", "-c", addresses.srcPodIP, "-p", fmt.Sprintf("%d", 5201+i), "-b", "1M", "-i", "1", "-t", "3", "&"} - _, err = infraprovider.Get().ExecExternalContainerCommand(container, cmd) - framework.ExpectNoError(err, "failed to start iperf client from external container") - networkInfo, err := infraprovider.Get().GetExternalContainerNetworkInterface(container, network) - framework.ExpectNoError(err, "failed to get external container network information") - // Trim leading 0s because conntrack dumped labels are just integers - // in hex without leading 0s. - macAddressGW[i] = strings.TrimLeft(strings.Replace(networkInfo.MAC, ":", "", -1), "0") - } - - ginkgo.By("Check if conntrack entries for ECMP routes are created for the 2 external gateways") - nodeName := getPod(f, srcPodName).Spec.NodeName - podConnEntriesWithMACLabelsSet := pokeConntrackEntries(nodeName, addresses.srcPodIP, protocol, macAddressGW) - gomega.Expect(podConnEntriesWithMACLabelsSet).To(gomega.Equal(2)) - totalPodConnEntries := pokeConntrackEntries(nodeName, addresses.srcPodIP, protocol, nil) - gomega.Expect(totalPodConnEntries).To(gomega.Equal(4)) // total conntrack entries for this pod/protocol - - cleanUpFn := handleGatewayPodRemoval(f, removalType, gatewayPodName2, servingNamespace, addresses.gatewayIPs[1], true) - if cleanUpFn != nil { - defer cleanUpFn() - } - - ginkgo.By("Check if conntrack entries for ECMP routes are removed for the deleted external gateway if traffic is UDP") - podConnEntriesWithMACLabelsSet = pokeConntrackEntries(nodeName, addresses.srcPodIP, protocol, macAddressGW) - totalPodConnEntries = pokeConntrackEntries(nodeName, addresses.srcPodIP, protocol, nil) - - gomega.Expect(podConnEntriesWithMACLabelsSet).To(gomega.Equal(1)) // we still have the conntrack entry for the remaining gateway - gomega.Expect(totalPodConnEntries).To(gomega.Equal(3)) // 4-1 - - ginkgo.By("Remove first external gateway pod's routing-namespace annotation") - annotatePodForGateway(gatewayPodName1, servingNamespace, "", addresses.gatewayIPs[0], false) - - ginkgo.By("Check if conntrack entries for ECMP routes are removed for the deleted external gateway if traffic is UDP") - podConnEntriesWithMACLabelsSet = pokeConntrackEntries(nodeName, addresses.srcPodIP, protocol, macAddressGW) - totalPodConnEntries = pokeConntrackEntries(nodeName, addresses.srcPodIP, protocol, nil) - - gomega.Expect(podConnEntriesWithMACLabelsSet).To(gomega.Equal(0)) // we don't have any remaining gateways left - gomega.Expect(totalPodConnEntries).To(gomega.Equal(2)) // 4-2 - }, - ginkgo.Entry("IPV4 udp + pod annotation update", &addressesv4, "udp", GatewayUpdate), - ginkgo.Entry("IPV4 tcp + pod annotation update", &addressesv4, "tcp", GatewayUpdate), - ginkgo.Entry("IPV6 udp + pod annotation update", &addressesv6, "udp", GatewayUpdate), - ginkgo.Entry("IPV6 tcp + pod annotation update", &addressesv6, "tcp", GatewayUpdate), - ginkgo.Entry("IPV4 udp + pod delete", &addressesv4, "udp", GatewayDelete), - ginkgo.Entry("IPV6 tcp + pod delete", &addressesv6, "tcp", GatewayDelete), - ginkgo.Entry("IPV4 udp + pod deletion timestamp", &addressesv4, "udp", GatewayDeletionTimestamp), - ginkgo.Entry("IPV4 tcp + pod deletion timestamp", &addressesv4, "tcp", GatewayDeletionTimestamp), - ginkgo.Entry("IPV6 udp + pod deletion timestamp", &addressesv6, "udp", GatewayDeletionTimestamp), - ginkgo.Entry("IPV6 tcp + pod deletion timestamp", &addressesv6, "tcp", GatewayDeletionTimestamp), - ginkgo.Entry("IPV4 udp + pod not ready", &addressesv4, "udp", GatewayNotReady), - ginkgo.Entry("IPV4 tcp + pod not ready", &addressesv4, "tcp", GatewayNotReady), - ginkgo.Entry("IPV6 udp + pod not ready", &addressesv6, "udp", GatewayNotReady), - ginkgo.Entry("IPV6 tcp + pod not ready", &addressesv6, "tcp", GatewayNotReady), - ) - }) - - // BFD Tests are dual of external gateway. The only difference is that they enable BFD on ovn and - // on the external containers, and after doing one round veryfing that the traffic reaches both containers, - // they delete one and verify that the traffic is always reaching the only alive container. - var _ = ginkgo.Context("BFD", func() { - var _ = ginkgo.Describe("e2e non-vxlan external gateway through an annotated gateway pod", func() { - const ( - svcname string = "externalgw-pod-novxlan" - gwContainer1Template string = "ex-gw-container1-%d" - gwContainer2Template string = "ex-gw-container2-%d" - srcPingPodName string = "e2e-exgw-src-ping-pod" - gatewayPodName1 string = "e2e-gateway-pod1" - gatewayPodName2 string = "e2e-gateway-pod2" - ecmpRetry int = 20 - testTimeout time.Duration = 20 * time.Second - ) - - var ( - sleepCommand = []string{"bash", "-c", "sleep 20000"} - addressesv4, addressesv6 gatewayTestIPs - servingNamespace string - gwContainers []infraapi.ExternalContainer - providerCtx infraapi.Context - ) - - f := wrappedTestFramework(svcname) - - ginkgo.BeforeEach(func() { - providerCtx = infraprovider.Get().NewTestContext() - // retrieve worker node names - nodes, err := e2enode.GetBoundedReadySchedulableNodes(context.TODO(), f.ClientSet, 3) - framework.ExpectNoError(err) - if len(nodes.Items) < 3 { - framework.Failf( - "Test requires >= 3 Ready nodes, but there are only %v nodes", - len(nodes.Items)) - } - - ns, err := f.CreateNamespace(context.TODO(), "exgw-bfd-serving", nil) - framework.ExpectNoError(err) - servingNamespace = ns.Name - network, err := infraprovider.Get().PrimaryNetwork() - framework.ExpectNoError(err, "failed to get primary network information") - if overrideNetworkName, _, _ := getOverrideNetwork(); overrideNetworkName != "" { - overrideNetwork, err := infraprovider.Get().GetNetwork(overrideNetworkName) - framework.ExpectNoError(err, "over ride network must exist") - network = overrideNetwork - } - gwContainers, addressesv4, addressesv6 = setupGatewayContainers(f, providerCtx, nodes, network, gwContainer1Template, - gwContainer2Template, srcPingPodName, gwUDPPort, gwTCPPort, podUDPPort, podTCPPort, ecmpRetry, true) - setupAnnotatedGatewayPods(f, nodes, network, gatewayPodName1, gatewayPodName2, servingNamespace, sleepCommand, addressesv4, addressesv6, true) - }) - - ginkgo.AfterEach(func() { - resetGatewayAnnotations(f) - }) - - ginkgo.DescribeTable("Should validate ICMP connectivity to an external gateway's loopback address via a pod with external gateway annotations enabled", - func(addresses *gatewayTestIPs, icmpCommand string) { - if addresses.srcPodIP == "" || addresses.nodeIP == "" { - skipper.Skipf("Skipping as pod ip / node ip are not set pod ip %s node ip %s", addresses.srcPodIP, addresses.nodeIP) - } - - ginkgo.By("Verifying connectivity to the pod from external gateways") - for _, gwContainer := range gwContainers { - // Ping from a common IP address that exists on both gateways to ensure test coverage where ingress reply goes back to the same host. - gomega.Eventually(infraprovider.Get().ExecExternalContainerCommand). - WithArguments(gwContainer, []string{"ping", "-B", "-c1", "-W1", "-I", addresses.targetIPs[0], addresses.srcPodIP}). - WithTimeout(testTimeout). - ShouldNot(gomega.BeEmpty(), "Failed to ping %s from container %s", addresses.srcPodIP, gwContainer.Name) - } - - for _, gwContainer := range gwContainers { - gomega.Eventually(isBFDPaired). - WithArguments(gwContainer, addresses.nodeIP). - WithTimeout(time.Minute). - WithPolling(5*time.Second). - Should(gomega.BeTrue(), "Bfd not paired") - } - - tcpDumpSync := sync.WaitGroup{} - tcpDumpSync.Add(len(gwContainers)) - for _, gwContainer := range gwContainers { - go checkReceivedPacketsOnExternalContainer(gwContainer, srcPingPodName, anyLink, []string{icmpCommand}, &tcpDumpSync) - } - - // Verify the external gateway loopback address running on the external container is reachable and - // that traffic from the source ping pod is proxied through the pod in the serving namespace - ginkgo.By("Verifying connectivity via the gateway namespace to the remote addresses") - - pingSync := sync.WaitGroup{} - // spawn a goroutine to asynchronously (to speed up the test) - // to ping the gateway loopbacks on both containers via ECMP. - for _, address := range addresses.targetIPs { - pingSync.Add(1) - go func(target string) { - defer ginkgo.GinkgoRecover() - defer pingSync.Done() - gomega.Eventually(e2ekubectl.RunKubectl). - WithArguments(f.Namespace.Name, "exec", srcPingPodName, "--", "ping", "-c1", "-W1", target). - WithTimeout(testTimeout). - ShouldNot(gomega.BeEmpty(), "Failed to ping remote gateway %s from pod %s", target, srcPingPodName) - }(address) - } - - pingSync.Wait() - tcpDumpSync.Wait() - - if len(gwContainers) > 1 { - ginkgo.By("Deleting one container") - err := providerCtx.DeleteExternalContainer(gwContainers[1]) - framework.ExpectNoError(err, "failed to delete external container %s", gwContainers[1].Name) - time.Sleep(bfdTimeout) - - tcpDumpSync = sync.WaitGroup{} - tcpDumpSync.Add(1) - go checkReceivedPacketsOnExternalContainer(gwContainers[0], srcPingPodName, anyLink, []string{icmpCommand}, &tcpDumpSync) - - // Verify the external gateway loopback address running on the external container is reachable and - // that traffic from the source ping pod is proxied through the pod in the serving namespace - ginkgo.By("Verifying connectivity via the gateway namespace to the remote addresses") - pingSync = sync.WaitGroup{} - - for _, t := range addresses.targetIPs { - pingSync.Add(1) - go func(target string) { - defer ginkgo.GinkgoRecover() - defer pingSync.Done() - gomega.Eventually(e2ekubectl.RunKubectl). - WithArguments(f.Namespace.Name, "exec", srcPingPodName, "--", "ping", "-c1", "-W1", target). - WithTimeout(testTimeout). - ShouldNot(gomega.BeEmpty(), "Failed to ping remote gateway %s from pod %s", target, srcPingPodName) - }(t) - } - pingSync.Wait() - tcpDumpSync.Wait() - } - }, - ginkgo.Entry("ipv4", &addressesv4, "icmp"), - ginkgo.Entry("ipv6", &addressesv6, "icmp6")) - - ginkgo.DescribeTable("Should validate TCP/UDP connectivity to an external gateway's loopback address via a pod with external gateway annotations enabled", - func(protocol string, addresses *gatewayTestIPs, gwPort int) { - if addresses.srcPodIP == "" || addresses.nodeIP == "" { - skipper.Skipf("Skipping as pod ip / node ip are not set pod ip %s node ip %s", addresses.srcPodIP, addresses.nodeIP) - } - - for _, gwContainer := range gwContainers { - gomega.Eventually(infraprovider.Get().ExecExternalContainerCommand). - WithArguments(gwContainer, []string{"ping", "-c1", "-W1", addresses.srcPodIP}). - WithTimeout(testTimeout). - ShouldNot(gomega.BeEmpty(), "Failed to ping %s from container %s", addresses.srcPodIP, gwContainer.Name) - } - - for _, gwContainer := range gwContainers { - gomega.Eventually(isBFDPaired). - WithArguments(gwContainer, addresses.nodeIP). - WithTimeout(time.Minute). - WithPolling(5*time.Second). - Should(gomega.BeTrue(), "Bfd not paired") - } - - expectedHostNames := hostNamesForExternalContainers(gwContainers) - framework.Logf("Expected hostnames are %v", expectedHostNames) - - returnedHostNames := make(map[string]struct{}) - gwIP := addresses.targetIPs[0] - success := false - for i := 0; i < singleTargetRetries; i++ { - hostname := pokeHostnameViaNC(srcPingPodName, f.Namespace.Name, protocol, gwIP, gwPort) - if hostname != "" { - returnedHostNames[hostname] = struct{}{} - } - - if cmp.Equal(returnedHostNames, expectedHostNames) { - success = true - break - } - } - framework.Logf("Received hostnames for protocol %s are %v ", protocol, returnedHostNames) - - if !success { - framework.Failf("Failed to hit all the external gateways via for protocol %s, diff %s", protocol, cmp.Diff(expectedHostNames, returnedHostNames)) - } - - if len(gwContainers) > 1 { - ginkgo.By("Deleting one container") - err := providerCtx.DeleteExternalContainer(gwContainers[1]) - framework.ExpectNoError(err, "failed to delete external container %s", gwContainers[1].Name) - ginkgo.By("Waiting for BFD to sync") - time.Sleep(bfdTimeout) - - // ECMP should direct all the traffic to the only container - expectedHostName := hostNameForExternalContainer(gwContainers[0]) - - ginkgo.By("Checking hostname multiple times") - for i := 0; i < 20; i++ { - hostname := pokeHostnameViaNC(srcPingPodName, f.Namespace.Name, protocol, gwIP, gwPort) - gomega.Expect(expectedHostName).To(gomega.Equal(hostname), "Hostname returned by nc not as expected") - } - } - }, - ginkgo.Entry("UDP ipv4", "udp", &addressesv4, gwUDPPort), - ginkgo.Entry("TCP ipv4", "tcp", &addressesv4, gwTCPPort), - ginkgo.Entry("UDP ipv6", "udp", &addressesv6, gwUDPPort), - ginkgo.Entry("TCP ipv6", "tcp", &addressesv6, gwTCPPort)) - }) - - // Validate pods can reach a network running in multiple container's loopback - // addresses via two external gateways running on eth0 of the container without - // any tunnel encap. This test defines two external gateways and validates ECMP - // functionality to the container loopbacks. To verify traffic reaches the - // gateways, tcpdump is running on the external gateways and will exit successfully - // once an ICMP packet is received from the annotated pod in the k8s cluster. - // Two additional gateways are added to verify the tcp / udp protocols. - // They run the netexec command, and the pod asks to return their hostname. - // The test checks that both hostnames are collected at least once. - var _ = ginkgo.Describe("e2e multiple external gateway validation", func() { - const ( - svcname string = "novxlan-externalgw-ecmp" - gwContainer1Template string = "gw-test-container1-%d" - gwContainer2Template string = "gw-test-container2-%d" - testTimeout time.Duration = 30 * time.Second - ecmpRetry int = 20 - srcPodName = "e2e-exgw-src-pod" - ) - - var ( - gwContainers []infraapi.ExternalContainer - providerCtx infraapi.Context - testContainer = fmt.Sprintf("%s-container", srcPodName) - testContainerFlag = fmt.Sprintf("--container=%s", testContainer) - addressesv4, addressesv6 gatewayTestIPs - ) - - f := wrappedTestFramework(svcname) - - ginkgo.BeforeEach(func() { - providerCtx = infraprovider.Get().NewTestContext() - nodes, err := e2enode.GetBoundedReadySchedulableNodes(context.TODO(), f.ClientSet, 3) - framework.ExpectNoError(err) - if len(nodes.Items) < 3 { - framework.Failf( - "Test requires >= 3 Ready nodes, but there are only %v nodes", - len(nodes.Items)) - } - network, err := infraprovider.Get().PrimaryNetwork() - framework.ExpectNoError(err, "failed to get primary network information") - if overrideNetworkName, _, _ := getOverrideNetwork(); overrideNetworkName != "" { - overrideNetwork, err := infraprovider.Get().GetNetwork(overrideNetworkName) - framework.ExpectNoError(err, "over ride network must exist") - network = overrideNetwork - } - if network.Name() == "host" { - skipper.Skipf("Skipping as host network doesn't support multiple external gateways") - } - gwContainers, addressesv4, addressesv6 = setupGatewayContainers(f, providerCtx, nodes, network, - gwContainer1Template, gwContainer2Template, srcPodName, gwUDPPort, gwTCPPort, podUDPPort, podTCPPort, ecmpRetry, true) - - }) - - ginkgo.AfterEach(func() { - resetGatewayAnnotations(f) - }) - - ginkgo.DescribeTable("Should validate ICMP connectivity to multiple external gateways for an ECMP scenario", func(addresses *gatewayTestIPs, icmpToDump string) { - if addresses.srcPodIP == "" || addresses.nodeIP == "" { - skipper.Skipf("Skipping as pod ip / node ip are not set pod ip %s node ip %s", addresses.srcPodIP, addresses.nodeIP) - } - - annotateNamespaceForGateway(f.Namespace.Name, true, addresses.gatewayIPs[:]...) - for _, gwContainer := range gwContainers { - // Ping from a common IP address that exists on both gateways to ensure test coverage where ingress reply goes back to the same host. - gomega.Eventually(infraprovider.Get().ExecExternalContainerCommand). - WithArguments(gwContainer, []string{"ping", "-B", "-c1", "-W1", "-I", addresses.targetIPs[0], addresses.srcPodIP}). - WithTimeout(testTimeout). - ShouldNot(gomega.BeEmpty(), "Failed to ping %s from container %s", addresses.srcPodIP, gwContainer.Name) - } - - for _, gwContainer := range gwContainers { - gomega.Eventually(isBFDPaired). - WithArguments(gwContainer, addresses.nodeIP). - WithTimeout(time.Minute). - WithPolling(5*time.Second). - Should(gomega.BeTrue(), "Bfd not paired") - } - - // Verify the gateways and remote loopback addresses are reachable from the pod. - // Iterate checking connectivity to the loopbacks on the gateways until tcpdump see - // the traffic or 20 attempts fail. Odds of a false negative here is ~ (1/2)^20 - ginkgo.By("Verifying ecmp connectivity to the external gateways by iterating through the targets") - - // Check for egress traffic to both gateway loopback addresses using tcpdump, since - // /proc/net/dev counters only record the ingress interface traffic is received on. - // The test will waits until an ICMP packet is matched on the gateways or fail the - // test if a packet to the loopback is not received within the timer interval. - // If an ICMP packet is never detected, return the error via the specified chanel. - - tcpDumpSync := sync.WaitGroup{} - tcpDumpSync.Add(len(gwContainers)) - for _, gwContainer := range gwContainers { - go checkReceivedPacketsOnExternalContainer(gwContainer, srcPodName, anyLink, []string{icmpToDump}, &tcpDumpSync) - } - - // spawn a goroutine to asynchronously (to speed up the test) - // to ping the gateway loopbacks on both containers via ECMP. - - pingSync := sync.WaitGroup{} - - // spawn a goroutine to asynchronously (to speed up the test) - // to ping the gateway loopbacks on both containers via ECMP. - for _, address := range addresses.targetIPs { - pingSync.Add(1) - go func(target string) { - defer ginkgo.GinkgoRecover() - defer pingSync.Done() - gomega.Eventually(e2ekubectl.RunKubectl). - WithArguments(f.Namespace.Name, "exec", srcPodName, testContainerFlag, "--", "ping", "-c1", "-W1", target). - WithTimeout(testTimeout). - ShouldNot(gomega.BeEmpty(), "Failed to ping remote gateway %s from pod %s", target, srcPodName) - }(address) - } - - pingSync.Wait() - tcpDumpSync.Wait() - - ginkgo.By("Deleting one container") - err := providerCtx.DeleteExternalContainer(gwContainers[1]) - framework.ExpectNoError(err, "failed to delete external container %s", gwContainers[1].Name) - time.Sleep(bfdTimeout) - - pingSync = sync.WaitGroup{} - tcpDumpSync = sync.WaitGroup{} - - tcpDumpSync.Add(1) - go checkReceivedPacketsOnExternalContainer(gwContainers[0], srcPodName, anyLink, []string{icmpToDump}, &tcpDumpSync) - - // spawn a goroutine to asynchronously (to speed up the test) - // to ping the gateway loopbacks on both containers via ECMP. - for _, address := range addresses.targetIPs { - pingSync.Add(1) - go func(target string) { - defer ginkgo.GinkgoRecover() - defer pingSync.Done() - gomega.Eventually(e2ekubectl.RunKubectl). - WithArguments(f.Namespace.Name, "exec", srcPodName, testContainerFlag, "--", "ping", "-c1", "-W1", target). - WithTimeout(testTimeout). - ShouldNot(gomega.BeEmpty(), "Failed to ping remote gateway %s from pod %s", target, srcPodName) - }(address) - } - - pingSync.Wait() - tcpDumpSync.Wait() - - }, ginkgo.Entry("IPV4", &addressesv4, "icmp"), - ginkgo.Entry("IPV6", &addressesv6, "icmp6")) - - // This test runs a listener on the external container, returning the host name both on tcp and udp. - // The src pod tries to hit the remote address until both the containers are hit. - ginkgo.DescribeTable("Should validate TCP/UDP connectivity to multiple external gateways for a UDP / TCP scenario", func(addresses *gatewayTestIPs, protocol string, gwPort int) { - if addresses.srcPodIP == "" || addresses.nodeIP == "" { - skipper.Skipf("Skipping as pod ip / node ip are not set pod ip %s node ip %s", addresses.srcPodIP, addresses.nodeIP) - } - - annotateNamespaceForGateway(f.Namespace.Name, true, addresses.gatewayIPs[:]...) - - for _, gwContainer := range gwContainers { - gomega.Eventually(infraprovider.Get().ExecExternalContainerCommand). - WithArguments(gwContainer, []string{"ping", "-c1", "-W1", addresses.srcPodIP}). - WithTimeout(testTimeout). - ShouldNot(gomega.BeEmpty(), "Failed to ping %s from container %s", addresses.srcPodIP, gwContainer.Name) - } - - for _, gwContainer := range gwContainers { - gomega.Eventually(isBFDPaired). - WithArguments(gwContainer, addresses.nodeIP). - WithTimeout(time.Minute). - WithPolling(5*time.Second). - Should(gomega.BeTrue(), "Bfd not paired") - } - - expectedHostNames := hostNamesForExternalContainers(gwContainers) - framework.Logf("Expected hostnames are %v", expectedHostNames) - - returnedHostNames := make(map[string]struct{}) - success := false - - // Picking only the first address, the one the udp listener is set for - gwIP := addresses.targetIPs[0] - for i := 0; i < singleTargetRetries; i++ { - hostname := pokeHostnameViaNC(srcPodName, f.Namespace.Name, protocol, gwIP, gwPort) - if hostname != "" { - returnedHostNames[hostname] = struct{}{} - } - if cmp.Equal(returnedHostNames, expectedHostNames) { - success = true - break - } - } - - framework.Logf("Received hostnames for protocol %s are %v ", protocol, returnedHostNames) - - if !success { - framework.Failf("Failed to hit all the external gateways via for protocol %s, diff %s", protocol, cmp.Diff(expectedHostNames, returnedHostNames)) - } - - ginkgo.By("Deleting one container") - err := providerCtx.DeleteExternalContainer(gwContainers[1]) - framework.ExpectNoError(err, "failed to delete external container %s", gwContainers[1].Name) - ginkgo.By("Waiting for BFD to sync") - time.Sleep(bfdTimeout) - - // ECMP should direct all the traffic to the only container - expectedHostName := hostNameForExternalContainer(gwContainers[0]) - - ginkgo.By("Checking hostname multiple times") - for i := 0; i < 20; i++ { - hostname := pokeHostnameViaNC(srcPodName, f.Namespace.Name, protocol, gwIP, gwPort) - gomega.Expect(expectedHostName).To(gomega.Equal(hostname), "Hostname returned by nc not as expected") - } - }, ginkgo.Entry("IPV4 udp", &addressesv4, "udp", gwUDPPort), - ginkgo.Entry("IPV4 tcp", &addressesv4, "tcp", gwTCPPort), - ginkgo.Entry("IPV6 udp", &addressesv6, "udp", gwUDPPort), - ginkgo.Entry("IPV6 tcp", &addressesv6, "tcp", gwTCPPort)) - }) - }) - - }) - var _ = ginkgo.Context("With Admin Policy Based External Route CRs", func() { // Validate pods can reach a network running in a container's looback address via @@ -2031,7 +727,7 @@ var _ = ginkgo.Describe("External Gateway", feature.ExternalGateway, func() { }, time.Minute, 5).Should(gomega.Equal(podConnEntriesWithMACLabelsSet)) gomega.Expect(pokeConntrackEntries(nodeName, addresses.srcPodIP, protocol, nil)).To(gomega.Equal(totalPodConnEntries)) // total conntrack entries for this pod/protocol - cleanUpFn := handleGatewayPodRemoval(f, removalType, gatewayPodName2, servingNamespace, addresses.gatewayIPs[1], false) + cleanUpFn := handleGatewayPodRemoval(f, removalType, gatewayPodName2, servingNamespace) if cleanUpFn != nil { defer cleanUpFn() } @@ -2410,428 +1106,108 @@ var _ = ginkgo.Describe("External Gateway", feature.ExternalGateway, func() { }(address) } - pingSync.Wait() - tcpDumpSync.Wait() - - ginkgo.By("Deleting one container") - err := providerCtx.DeleteExternalContainer(gwContainers[1]) - framework.ExpectNoError(err, "failed to delete external container %s", gwContainers[1].Name) - time.Sleep(bfdTimeout) - - pingSync = sync.WaitGroup{} - tcpDumpSync = sync.WaitGroup{} - - tcpDumpSync.Add(1) - go checkReceivedPacketsOnExternalContainer(gwContainers[0], srcPodName, anyLink, []string{icmpToDump}, &tcpDumpSync) - - // spawn a goroutine to asynchronously (to speed up the test) - // to ping the gateway loopbacks on both containers via ECMP. - for _, address := range addresses.targetIPs { - pingSync.Add(1) - go func(target string) { - defer ginkgo.GinkgoRecover() - defer pingSync.Done() - gomega.Eventually(e2ekubectl.RunKubectl). - WithArguments(f.Namespace.Name, "exec", srcPodName, testContainerFlag, "--", "ping", "-c1", "-W1", target). - WithTimeout(testTimeout). - ShouldNot(gomega.BeEmpty(), "Failed to ping remote gateway %s from pod %s", target, srcPodName) - }(address) - } - - pingSync.Wait() - tcpDumpSync.Wait() - - }, ginkgo.Entry("IPV4", &addressesv4, "icmp"), - ginkgo.Entry("IPV6", &addressesv6, "icmp6")) - - // This test runs a listener on the external container, returning the host name both on tcp and udp. - // The src pod tries to hit the remote address until both the containers are hit. - ginkgo.DescribeTable("Should validate TCP/UDP connectivity to multiple external gateways for a UDP / TCP scenario", func(addresses *gatewayTestIPs, protocol string, gwPort int) { - if addresses.srcPodIP == "" || addresses.nodeIP == "" { - skipper.Skipf("Skipping as pod ip / node ip are not set pod ip %s node ip %s", addresses.srcPodIP, addresses.nodeIP) - } - createAPBExternalRouteCRWithStaticHop(defaultPolicyName, f.Namespace.Name, true, addresses.gatewayIPs...) - - for _, gwContainer := range gwContainers { - gomega.Eventually(infraprovider.Get().ExecExternalContainerCommand). - WithArguments(gwContainer, []string{"ping", "-c1", "-W1", addresses.srcPodIP}). - WithTimeout(testTimeout). - ShouldNot(gomega.BeEmpty(), "Failed to ping %s from container %s", addresses.srcPodIP, gwContainer.Name) - } - - for _, gwContainer := range gwContainers { - gomega.Eventually(isBFDPaired). - WithArguments(gwContainer, addresses.nodeIP). - WithTimeout(time.Minute). - WithPolling(5*time.Second). - Should(gomega.BeTrue(), "Bfd not paired") - } - - expectedHostNames := hostNamesForExternalContainers(gwContainers) - framework.Logf("Expected hostnames are %v", expectedHostNames) - - returnedHostNames := make(map[string]struct{}) - success := false - - // Picking only the first address, the one the udp listener is set for - gwIP := addresses.targetIPs[0] - for i := 0; i < singleTargetRetries; i++ { - hostname := pokeHostnameViaNC(srcPodName, f.Namespace.Name, protocol, gwIP, gwPort) - if hostname != "" { - returnedHostNames[hostname] = struct{}{} - } - if cmp.Equal(returnedHostNames, expectedHostNames) { - success = true - break - } - } - - framework.Logf("Received hostnames for protocol %s are %v ", protocol, returnedHostNames) - - if !success { - framework.Failf("Failed to hit all the external gateways via for protocol %s, diff %s", protocol, cmp.Diff(expectedHostNames, returnedHostNames)) - } - - ginkgo.By("Deleting one container") - err := providerCtx.DeleteExternalContainer(gwContainers[1]) - framework.ExpectNoError(err, "failed to delete external container %s", gwContainers[1].Name) - ginkgo.By("Waiting for BFD to sync") - time.Sleep(bfdTimeout) - - // ECMP should direct all the traffic to the only container - expectedHostName := hostNameForExternalContainer(gwContainers[0]) - - ginkgo.By("Checking hostname multiple times") - for i := 0; i < 20; i++ { - hostname := pokeHostnameViaNC(srcPodName, f.Namespace.Name, protocol, gwIP, gwPort) - gomega.Expect(expectedHostName).To(gomega.Equal(hostname), "Hostname returned by nc not as expected") - } - }, ginkgo.Entry("IPV4 udp", &addressesv4, "udp", gwUDPPort), - ginkgo.Entry("IPV4 tcp", &addressesv4, "tcp", gwTCPPort), - ginkgo.Entry("IPV6 udp", &addressesv6, "udp", gwUDPPort), - ginkgo.Entry("IPV6 tcp", &addressesv6, "tcp", gwTCPPort)) - }) - }) - }) - - var _ = ginkgo.Context("When migrating from Annotations to Admin Policy Based External Route CRs", func() { - // Validate pods can reach a network running in a container's looback address via - // an external gateway running on eth0 of the container without any tunnel encap. - // The traffic will get proxied through an annotated pod in the serving namespace. - var _ = ginkgo.Describe("e2e non-vxlan external gateway through a gateway pod", func() { - const ( - svcname string = "externalgw-pod-novxlan" - gwContainer1Template string = "ex-gw-container1-%d" - gwContainer2Template string = "ex-gw-container2-%d" - srcPingPodName string = "e2e-exgw-src-ping-pod" - gatewayPodName1 string = "e2e-gateway-pod1" - gatewayPodName2 string = "e2e-gateway-pod2" - ecmpRetry int = 20 - testTimeout time.Duration = 20 * time.Second - ) - - var ( - sleepCommand = []string{"bash", "-c", "sleep 20000"} - addressesv4, addressesv6 gatewayTestIPs - servingNamespace string - gwContainers []infraapi.ExternalContainer - providerCtx infraapi.Context - ) - - f := wrappedTestFramework(svcname) - - ginkgo.BeforeEach(func() { - providerCtx = infraprovider.Get().NewTestContext() - // retrieve worker node names - nodes, err := e2enode.GetBoundedReadySchedulableNodes(context.TODO(), f.ClientSet, 3) - framework.ExpectNoError(err) - if len(nodes.Items) < 3 { - framework.Failf( - "Test requires >= 3 Ready nodes, but there are only %v nodes", - len(nodes.Items)) - } - - ns, err := f.CreateNamespace(context.TODO(), "exgw-serving", nil) - framework.ExpectNoError(err) - servingNamespace = ns.Name - network, err := infraprovider.Get().PrimaryNetwork() - framework.ExpectNoError(err, "failed to get primary network info") - if overrideNetworkName, _, _ := getOverrideNetwork(); overrideNetworkName != "" { - overrideNetwork, err := infraprovider.Get().GetNetwork(overrideNetworkName) - framework.ExpectNoError(err, "over ride network must exist") - network = overrideNetwork - } - gwContainers, addressesv4, addressesv6 = setupGatewayContainers(f, providerCtx, nodes, network, - gwContainer1Template, gwContainer2Template, srcPingPodName, gwUDPPort, gwTCPPort, podUDPPort, podTCPPort, ecmpRetry, false) - setupAnnotatedGatewayPods(f, nodes, network, gatewayPodName1, gatewayPodName2, servingNamespace, sleepCommand, addressesv4, addressesv6, false) - }) - - ginkgo.AfterEach(func() { - deleteAPBExternalRouteCR(defaultPolicyName) - resetGatewayAnnotations(f) - }) - - ginkgo.DescribeTable("Should validate ICMP connectivity to an external gateway's loopback address via a pod with external gateway annotations and a policy CR and after the annotations are removed", - func(addresses *gatewayTestIPs, icmpCommand string) { - if addresses.srcPodIP == "" || addresses.nodeIP == "" { - skipper.Skipf("Skipping as pod ip / node ip are not set pod ip %s node ip %s", addresses.srcPodIP, addresses.nodeIP) - } - - createAPBExternalRouteCRWithDynamicHop(defaultPolicyName, f.Namespace.Name, servingNamespace, false, addressesv4.gatewayIPs) - ginkgo.By("Remove gateway annotations in pods") - annotatePodForGateway(gatewayPodName2, servingNamespace, "", addresses.gatewayIPs[1], false) - annotatePodForGateway(gatewayPodName1, servingNamespace, "", addresses.gatewayIPs[0], false) - ginkgo.By("Validate ICMP connectivity again with only CR policy to support it") - ginkgo.By(fmt.Sprintf("Verifying connectivity to the pod [%s] from external gateways", addresses.srcPodIP)) - for _, gwContainer := range gwContainers { - // Ping from a common IP address that exists on both gateways to ensure test coverage where ingress reply goes back to the same host. - gomega.Eventually(infraprovider.Get().ExecExternalContainerCommand). - WithArguments(gwContainer, []string{"ping", "-B", "-c1", "-W1", "-I", addresses.targetIPs[0], addresses.srcPodIP}). - WithTimeout(testTimeout). - ShouldNot(gomega.BeEmpty(), "Failed to ping %s from container %s", addresses.srcPodIP, gwContainer.Name) - } - tcpDumpSync := sync.WaitGroup{} - tcpDumpSync.Add(len(gwContainers)) - - for _, gwContainer := range gwContainers { - go checkReceivedPacketsOnExternalContainer(gwContainer, srcPingPodName, anyLink, []string{icmpCommand}, &tcpDumpSync) - } - - // Verify the external gateway loopback address running on the external container is reachable and - // that traffic from the source ping pod is proxied through the pod in the serving namespace - ginkgo.By("Verifying connectivity via the gateway namespace to the remote addresses") - pingSync := sync.WaitGroup{} - for _, t := range addresses.targetIPs { + pingSync.Wait() + tcpDumpSync.Wait() + + ginkgo.By("Deleting one container") + err := providerCtx.DeleteExternalContainer(gwContainers[1]) + framework.ExpectNoError(err, "failed to delete external container %s", gwContainers[1].Name) + time.Sleep(bfdTimeout) + + pingSync = sync.WaitGroup{} + tcpDumpSync = sync.WaitGroup{} + + tcpDumpSync.Add(1) + go checkReceivedPacketsOnExternalContainer(gwContainers[0], srcPodName, anyLink, []string{icmpToDump}, &tcpDumpSync) + + // spawn a goroutine to asynchronously (to speed up the test) + // to ping the gateway loopbacks on both containers via ECMP. + for _, address := range addresses.targetIPs { pingSync.Add(1) - go func(gwIP string) { + go func(target string) { defer ginkgo.GinkgoRecover() defer pingSync.Done() gomega.Eventually(e2ekubectl.RunKubectl). - WithArguments(f.Namespace.Name, "exec", srcPingPodName, "--", "ping", "-c1", "-W1", gwIP). + WithArguments(f.Namespace.Name, "exec", srcPodName, testContainerFlag, "--", "ping", "-c1", "-W1", target). WithTimeout(testTimeout). - ShouldNot(gomega.BeEmpty(), "Failed to ping remote gateway %s from pod %s", gwIP, srcPingPodName) - }(t) + ShouldNot(gomega.BeEmpty(), "Failed to ping remote gateway %s from pod %s", target, srcPodName) + }(address) } + pingSync.Wait() tcpDumpSync.Wait() - checkAPBExternalRouteStatus(defaultPolicyName) - }, - ginkgo.Entry("ipv4", &addressesv4, "icmp")) - ginkgo.DescribeTable("Should validate TCP/UDP connectivity to an external gateway's loopback "+ - "address via a pod when deleting the annotation and supported by a CR with the same gateway IPs", - func(protocol string, addresses *gatewayTestIPs, gwPort, podPort int) { + }, ginkgo.Entry("IPV4", &addressesv4, "icmp"), + ginkgo.Entry("IPV6", &addressesv6, "icmp6")) + + // This test runs a listener on the external container, returning the host name both on tcp and udp. + // The src pod tries to hit the remote address until both the containers are hit. + ginkgo.DescribeTable("Should validate TCP/UDP connectivity to multiple external gateways for a UDP / TCP scenario", func(addresses *gatewayTestIPs, protocol string, gwPort int) { if addresses.srcPodIP == "" || addresses.nodeIP == "" { skipper.Skipf("Skipping as pod ip / node ip are not set pod ip %s node ip %s", addresses.srcPodIP, addresses.nodeIP) } - createAPBExternalRouteCRWithDynamicHop(defaultPolicyName, f.Namespace.Name, servingNamespace, false, addressesv4.gatewayIPs) - ginkgo.By("removing the annotations in the pod gateways") - annotatePodForGateway(gatewayPodName2, servingNamespace, "", addresses.gatewayIPs[1], false) - annotatePodForGateway(gatewayPodName1, servingNamespace, "", addresses.gatewayIPs[0], false) + createAPBExternalRouteCRWithStaticHop(defaultPolicyName, f.Namespace.Name, true, addresses.gatewayIPs...) - for _, container := range gwContainers { - reachPodFromGateway(container, addresses.srcPodIP, strconv.Itoa(podPort), srcPingPodName, protocol) + for _, gwContainer := range gwContainers { + gomega.Eventually(infraprovider.Get().ExecExternalContainerCommand). + WithArguments(gwContainer, []string{"ping", "-c1", "-W1", addresses.srcPodIP}). + WithTimeout(testTimeout). + ShouldNot(gomega.BeEmpty(), "Failed to ping %s from container %s", addresses.srcPodIP, gwContainer.Name) } - expectedHostNames := make(map[string]struct{}) - for _, c := range gwContainers { - res, err := infraprovider.Get().ExecExternalContainerCommand(c, []string{"hostname"}) - framework.ExpectNoError(err, "failed to run hostname in %s", c) - hostname := strings.TrimSuffix(res, "\n") - framework.Logf("Hostname for %s is %s", c, hostname) - expectedHostNames[hostname] = struct{}{} + for _, gwContainer := range gwContainers { + gomega.Eventually(isBFDPaired). + WithArguments(gwContainer, addresses.nodeIP). + WithTimeout(time.Minute). + WithPolling(5*time.Second). + Should(gomega.BeTrue(), "Bfd not paired") } + + expectedHostNames := hostNamesForExternalContainers(gwContainers) framework.Logf("Expected hostnames are %v", expectedHostNames) - ginkgo.By("Checking that external ips are reachable with both gateways") returnedHostNames := make(map[string]struct{}) - gwIP := addresses.targetIPs[0] success := false + + // Picking only the first address, the one the udp listener is set for + gwIP := addresses.targetIPs[0] for i := 0; i < singleTargetRetries; i++ { - args := []string{"exec", srcPingPodName, "--"} - if protocol == "tcp" { - args = append(args, "bash", "-c", fmt.Sprintf("echo | nc -w 1 %s %d", gwIP, gwPort)) - } else { - args = append(args, "bash", "-c", fmt.Sprintf("echo | nc -w 1 -u %s %d", gwIP, gwPort)) - } - res, err := e2ekubectl.RunKubectl(f.Namespace.Name, args...) - framework.ExpectNoError(err, "failed to reach %s (%s)", gwIP, protocol) - hostname := strings.TrimSuffix(res, "\n") + hostname := pokeHostnameViaNC(srcPodName, f.Namespace.Name, protocol, gwIP, gwPort) if hostname != "" { returnedHostNames[hostname] = struct{}{} } - if cmp.Equal(returnedHostNames, expectedHostNames) { success = true break } } + framework.Logf("Received hostnames for protocol %s are %v ", protocol, returnedHostNames) if !success { framework.Failf("Failed to hit all the external gateways via for protocol %s, diff %s", protocol, cmp.Diff(expectedHostNames, returnedHostNames)) } - checkAPBExternalRouteStatus(defaultPolicyName) - }, - ginkgo.Entry("UDP ipv4", "udp", &addressesv4, gwUDPPort, podUDPPort), - ginkgo.Entry("TCP ipv4", "tcp", &addressesv4, gwTCPPort, podTCPPort), - ginkgo.Entry("UDP ipv6", "udp", &addressesv6, gwUDPPort, podUDPPort), - ginkgo.Entry("TCP ipv6", "tcp", &addressesv6, gwTCPPort, podTCPPort)) - }) - - var _ = ginkgo.Describe("e2e multiple external gateway stale conntrack entry deletion validation", func() { - const ( - svcname string = "novxlan-externalgw-ecmp" - gwContainer1Template string = "gw-test-container1-%d" - gwContainer2Template string = "gw-test-container2-%d" - srcPodName string = "e2e-exgw-src-pod" - gatewayPodName1 string = "e2e-gateway-pod1" - gatewayPodName2 string = "e2e-gateway-pod2" - ) - - var ( - servingNamespace string - addressesv4, addressesv6 gatewayTestIPs - sleepCommand []string - nodes *corev1.NodeList - err error - gwContainers []infraapi.ExternalContainer - providerCtx infraapi.Context - network infraapi.Network - ) - - f := wrappedTestFramework(svcname) - - ginkgo.BeforeEach(func() { - providerCtx = infraprovider.Get().NewTestContext() - // retrieve worker node names - nodes, err = e2enode.GetBoundedReadySchedulableNodes(context.TODO(), f.ClientSet, 3) - framework.ExpectNoError(err) - if len(nodes.Items) < 3 { - framework.Failf( - "Test requires >= 3 Ready nodes, but there are only %v nodes", - len(nodes.Items)) - } - network, err = infraprovider.Get().PrimaryNetwork() - framework.ExpectNoError(err, "failed to get primary network information") - if overrideNetworkName, _, _ := getOverrideNetwork(); overrideNetworkName != "" { - overrideNetwork, err := infraprovider.Get().GetNetwork(overrideNetworkName) - framework.ExpectNoError(err, "over ride network must exist") - network = overrideNetwork - } - if network.Name() == "host" { - skipper.Skipf("Skipping as host network doesn't support multiple external gateways") - } - - ns, err := f.CreateNamespace(context.TODO(), "exgw-conntrack-serving", nil) - framework.ExpectNoError(err) - servingNamespace = ns.Name - - gwContainers, addressesv4, addressesv6 = setupGatewayContainersForConntrackTest(f, providerCtx, nodes, network, gwContainer1Template, gwContainer2Template, srcPodName) - sleepCommand = []string{"bash", "-c", "sleep 20000"} - _, err = createGenericPodWithLabel(f, gatewayPodName1, nodes.Items[0].Name, servingNamespace, sleepCommand, map[string]string{"gatewayPod": "true"}) - framework.ExpectNoError(err, "Create and annotate the external gw pods to manage the src app pod namespace, failed: %v", err) - _, err = createGenericPodWithLabel(f, gatewayPodName2, nodes.Items[1].Name, servingNamespace, sleepCommand, map[string]string{"gatewayPod": "true"}) - framework.ExpectNoError(err, "Create and annotate the external gw pods to manage the src app pod namespace, failed: %v", err) - }) - - ginkgo.AfterEach(func() { - deleteAPBExternalRouteCR(defaultPolicyName) - resetGatewayAnnotations(f) - }) - - ginkgo.DescribeTable("Namespace annotation: Should validate conntrack entry remains unchanged when deleting the annotation in the namespace while the CR static hop still references the same namespace in the policy", func(addresses *gatewayTestIPs, protocol string) { - if addresses.srcPodIP == "" || addresses.nodeIP == "" { - skipper.Skipf("Skipping as pod ip / node ip are not set pod ip %s node ip %s", addresses.srcPodIP, addresses.nodeIP) - } - ginkgo.By("Annotate the app namespace to get managed by external gateways") - annotateNamespaceForGateway(f.Namespace.Name, false, addresses.gatewayIPs...) - createAPBExternalRouteCRWithStaticHop(defaultPolicyName, f.Namespace.Name, false, addresses.gatewayIPs...) - macAddressGW := make([]string, 2) - for i, container := range gwContainers { - ginkgo.By("Start iperf3 client from external container to connect to iperf3 server running at the src pod") - _, err = infraprovider.Get().ExecExternalContainerCommand(container, []string{"iperf3", "-u", "-c", addresses.srcPodIP, - "-p", fmt.Sprintf("%d", 5201+i), "-b", "1M", "-i", "1", "-t", "3", "&"}) - networkInfo, err := infraprovider.Get().GetExternalContainerNetworkInterface(container, network) - framework.ExpectNoError(err, "failed to get network %s information for external container %s", network.Name(), container.Name) - // Trim leading 0s because conntrack dumped labels are just integers - // in hex without leading 0s. - macAddressGW[i] = strings.TrimLeft(strings.Replace(networkInfo.MAC, ":", "", -1), "0") - } - - nodeName := getPod(f, srcPodName).Spec.NodeName - expectedTotalEntries := 4 - expectedMACEntries := 2 - ginkgo.By("Check to ensure initial conntrack entries are 2 mac address label, and 4 total entries") - gomega.Eventually(func() int { - n := pokeConntrackEntries(nodeName, addresses.srcPodIP, protocol, macAddressGW) - klog.Infof("Number of entries with macAddressGW %s:%d", macAddressGW, n) - return n - }, time.Minute, 5).Should(gomega.Equal(expectedMACEntries)) - gomega.Expect(pokeConntrackEntries(nodeName, addresses.srcPodIP, protocol, nil)).To(gomega.Equal(expectedTotalEntries)) // total conntrack entries for this pod/protocol - - ginkgo.By("Removing the namespace annotations to leave only the CR policy active") - annotateNamespaceForGateway(f.Namespace.Name, false, "") - ginkgo.By("Check if conntrack entries for ECMP routes still exist for the 2 external gateways") - gomega.Eventually(func() int { - n := pokeConntrackEntries(nodeName, addresses.srcPodIP, protocol, macAddressGW) - klog.Infof("Number of entries with macAddressGW %s:%d", macAddressGW, n) - return n - }, time.Minute, 5).Should(gomega.Equal(expectedMACEntries)) - - totalPodConnEntries := pokeConntrackEntries(nodeName, addresses.srcPodIP, protocol, nil) - gomega.Expect(totalPodConnEntries).To(gomega.Equal(expectedTotalEntries)) // total conntrack entries for this pod/protocol + ginkgo.By("Deleting one container") + err := providerCtx.DeleteExternalContainer(gwContainers[1]) + framework.ExpectNoError(err, "failed to delete external container %s", gwContainers[1].Name) + ginkgo.By("Waiting for BFD to sync") + time.Sleep(bfdTimeout) - }, - ginkgo.Entry("IPV4 udp", &addressesv4, "udp"), - ginkgo.Entry("IPV4 tcp", &addressesv4, "tcp"), - ginkgo.Entry("IPV6 udp", &addressesv6, "udp"), - ginkgo.Entry("IPV6 tcp", &addressesv6, "tcp")) + // ECMP should direct all the traffic to the only container + expectedHostName := hostNameForExternalContainer(gwContainers[0]) - ginkgo.DescribeTable("ExternalGWPod annotation: Should validate conntrack entry remains unchanged when deleting the annotation in the pods while the CR dynamic hop still "+ - "references the same pods with the pod selector", func(addresses *gatewayTestIPs, protocol string) { - if addresses.srcPodIP == "" || addresses.nodeIP == "" { - skipper.Skipf("Skipping as pod ip / node ip are not set pod ip %s node ip %s", addresses.srcPodIP, addresses.nodeIP) - } - ginkgo.By("Annotate the external gw pods to manage the src app pod namespace") - for i, gwPod := range []string{gatewayPodName1, gatewayPodName2} { - networkIPs := fmt.Sprintf("\"%s\"", addresses.gatewayIPs[i]) - if addresses.srcPodIP != "" && addresses.nodeIP != "" { - networkIPs = fmt.Sprintf("\"%s\", \"%s\"", addresses.gatewayIPs[i], addresses.gatewayIPs[i]) + ginkgo.By("Checking hostname multiple times") + for i := 0; i < 20; i++ { + hostname := pokeHostnameViaNC(srcPodName, f.Namespace.Name, protocol, gwIP, gwPort) + gomega.Expect(expectedHostName).To(gomega.Equal(hostname), "Hostname returned by nc not as expected") } - annotatePodForGateway(gwPod, servingNamespace, f.Namespace.Name, networkIPs, false) - } - createAPBExternalRouteCRWithDynamicHop(defaultPolicyName, f.Namespace.Name, servingNamespace, false, addresses.gatewayIPs) - annotatePodForGateway(gatewayPodName2, servingNamespace, "", addresses.gatewayIPs[1], false) - annotatePodForGateway(gatewayPodName1, servingNamespace, "", addresses.gatewayIPs[0], false) - macAddressGW := make([]string, 2) - for i, container := range gwContainers { - ginkgo.By("Start iperf3 client from external container to connect to iperf3 server running at the src pod") - _, err = infraprovider.Get().ExecExternalContainerCommand(container, []string{"iperf3", "-u", "-c", addresses.srcPodIP, - "-p", fmt.Sprintf("%d", 5201+i), "-b", "1M", "-i", "1", "-t", "3", "&"}) - framework.ExpectNoError(err, "failed to execute iperf client command from external container") - networkInfo, err := infraprovider.Get().GetExternalContainerNetworkInterface(container, network) - framework.ExpectNoError(err, "failed to get network %s information for external container %s", network.Name(), container.Name) - // Trim leading 0s because conntrack dumped labels are just integers - // in hex without leading 0s. - macAddressGW[i] = strings.TrimLeft(strings.Replace(networkInfo.MAC, ":", "", -1), "0") - } - - ginkgo.By("Check if conntrack entries for ECMP routes are created for the 2 external gateways") - nodeName := getPod(f, srcPodName).Spec.NodeName - podConnEntriesWithMACLabelsSet := pokeConntrackEntries(nodeName, addresses.srcPodIP, protocol, macAddressGW) - gomega.Expect(podConnEntriesWithMACLabelsSet).To(gomega.Equal(2)) - totalPodConnEntries := pokeConntrackEntries(nodeName, addresses.srcPodIP, protocol, nil) - gomega.Expect(totalPodConnEntries).To(gomega.Equal(4)) // total conntrack entries for this pod/protocol - checkAPBExternalRouteStatus(defaultPolicyName) - }, - ginkgo.Entry("IPV4 udp", &addressesv4, "udp"), - ginkgo.Entry("IPV4 tcp", &addressesv4, "tcp"), - ginkgo.Entry("IPV6 udp", &addressesv6, "udp"), - ginkgo.Entry("IPV6 tcp", &addressesv6, "tcp")) + }, ginkgo.Entry("IPV4 udp", &addressesv4, "udp", gwUDPPort), + ginkgo.Entry("IPV4 tcp", &addressesv4, "tcp", gwTCPPort), + ginkgo.Entry("IPV6 udp", &addressesv6, "udp", gwUDPPort), + ginkgo.Entry("IPV6 tcp", &addressesv6, "tcp", gwTCPPort)) + }) }) - }) var _ = ginkgo.Context("When validating the Admin Policy Based External Route status", func() { @@ -3077,38 +1453,6 @@ func setupGatewayContainers(f *framework.Framework, providerCtx infraapi.Context return gwContainers, addressesv4, addressesv6 } -func setupAnnotatedGatewayPods(f *framework.Framework, nodes *corev1.NodeList, network infraapi.Network, pod1, pod2, ns string, cmd []string, addressesv4, addressesv6 gatewayTestIPs, bfd bool) []string { - gwPods := []string{pod1, pod2} - if network.Name() == "host" { - gwPods = []string{pod1} - } - - for i, gwPod := range gwPods { - _, err := createGenericPodWithLabel(f, gwPod, nodes.Items[i].Name, ns, cmd, map[string]string{"gatewayPod": "true"}) - framework.ExpectNoError(err) - } - - for i, gwPod := range gwPods { - var networkIPs string - if len(addressesv4.gatewayIPs) > 0 { - // IPv4 - networkIPs = fmt.Sprintf("\"%s\"", addressesv4.gatewayIPs[i]) - } - if addressesv6.srcPodIP != "" && addressesv6.nodeIP != "" { - if len(networkIPs) > 0 { - // IPv4 and IPv6 - networkIPs = fmt.Sprintf("%s, \"%s\"", networkIPs, addressesv6.gatewayIPs[i]) - } else { - // IPv6 only - networkIPs = fmt.Sprintf("\"%s\"", addressesv6.gatewayIPs[i]) - } - } - annotatePodForGateway(gwPod, ns, f.Namespace.Name, networkIPs, bfd) - } - - return gwPods -} - func setupPolicyBasedGatewayPods(f *framework.Framework, nodes *corev1.NodeList, network infraapi.Network, pod1, pod2, ns string, cmd []string, addressesv4, addressesv6 gatewayTestIPs) []string { gwPods := []string{pod1, pod2} if network.Name() == "host" { @@ -3251,25 +1595,6 @@ func reachPodFromGateway(srcContainer infraapi.ExternalContainer, targetAddress, gomega.Expect(strings.Trim(res, "\n")).To(gomega.Equal(targetPodName)) } -func annotatePodForGateway(podName, podNS, targetNamespace, networkIPs string, bfd bool) { - if !strings.HasPrefix(networkIPs, "\"") { - networkIPs = fmt.Sprintf("\"%s\"", networkIPs) - } - // add the annotations to the pod to enable the gateway forwarding. - // this fakes out the multus annotation so that the pod IP is - // actually an IP of an external container for testing purposes - annotateArgs := []string{ - fmt.Sprintf("k8s.v1.cni.cncf.io/network-status=[{\"name\":\"%s\",\"interface\":"+ - "\"net1\",\"ips\":[%s],\"mac\":\"%s\"}]", "foo", networkIPs, "01:23:45:67:89:10"), - fmt.Sprintf("k8s.ovn.org/routing-namespaces=%s", targetNamespace), - fmt.Sprintf("k8s.ovn.org/routing-network=%s", "foo"), - } - if bfd { - annotateArgs = append(annotateArgs, "k8s.ovn.org/bfd-enabled=\"\"") - } - annotatePodForGatewayWithAnnotations(podName, podNS, annotateArgs) -} - func annotateMultusNetworkStatusInPodGateway(podName, podNS string, networkIPs []string) { // add the annotations to the pod to enable the gateway forwarding. // this fakes out the multus annotation so that the pod IP is @@ -3296,24 +1621,6 @@ func annotatePodForGatewayWithAnnotations(podName, podNS string, annotations []s e2ekubectl.RunKubectlOrDie(podNS, annotateArgs...) } -func annotateNamespaceForGateway(namespace string, bfd bool, gateways ...string) { - - externalGateways := strings.Join(gateways, ",") - // annotate the test namespace with multiple gateways defined - annotateArgs := []string{ - "annotate", - "namespace", - namespace, - fmt.Sprintf("k8s.ovn.org/routing-external-gws=%s", externalGateways), - "--overwrite", - } - if bfd { - annotateArgs = append(annotateArgs, "k8s.ovn.org/bfd-enabled=\"\"") - } - framework.Logf("Annotating the external gateway test namespace to container gateways: %s", externalGateways) - e2ekubectl.RunKubectlOrDie(namespace, annotateArgs...) -} - func createAPBExternalRouteCRWithDynamicHop(policyName, targetNamespace, servingNamespace string, bfd bool, gateways []string) { data := fmt.Sprintf(`apiVersion: k8s.ovn.org/v1 kind: AdminPolicyBasedExternalRoute @@ -3581,25 +1888,6 @@ func checkReceivedPacketsOnExternalContainer(container infraapi.ExternalContaine framework.Logf("Packet successfully detected on gateway %s", container) } -func resetGatewayAnnotations(f *framework.Framework) { - // remove the routing external annotation - if f == nil || f.Namespace == nil { - return - } - annotations := []string{ - "k8s.ovn.org/routing-external-gws-", - "k8s.ovn.org/bfd-enabled-", - } - ginkgo.By("Resetting the gw annotations") - for _, annotation := range annotations { - e2ekubectl.RunKubectlOrDie("", []string{ - "annotate", - "namespace", - f.Namespace.Name, - annotation}...) - } -} - func setupPodWithReadinessProbe(f *framework.Framework, podName, nodeSelector, namespace string, command []string, labels map[string]string) (*corev1.Pod, error) { // Handle bash -c commands specially to preserve argument structure if len(command) >= 3 && command[0] == "bash" && command[1] == "-c" { @@ -3653,7 +1941,7 @@ func recreatePodWithReadinessProbe(f *framework.Framework, podName, nodeSelector }).Should(gomega.Equal(true), fmt.Sprintf("Readiness probe for second external gateway pod %s from ns %s, failed: %v", podName, namespace, err)) } -func handleGatewayPodRemoval(f *framework.Framework, removalType GatewayRemovalType, gatewayPodName, servingNamespace, gatewayIP string, isAnnotated bool) func() { +func handleGatewayPodRemoval(f *framework.Framework, removalType GatewayRemovalType, gatewayPodName, servingNamespace string) func() { var err error switch removalType { case GatewayDelete: @@ -3662,12 +1950,6 @@ func handleGatewayPodRemoval(f *framework.Framework, removalType GatewayRemovalT framework.ExpectNoError(err, "Delete the gateway pod failed: %v", err) return nil case GatewayUpdate: - if isAnnotated { - ginkgo.By("Remove second external gateway pod's routing-namespace annotation") - annotatePodForGateway(gatewayPodName, servingNamespace, "", gatewayIP, false) - return nil - } - ginkgo.By("Updating external gateway pod labels") p := getGatewayPod(f, servingNamespace, gatewayPodName) p.Labels = map[string]string{"name": gatewayPodName} From c824eb88ed68eee041386321bec7ac3add574d54 Mon Sep 17 00:00:00 2001 From: Mykola Yurchenko Date: Thu, 4 Jun 2026 09:45:23 -0700 Subject: [PATCH 11/51] node: run periodic external-gateway conntrack cleanup for APB namespaces The periodic stale-conntrack cleanup was gated on the now-unwritten external-gw-pod-ips annotation, so it never ran. Gate it on the namespace's AdminPolicyBasedExternalRoute gateway IPs instead (nil-safe; no-op when the feature is disabled). Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Mykola Yurchenko --- .../node/default_node_network_controller.go | 37 +++++++++++-------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/go-controller/pkg/node/default_node_network_controller.go b/go-controller/pkg/node/default_node_network_controller.go index 8ce2263f91..1abb2dc2bb 100644 --- a/go-controller/pkg/node/default_node_network_controller.go +++ b/go-controller/pkg/node/default_node_network_controller.go @@ -1318,20 +1318,28 @@ func exGatewayPodsAnnotationsChanged(oldNs, newNs *corev1.Namespace) bool { func (nc *DefaultNodeNetworkController) checkAndDeleteStaleConntrackEntries() { namespaces, err := nc.watchFactory.GetNamespaces() if err != nil { - klog.Errorf("Unable to get pods from informer: %v", err) + klog.Errorf("Unable to get namespaces from informer: %v", err) } for _, namespace := range namespaces { - _, foundExternalGatewayPodIPsAnnotation := namespace.Annotations[util.ExternalGatewayPodIPsAnnotation] - if foundExternalGatewayPodIPsAnnotation { - pods, err := nc.watchFactory.GetPods(namespace.Name) - if err != nil { - klog.Warningf("Unable to get pods from informer for namespace %s: %v", namespace.Name, err) - } - if len(pods) > 0 || err != nil { - // we only need to proceed if there is at least one pod in this namespace on this node - // OR if we couldn't fetch the pods for some reason at this juncture - _ = nc.syncConntrackForExternalGateways(namespace) - } + // Only namespaces targeted by an AdminPolicyBasedExternalRoute can have + // external-gateway ECMP conntrack entries to reconcile (the legacy + // routing-external-gws annotation is no longer supported). + gatewayIPs, err := nc.apbExternalRouteNodeController.GetAdminPolicyBasedExternalRouteIPsForTargetNamespace(namespace.Name) + if err != nil { + klog.Errorf("Unable to retrieve gateway IPs for Admin Policy Based External Route objects for namespace %s: %v", namespace.Name, err) + continue + } + if gatewayIPs.Len() == 0 { + continue + } + pods, err := nc.watchFactory.GetPods(namespace.Name) + if err != nil { + klog.Warningf("Unable to get pods from informer for namespace %s: %v", namespace.Name, err) + } + if len(pods) > 0 || err != nil { + // we only need to proceed if there is at least one pod in this namespace on this node + // OR if we couldn't fetch the pods for some reason at this juncture + _ = nc.syncConntrackForExternalGateways(namespace) } } } @@ -1341,9 +1349,8 @@ func (nc *DefaultNodeNetworkController) syncConntrackForExternalGateways(newNs * if err != nil { return fmt.Errorf("unable to retrieve gateway IPs for Admin Policy Based External Route objects: %w", err) } - // loop through all the IPs on the annotations; ARP for their MACs and form an allowlist - gatewayIPs = gatewayIPs.Insert(strings.Split(newNs.Annotations[util.ExternalGatewayPodIPsAnnotation], ",")...) - + // ARP for the gateway IPs' MACs to form an allowlist; conntrack entries whose + // destination MAC is no longer valid (e.g. after a gateway MAC change) are removed. return util.SyncConntrackForExternalGateways(gatewayIPs, nil, func() ([]*corev1.Pod, error) { return nc.watchFactory.GetPods(newNs.Name) }) From 1355df2772fe2f49d803a9a27aa5a5f51c312c17 Mon Sep 17 00:00:00 2001 From: Mykola Yurchenko Date: Thu, 4 Jun 2026 09:45:23 -0700 Subject: [PATCH 12/51] tests: cover external-gateway route deletion for APB-managed routes Seed the shared route cache and assert the ECMP route is torn down on pod and namespace deletion, restoring unit coverage of deleteGWRoutesForPod/Namespace lost when the annotation tests were removed. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Mykola Yurchenko --- go-controller/pkg/ovn/egressgw_test.go | 64 ++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/go-controller/pkg/ovn/egressgw_test.go b/go-controller/pkg/ovn/egressgw_test.go index 4184b614c0..655ca63407 100644 --- a/go-controller/pkg/ovn/egressgw_test.go +++ b/go-controller/pkg/ovn/egressgw_test.go @@ -12,6 +12,7 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + ktypes "k8s.io/apimachinery/pkg/types" "github.com/ovn-kubernetes/ovn-kubernetes/go-controller/pkg/config" "github.com/ovn-kubernetes/ovn-kubernetes/go-controller/pkg/kube" @@ -383,6 +384,69 @@ var _ = ginkgo.Describe("OVN Egress Gateway Operations", func() { gomega.Expect(err).NotTo(gomega.HaveOccurred()) }) }) + ginkgo.Context("external gateway route cleanup on pod and namespace deletion", func() { + seedRoute := func() ktypes.NamespacedName { + podNsName := ktypes.NamespacedName{Namespace: namespaceName, Name: "myPod"} + fakeOvn.startWithDBSetup( + libovsdbtest.TestSetup{ + NBData: []libovsdbtest.TestData{ + &nbdb.LogicalRouterStaticRoute{ + UUID: "static-route-1-UUID", + IPPrefix: "10.128.1.3/32", + Nexthop: "9.0.0.1", + Options: map[string]string{"ecmp_symmetric_reply": "true"}, + OutputPort: &logicalRouterPort, + Policy: &nbdb.LogicalRouterStaticRoutePolicySrcIP, + }, + &nbdb.LogicalRouter{ + UUID: "GR_node1-UUID", + Name: "GR_node1", + StaticRoutes: []string{"static-route-1-UUID"}, + }, + }, + }, + ) + injectNode(fakeOvn) + // Seed the shared route cache as if the APB controller had programmed an + // external-gateway ECMP route for this pod. + err := fakeOvn.controller.externalGatewayRouteInfo.CreateOrLoad(podNsName, func(routeInfo *apbroute.RouteInfo) error { + routeInfo.PodExternalRoutes["10.128.1.3"] = map[string]string{"9.0.0.1": "GR_node1"} + return nil + }) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + return podNsName + } + routeDeletedNB := []libovsdbtest.TestData{ + &nbdb.LogicalRouter{ + UUID: "GR_node1-UUID", + Name: "GR_node1", + StaticRoutes: []string{}, + }, + } + ginkgo.It("deletes external gateway ECMP routes for a pod with no matching policy", func() { + app.Action = func(*cli.Context) error { + config.Gateway.Mode = config.GatewayModeShared + podNsName := seedRoute() + err := fakeOvn.controller.deleteGWRoutesForPod(podNsName, []*net.IPNet{{IP: net.ParseIP("10.128.1.3")}}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Eventually(fakeOvn.nbClient).Should(libovsdbtest.HaveData(routeDeletedNB)) + return nil + } + gomega.Expect(app.Run([]string{app.Name})).To(gomega.Succeed()) + }) + ginkgo.It("deletes external gateway ECMP routes when the namespace is removed", func() { + app.Action = func(*cli.Context) error { + config.Gateway.Mode = config.GatewayModeShared + seedRoute() + err := fakeOvn.controller.deleteGWRoutesForNamespace(namespaceName, nil) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Eventually(fakeOvn.nbClient).Should(libovsdbtest.HaveData(routeDeletedNB)) + return nil + } + gomega.Expect(app.Run([]string{app.Name})).To(gomega.Succeed()) + }) + }) + ginkgo.Context("SNAT on gateway router operations", func() { ginkgo.It("add/delete SNAT per pod on gateway router", func() { app.Action = func(*cli.Context) error { From ebc865ca9c12f12888066278d22716f497f1f955 Mon Sep 17 00:00:00 2001 From: Mykola Yurchenko Date: Thu, 4 Jun 2026 09:45:23 -0700 Subject: [PATCH 13/51] tests,e2e: drop orphaned external-gw-pod-ips constant Only the removed legacy annotation specs and helpers referenced it. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Mykola Yurchenko --- test/e2e/external_gateways.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/test/e2e/external_gateways.go b/test/e2e/external_gateways.go index 3201c7f305..568dafd09e 100644 --- a/test/e2e/external_gateways.go +++ b/test/e2e/external_gateways.go @@ -38,10 +38,9 @@ import ( // This is the image used for the containers acting as externalgateways, built // out from the e2e/images/Dockerfile.frr dockerfile const ( - externalContainerImage = "quay.io/trozet/ovnkbfdtest:0.3" - externalGatewayPodIPsAnnotation = "k8s.ovn.org/external-gw-pod-ips" - defaultPolicyName = "default-route-policy" - anyLink = "any" + externalContainerImage = "quay.io/trozet/ovnkbfdtest:0.3" + defaultPolicyName = "default-route-policy" + anyLink = "any" ) // GatewayRemovalType defines ways to remove pod as external gateway From fc34b18ff2f76827acce0f7f30b78817b9758be6 Mon Sep 17 00:00:00 2001 From: Mykola Yurchenko Date: Thu, 4 Jun 2026 09:45:23 -0700 Subject: [PATCH 14/51] tests,e2e: add BFD route-cleanup coverage and fix stale wording Add a unit test asserting deleteGWRoutesForPod also reaps the route's BFD entry (cleanUpBFDEntry), and fix e2e descriptions that said "annotation" where the code now updates pod labels. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Mykola Yurchenko --- go-controller/pkg/ovn/egressgw_test.go | 52 ++++++++++++++++++++++++++ test/e2e/external_gateways.go | 10 ++--- 2 files changed, 57 insertions(+), 5 deletions(-) diff --git a/go-controller/pkg/ovn/egressgw_test.go b/go-controller/pkg/ovn/egressgw_test.go index 655ca63407..64cedc3189 100644 --- a/go-controller/pkg/ovn/egressgw_test.go +++ b/go-controller/pkg/ovn/egressgw_test.go @@ -445,6 +445,58 @@ var _ = ginkgo.Describe("OVN Egress Gateway Operations", func() { } gomega.Expect(app.Run([]string{app.Name})).To(gomega.Succeed()) }) + ginkgo.It("deletes the BFD entry along with the external gateway route on pod deletion", func() { + app.Action = func(*cli.Context) error { + config.Gateway.Mode = config.GatewayModeShared + podNsName := ktypes.NamespacedName{Namespace: namespaceName, Name: "myPod"} + bfdUUID := "bfd-1-UUID" + fakeOvn.startWithDBSetup( + libovsdbtest.TestSetup{ + NBData: []libovsdbtest.TestData{ + &nbdb.BFD{ + UUID: bfdUUID, + LogicalPort: "rtoe-GR_node1", + DstIP: "9.0.0.1", + }, + &nbdb.LogicalRouterStaticRoute{ + UUID: "static-route-1-UUID", + IPPrefix: "10.128.1.3/32", + Nexthop: "9.0.0.1", + BFD: &bfdUUID, + Options: map[string]string{"ecmp_symmetric_reply": "true"}, + OutputPort: &logicalRouterPort, + Policy: &nbdb.LogicalRouterStaticRoutePolicySrcIP, + }, + &nbdb.LogicalRouter{ + UUID: "GR_node1-UUID", + Name: "GR_node1", + StaticRoutes: []string{"static-route-1-UUID"}, + }, + }, + }, + ) + injectNode(fakeOvn) + err := fakeOvn.controller.externalGatewayRouteInfo.CreateOrLoad(podNsName, func(routeInfo *apbroute.RouteInfo) error { + routeInfo.PodExternalRoutes["10.128.1.3"] = map[string]string{"9.0.0.1": "GR_node1"} + return nil + }) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + // Both the ECMP route and its now-dangling BFD row must be removed. + finalNB := []libovsdbtest.TestData{ + &nbdb.LogicalRouter{ + UUID: "GR_node1-UUID", + Name: "GR_node1", + StaticRoutes: []string{}, + }, + } + err = fakeOvn.controller.deleteGWRoutesForPod(podNsName, []*net.IPNet{{IP: net.ParseIP("10.128.1.3")}}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Eventually(fakeOvn.nbClient).Should(libovsdbtest.HaveData(finalNB)) + return nil + } + gomega.Expect(app.Run([]string{app.Name})).To(gomega.Succeed()) + }) + }) ginkgo.Context("SNAT on gateway router operations", func() { diff --git a/test/e2e/external_gateways.go b/test/e2e/external_gateways.go index 568dafd09e..e1e9853aa1 100644 --- a/test/e2e/external_gateways.go +++ b/test/e2e/external_gateways.go @@ -743,7 +743,7 @@ var _ = ginkgo.Describe("External Gateway", feature.ExternalGateway, func() { }, 10).Should(gomega.Equal(podConnEntriesWithMACLabelsSet)) gomega.Expect(pokeConntrackEntries(nodeName, addresses.srcPodIP, protocol, nil)).To(gomega.Equal(totalPodConnEntries)) - ginkgo.By("Remove first external gateway pod's routing-namespace annotation") + ginkgo.By("Update first external gateway pod's labels so it no longer matches the policy") p := getGatewayPod(f, servingNamespace, gatewayPodName1) p.Labels = map[string]string{"name": gatewayPodName1} updatePod(f, p) @@ -760,10 +760,10 @@ var _ = ginkgo.Describe("External Gateway", feature.ExternalGateway, func() { gomega.Expect(pokeConntrackEntries(nodeName, addresses.srcPodIP, protocol, nil)).To(gomega.Equal(totalPodConnEntries)) checkAPBExternalRouteStatus(defaultPolicyName) }, - ginkgo.Entry("IPV4 udp + pod annotation update", &addressesv4, "udp", GatewayUpdate), - ginkgo.Entry("IPV4 tcp + pod annotation update", &addressesv4, "tcp", GatewayUpdate), - ginkgo.Entry("IPV6 udp + pod annotation update", &addressesv6, "udp", GatewayUpdate), - ginkgo.Entry("IPV6 tcp + pod annotation update", &addressesv6, "tcp", GatewayUpdate), + ginkgo.Entry("IPV4 udp + pod label update", &addressesv4, "udp", GatewayUpdate), + ginkgo.Entry("IPV4 tcp + pod label update", &addressesv4, "tcp", GatewayUpdate), + ginkgo.Entry("IPV6 udp + pod label update", &addressesv6, "udp", GatewayUpdate), + ginkgo.Entry("IPV6 tcp + pod label update", &addressesv6, "tcp", GatewayUpdate), ginkgo.Entry("IPV4 udp + pod deletion timestamp", &addressesv4, "udp", GatewayDeletionTimestamp), ginkgo.Entry("IPV4 tcp + pod deletion timestamp", &addressesv4, "tcp", GatewayDeletionTimestamp), ginkgo.Entry("IPV6 udp + pod deletion timestamp", &addressesv6, "udp", GatewayDeletionTimestamp), From a0c9fa1f809826065fc862b103202a2b214f4658 Mon Sep 17 00:00:00 2001 From: Tim Rozet Date: Mon, 18 May 2026 16:58:39 -0400 Subject: [PATCH 15/51] Set DPU host BGP next-hop for no-overlay routes Generated FRRConfiguration objects for DPU-host nodes need to advertise no-overlay pod CIDRs with the host shared gateway IP as the BGP next hop. This keeps the DPU transparent for routes learned from the host side while FRR runs in the DPU cluster. Use the typed frr-k8s next-hop API on advertised prefixes and reconcile node updates when gateway or chassis annotations change so the generated next-hop config follows node state. Signed-off-by: Tim Rozet --- .../routeadvertisements/controller.go | 56 +++++++++++ .../routeadvertisements/controller_test.go | 99 ++++++++++++++++--- go-controller/pkg/types/const.go | 1 + 3 files changed, 145 insertions(+), 11 deletions(-) diff --git a/go-controller/pkg/clustermanager/routeadvertisements/controller.go b/go-controller/pkg/clustermanager/routeadvertisements/controller.go index be1784b969..43c4c474ed 100644 --- a/go-controller/pkg/clustermanager/routeadvertisements/controller.go +++ b/go-controller/pkg/clustermanager/routeadvertisements/controller.go @@ -841,6 +841,11 @@ func (c *Controller) generateFRRConfiguration( nodeV4, _, _ := strings.Cut(nodeIfAddr.IPv4, "/") nodeV6, _, _ := strings.Cut(nodeIfAddr.IPv6, "/") + dpuHostGatewayNextHops, err := getDPUHostGatewayNextHops(node) + if err != nil { + return nil, err + } + targetRouter.Neighbors = make([]frrtypes.Neighbor, 0, len(source.Spec.BGP.Routers[i].Neighbors)) for _, neighbor := range source.Spec.BGP.Routers[i].Neighbors { // Skip neighbors that are the node itself @@ -877,6 +882,13 @@ func (c *Controller) generateFRRConfiguration( Prefixes: advertisePrefixes, }, } + if nextHop := dpuHostGatewayNextHops[isIPV6]; nextHop != "" { + if isIPV6 { + neighbor.ToAdvertise.NextHop.IPv6 = nextHop + } else { + neighbor.ToAdvertise.NextHop.IPv4 = nextHop + } + } // For no-overlay networks, add routes to pod subnets to the accepted routes list // frr-k8s will merge the prefixes from both the generated and the base FRRConfiguration @@ -1145,6 +1157,48 @@ func (c *Controller) generateFRRConfiguration( return new, nil } +func getDPUHostGatewayNextHops(node *corev1.Node) (map[bool]string, error) { + if config.Gateway.Mode != config.GatewayModeShared { + return nil, nil + } + if _, ok := node.Labels[types.OvnDPUHostNodeLabel]; !ok { + return nil, nil + } + + // ParseNodeL3GatewayAnnotation also requires the chassis ID for enabled + // gateways, but reports a missing chassis ID as a config error. Check it + // first so a DPU host that is still initializing leaves the RA pending. + if _, err := util.ParseNodeChassisIDAnnotation(node); err != nil { + if util.IsAnnotationNotSetError(err) { + return nil, fmt.Errorf("%w: waiting for chassis ID annotation to be set for DPU host node %q: %w", + errPending, node.Name, err) + } + return nil, fmt.Errorf("%w: failed to parse chassis ID annotation for DPU host node %q: %w", + errConfig, node.Name, err) + } + + gatewayConfig, err := util.ParseNodeL3GatewayAnnotation(node) + if err != nil { + if util.IsAnnotationNotSetError(err) { + return nil, fmt.Errorf("%w: waiting for L3 gateway annotation to be set for DPU host node %q: %w", + errPending, node.Name, err) + } + return nil, fmt.Errorf("%w: failed to parse L3 gateway annotation for DPU host node %q: %w", + errConfig, node.Name, err) + } + nextHops := map[bool]string{} + for _, ipNet := range gatewayConfig.IPAddresses { + if ipNet == nil || ipNet.IP == nil { + continue + } + nextHops[ipNet.IP.To4() == nil] = ipNet.IP.String() + } + if len(nextHops) == 0 { + return nil, fmt.Errorf("%w: no shared gateway IP addresses found for DPU host node %q", errConfig, node.Name) + } + return nextHops, nil +} + // vtepCIDRPrefixSelectors converts VTEP CIDRs into FRR PrefixSelectors that // accept host routes (/32 for IPv4, /128 for IPv6) within each CIDR range. func vtepCIDRPrefixSelectors(cidrs []string) []frrtypes.PrefixSelector { @@ -1547,6 +1601,8 @@ func nodeNeedsUpdate(oldObj, newObj *corev1.Node) bool { !reflect.DeepEqual(oldObj.Labels, newObj.Labels) || util.NodeSubnetAnnotationChanged(oldObj, newObj) || oldObj.Annotations[util.OvnNodeIfAddr] != newObj.Annotations[util.OvnNodeIfAddr] || + util.NodeL3GatewayAnnotationChanged(oldObj, newObj) || + util.NodeChassisIDAnnotationChanged(oldObj, newObj) || util.NodeVTEPsAnnotationChanged(oldObj, newObj) } diff --git a/go-controller/pkg/clustermanager/routeadvertisements/controller_test.go b/go-controller/pkg/clustermanager/routeadvertisements/controller_test.go index 37e6938abb..6c5c505055 100644 --- a/go-controller/pkg/clustermanager/routeadvertisements/controller_test.go +++ b/go-controller/pkg/clustermanager/routeadvertisements/controller_test.go @@ -133,13 +133,15 @@ func (tn testNamespace) Namespace() *corev1.Namespace { } type testNode struct { - Name string - Generation int - Labels map[string]string - PrimaryAddressAnnotation string - SubnetsAnnotation string - VTEPIPs map[string][]string - RawVTEPAnnotation *string + Name string + Generation int + Labels map[string]string + PrimaryAddressAnnotation string + SubnetsAnnotation string + L3GatewayConfigAnnotation string + ChassisIDAnnotation string + VTEPIPs map[string][]string + RawVTEPAnnotation *string } func (tn testNode) Node() *corev1.Node { @@ -151,6 +153,13 @@ func (tn testNode) Node() *corev1.Node { "k8s.ovn.org/node-subnets": tn.SubnetsAnnotation, util.OvnNodeIfAddr: primaryAddressAnnotation, } + if tn.L3GatewayConfigAnnotation != "" { + annotations[util.OvnNodeL3GatewayConfig] = tn.L3GatewayConfigAnnotation + annotations[util.OvnNodeChassisID] = "chassis-" + tn.Name + } + if tn.ChassisIDAnnotation != "" { + annotations[util.OvnNodeChassisID] = tn.ChassisIDAnnotation + } if tn.RawVTEPAnnotation != nil { annotations[util.OVNNodeVTEPs] = *tn.RawVTEPAnnotation } else if len(tn.VTEPIPs) > 0 { @@ -182,6 +191,8 @@ type testNeighbor struct { Address string DisableMP *bool Advertise []string + NextHopV4 string + NextHopV6 string Receive []testPrefixSelector } @@ -195,6 +206,10 @@ func (tn testNeighbor) Neighbor() frrapi.Neighbor { Mode: frrapi.AllowRestricted, Prefixes: tn.Advertise, }, + NextHop: frrapi.NextHop{ + IPv4: tn.NextHopV4, + IPv6: tn.NextHopV6, + }, }, } if tn.DisableMP != nil { @@ -469,6 +484,7 @@ func TestController_reconcile(t *testing.T) { vteps []*vtepv1.VTEP reconcile string transport string + gatewayMode config.GatewayMode ipv6 bool wantErr bool expectAcceptedStatus metav1.ConditionStatus @@ -910,6 +926,44 @@ func TestController_reconcile(t *testing.T) { }, expectNADAnnotations: map[string]map[string]string{"default": {types.OvnRouteAdvertisementsKey: "[\"ra\"]"}}, }, + { + name: "reconciles pod RouteAdvertisement for DPU host default network with next-hop", + ra: &testRA{Name: "ra", AdvertisePods: true, SelectsDefault: true}, + gatewayMode: config.GatewayModeShared, + frrConfigs: []*testFRRConfig{ + { + Name: "frrConfig", + Namespace: frrNamespace, + Routers: []*testRouter{ + {ASN: 1, Neighbors: []*testNeighbor{ + {ASN: 1, Address: "1.0.0.100"}, + }}, + }, + }, + }, + nodes: []*testNode{ + { + Name: "node", + Labels: map[string]string{types.OvnDPUHostNodeLabel: ""}, + SubnetsAnnotation: "{\"default\":\"1.1.0.0/24\"}", + L3GatewayConfigAnnotation: `{"default":{"mode":"shared","mac-address":"52:54:00:4c:e6:00","ip-addresses":["172.18.255.254/16"],"next-hops":["172.18.0.1"]}}`, + }, + }, + reconcile: "ra", + expectAcceptedStatus: metav1.ConditionTrue, + expectFRRConfigs: []*testFRRConfig{ + { + Labels: map[string]string{types.OvnRouteAdvertisementsKey: "ra"}, + Annotations: map[string]string{types.OvnRouteAdvertisementsKey: "ra/frrConfig/node"}, + NodeSelector: map[string]string{"kubernetes.io/hostname": "node"}, + Routers: []*testRouter{ + {ASN: 1, Prefixes: []string{"1.1.0.0/24"}, Neighbors: []*testNeighbor{ + {ASN: 1, Address: "1.0.0.100", Advertise: []string{"1.1.0.0/24"}, NextHopV4: "172.18.255.254"}, + }}, + }}, + }, + expectNADAnnotations: map[string]map[string]string{"default": {types.OvnRouteAdvertisementsKey: "[\"ra\"]"}}, + }, { name: "reconciles pod RouteAdvertisement for CUDN in no-overlay mode with ToReceive routes including CUDN pod subnets", ra: &testRA{Name: "ra", AdvertisePods: true, SelectsDefault: true, NetworkSelector: map[string]string{"selected": "true"}}, @@ -2157,6 +2211,9 @@ exit config.OVNKubernetesFeature.EnableEVPN = true // satisfy EVPN LGW restriction, otherwise no effect config.Gateway.Mode = config.GatewayModeLocal + if tt.gatewayMode != "" { + config.Gateway.Mode = tt.gatewayMode + } fakeClientset := util.GetOVNClientset().GetClusterManagerClientset() addGenerateNameReactor[*frrfake.Clientset](fakeClientset.FRRClient) @@ -2232,15 +2289,17 @@ exit defer wf.Shutdown() // wait for caches to sync - cache.WaitForCacheSync( - context.Background().Done(), + hasSynced := []cache.InformerSynced{ wf.RouteAdvertisementsInformer().Informer().HasSynced, wf.FRRConfigurationsInformer().Informer().HasSynced, wf.NADInformer().Informer().HasSynced, wf.NodeCoreInformer().Informer().HasSynced, wf.EgressIPInformer().Informer().HasSynced, - wf.VTEPInformer().Informer().HasSynced, - ) + } + if config.Gateway.Mode == config.GatewayModeLocal { + hasSynced = append(hasSynced, wf.VTEPInformer().Informer().HasSynced) + } + cache.WaitForCacheSync(context.Background().Done(), hasSynced...) err = nm.Start() // some test cases start with a bad RA status, avoid asserting @@ -2499,6 +2558,24 @@ func TestUpdates(t *testing.T) { newObject: &testNode{Name: "eip", PrimaryAddressAnnotation: "new"}, expectedReconcile: []string{"ra1", "ra2", "ra3"}, }, + { + name: "reconciles all RAs on updated Node L3 gateway annotation", + oldObject: &testNode{ + Name: "eip", + L3GatewayConfigAnnotation: `{"default":{"mode":"shared","mac-address":"52:54:00:4c:e6:00","ip-addresses":["172.18.255.254/16"],"next-hops":["172.18.0.1"]}}`, + }, + newObject: &testNode{ + Name: "eip", + L3GatewayConfigAnnotation: `{"default":{"mode":"shared","mac-address":"52:54:00:d4:ac:00","ip-addresses":["172.18.255.253/16"],"next-hops":["172.18.0.1"]}}`, + }, + expectedReconcile: []string{"ra1", "ra2", "ra3"}, + }, + { + name: "reconciles all RAs on updated Node chassis ID annotation", + oldObject: &testNode{Name: "eip", ChassisIDAnnotation: "old"}, + newObject: &testNode{Name: "eip", ChassisIDAnnotation: "new"}, + expectedReconcile: []string{"ra1", "ra2", "ra3"}, + }, { name: "reconciles all RAs on new Node VTEP annotation", oldObject: &testNode{Name: "eip"}, diff --git a/go-controller/pkg/types/const.go b/go-controller/pkg/types/const.go index 1945f144ba..f5e41d5f4e 100644 --- a/go-controller/pkg/types/const.go +++ b/go-controller/pkg/types/const.go @@ -194,6 +194,7 @@ const ( OvnK8sTopoAnno = OvnK8sPrefix + "/" + "topology-version" OvnK8sSmallMTUTaintKey = OvnK8sPrefix + "/" + "mtu-too-small" OvnRouteAdvertisementsKey = OvnK8sPrefix + "/route-advertisements" + OvnDPUHostNodeLabel = OvnK8sPrefix + "/dpu-host" // name of the configmap used to synchronize status (e.g. watch for topology changes) OvnK8sStatusCMName = "control-plane-status" From fa24ddb50773fffb5a263fd62acb56523019b06e Mon Sep 17 00:00:00 2001 From: Tim Rozet Date: Thu, 4 Jun 2026 13:28:56 -0400 Subject: [PATCH 16/51] Fix using docker for dpu-sim CI lane The CI lane was setting up docker, but then executing dpu-sim with podman. We need to use docker anyway in anticipation of landing https://github.com/ovn-kubernetes/dpu-simulator/pull/26 Signed-off-by: Tim Rozet --- .github/workflows/kind-dpu-offload.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/kind-dpu-offload.yml b/.github/workflows/kind-dpu-offload.yml index c3a67589f2..1a4e0bd7ff 100644 --- a/.github/workflows/kind-dpu-offload.yml +++ b/.github/workflows/kind-dpu-offload.yml @@ -24,6 +24,7 @@ concurrency: env: DPU_SIM_REF: main OVN_KUBERNETES_PATH: ${{ github.workspace }}/ovn-kubernetes + KIND_EXPERIMENTAL_PROVIDER: docker jobs: kind-dpu-offload: From cc84a978df6ad0763da642c9aabdfc861ac14ac0 Mon Sep 17 00:00:00 2001 From: Tim Rozet Date: Thu, 4 Jun 2026 10:14:19 -0400 Subject: [PATCH 17/51] Stop using deprecated frr-k8s DisableMP field The frr-k8s API now deprecates DisableMP in favor of the inverse DualStackAddressFamily field. Stop writing and reading the deprecated field so staticcheck does not fail after the vendor bump. Keep the existing address-family-specific session behavior by leaving DualStackAddressFamily unset in generated managed BGP configs, and reject selected FRRConfigurations that enable it in route advertisements. Signed-off-by: Tim Rozet --- .../clustermanager/managedbgp/controller.go | 10 ++-- .../managedbgp/controller_test.go | 12 ++-- .../routeadvertisements/controller.go | 29 +++++---- .../routeadvertisements/controller_test.go | 59 ++++++++++++++----- 4 files changed, 72 insertions(+), 38 deletions(-) diff --git a/go-controller/pkg/clustermanager/managedbgp/controller.go b/go-controller/pkg/clustermanager/managedbgp/controller.go index 8d82a047fe..619e3167a0 100644 --- a/go-controller/pkg/clustermanager/managedbgp/controller.go +++ b/go-controller/pkg/clustermanager/managedbgp/controller.go @@ -258,16 +258,14 @@ func (c *Controller) ensureBaseFRRConfiguration(allNodes []*corev1.Node) error { v4, v6 := util.GetNodeInternalAddrs(node) if v4 != nil { neighbors = append(neighbors, frrtypes.Neighbor{ - Address: v4.String(), - ASN: config.ManagedBGP.ASNumber, - DisableMP: true, + Address: v4.String(), + ASN: config.ManagedBGP.ASNumber, }) } if v6 != nil { neighbors = append(neighbors, frrtypes.Neighbor{ - Address: v6.String(), - ASN: config.ManagedBGP.ASNumber, - DisableMP: true, + Address: v6.String(), + ASN: config.ManagedBGP.ASNumber, }) } } diff --git a/go-controller/pkg/clustermanager/managedbgp/controller_test.go b/go-controller/pkg/clustermanager/managedbgp/controller_test.go index 17b171539c..0227d47312 100644 --- a/go-controller/pkg/clustermanager/managedbgp/controller_test.go +++ b/go-controller/pkg/clustermanager/managedbgp/controller_test.go @@ -159,9 +159,9 @@ var _ = ginkgo.Describe("Managed BGP Controller", func() { addresses := []string{baseConfig.Spec.BGP.Routers[0].Neighbors[0].Address, baseConfig.Spec.BGP.Routers[0].Neighbors[1].Address} gomega.Expect(addresses).To(gomega.ConsistOf("10.0.0.1", "10.0.0.2")) - // Verify DisableMP is set + // Verify neighbors use address-family-specific sessions. for _, neighbor := range baseConfig.Spec.BGP.Routers[0].Neighbors { - gomega.Expect(neighbor.DisableMP).To(gomega.BeTrue()) + gomega.Expect(neighbor.DualStackAddressFamily).To(gomega.BeFalse()) gomega.Expect(neighbor.ASN).To(gomega.Equal(uint32(64512))) } }) @@ -874,9 +874,9 @@ var _ = ginkgo.Describe("Managed BGP Controller", func() { drifted := baseConfig.DeepCopy() drifted.Labels = map[string]string{"drifted": "true"} drifted.Spec.BGP.Routers[0].Neighbors = []frrtypes.Neighbor{{ - Address: "10.0.0.99", - ASN: config.ManagedBGP.ASNumber, - DisableMP: false, + Address: "10.0.0.99", + ASN: config.ManagedBGP.ASNumber, + DualStackAddressFamily: true, }} _, err = frrFakeClient.ApiV1beta1().FRRConfigurations(config.ManagedBGP.FRRNamespace).Update(context.TODO(), drifted, metav1.UpdateOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -893,7 +893,7 @@ var _ = ginkgo.Describe("Managed BGP Controller", func() { return fmt.Errorf("expected single-node full-mesh config to be restored") } neighbor := cfg.Spec.BGP.Routers[0].Neighbors[0] - if neighbor.Address != "10.0.0.1" || !neighbor.DisableMP { + if neighbor.Address != "10.0.0.1" || neighbor.DualStackAddressFamily { return fmt.Errorf("base FRRConfiguration not restored") } return nil diff --git a/go-controller/pkg/clustermanager/routeadvertisements/controller.go b/go-controller/pkg/clustermanager/routeadvertisements/controller.go index 43c4c474ed..7173b0f87d 100644 --- a/go-controller/pkg/clustermanager/routeadvertisements/controller.go +++ b/go-controller/pkg/clustermanager/routeadvertisements/controller.go @@ -853,16 +853,16 @@ func (c *Controller) generateFRRConfiguration( continue } - // If MultiProtocol is enabled (default) then a BGP session carries - // prefixes of both IPv4 and IPv6 families. Our problem is that with - // an IPv4 session, FRR can incorrectly pick the masquerade IPv6 - // address (instead of the real address) as next hop for IPv6 - // prefixes and that won't work. Note that with a dedicated IPv6 - // session that can't happen since FRR will use the same address - // that was used to establish the session. Let's enforce the use of - // DisableMP for now. - if !neighbor.DisableMP { - return nil, fmt.Errorf("%w: DisableMP==false not supported, seen on FRRConfiguration %s/%s neighbor %s", + // If the dual-stack address family is enabled then a BGP session + // carries prefixes of both IPv4 and IPv6 families. Our problem is + // that with an IPv4 session, FRR can incorrectly pick the + // masquerade IPv6 address (instead of the real address) as next + // hop for IPv6 prefixes and that won't work. Note that with a + // dedicated IPv6 session that can't happen since FRR will use the + // same address that was used to establish the session. Enforce + // address-family-specific sessions for now. + if neighbor.DualStackAddressFamily { + return nil, fmt.Errorf("%w: DualStackAddressFamily==true not supported, seen on FRRConfiguration %s/%s neighbor %s", errConfig, source.Namespace, source.Name, @@ -1077,8 +1077,13 @@ func (c *Controller) generateFRRConfiguration( vtepRouter.Prefixes = vtepIPs vtepRouter.Neighbors = nil // will rebuild below for _, neighbor := range router.Neighbors { - if !neighbor.DisableMP { - continue + if neighbor.DualStackAddressFamily { + return nil, fmt.Errorf("%w: DualStackAddressFamily==true not supported, seen on FRRConfiguration %s/%s neighbor %s", + errConfig, + source.Namespace, + source.Name, + neighbor.Address, + ) } isIPV6 := utilnet.IsIPv6String(neighbor.Address) filteredVTEPIPs := util.MatchAllIPNetsStringFamily(isIPV6, vtepIPs) diff --git a/go-controller/pkg/clustermanager/routeadvertisements/controller_test.go b/go-controller/pkg/clustermanager/routeadvertisements/controller_test.go index 6c5c505055..2f7b78830b 100644 --- a/go-controller/pkg/clustermanager/routeadvertisements/controller_test.go +++ b/go-controller/pkg/clustermanager/routeadvertisements/controller_test.go @@ -187,20 +187,19 @@ type testPrefixSelector struct { } type testNeighbor struct { - ASN uint32 - Address string - DisableMP *bool - Advertise []string - NextHopV4 string - NextHopV6 string - Receive []testPrefixSelector + ASN uint32 + Address string + DualStackAddressFamily *bool + Advertise []string + NextHopV4 string + NextHopV6 string + Receive []testPrefixSelector } func (tn testNeighbor) Neighbor() frrapi.Neighbor { n := frrapi.Neighbor{ - ASN: tn.ASN, - Address: tn.Address, - DisableMP: true, + ASN: tn.ASN, + Address: tn.Address, ToAdvertise: frrapi.Advertise{ Allowed: frrapi.AllowedOutPrefixes{ Mode: frrapi.AllowRestricted, @@ -212,8 +211,8 @@ func (tn testNeighbor) Neighbor() frrapi.Neighbor { }, }, } - if tn.DisableMP != nil { - n.DisableMP = *tn.DisableMP + if tn.DualStackAddressFamily != nil { + n.DualStackAddressFamily = *tn.DualStackAddressFamily } if len(tn.Receive) > 0 { prefixSelectors := make([]frrapi.PrefixSelector, 0, len(tn.Receive)) @@ -1150,7 +1149,7 @@ func TestController_reconcile(t *testing.T) { expectAcceptedStatus: metav1.ConditionFalse, }, { - name: "fails to reconcile if DisableMP is unset", + name: "fails to reconcile if dual-stack address family is set", ra: &testRA{Name: "ra", AdvertisePods: true}, frrConfigs: []*testFRRConfig{ { @@ -1158,7 +1157,7 @@ func TestController_reconcile(t *testing.T) { Namespace: frrNamespace, Routers: []*testRouter{ {ASN: 1, Prefixes: []string{"1.1.1.0/24"}, Neighbors: []*testNeighbor{ - {ASN: 1, Address: "1.0.0.100", DisableMP: ptr.To(false)}, + {ASN: 1, Address: "1.0.0.100", DualStackAddressFamily: ptr.To(true)}, }}, }, }, @@ -1716,6 +1715,38 @@ exit }, expectNADAnnotations: map[string]map[string]string{"blue6": {types.OvnRouteAdvertisementsKey: "[\"ra\"]"}}, }, + { + name: "fails to reconcile EVPN target VRF when cloned default-VRF neighbor has dual-stack address family set", + ra: &testRA{Name: "ra", TargetVRF: "red", AdvertisePods: true, NetworkSelector: map[string]string{"selected": "true"}}, + frrConfigs: []*testFRRConfig{ + { + Name: "frrConfig", + Namespace: frrNamespace, + Routers: []*testRouter{ + {ASN: 65000, Neighbors: []*testNeighbor{ + {ASN: 65000, Address: "192.168.1.1", DualStackAddressFamily: ptr.To(true)}, + }}, + {ASN: 65000, VRF: "red", Prefixes: []string{"10.1.0.0/16"}, Neighbors: []*testNeighbor{ + {ASN: 65000, Address: "192.168.1.1"}, + }}, + }, + }, + }, + nads: []*testNAD{ + {Name: "red", Namespace: "red", Network: util.GenerateCUDNNetworkName("red"), + Topology: "layer2", Subnet: "10.1.0.0/16", Labels: map[string]string{"selected": "true"}, + EVPNVTEPName: "my-vtep", EVPNMACVRFVNI: 1000, EVPNMACVRFRouteTarget: "65000:1000"}, + }, + vteps: []*vtepv1.VTEP{ + { + ObjectMeta: metav1.ObjectMeta{Name: "my-vtep"}, + Spec: vtepv1.VTEPSpec{CIDRs: []vtepv1.CIDR{"100.64.0.0/16"}}, + }, + }, + nodes: []*testNode{{Name: "node", SubnetsAnnotation: "{\"default\":\"1.1.0.0/24\"}", VTEPIPs: map[string][]string{"my-vtep": {"100.64.0.1"}}}}, + reconcile: "ra", + expectAcceptedStatus: metav1.ConditionFalse, + }, { name: "advertises VTEP IP /32 in default-VRF router prefixes for EVPN MAC-VRF with target VRF", ra: &testRA{Name: "ra", TargetVRF: "red", AdvertisePods: true, NetworkSelector: map[string]string{"selected": "true"}}, From 351aa103bb50480077be517fad6cf6f2078b1d2a Mon Sep 17 00:00:00 2001 From: Tim Rozet Date: Thu, 4 Jun 2026 10:58:53 -0400 Subject: [PATCH 18/51] Bump OpenTelemetry to v1.41.0 Update go.opentelemetry.io/otel and otel/trace to v1.41.0 to pick up the fix for GHSA-mh2q-q3fh-2475. Signed-off-by: Tim Rozet --- .../otel/metric/{v1.40.0 => v1.41.0}/LICENSE | 0 .../otel/trace/{v1.40.0 => v1.41.0}/LICENSE | 0 .../otel/{v1.40.0 => v1.41.0}/LICENSE | 0 go-controller/go.mod | 4 ++-- go-controller/go.sum | 12 ++++++------ .../go.opentelemetry.io/otel/trace/tracestate.go | 3 +++ go-controller/vendor/modules.txt | 4 ++-- 7 files changed, 13 insertions(+), 10 deletions(-) rename LICENSES/packages/go.opentelemetry.io/otel/metric/{v1.40.0 => v1.41.0}/LICENSE (100%) rename LICENSES/packages/go.opentelemetry.io/otel/trace/{v1.40.0 => v1.41.0}/LICENSE (100%) rename LICENSES/packages/go.opentelemetry.io/otel/{v1.40.0 => v1.41.0}/LICENSE (100%) diff --git a/LICENSES/packages/go.opentelemetry.io/otel/metric/v1.40.0/LICENSE b/LICENSES/packages/go.opentelemetry.io/otel/metric/v1.41.0/LICENSE similarity index 100% rename from LICENSES/packages/go.opentelemetry.io/otel/metric/v1.40.0/LICENSE rename to LICENSES/packages/go.opentelemetry.io/otel/metric/v1.41.0/LICENSE diff --git a/LICENSES/packages/go.opentelemetry.io/otel/trace/v1.40.0/LICENSE b/LICENSES/packages/go.opentelemetry.io/otel/trace/v1.41.0/LICENSE similarity index 100% rename from LICENSES/packages/go.opentelemetry.io/otel/trace/v1.40.0/LICENSE rename to LICENSES/packages/go.opentelemetry.io/otel/trace/v1.41.0/LICENSE diff --git a/LICENSES/packages/go.opentelemetry.io/otel/v1.40.0/LICENSE b/LICENSES/packages/go.opentelemetry.io/otel/v1.41.0/LICENSE similarity index 100% rename from LICENSES/packages/go.opentelemetry.io/otel/v1.40.0/LICENSE rename to LICENSES/packages/go.opentelemetry.io/otel/v1.41.0/LICENSE diff --git a/go-controller/go.mod b/go-controller/go.mod index 02962ab37a..4cafa209d1 100644 --- a/go-controller/go.mod +++ b/go-controller/go.mod @@ -135,8 +135,8 @@ require ( github.com/x448/float16 v0.8.4 // indirect github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 // indirect go.opencensus.io v0.24.0 // indirect - go.opentelemetry.io/otel v1.40.0 // indirect - go.opentelemetry.io/otel/trace v1.40.0 // indirect + go.opentelemetry.io/otel v1.41.0 // indirect + go.opentelemetry.io/otel/trace v1.41.0 // indirect go.yaml.in/yaml/v2 v2.4.3 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/crypto v0.46.0 // indirect diff --git a/go-controller/go.sum b/go-controller/go.sum index 4978069424..daf2634ee9 100644 --- a/go-controller/go.sum +++ b/go-controller/go.sum @@ -833,16 +833,16 @@ go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= -go.opentelemetry.io/otel v1.40.0 h1:oA5YeOcpRTXq6NN7frwmwFR0Cn3RhTVZvXsP4duvCms= -go.opentelemetry.io/otel v1.40.0/go.mod h1:IMb+uXZUKkMXdPddhwAHm6UfOwJyh4ct1ybIlV14J0g= -go.opentelemetry.io/otel/metric v1.40.0 h1:rcZe317KPftE2rstWIBitCdVp89A2HqjkxR3c11+p9g= -go.opentelemetry.io/otel/metric v1.40.0/go.mod h1:ib/crwQH7N3r5kfiBZQbwrTge743UDc7DTFVZrrXnqc= +go.opentelemetry.io/otel v1.41.0 h1:YlEwVsGAlCvczDILpUXpIpPSL/VPugt7zHThEMLce1c= +go.opentelemetry.io/otel v1.41.0/go.mod h1:Yt4UwgEKeT05QbLwbyHXEwhnjxNO6D8L5PQP51/46dE= +go.opentelemetry.io/otel/metric v1.41.0 h1:rFnDcs4gRzBcsO9tS8LCpgR0dxg4aaxWlJxCno7JlTQ= +go.opentelemetry.io/otel/metric v1.41.0/go.mod h1:xPvCwd9pU0VN8tPZYzDZV/BMj9CM9vs00GuBjeKhJps= go.opentelemetry.io/otel/sdk v1.40.0 h1:KHW/jUzgo6wsPh9At46+h4upjtccTmuZCFAc9OJ71f8= go.opentelemetry.io/otel/sdk v1.40.0/go.mod h1:Ph7EFdYvxq72Y8Li9q8KebuYUr2KoeyHx0DRMKrYBUE= go.opentelemetry.io/otel/sdk/metric v1.39.0 h1:cXMVVFVgsIf2YL6QkRF4Urbr/aMInf+2WKg+sEJTtB8= go.opentelemetry.io/otel/sdk/metric v1.39.0/go.mod h1:xq9HEVH7qeX69/JnwEfp6fVq5wosJsY1mt4lLfYdVew= -go.opentelemetry.io/otel/trace v1.40.0 h1:WA4etStDttCSYuhwvEa8OP8I5EWu24lkOzp+ZYblVjw= -go.opentelemetry.io/otel/trace v1.40.0/go.mod h1:zeAhriXecNGP/s2SEG3+Y8X9ujcJOTqQ5RgdEJcawiA= +go.opentelemetry.io/otel/trace v1.41.0 h1:Vbk2co6bhj8L59ZJ6/xFTskY+tGAbOnCtQGVVa9TIN0= +go.opentelemetry.io/otel/trace v1.41.0/go.mod h1:U1NU4ULCoxeDKc09yCWdWe+3QoyweJcISEVa1RBzOis= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= diff --git a/go-controller/vendor/go.opentelemetry.io/otel/trace/tracestate.go b/go-controller/vendor/go.opentelemetry.io/otel/trace/tracestate.go index 073adae2fa..df65c5cc72 100644 --- a/go-controller/vendor/go.opentelemetry.io/otel/trace/tracestate.go +++ b/go-controller/vendor/go.opentelemetry.io/otel/trace/tracestate.go @@ -61,6 +61,9 @@ func checkValue(val string) bool { func checkKeyRemain(key string) bool { // ( lcalpha / DIGIT / "_" / "-"/ "*" / "/" ) for _, v := range key { + if v > 127 { + return false + } if isAlphaNum(byte(v)) { continue } diff --git a/go-controller/vendor/modules.txt b/go-controller/vendor/modules.txt index c2c9d6d019..4001755d55 100644 --- a/go-controller/vendor/modules.txt +++ b/go-controller/vendor/modules.txt @@ -523,14 +523,14 @@ go.opencensus.io/internal go.opencensus.io/trace go.opencensus.io/trace/internal go.opencensus.io/trace/tracestate -# go.opentelemetry.io/otel v1.40.0 +# go.opentelemetry.io/otel v1.41.0 ## explicit; go 1.24.0 go.opentelemetry.io/otel/attribute go.opentelemetry.io/otel/attribute/internal go.opentelemetry.io/otel/attribute/internal/xxhash go.opentelemetry.io/otel/codes go.opentelemetry.io/otel/semconv/v1.39.0 -# go.opentelemetry.io/otel/trace v1.40.0 +# go.opentelemetry.io/otel/trace v1.41.0 ## explicit; go 1.24.0 go.opentelemetry.io/otel/trace go.opentelemetry.io/otel/trace/embedded From 4ab105d4582f37d32a6c4d5d3e5fefa80ddbf344 Mon Sep 17 00:00:00 2001 From: Riccardo Ravaioli Date: Thu, 4 Jun 2026 14:38:02 +0200 Subject: [PATCH 19/51] ci: wait for service reachability after dual-stack conversion The dual-stack conversion setup validates service reachability immediately after restarting OVN-Kubernetes and recreating pods. In a failed CI job (https://github.com/ovn-kubernetes/ovn-kubernetes/actions/runs/26942442261/job/79488476191?pr=6498), the IPv4 service curl succeeded, but the first IPv6 service curl timed out within its single 5-second connect window. The log only proves that the IPv6 service was not reachable immediately after the setup readiness checks completed. The exact failed-attempt artifacts show that OVNK had transient IPv6 setup errors during conversion, but the OVN DB later had the IPv6 service VIP and replacement backends before the curl ran. So the failure does not prove whether IPv6 service connectivity would have converged shortly afterward or whether the cluster had a persistent IPv6 connectivity problem. Add a bounded retry around the post-conversion service curl checks, so transient post-conversion service/datapath convergence does not fail the setup step, while still failing if IPv4 or IPv6 service connectivity never becomes available. Assisted-by: Codex Signed-off-by: Riccardo Ravaioli --- contrib/kind-dual-stack-conversion.sh | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/contrib/kind-dual-stack-conversion.sh b/contrib/kind-dual-stack-conversion.sh index 0b885440a0..c3cd14701b 100755 --- a/contrib/kind-dual-stack-conversion.sh +++ b/contrib/kind-dual-stack-conversion.sh @@ -46,6 +46,26 @@ convert_cni() { echo "Updated CNI" } +curl_service_from_node() { + local node=$1 + local target=$2 + local attempts=${3:-6} + local interval=${4:-2} + local timeout=${5:-5} + + for i in $(seq 1 "${attempts}"); do + if docker exec "${node}" curl --max-time "${timeout}" --silent --show-error --fail "${target}" >/dev/null; then + echo "Validated ${target} from node ${node} after ${i} attempt(s)" + return 0 + fi + if [ "${i}" -eq "${attempts}" ]; then + return 1 + fi + echo "Waiting for ${target} to be reachable from node ${node} (${i}/${attempts})" + sleep "${interval}" + done +} + convert_k8s_control_plane(){ # Kubeadm installs apiserver and controller-manager as static pods # one the configuration has changed kubelet restart them @@ -288,11 +308,11 @@ IPS=() CLUSTER_IPV4=$(kubectl get services my-service-v4 -o jsonpath='{.spec.clusterIPs[0]}') IPS+=("${CLUSTER_IPV4}:8081") CLUSTER_IPV6=$(kubectl get services my-service-v6 -o jsonpath='{.spec.clusterIPs[0]}') -IPS+=("\[${CLUSTER_IPV6}\]:8080") +IPS+=("[${CLUSTER_IPV6}]:8080") for n in $NODES; do - for ip in ${IPS[@]}; do - docker exec $n curl --connect-timeout 5 $ip + for ip in "${IPS[@]}"; do + curl_service_from_node "${n}" "${ip}" done done From d9cefde5f57122c1a71e1c895b08e998a4c04168 Mon Sep 17 00:00:00 2001 From: Tim Rozet Date: Thu, 4 Jun 2026 16:41:29 -0400 Subject: [PATCH 20/51] Use FRR-K8S next-hop API in kind installs Pin the kind FRR-K8S checkout to the same upstream commit used by the vendored dependency. This lets the installed CRDs preserve toAdvertise.nextHop. Check out that commit explicitly and normalize the demo image context before applying the existing OVN-K demo patch. Signed-off-by: Tim Rozet --- contrib/kind-common.sh | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/contrib/kind-common.sh b/contrib/kind-common.sh index a0cd24a5aa..3f590eb750 100644 --- a/contrib/kind-common.sh +++ b/contrib/kind-common.sh @@ -1155,8 +1155,10 @@ get_kubevirt_release_url() { echo "$kubevirt_release_url" } -readonly FRR_K8S_VERSION=v0.0.21 +readonly FRR_K8S_VERSION=v0.0.0-20260603082256-b43efcb206be +readonly FRR_K8S_GIT_REF=b43efcb206be readonly FRR_K8S_UPSTREAM_FRR_IMAGE=quay.io/frrouting/frr:10.4.1 +readonly FRR_K8S_ALL_IN_ONE_UPSTREAM_FRR_IMAGE=quay.io/frrouting/frr:10.4.3 readonly FRR_DEPLOYED_IMAGE=quay.io/frrouting/frr:10.6.0 # Override to test newer FRR builds in the in-cluster frr-k8s daemonset # without changing the pinned frr-k8s release. @@ -1168,13 +1170,20 @@ clone_frr() { [ -d "$FRR_TMP_DIR" ] || { mkdir -p "$FRR_TMP_DIR" && trap 'rm -rf $FRR_TMP_DIR' EXIT pushd "$FRR_TMP_DIR" || exit 1 - git clone --depth 1 --branch $FRR_K8S_VERSION https://github.com/metallb/frr-k8s + git clone --no-tags --single-branch --branch main https://github.com/metallb/frr-k8s + pushd frr-k8s + git checkout --detach "$FRR_K8S_GIT_REF" + popd # Download the patches curl -Ls https://github.com/jcaamano/frr-k8s/archive/refs/heads/ovnk-bgp-v0.0.21.tar.gz | tar xzvf - frr-k8s-ovnk-bgp-v0.0.21/patches --strip-components 1 # Change into the cloned repo directory before applying patches pushd frr-k8s + # The OVN-K demo patch was authored before upstream bumped the demo + # image. Normalize that context before applying the patch; the image is + # bumped to FRR_EXTERNAL_DEMO_IMAGE below. + sed -i 's|quay.io/frrouting/frr:10.4.3|quay.io/frrouting/frr:9.1.0|g' hack/demo/demo.sh git apply ../patches/* # The upstream frr-k8s demo.sh hardcodes quay.io/frrouting/frr:9.1.0, @@ -1208,7 +1217,7 @@ deploy_frr_external_container() { clone_frr pushd "$FRR_TMP_DIR" || exit 1 - run_kubectl apply -f frr-k8s/charts/frr-k8s/charts/crds/templates/frrk8s.metallb.io_frrconfigurations.yaml + run_kubectl apply -f frr-k8s/charts/frr-k8s/charts/crds/templates/ popd || exit 1 # apply the demo which will deploy an external FRR container that the cluster @@ -1391,7 +1400,7 @@ install_frr_k8s() { replace_in_file_or_exit \ "${FRR_TMP_DIR}"/frr-k8s/config/all-in-one/frr-k8s.yaml \ - "${FRR_K8S_UPSTREAM_FRR_IMAGE}" \ + "${FRR_K8S_ALL_IN_ONE_UPSTREAM_FRR_IMAGE}" \ "${FRR_K8S_FRR_IMAGE}" if [ "${bgp_port}" -ne 0 ]; then From 09cf49448fe8fb54076b75a8da0bd83863134024 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Rigault?= Date: Fri, 5 Jun 2026 17:43:18 +0200 Subject: [PATCH 21/51] [FYI] prevent duplicate logical_router_static_route MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On some OpenShift platforms I notice logical_router_static_route from previous deleted nodes are sometimes leaked and never cleaned, leading to traffic loss between nodes. ovn-nbctl lr-route-list ovn_cluster_router | grep ecmp 10.224.67.0/24 100.88.10.33 dst-ip ecmp 10.224.67.0/24 100.88.3.148 dst-ip ecmp 10.224.192.0/24 100.88.1.94 dst-ip ecmp 10.224.192.0/24 100.88.4.222 dst-ip ecmp Proposal here to make sure no 2 routes with same prefix are created. Tested against a kind cluster in this fashion, where we simulate the addition of a node while a prior route already exists: k exec -ti -n ovn-kubernetes ovnkube-node-btx87 -c nb-ovsdb -- bash [root@ovn-worker ~]# ovn-nbctl lr-route-list ovn_cluster_router IPv4 Routes Route Table
: 100.64.0.2 100.88.0.2 dst-ip 100.64.0.3 100.88.0.3 dst-ip 100.64.0.4 100.64.0.4 dst-ip 10.244.0.0/24 100.88.0.2 dst-ip 10.244.1.0/24 100.88.0.3 dst-ip 10.244.2.0/24 100.64.0.4 src-ip 10.244.0.0/16 100.64.0.4 src-ip [root@ovn-worker ~]# ovn-nbctl lr-route-del ovn_cluster_router 10.244.0.0/24 100.88.0.2 [root@ovn-worker ~]# ovn-nbctl lr-route-add ovn_cluster_router 10.244.0.0/24 1.2.3.4 [root@ovn-worker ~]# ovn-nbctl lr-route-list ovn_cluster_router IPv4 Routes Route Table
: 100.64.0.2 100.88.0.2 dst-ip 100.64.0.3 100.88.0.3 dst-ip 100.64.0.4 100.64.0.4 dst-ip 10.244.0.0/24 1.2.3.4 dst-ip 10.244.1.0/24 100.88.0.3 dst-ip 10.244.2.0/24 100.64.0.4 src-ip 10.244.0.0/16 100.64.0.4 src-ip k exec -ti -n ovn-kubernetes ovnkube-node-btx87 -c ovnkube-controller -- bash pkill -f "bash /root/ovnkube.sh ovnkube-controller-with-node" after restart check there is no duplicate: [root@ovn-worker ~]# ovn-nbctl lr-route-list ovn_cluster_router IPv4 Routes Route Table
: 100.64.0.2 100.88.0.2 dst-ip 100.64.0.3 100.88.0.3 dst-ip 100.64.0.4 100.64.0.4 dst-ip 10.244.0.0/24 100.88.0.2 dst-ip 10.244.1.0/24 100.88.0.3 dst-ip 10.244.2.0/24 100.64.0.4 src-ip 10.244.0.0/16 100.64.0.4 src-ip Signed-off-by: François Rigault --- go-controller/pkg/ovn/zone_interconnect/zone_ic_handler.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/go-controller/pkg/ovn/zone_interconnect/zone_ic_handler.go b/go-controller/pkg/ovn/zone_interconnect/zone_ic_handler.go index 6709520ee9..76f4d42865 100644 --- a/go-controller/pkg/ovn/zone_interconnect/zone_ic_handler.go +++ b/go-controller/pkg/ovn/zone_interconnect/zone_ic_handler.go @@ -665,9 +665,7 @@ func (zic *ZoneInterconnectHandler) addRemoteNodeStaticRoutes(node *corev1.Node, // external-ids, skip types.NetworkExternalID check in the predicate function to replace existing static route // with correct external-ids on an upgrade scenario. p := func(lrsr *nbdb.LogicalRouterStaticRoute) bool { - return lrsr.IPPrefix == prefix && - lrsr.Nexthop == nexthop && - lrsr.ExternalIDs["ic-node"] == node.Name + return lrsr.IPPrefix == prefix } var err error ops, err = libovsdbops.CreateOrReplaceLogicalRouterStaticRouteWithPredicateOps(zic.nbClient, ops, zic.networkClusterRouterName, &logicalRouterStaticRoute, p) From 0fe6e3d969cd2effaac9a37b7721a7aeaffb5692 Mon Sep 17 00:00:00 2001 From: Igal Tsoiref Date: Sat, 6 Jun 2026 08:06:45 -0400 Subject: [PATCH 22/51] dpulease: fix nil pointer panic in CNI health check When DPUNodeLeaseRenewInterval is set to 0 the lease manager is not created, but nc.dpuNodeLeaseManager (a typed nil *Manager) was still passed to NewCNIServer. Go wraps the nil pointer in a non-nil interface value, so the existing `if s.dpuHealth == nil` check doesn't catch it, and calling Ready() on the nil receiver panics. Fix by explicitly passing a nil interface when the manager is not initialized, so the existing nil check works correctly. Signed-off-by: Igal Tsoiref Co-authored-by: Cursor --- go-controller/pkg/node/default_node_network_controller.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/go-controller/pkg/node/default_node_network_controller.go b/go-controller/pkg/node/default_node_network_controller.go index d36607d417..14dd69bcdf 100644 --- a/go-controller/pkg/node/default_node_network_controller.go +++ b/go-controller/pkg/node/default_node_network_controller.go @@ -864,7 +864,11 @@ func (nc *DefaultNodeNetworkController) Init(ctx context.Context) error { if !ok { return fmt.Errorf("cannot get kubeclient for starting CNI server") } - cniServer, err = cni.NewCNIServer(nc.watchFactory, kclient.KClient, nc.networkManager, nc.ovsClient, nc.dpuNodeLeaseManager) + var dpuHealth cni.DPUStatusProvider + if nc.dpuNodeLeaseManager != nil { + dpuHealth = nc.dpuNodeLeaseManager + } + cniServer, err = cni.NewCNIServer(nc.watchFactory, kclient.KClient, nc.networkManager, nc.ovsClient, dpuHealth) if err != nil { return err } From c83b6f3ddc15b69e067795bfc7fad0adc387ccf9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Caama=C3=B1o=20Ruiz?= Date: Mon, 8 Jun 2026 13:19:21 +0000 Subject: [PATCH 23/51] e2e: configure framework TestContext to verify service accounts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jaime Caamaño Ruiz --- test/e2e/testcontext.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/e2e/testcontext.go b/test/e2e/testcontext.go index 3dfaacb7eb..cdc68d207a 100644 --- a/test/e2e/testcontext.go +++ b/test/e2e/testcontext.go @@ -66,6 +66,9 @@ func ProcessTestContextAndSetupLogging() { t.Provider = "skeleton" } + // verify service accounts are created for namespaces before tests run + t.VerifyServiceAccount = true + var err error t.CloudConfig.Provider, err = framework.SetupProviderConfig(t.Provider) if err != nil { From 4b4d0b9a76e0b595215195bfd8848d8324e52916 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Caama=C3=B1o=20Ruiz?= Date: Fri, 29 May 2026 12:05:50 +0000 Subject: [PATCH 24/51] RA controller: rename evpn_rawconfig to rawconfig for future broader use MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jaime Caamaño Ruiz --- .../routeadvertisements/{evpn_rawconfig.go => rawconfig.go} | 0 .../{evpn_rawconfig_test.go => rawconfig_test.go} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename go-controller/pkg/clustermanager/routeadvertisements/{evpn_rawconfig.go => rawconfig.go} (100%) rename go-controller/pkg/clustermanager/routeadvertisements/{evpn_rawconfig_test.go => rawconfig_test.go} (100%) diff --git a/go-controller/pkg/clustermanager/routeadvertisements/evpn_rawconfig.go b/go-controller/pkg/clustermanager/routeadvertisements/rawconfig.go similarity index 100% rename from go-controller/pkg/clustermanager/routeadvertisements/evpn_rawconfig.go rename to go-controller/pkg/clustermanager/routeadvertisements/rawconfig.go diff --git a/go-controller/pkg/clustermanager/routeadvertisements/evpn_rawconfig_test.go b/go-controller/pkg/clustermanager/routeadvertisements/rawconfig_test.go similarity index 100% rename from go-controller/pkg/clustermanager/routeadvertisements/evpn_rawconfig_test.go rename to go-controller/pkg/clustermanager/routeadvertisements/rawconfig_test.go From e08225fc31eb4330cdbde288ba36a8a9f5c5c356 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Caama=C3=B1o=20Ruiz?= Date: Thu, 28 May 2026 17:26:19 +0000 Subject: [PATCH 25/51] RA contrioller: add unicast allowas-in origin to raw FRR config MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When all cluster nodes share the same ASN, as eBGP peers, they need allowas-in origin to accept routes containing their own ASN. from other nodes. This was already configured for l2vpn evpn, but unicast address-families lacked it, preventing nodes from learning each other's routes. For EVPN specifically, this also means VTEP IPs advertised via unicast were rejected, breaking overlay connectivity. Assisted-By: Claude Opus 4.6 Signed-off-by: Jaime Caamaño Ruiz --- .../routeadvertisements/controller.go | 61 +++-- .../routeadvertisements/controller_test.go | 213 ++++++++++++------ .../routeadvertisements/rawconfig.go | 188 ++++++++++++---- .../routeadvertisements/rawconfig_test.go | 131 +++++++++-- go-controller/pkg/util/net.go | 13 ++ 5 files changed, 437 insertions(+), 169 deletions(-) diff --git a/go-controller/pkg/clustermanager/routeadvertisements/controller.go b/go-controller/pkg/clustermanager/routeadvertisements/controller.go index 7173b0f87d..9fa4a60ee8 100644 --- a/go-controller/pkg/clustermanager/routeadvertisements/controller.go +++ b/go-controller/pkg/clustermanager/routeadvertisements/controller.go @@ -54,8 +54,9 @@ import ( const ( generateName = "ovnk-generated-" fieldManager = "clustermanager-routeadvertisements-controller" - // evpnRawConfigPriority is set to an arbitrary value that still allows users to override EVPN config if needed. - evpnRawConfigPriority = 10 + // rawConfigPriority is set to an arbitrary value that still allows users to + // override if needed. + rawConfigPriority = 10 ) var ( @@ -767,6 +768,10 @@ func (c *Controller) generateFRRConfiguration( ) (*frrtypes.FRRConfiguration, error) { var routers []frrtypes.Router + // track neighbors and ASNs to generate raw config later on + vrfNeighbors := map[string][]string{} + vrfASNs := map[string]uint32{} + // go over the source routers for i, router := range source.Spec.BGP.Routers { @@ -911,6 +916,7 @@ func (c *Controller) generateFRRConfiguration( } } + vrfNeighbors[matchedVRF] = append(vrfNeighbors[matchedVRF], neighbor.Address) targetRouter.Neighbors = append(targetRouter.Neighbors, neighbor) } if len(targetRouter.Neighbors) == 0 { @@ -921,6 +927,7 @@ func (c *Controller) generateFRRConfiguration( // append this router to the list of routers we will include in the // generated FRR config and track its index as we might need to add // imports to it + vrfASNs[matchedVRF] = router.ASN routers = append(routers, targetRouter) targetRouterIndex := len(routers) - 1 @@ -961,17 +968,16 @@ func (c *Controller) generateFRRConfiguration( routers = append(routers, importRouter) } } - var globalRouterASN uint32 - var neighbors []string - vrfASNs := map[string]uint32{} - if len(selectedNetworks.macVRFConfigs) > 0 || len(selectedNetworks.ipVRFConfigs) > 0 { + hasEVPN := len(selectedNetworks.macVRFConfigs) > 0 || len(selectedNetworks.ipVRFConfigs) > 0 + if hasEVPN && vrfASNs[""] == 0 { // Look for global router in the source FRRConfiguration, not in the filtered routers for _, router := range source.Spec.BGP.Routers { if router.VRF == "" { // default VRF - globalRouterASN = router.ASN + vrfASNs[""] = router.ASN + vrfNeighbors[""] = make([]string, 0, len(router.Neighbors)) for _, neighbor := range router.Neighbors { - neighbors = append(neighbors, neighbor.Address) + vrfNeighbors[""] = append(vrfNeighbors[""], neighbor.Address) } break } @@ -999,14 +1005,14 @@ func (c *Controller) generateFRRConfiguration( } } // If not in current source, another source will handle it - } else if globalRouterASN > 0 { + } else if vrfASNs[""] > 0 { // VRF router doesn't exist anywhere - create with global ASN klog.Infof("Creating router for EVPN network %q VRF %q with ASN=%d, prefixes=%v", - cfg.NetworkName, cfg.VRFName, globalRouterASN, selectedNetworks.hostNetworkSubnets[cfg.NetworkName]) + cfg.NetworkName, cfg.VRFName, vrfASNs[""], selectedNetworks.hostNetworkSubnets[cfg.NetworkName]) matchedNetworks.Insert(cfg.NetworkName) - vrfASNs[cfg.VRFName] = globalRouterASN + vrfASNs[cfg.VRFName] = vrfASNs[""] routers = append(routers, frrtypes.Router{ - ASN: globalRouterASN, + ASN: vrfASNs[""], VRF: cfg.VRFName, Prefixes: selectedNetworks.hostNetworkSubnets[cfg.NetworkName], }) @@ -1017,7 +1023,7 @@ func (c *Controller) generateFRRConfiguration( // by the global router's EVPN raw config (advertise-all-vni) rather // than by a VRF-specific router. Mark them as matched when a global // router with neighbors is present. - if ra.Spec.TargetVRF == "auto" && globalRouterASN > 0 && len(neighbors) > 0 { + if ra.Spec.TargetVRF == "auto" && vrfASNs[""] > 0 && len(vrfNeighbors[""]) > 0 { for _, cfg := range selectedNetworks.macVRFConfigs { if !ipVRFNetworks.Has(cfg.NetworkName) { matchedNetworks.Insert(cfg.NetworkName) @@ -1035,7 +1041,7 @@ func (c *Controller) generateFRRConfiguration( // router is not included even though it exists in the source (confirmed by globalRouterASN > 0 above). // In that case we create a new default-VRF router from the source // to carry the VTEP IPs. - if vtepIPs := selectedNetworks.vtepIPsByNode[nodeName]; len(vtepIPs) > 0 && globalRouterASN > 0 { + if vtepIPs := selectedNetworks.vtepIPsByNode[nodeName]; len(vtepIPs) > 0 && vrfASNs[""] > 0 { // Build ToReceive prefix selectors from the VTEP CIDRs so each // node accepts routes for all VTEP IPs within these ranges. vtepReceiveSelectors := vtepCIDRPrefixSelectors(selectedNetworks.vtepCIDRs) @@ -1115,15 +1121,14 @@ func (c *Controller) generateFRRConfiguration( } } - // Check if we have anything to generate: routers or EVPN raw config. - // EVPN raw config is generated when we have: - // - A global router (globalRouterASN > 0 && len(neighbors) > 0) for the global EVPN section - // - IP-VRF configs for VRF VNI and VRF EVPN sections - hasEVPNRawConfig := (globalRouterASN > 0 && len(neighbors) > 0) || len(selectedNetworks.ipVRFConfigs) > 0 - if len(routers) == 0 && !hasEVPNRawConfig { - // we ended up with no routers and no EVPN raw config to generate, bail out + // Generate raw config, if any. + // TODO: once frr-k8s provides a typed API for this config, we can use that instead of raw config + rawConfig := generateRawConfig(selectedNetworks, vrfNeighbors, vrfASNs) + if len(routers) == 0 && rawConfig == "" { + // we ended up with no routers and no raw config to generate, bail out return nil, nil } + new := &frrtypes.FRRConfiguration{} new.GenerateName = generateName new.Namespace = source.Namespace @@ -1146,16 +1151,10 @@ func (c *Controller) generateFRRConfiguration( "kubernetes.io/hostname": nodeName, }, } - - // Generate EVPN raw config for the EVPN-specific parts. - // TODO: once frr-k8s provides a typed EVPN API, we can use that instead of raw config - if len(selectedNetworks.macVRFConfigs) > 0 || len(selectedNetworks.ipVRFConfigs) > 0 { - rawConfig := generateEVPNRawConfig(selectedNetworks, globalRouterASN, neighbors, vrfASNs) - if rawConfig != "" { - new.Spec.Raw = frrtypes.RawConfig{ - Priority: evpnRawConfigPriority, - Config: rawConfig, - } + if rawConfig != "" { + new.Spec.Raw = frrtypes.RawConfig{ + Priority: rawConfigPriority, + Config: rawConfig, } } diff --git a/go-controller/pkg/clustermanager/routeadvertisements/controller_test.go b/go-controller/pkg/clustermanager/routeadvertisements/controller_test.go index 2f7b78830b..3f49a73f2f 100644 --- a/go-controller/pkg/clustermanager/routeadvertisements/controller_test.go +++ b/go-controller/pkg/clustermanager/routeadvertisements/controller_test.go @@ -7,7 +7,9 @@ import ( "context" "encoding/json" "fmt" + "maps" "os" + "slices" "strings" "sync" "sync/atomic" @@ -27,6 +29,7 @@ import ( ctesting "k8s.io/client-go/testing" "k8s.io/client-go/tools/cache" "k8s.io/client-go/util/workqueue" + utilnet "k8s.io/utils/net" "k8s.io/utils/ptr" "github.com/ovn-kubernetes/ovn-kubernetes/go-controller/pkg/allocator/id" @@ -258,16 +261,15 @@ func (tr testRouter) Router() frrapi.Router { } type testFRRConfig struct { - Name string - Namespace string - Generation int - Labels map[string]string - Annotations map[string]string - Routers []*testRouter - NodeSelector map[string]string - OwnUpdate bool - RawConfig string - RawConfigPriority int + Name string + Namespace string + Generation int + Labels map[string]string + Annotations map[string]string + Routers []*testRouter + NodeSelector map[string]string + OwnUpdate bool + RawConfig string } func (tf testFRRConfig) FRRConfiguration() *frrapi.FRRConfiguration { @@ -288,9 +290,13 @@ func (tf testFRRConfig) FRRConfiguration() *frrapi.FRRConfiguration { for _, r := range tf.Routers { f.Spec.BGP.Routers = append(f.Spec.BGP.Routers, r.Router()) } - if tf.RawConfig != "" { - f.Spec.Raw.Config = tf.RawConfig - f.Spec.Raw.Priority = tf.RawConfigPriority + rawConfig := tf.RawConfig + if rawConfig == "" { + rawConfig = tf.generateUnicastRawConfig() + } + if rawConfig != "" { + f.Spec.Raw.Config = rawConfig + f.Spec.Raw.Priority = rawConfigPriority } if tf.OwnUpdate { f.ManagedFields = append(f.ManagedFields, metav1.ManagedFieldsEntry{ @@ -301,6 +307,34 @@ func (tf testFRRConfig) FRRConfiguration() *frrapi.FRRConfiguration { return f } +func (tf testFRRConfig) generateUnicastRawConfig() string { + var buf strings.Builder + routers := make(map[string]*testRouter, len(tf.Routers)) + for _, r := range tf.Routers { + routers[r.VRF] = r + } + for _, vrf := range slices.Sorted(maps.Keys(routers)) { + r := routers[vrf] + if len(r.Neighbors) == 0 { + continue + } + if r.VRF == "" { + fmt.Fprintf(&buf, "router bgp %d\n", r.ASN) + } else { + fmt.Fprintf(&buf, "router bgp %d vrf %s\n", r.ASN, r.VRF) + } + for _, n := range r.Neighbors { + if utilnet.IsIPv6String(n.Address) { + fmt.Fprintf(&buf, " address-family ipv6 unicast\n neighbor %s allowas-in origin\n exit-address-family\n", n.Address) + } else { + fmt.Fprintf(&buf, " address-family ipv4 unicast\n neighbor %s allowas-in origin\n exit-address-family\n", n.Address) + } + } + buf.WriteString("exit\n!\n") + } + return buf.String() +} + type testEIP struct { Name string Generation int @@ -1250,11 +1284,13 @@ func TestController_reconcile(t *testing.T) { expectAcceptedStatus: metav1.ConditionTrue, expectFRRConfigs: []*testFRRConfig{ { - Labels: map[string]string{types.OvnRouteAdvertisementsKey: "ra"}, - Annotations: map[string]string{types.OvnRouteAdvertisementsKey: "ra/frrConfig/node"}, - NodeSelector: map[string]string{"kubernetes.io/hostname": "node"}, - RawConfigPriority: 10, + Labels: map[string]string{types.OvnRouteAdvertisementsKey: "ra"}, + Annotations: map[string]string{types.OvnRouteAdvertisementsKey: "ra/frrConfig/node"}, + NodeSelector: map[string]string{"kubernetes.io/hostname": "node"}, RawConfig: `router bgp 65000 + address-family ipv4 unicast + neighbor 192.168.1.1 allowas-in origin + exit-address-family address-family l2vpn evpn neighbor 192.168.1.1 activate neighbor 192.168.1.1 allowas-in origin @@ -1295,11 +1331,13 @@ exit expectAcceptedStatus: metav1.ConditionTrue, expectFRRConfigs: []*testFRRConfig{ { - Labels: map[string]string{types.OvnRouteAdvertisementsKey: "ra"}, - Annotations: map[string]string{types.OvnRouteAdvertisementsKey: "ra/frrConfig/node"}, - NodeSelector: map[string]string{"kubernetes.io/hostname": "node"}, - RawConfigPriority: 10, + Labels: map[string]string{types.OvnRouteAdvertisementsKey: "ra"}, + Annotations: map[string]string{types.OvnRouteAdvertisementsKey: "ra/frrConfig/node"}, + NodeSelector: map[string]string{"kubernetes.io/hostname": "node"}, RawConfig: `router bgp 65000 + address-family ipv4 unicast + neighbor 192.168.1.1 allowas-in origin + exit-address-family address-family l2vpn evpn neighbor 192.168.1.1 activate neighbor 192.168.1.1 allowas-in origin @@ -1340,11 +1378,13 @@ exit expectAcceptedStatus: metav1.ConditionTrue, expectFRRConfigs: []*testFRRConfig{ { - Labels: map[string]string{types.OvnRouteAdvertisementsKey: "ra"}, - Annotations: map[string]string{types.OvnRouteAdvertisementsKey: "ra/frrConfig/node"}, - NodeSelector: map[string]string{"kubernetes.io/hostname": "node"}, - RawConfigPriority: 10, + Labels: map[string]string{types.OvnRouteAdvertisementsKey: "ra"}, + Annotations: map[string]string{types.OvnRouteAdvertisementsKey: "ra/frrConfig/node"}, + NodeSelector: map[string]string{"kubernetes.io/hostname": "node"}, RawConfig: `router bgp 65000 + address-family ipv4 unicast + neighbor 192.168.1.1 allowas-in origin + exit-address-family address-family l2vpn evpn neighbor 192.168.1.1 activate neighbor 192.168.1.1 allowas-in origin @@ -1403,11 +1443,13 @@ exit expectAcceptedStatus: metav1.ConditionTrue, expectFRRConfigs: []*testFRRConfig{ { - Labels: map[string]string{types.OvnRouteAdvertisementsKey: "ra"}, - Annotations: map[string]string{types.OvnRouteAdvertisementsKey: "ra/frrConfigGlobal/node"}, - NodeSelector: map[string]string{"kubernetes.io/hostname": "node"}, - RawConfigPriority: 10, + Labels: map[string]string{types.OvnRouteAdvertisementsKey: "ra"}, + Annotations: map[string]string{types.OvnRouteAdvertisementsKey: "ra/frrConfigGlobal/node"}, + NodeSelector: map[string]string{"kubernetes.io/hostname": "node"}, RawConfig: `router bgp 65000 + address-family ipv4 unicast + neighbor 192.168.1.1 allowas-in origin + exit-address-family address-family l2vpn evpn neighbor 192.168.1.1 activate neighbor 192.168.1.1 allowas-in origin @@ -1427,10 +1469,9 @@ exit-vrf }, }, { - Labels: map[string]string{types.OvnRouteAdvertisementsKey: "ra"}, - Annotations: map[string]string{types.OvnRouteAdvertisementsKey: "ra/frrConfigVRF/node"}, - NodeSelector: map[string]string{"kubernetes.io/hostname": "node"}, - RawConfigPriority: 10, + Labels: map[string]string{types.OvnRouteAdvertisementsKey: "ra"}, + Annotations: map[string]string{types.OvnRouteAdvertisementsKey: "ra/frrConfigVRF/node"}, + NodeSelector: map[string]string{"kubernetes.io/hostname": "node"}, RawConfig: `vrf blue vni 2000 exit-vrf @@ -1553,11 +1594,13 @@ exit expectAcceptedStatus: metav1.ConditionTrue, expectFRRConfigs: []*testFRRConfig{ { - Labels: map[string]string{types.OvnRouteAdvertisementsKey: "ra"}, - Annotations: map[string]string{types.OvnRouteAdvertisementsKey: "ra/frrConfigGlobal/node"}, - NodeSelector: map[string]string{"kubernetes.io/hostname": "node"}, - RawConfigPriority: 10, + Labels: map[string]string{types.OvnRouteAdvertisementsKey: "ra"}, + Annotations: map[string]string{types.OvnRouteAdvertisementsKey: "ra/frrConfigGlobal/node"}, + NodeSelector: map[string]string{"kubernetes.io/hostname": "node"}, RawConfig: `router bgp 65000 + address-family ipv4 unicast + neighbor 192.168.1.1 allowas-in origin + exit-address-family address-family l2vpn evpn neighbor 192.168.1.1 activate neighbor 192.168.1.1 allowas-in origin @@ -1614,11 +1657,13 @@ exit expectAcceptedStatus: metav1.ConditionTrue, expectFRRConfigs: []*testFRRConfig{ { - Labels: map[string]string{types.OvnRouteAdvertisementsKey: "ra"}, - Annotations: map[string]string{types.OvnRouteAdvertisementsKey: "ra/frrConfig/node"}, - NodeSelector: map[string]string{"kubernetes.io/hostname": "node"}, - RawConfigPriority: 10, + Labels: map[string]string{types.OvnRouteAdvertisementsKey: "ra"}, + Annotations: map[string]string{types.OvnRouteAdvertisementsKey: "ra/frrConfig/node"}, + NodeSelector: map[string]string{"kubernetes.io/hostname": "node"}, RawConfig: `router bgp 65000 + address-family ipv4 unicast + neighbor 192.168.1.1 allowas-in origin + exit-address-family address-family l2vpn evpn neighbor 192.168.1.1 activate neighbor 192.168.1.1 allowas-in origin @@ -1680,11 +1725,13 @@ exit expectAcceptedStatus: metav1.ConditionTrue, expectFRRConfigs: []*testFRRConfig{ { - Labels: map[string]string{types.OvnRouteAdvertisementsKey: "ra"}, - Annotations: map[string]string{types.OvnRouteAdvertisementsKey: "ra/frrConfig/node"}, - NodeSelector: map[string]string{"kubernetes.io/hostname": "node"}, - RawConfigPriority: 10, + Labels: map[string]string{types.OvnRouteAdvertisementsKey: "ra"}, + Annotations: map[string]string{types.OvnRouteAdvertisementsKey: "ra/frrConfig/node"}, + NodeSelector: map[string]string{"kubernetes.io/hostname": "node"}, RawConfig: `router bgp 65000 + address-family ipv6 unicast + neighbor fd00::1 allowas-in origin + exit-address-family address-family l2vpn evpn neighbor fd00::1 activate neighbor fd00::1 allowas-in origin @@ -1780,11 +1827,13 @@ exit expectAcceptedStatus: metav1.ConditionTrue, expectFRRConfigs: []*testFRRConfig{ { - Labels: map[string]string{types.OvnRouteAdvertisementsKey: "ra"}, - Annotations: map[string]string{types.OvnRouteAdvertisementsKey: "ra/frrConfig/node"}, - NodeSelector: map[string]string{"kubernetes.io/hostname": "node"}, - RawConfigPriority: 10, + Labels: map[string]string{types.OvnRouteAdvertisementsKey: "ra"}, + Annotations: map[string]string{types.OvnRouteAdvertisementsKey: "ra/frrConfig/node"}, + NodeSelector: map[string]string{"kubernetes.io/hostname": "node"}, RawConfig: `router bgp 65000 + address-family ipv4 unicast + neighbor 192.168.1.1 allowas-in origin + exit-address-family address-family l2vpn evpn neighbor 192.168.1.1 activate neighbor 192.168.1.1 allowas-in origin @@ -1796,6 +1845,12 @@ exit exit-address-family exit ! +router bgp 65000 vrf red + address-family ipv4 unicast + neighbor 192.168.1.1 allowas-in origin + exit-address-family +exit +! `, Routers: []*testRouter{ {ASN: 65000, VRF: "red", Prefixes: []string{"10.1.0.0/16"}, Neighbors: []*testNeighbor{ @@ -1842,11 +1897,13 @@ exit expectAcceptedStatus: metav1.ConditionTrue, expectFRRConfigs: []*testFRRConfig{ { - Labels: map[string]string{types.OvnRouteAdvertisementsKey: "ra"}, - Annotations: map[string]string{types.OvnRouteAdvertisementsKey: "ra/frrConfig/node"}, - NodeSelector: map[string]string{"kubernetes.io/hostname": "node"}, - RawConfigPriority: 10, + Labels: map[string]string{types.OvnRouteAdvertisementsKey: "ra"}, + Annotations: map[string]string{types.OvnRouteAdvertisementsKey: "ra/frrConfig/node"}, + NodeSelector: map[string]string{"kubernetes.io/hostname": "node"}, RawConfig: `router bgp 65000 + address-family ipv4 unicast + neighbor 192.168.1.1 allowas-in origin + exit-address-family address-family l2vpn evpn neighbor 192.168.1.1 activate neighbor 192.168.1.1 allowas-in origin @@ -1926,11 +1983,13 @@ exit expectAcceptedStatus: metav1.ConditionTrue, expectFRRConfigs: []*testFRRConfig{ { - Labels: map[string]string{types.OvnRouteAdvertisementsKey: "ra"}, - Annotations: map[string]string{types.OvnRouteAdvertisementsKey: "ra/frrConfig/node1"}, - NodeSelector: map[string]string{"kubernetes.io/hostname": "node1"}, - RawConfigPriority: 10, + Labels: map[string]string{types.OvnRouteAdvertisementsKey: "ra"}, + Annotations: map[string]string{types.OvnRouteAdvertisementsKey: "ra/frrConfig/node1"}, + NodeSelector: map[string]string{"kubernetes.io/hostname": "node1"}, RawConfig: `router bgp 65000 + address-family ipv4 unicast + neighbor 192.168.1.1 allowas-in origin + exit-address-family address-family l2vpn evpn neighbor 192.168.1.1 activate neighbor 192.168.1.1 allowas-in origin @@ -1959,11 +2018,13 @@ exit }, }, { - Labels: map[string]string{types.OvnRouteAdvertisementsKey: "ra"}, - Annotations: map[string]string{types.OvnRouteAdvertisementsKey: "ra/frrConfig/node2"}, - NodeSelector: map[string]string{"kubernetes.io/hostname": "node2"}, - RawConfigPriority: 10, + Labels: map[string]string{types.OvnRouteAdvertisementsKey: "ra"}, + Annotations: map[string]string{types.OvnRouteAdvertisementsKey: "ra/frrConfig/node2"}, + NodeSelector: map[string]string{"kubernetes.io/hostname": "node2"}, RawConfig: `router bgp 65000 + address-family ipv4 unicast + neighbor 192.168.1.1 allowas-in origin + exit-address-family address-family l2vpn evpn neighbor 192.168.1.1 activate neighbor 192.168.1.1 allowas-in origin @@ -2031,11 +2092,13 @@ exit expectAcceptedStatus: metav1.ConditionTrue, expectFRRConfigs: []*testFRRConfig{ { - Labels: map[string]string{types.OvnRouteAdvertisementsKey: "ra"}, - Annotations: map[string]string{types.OvnRouteAdvertisementsKey: "ra/frrConfig/node"}, - NodeSelector: map[string]string{"kubernetes.io/hostname": "node"}, - RawConfigPriority: 10, + Labels: map[string]string{types.OvnRouteAdvertisementsKey: "ra"}, + Annotations: map[string]string{types.OvnRouteAdvertisementsKey: "ra/frrConfig/node"}, + NodeSelector: map[string]string{"kubernetes.io/hostname": "node"}, RawConfig: `router bgp 65000 + address-family ipv4 unicast + neighbor 192.168.1.1 allowas-in origin + exit-address-family address-family l2vpn evpn neighbor 192.168.1.1 activate neighbor 192.168.1.1 allowas-in origin @@ -2115,11 +2178,13 @@ exit expectAcceptedStatus: metav1.ConditionTrue, expectFRRConfigs: []*testFRRConfig{ { - Labels: map[string]string{types.OvnRouteAdvertisementsKey: "ra"}, - Annotations: map[string]string{types.OvnRouteAdvertisementsKey: "ra/frrConfig/node1"}, - NodeSelector: map[string]string{"kubernetes.io/hostname": "node1"}, - RawConfigPriority: 10, + Labels: map[string]string{types.OvnRouteAdvertisementsKey: "ra"}, + Annotations: map[string]string{types.OvnRouteAdvertisementsKey: "ra/frrConfig/node1"}, + NodeSelector: map[string]string{"kubernetes.io/hostname": "node1"}, RawConfig: `router bgp 65000 + address-family ipv4 unicast + neighbor 192.168.1.1 allowas-in origin + exit-address-family address-family l2vpn evpn neighbor 192.168.1.1 activate neighbor 192.168.1.1 allowas-in origin @@ -2148,11 +2213,13 @@ exit }, }, { - Labels: map[string]string{types.OvnRouteAdvertisementsKey: "ra"}, - Annotations: map[string]string{types.OvnRouteAdvertisementsKey: "ra/frrConfig/node2"}, - NodeSelector: map[string]string{"kubernetes.io/hostname": "node2"}, - RawConfigPriority: 10, + Labels: map[string]string{types.OvnRouteAdvertisementsKey: "ra"}, + Annotations: map[string]string{types.OvnRouteAdvertisementsKey: "ra/frrConfig/node2"}, + NodeSelector: map[string]string{"kubernetes.io/hostname": "node2"}, RawConfig: `router bgp 65000 + address-family ipv4 unicast + neighbor 192.168.1.1 allowas-in origin + exit-address-family address-family l2vpn evpn neighbor 192.168.1.1 activate neighbor 192.168.1.1 allowas-in origin diff --git a/go-controller/pkg/clustermanager/routeadvertisements/rawconfig.go b/go-controller/pkg/clustermanager/routeadvertisements/rawconfig.go index 4e705c2298..6aac8f6aaf 100644 --- a/go-controller/pkg/clustermanager/routeadvertisements/rawconfig.go +++ b/go-controller/pkg/clustermanager/routeadvertisements/rawconfig.go @@ -5,20 +5,32 @@ package routeadvertisements import ( "fmt" + "maps" + "slices" "strings" + + "github.com/ovn-kubernetes/ovn-kubernetes/go-controller/pkg/util" ) -// generateEVPNRawConfig generates raw FRR configuration for EVPN. -// If asn/neighbors aren't provided the related sections are skipped. +// generateRawConfig generates raw FRR configuration. Main purpose is to +// generates EVPN config sections based on the provided selected networks +// IP-VRFs and MAC-VRFs. Also adds allowas-in origin to all neighbors, which is +// needed for eBGP peers sharing our ASN and is a no-op for iBGP. // // Generated config structure: // -// router bgp <- genGlobalEVPNSection -// address-family l2vpn evpn +// router bgp <- genDefaultVRFSection +// address-family ipv4 unicast <- genUnicastSection (only if default VRF has neighbors) +// neighbor allowas-in origin +// exit-address-family +// address-family ipv6 unicast +// neighbor allowas-in origin +// exit-address-family +// address-family l2vpn evpn <- genDefaultVRFEVPNSection (only if networks have MAC-VRFs/IP-VRFs) // neighbor activate // neighbor allowas-in origin // advertise-all-vni -// vni <- (one per MAC-VRF with RT, section only added when MAC-VRF RT is set) +// vni <- genVRFVNISection (one per MAC-VRF with RT, section only added when MAC-VRF RT is set) // route-target import // route-target export // exit-vni @@ -29,8 +41,14 @@ import ( // vni // exit-vrf // ! -// router bgp vrf <- genVRFEVPNSection (one per IP-VRF) -// address-family l2vpn evpn +// router bgp vrf <- genNonDefaultVRFSection (one per non-default VRF) +// address-family ipv4 unicast <- genUnicastSection (only if VRF has neighbors i.e. VRF-Lite) +// neighbor allowas-in origin +// exit-address-family +// address-family ipv6 unicast +// neighbor allowas-in origin +// exit-address-family +// address-family l2vpn evpn <- genNonDefaultVRFEVPNSection (only if network has IP-VRF) // advertise ipv4 unicast // advertise ipv6 unicast // route-target import @@ -38,21 +56,31 @@ import ( // exit-address-family // exit // ! -func generateEVPNRawConfig(selected *selectedNetworks, asn uint32, neighbors []string, vrfASNs map[string]uint32) string { +func generateRawConfig(selected *selectedNetworks, vrfNeighbors map[string][]string, vrfASNs map[string]uint32) string { var buf strings.Builder - if asn > 0 && len(neighbors) > 0 { - buf.WriteString(genGlobalEVPNSection(asn, neighbors, selected.macVRFConfigs)) - } + // handle default VRF router, neighbors sorted for deterministic config + // generation + neighbors := slices.Sorted(slices.Values(vrfNeighbors[""])) + buf.WriteString(genDefaultVRFSection(vrfASNs[""], neighbors, selected)) + + // handle VRF<->VNI mappings + ipVRFConfigMap := make(map[string]*ipVRFConfig, len(selected.ipVRFConfigs)) for _, cfg := range selected.ipVRFConfigs { + ipVRFConfigMap[cfg.VRFName] = cfg buf.WriteString(genVRFVNISection(cfg)) } - // Generate VRF-specific EVPN sections using each config's ASN - for _, cfg := range selected.ipVRFConfigs { - if vrfASN := vrfASNs[cfg.VRFName]; vrfASN > 0 { - buf.WriteString(genVRFEVPNSection(vrfASN, cfg)) + + // handle non default VRFs + for _, vrf := range slices.Sorted(maps.Keys(vrfASNs)) { + if vrf == "" { + continue } + // neighbors sorted for deterministic config generation + neighbors := slices.Sorted(slices.Values(vrfNeighbors[vrf])) + buf.WriteString(genNonDefaultVRFSection(vrf, vrfASNs[vrf], neighbors, ipVRFConfigMap[vrf])) } + return buf.String() } @@ -70,24 +98,104 @@ exit-vrf `, cfg.VRFName, cfg.VNI) } -// genGlobalEVPNSection generates the global router's EVPN address-family. +// genDefaultVRFSection generates the default VRF router's unicast and EVPN address-families. // // router bgp -// address-family l2vpn evpn -// neighbor activate -// neighbor allowas-in origin -// advertise-all-vni -// vni <- (Section only added when MAC-VRF RT is set) -// route-target import -// route-target export -// exit-vni -// exit-address-family +// ... // exit // ! -func genGlobalEVPNSection(asn uint32, neighbors []string, macVRFs []*vrfConfig) string { +func genDefaultVRFSection(asn uint32, neighbors []string, selected *selectedNetworks) string { + if asn == 0 || len(neighbors) == 0 { + return "" + } + var buf strings.Builder fmt.Fprintf(&buf, "router bgp %d\n", asn) + + neighbors4, neighbors6 := util.SplitIPStringByIPFamily(neighbors) + buf.WriteString(genUnicastSection(neighbors4, neighbors6)) + buf.WriteString(genDefaultVRFEVPNSection(neighbors, selected)) + + buf.WriteString("exit\n!\n") + + return buf.String() +} + +// genNonDefaultVRFSection generates a non-default VRF router's unicast and EVPN address-families. +// +// router bgp vrf red +// ... +// exit +// ! +func genNonDefaultVRFSection(vrf string, asn uint32, neighbors []string, cfg *ipVRFConfig) string { + if vrf == "" || asn == 0 { + return "" + } + if len(neighbors) == 0 && cfg == nil { + return "" + } + + var buf strings.Builder + + fmt.Fprintf(&buf, "router bgp %d vrf %s\n", asn, vrf) + + neighbors4, neighbors6 := util.SplitIPStringByIPFamily(neighbors) + buf.WriteString(genUnicastSection(neighbors4, neighbors6)) + buf.WriteString(genNonDefaultVRFEVPNSection(cfg)) + + buf.WriteString("exit\n!\n") + + return buf.String() +} + +// genUnicastSection generates unicast address-family sections for IPv4 and/or IPv6 neighbors. +// +// address-family ipv4 unicast +// neighbor allowas-in origin +// exit-address-family +// address-family ipv6 unicast +// neighbor allowas-in origin +// exit-address-family +func genUnicastSection(neighbors4, neighbors6 []string) string { + var buf strings.Builder + + if len(neighbors4) > 0 { + buf.WriteString(" address-family ipv4 unicast\n") + for _, neighbor := range neighbors4 { + fmt.Fprintf(&buf, " neighbor %s allowas-in origin\n", neighbor) + } + buf.WriteString(" exit-address-family\n") + } + if len(neighbors6) > 0 { + buf.WriteString(" address-family ipv6 unicast\n") + for _, neighbor := range neighbors6 { + fmt.Fprintf(&buf, " neighbor %s allowas-in origin\n", neighbor) + } + buf.WriteString(" exit-address-family\n") + } + return buf.String() +} + +// genDefaultVRFEVPNSection generates the l2vpn evpn address-family for the default VRF. +// +// address-family l2vpn evpn +// neighbor activate +// neighbor allowas-in origin +// advertise-all-vni +// vni <- (Section only added when MAC-VRF RT is set) +// route-target import +// route-target export +// exit-vni +// exit-address-family +func genDefaultVRFEVPNSection(neighbors []string, selected *selectedNetworks) string { + hasEVPN := len(selected.ipVRFConfigs) > 0 || len(selected.macVRFConfigs) > 0 + if !hasEVPN { + return "" + } + + var buf strings.Builder + buf.WriteString(" address-family l2vpn evpn\n") for _, neighbor := range neighbors { @@ -99,7 +207,7 @@ func genGlobalEVPNSection(asn uint32, neighbors []string, macVRFs []*vrfConfig) } buf.WriteString(" advertise-all-vni\n") - for _, cfg := range macVRFs { + for _, cfg := range selected.macVRFConfigs { if cfg.RouteTarget == "" { continue } @@ -110,25 +218,24 @@ func genGlobalEVPNSection(asn uint32, neighbors []string, macVRFs []*vrfConfig) } buf.WriteString(" exit-address-family\n") - buf.WriteString("exit\n!\n") return buf.String() } -// genVRFEVPNSection generates a VRF router's EVPN address-family. +// genNonDefaultVRFEVPNSection generates the l2vpn evpn address-family for a non-default VRF. // -// router bgp 65000 vrf red -// address-family l2vpn evpn -// advertise ipv4 unicast -// advertise ipv6 unicast -// route-target import 65000:100 -// route-target export 65000:100 -// exit-address-family -// exit -// ! -func genVRFEVPNSection(asn uint32, cfg *ipVRFConfig) string { +// address-family l2vpn evpn +// advertise ipv4 unicast +// advertise ipv6 unicast +// route-target import +// route-target export +// exit-address-family +func genNonDefaultVRFEVPNSection(cfg *ipVRFConfig) string { + if cfg == nil { + return "" + } + var buf strings.Builder - fmt.Fprintf(&buf, "router bgp %d vrf %s\n", asn, cfg.VRFName) buf.WriteString(" address-family l2vpn evpn\n") if cfg.HasIPv4 { @@ -143,7 +250,6 @@ func genVRFEVPNSection(asn uint32, cfg *ipVRFConfig) string { } buf.WriteString(" exit-address-family\n") - buf.WriteString("exit\n!\n") return buf.String() } diff --git a/go-controller/pkg/clustermanager/routeadvertisements/rawconfig_test.go b/go-controller/pkg/clustermanager/routeadvertisements/rawconfig_test.go index f5618d363b..e14445be8e 100644 --- a/go-controller/pkg/clustermanager/routeadvertisements/rawconfig_test.go +++ b/go-controller/pkg/clustermanager/routeadvertisements/rawconfig_test.go @@ -7,14 +7,46 @@ import ( "testing" ) -func TestGenerateEVPNRawConfig(t *testing.T) { +func TestGenerateRawConfig(t *testing.T) { tests := []struct { - name string - selected *selectedNetworks - asn uint32 - neighbors []string - want string + name string + selected *selectedNetworks + vrfNeighbors map[string][]string + vrfASNs map[string]uint32 + want string }{ + { + name: "empty input", + selected: &selectedNetworks{}, + vrfNeighbors: map[string][]string{}, + vrfASNs: map[string]uint32{}, + want: "", + }, + { + name: "default and non-default VRF unicast only", + selected: &selectedNetworks{}, + vrfNeighbors: map[string][]string{"": {"192.168.1.1", "fd00::1"}, "red": {"10.0.0.1", "fd00::2"}}, + vrfASNs: map[string]uint32{"": 65000, "red": 65000}, + want: `router bgp 65000 + address-family ipv4 unicast + neighbor 192.168.1.1 allowas-in origin + exit-address-family + address-family ipv6 unicast + neighbor fd00::1 allowas-in origin + exit-address-family +exit +! +router bgp 65000 vrf red + address-family ipv4 unicast + neighbor 10.0.0.1 allowas-in origin + exit-address-family + address-family ipv6 unicast + neighbor fd00::2 allowas-in origin + exit-address-family +exit +! +`, + }, { name: "MAC-VRF without route target", selected: &selectedNetworks{ @@ -22,9 +54,12 @@ func TestGenerateEVPNRawConfig(t *testing.T) { {VNI: 1000}, }, }, - asn: 65000, - neighbors: []string{"192.168.1.1"}, + vrfNeighbors: map[string][]string{"": {"192.168.1.1"}}, + vrfASNs: map[string]uint32{"": 65000}, want: `router bgp 65000 + address-family ipv4 unicast + neighbor 192.168.1.1 allowas-in origin + exit-address-family address-family l2vpn evpn neighbor 192.168.1.1 activate neighbor 192.168.1.1 allowas-in origin @@ -41,9 +76,12 @@ exit {VNI: 1000, RouteTarget: "65000:1000"}, }, }, - asn: 65000, - neighbors: []string{"192.168.1.1"}, + vrfNeighbors: map[string][]string{"": {"192.168.1.1"}}, + vrfASNs: map[string]uint32{"": 65000}, want: `router bgp 65000 + address-family ipv4 unicast + neighbor 192.168.1.1 allowas-in origin + exit-address-family address-family l2vpn evpn neighbor 192.168.1.1 activate neighbor 192.168.1.1 allowas-in origin @@ -68,9 +106,12 @@ exit }, }, }, - asn: 65000, - neighbors: []string{"192.168.1.1"}, + vrfNeighbors: map[string][]string{"": {"192.168.1.1"}}, + vrfASNs: map[string]uint32{"": 65000, "blue": 65000}, want: `router bgp 65000 + address-family ipv4 unicast + neighbor 192.168.1.1 allowas-in origin + exit-address-family address-family l2vpn evpn neighbor 192.168.1.1 activate neighbor 192.168.1.1 allowas-in origin @@ -104,9 +145,12 @@ exit }, }, }, - asn: 65000, - neighbors: []string{"192.168.1.1"}, + vrfNeighbors: map[string][]string{"": {"192.168.1.1"}}, + vrfASNs: map[string]uint32{"": 65000, "blue": 65000}, want: `router bgp 65000 + address-family ipv4 unicast + neighbor 192.168.1.1 allowas-in origin + exit-address-family address-family l2vpn evpn neighbor 192.168.1.1 activate neighbor 192.168.1.1 allowas-in origin @@ -143,9 +187,13 @@ exit }, }, }, - asn: 65000, - neighbors: []string{"192.168.1.1", "192.168.1.2"}, + vrfNeighbors: map[string][]string{"": {"192.168.1.2", "192.168.1.1"}}, + vrfASNs: map[string]uint32{"": 65000, "blue": 65000}, want: `router bgp 65000 + address-family ipv4 unicast + neighbor 192.168.1.1 allowas-in origin + neighbor 192.168.1.2 allowas-in origin + exit-address-family address-family l2vpn evpn neighbor 192.168.1.1 activate neighbor 192.168.1.1 allowas-in origin @@ -171,21 +219,56 @@ router bgp 65000 vrf blue exit-address-family exit ! +`, + }, + { + name: "non-default VRF with neighbors and IP-VRF", + selected: &selectedNetworks{ + ipVRFConfigs: []*ipVRFConfig{ + { + vrfConfig: vrfConfig{VNI: 3000, RouteTarget: "65000:3000"}, + VRFName: "green", + HasIPv4: true, + }, + }, + }, + vrfNeighbors: map[string][]string{"": {"192.168.1.1"}, "green": {"10.0.0.1"}}, + vrfASNs: map[string]uint32{"": 65000, "green": 65000}, + want: `router bgp 65000 + address-family ipv4 unicast + neighbor 192.168.1.1 allowas-in origin + exit-address-family + address-family l2vpn evpn + neighbor 192.168.1.1 activate + neighbor 192.168.1.1 allowas-in origin + advertise-all-vni + exit-address-family +exit +! +vrf green + vni 3000 +exit-vrf +! +router bgp 65000 vrf green + address-family ipv4 unicast + neighbor 10.0.0.1 allowas-in origin + exit-address-family + address-family l2vpn evpn + advertise ipv4 unicast + route-target import 65000:3000 + route-target export 65000:3000 + exit-address-family +exit +! `, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - vrfASNs := map[string]uint32{} - for _, cfg := range tt.selected.ipVRFConfigs { - if cfg.VRFName != "" { - vrfASNs[cfg.VRFName] = tt.asn - } - } - got := generateEVPNRawConfig(tt.selected, tt.asn, tt.neighbors, vrfASNs) + got := generateRawConfig(tt.selected, tt.vrfNeighbors, tt.vrfASNs) if got != tt.want { - t.Errorf("generateEVPNRawConfig() mismatch\nGot:\n%s\nWant:\n%s", got, tt.want) + t.Errorf("generateRawConfig() mismatch\nGot:\n%s\nWant:\n%s", got, tt.want) } }) } diff --git a/go-controller/pkg/util/net.go b/go-controller/pkg/util/net.go index 7714889158..b90c533ed9 100644 --- a/go-controller/pkg/util/net.go +++ b/go-controller/pkg/util/net.go @@ -309,6 +309,19 @@ func MatchAllIPNetsStringFamily(isIPv6 bool, ipnets []string) []string { return out } +// SplitIPStringByIPFamily splits a slice of IP address strings into IPv4 and IPv6 slices. +func SplitIPStringByIPFamily(ips []string) (ipsv4, ipsv6 []string) { + for _, ip := range ips { + switch { + case utilnet.IsIPv6String(ip): + ipsv6 = append(ipsv6, ip) + default: + ipsv4 = append(ipsv4, ip) + } + } + return +} + // IsIPContainedInAnyCIDR returns true if ip is contained in any of the given ipnets func IsIPContainedInAnyCIDR(ip net.IP, ipnets ...*net.IPNet) bool { for _, ipnet := range ipnets { From 047d5d369f236a3667e204aca6b5f536e54da274 Mon Sep 17 00:00:00 2001 From: Yun Zhou Date: Fri, 5 Jun 2026 09:15:41 -0700 Subject: [PATCH 26/51] Sync reconcile pod-selector address sets at creation time When EnsureAddressSet creates a new in-memory entry for a new DB ID, it returns hashes before async reconcile populates the NB address set. ACLs wired during that window can reference an empty set. Populate from the informer cache synchronously before registration. Signed-off-by: Yun Zhou --- .../addresssetmanager/addressset_manager.go | 22 ++++++++++++++++--- .../addressset_manager_test.go | 20 ++++++++++++++++- 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/go-controller/pkg/ovn/addresssetmanager/addressset_manager.go b/go-controller/pkg/ovn/addresssetmanager/addressset_manager.go index e3af437199..5f63e15487 100644 --- a/go-controller/pkg/ovn/addresssetmanager/addressset_manager.go +++ b/go-controller/pkg/ovn/addresssetmanager/addressset_manager.go @@ -330,8 +330,10 @@ func (m *AddressSetManager) EnsureAddressSet(podSelector, namespaceSelector, nod } addrSetKey = getInternalKey(podSelector, namespaceSelector, nodeSelector, namespace, controllerName, legacyNetpolMode) + var found bool err = m.addressSets.DoWithLock(addrSetKey, func(key string) error { - psAddrSet, found := m.addressSets.Load(key) + var psAddrSet *podSelectorAddressSet + psAddrSet, found = m.addressSets.Load(key) if !found { addrSetDbIDs := GetPodSelectorAddrSetDbIDs(podSelector, namespaceSelector, nodeSelector, namespace, controllerName, legacyNetpolMode) ipv4Mode, ipv6Mode := netInfo.IPMode() @@ -343,6 +345,8 @@ func (m *AddressSetManager) EnsureAddressSet(podSelector, namespaceSelector, nod addrSet, err = m.addressSetFactoryV6.EnsureAddressSet(addrSetDbIDs) case ipv4Mode && ipv6Mode: addrSet, err = m.addressSetFactoryDualstack.EnsureAddressSet(addrSetDbIDs) + default: + return fmt.Errorf("neither IPv4 nor IPv6 mode is enabled") } // if the first step of creating address set fails, return error since there is nothing to cleanup if err != nil { @@ -365,14 +369,26 @@ func (m *AddressSetManager) EnsureAddressSet(podSelector, namespaceSelector, nod }, } m.addressSets.LoadOrStore(key, psAddrSet) - // this only puts key to the queue, no lock - m.addressSetReconciler.Reconcile(key) } // psAddrSet is successfully init-ed psAddrSet.backRefs[backRef] = true psAddrSetHashV4, psAddrSetHashV6 = psAddrSet.addressSet.GetASHashNames() return nil }) + if err != nil { + return + } + if !found { + // Populate a newly created address set before returning hashes to the caller. + // Until reconcile runs, the NB object is empty while ACLs may already reference it + // (e.g. on DB ID change or first ensure). This is a one-time sync on creation; + // existing sets are kept up to date by the async reconciler on pod/namespace/node events. + if reconcileErr := m.reconcileAddressSet(addrSetKey); reconcileErr != nil { + klog.Errorf("Failed to reconcile address set %s on ensure: %v", addrSetKey, reconcileErr) + // this only puts key to the queue, no lock + m.addressSetReconciler.Reconcile(addrSetKey) + } + } return } diff --git a/go-controller/pkg/ovn/addresssetmanager/addressset_manager_test.go b/go-controller/pkg/ovn/addresssetmanager/addressset_manager_test.go index 77e695a531..6fd7bf537f 100644 --- a/go-controller/pkg/ovn/addresssetmanager/addressset_manager_test.go +++ b/go-controller/pkg/ovn/addresssetmanager/addressset_manager_test.go @@ -197,6 +197,24 @@ var _ = ginkgo.Describe("OVN podSelectorAddressSet", func() { gomega.Consistently(addressSetManager.nbClient, 100*time.Millisecond, 20*time.Millisecond).Should(libovsdbtest.HaveData([]libovsdbtest.TestData{})) }) + ginkgo.It("populates address set synchronously on first ensure", func() { + namespace1 := *testing.NewNamespace(namespaceName1) + podIP := "10.128.1.3" + peer := knet.NetworkPolicyPeer{ + PodSelector: &metav1.LabelSelector{}, + } + pod := testing.NewPod(namespace1.Name, "pod1", nodeName, podIP) + startAddrSetManager(initialDB, []corev1.Namespace{namespace1}, []corev1.Pod{*pod}) + + _, _, _, err := addressSetManager.EnsureAddressSet( + peer.PodSelector, peer.NamespaceSelector, nil, namespace1.Name, "backRef", controllerName, &util.DefaultNetInfo{}, false) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + dbIDs := GetPodSelectorAddrSetDbIDs(peer.PodSelector, peer.NamespaceSelector, nil, namespace1.Name, controllerName, false) + expectedAS, _ := addressset.GetTestDbAddrSets(dbIDs, []string{podIP}) + gomega.Expect(addressSetManager.nbClient).Should(libovsdbtest.HaveData([]libovsdbtest.TestData{expectedAS})) + }) + ginkgo.It("reconciles the registered cluster node IP address set on node events", func() { node1 := corev1.Node{ ObjectMeta: metav1.ObjectMeta{ @@ -881,7 +899,7 @@ var _ = ginkgo.Describe("OVN podSelectorAddressSet", func() { "", "backRef", controllerName, &util.DefaultNetInfo{}, true) gomega.Expect(err).NotTo(gomega.HaveOccurred()) restartedAS, _ := addressset.GetTestDbAddrSets(hostNSASIDs, []string{node2MgmtIP}) - gomega.Eventually(addressSetManager.nbClient).Should(libovsdbtest.HaveData([]libovsdbtest.TestData{restartedAS})) + gomega.Expect(addressSetManager.nbClient).Should(libovsdbtest.HaveData([]libovsdbtest.TestData{restartedAS})) }) ginkgo.It("updates IPs for existing nodes when the host network traffic namespace is created", func() { config.Kubernetes.HostNetworkNamespace = "ovn-host-network" From 7a7f4caa3513e8b7740a7117d804825cb9edb594 Mon Sep 17 00:00:00 2001 From: Yun Zhou Date: Tue, 19 May 2026 09:17:17 -0700 Subject: [PATCH 27/51] ovn: drop dead IPAMClaim watch after non-IC removal Non-IC is no longer supported; cluster-manager owns IPAMClaim lifecycle for L2/Localnet UDNs in IC mode. The ovnkube-controller watch and delete handler for IPAMClaims never register and are dead code. Signed-off-by: Yun Zhou --- go-controller/pkg/ovn/base_event_handler.go | 18 +-------- .../pkg/ovn/base_network_controller.go | 4 -- .../base_network_controller_user_defined.go | 39 ------------------- ...ase_secondary_layer2_network_controller.go | 26 ------------- .../layer2_user_defined_network_controller.go | 6 --- ...ocalnet_user_defined_network_controller.go | 6 --- 6 files changed, 1 insertion(+), 98 deletions(-) diff --git a/go-controller/pkg/ovn/base_event_handler.go b/go-controller/pkg/ovn/base_event_handler.go index 7cf1e884db..8d37b14550 100644 --- a/go-controller/pkg/ovn/base_event_handler.go +++ b/go-controller/pkg/ovn/base_event_handler.go @@ -7,7 +7,6 @@ import ( "fmt" "reflect" - ipamclaimsapi "github.com/k8snetworkplumbingwg/ipamclaims/pkg/crd/ipamclaims/v1alpha1" mnpapi "github.com/k8snetworkplumbingwg/multi-networkpolicy/pkg/apis/k8s.cni.cncf.io/v1beta1" corev1 "k8s.io/api/core/v1" @@ -36,8 +35,7 @@ func hasResourceAnUpdateFunc(objType reflect.Type) bool { factory.EgressIPPodType, factory.EgressNodeType, factory.NamespaceType, - factory.MultiNetworkPolicyType, - factory.IPAMClaimsType: + factory.MultiNetworkPolicyType: return true } return false @@ -116,17 +114,6 @@ func (h *baseNetworkControllerEventHandler) areResourcesEqual(objType reflect.Ty mnp1.Annotations[ovnStatelessNetPolAnnotationName] == mnp2.Annotations[ovnStatelessNetPolAnnotationName] && mnp1.Annotations[PolicyForAnnotation] == mnp2.Annotations[PolicyForAnnotation] return areEqual, nil - - case factory.IPAMClaimsType: - ipamClaim1, ok := obj1.(*ipamclaimsapi.IPAMClaim) - if !ok { - return false, fmt.Errorf("could not cast obj1 of type %T to *ipamclaimsapi.IPAMClaim", obj1) - } - ipamClaim2, ok := obj2.(*ipamclaimsapi.IPAMClaim) - if !ok { - return false, fmt.Errorf("could not cast obj2 of type %T to *ipamclaimsapi.IPAMClaim", obj2) - } - return reflect.DeepEqual(ipamClaim1, ipamClaim2), nil } return false, fmt.Errorf("no object comparison for type %s", objType) @@ -170,9 +157,6 @@ func (h *baseNetworkControllerEventHandler) getResourceFromInformerCache(objType case factory.MultiNetworkPolicyType: obj, err = watchFactory.GetMultiNetworkPolicy(namespace, name) - case factory.IPAMClaimsType: - obj, err = watchFactory.GetIPAMClaim(namespace, name) - default: err = fmt.Errorf("object type %s not supported, cannot retrieve it from informers cache", objType) diff --git a/go-controller/pkg/ovn/base_network_controller.go b/go-controller/pkg/ovn/base_network_controller.go index 79f585fdec..123995e0da 100644 --- a/go-controller/pkg/ovn/base_network_controller.go +++ b/go-controller/pkg/ovn/base_network_controller.go @@ -101,8 +101,6 @@ type BaseNetworkController struct { retryNetworkPolicies *ovnretry.RetryFramework // retry framework for network policies retryMultiNetworkPolicies *ovnretry.RetryFramework - // retry framework for IPAMClaims - retryIPAMClaims *ovnretry.RetryFramework // nodeReconciler is the shared node controller used by controllers that // reconcile node topology through pkg/controllers/node. @@ -114,8 +112,6 @@ type BaseNetworkController struct { podHandler *factory.Handler // namespace events factory Handler namespaceHandler *factory.Handler - // ipam claims events factory Handler - ipamClaimsHandler *factory.Handler // A cache of all logical switches seen by the watcher and their subnets lsManager *lsm.LogicalSwitchManager diff --git a/go-controller/pkg/ovn/base_network_controller_user_defined.go b/go-controller/pkg/ovn/base_network_controller_user_defined.go index 77a740369b..8825b4ed1e 100644 --- a/go-controller/pkg/ovn/base_network_controller_user_defined.go +++ b/go-controller/pkg/ovn/base_network_controller_user_defined.go @@ -4,14 +4,12 @@ package ovn import ( - "errors" "fmt" "net" "reflect" "strings" "time" - ipamclaimsapi "github.com/k8snetworkplumbingwg/ipamclaims/pkg/crd/ipamclaims/v1alpha1" mnpapi "github.com/k8snetworkplumbingwg/multi-networkpolicy/pkg/apis/k8s.cni.cncf.io/v1beta1" nadapi "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" @@ -35,7 +33,6 @@ import ( addressset "github.com/ovn-kubernetes/ovn-kubernetes/go-controller/pkg/ovn/address_set" "github.com/ovn-kubernetes/ovn-kubernetes/go-controller/pkg/ovn/addresssetmanager" "github.com/ovn-kubernetes/ovn-kubernetes/go-controller/pkg/ovn/controller/udnenabledsvc" - "github.com/ovn-kubernetes/ovn-kubernetes/go-controller/pkg/persistentips" "github.com/ovn-kubernetes/ovn-kubernetes/go-controller/pkg/types" "github.com/ovn-kubernetes/ovn-kubernetes/go-controller/pkg/util" utilerrors "github.com/ovn-kubernetes/ovn-kubernetes/go-controller/pkg/util/errors" @@ -98,8 +95,6 @@ func (bsnc *BaseUserDefinedNetworkController) AddUserDefinedNetworkResourceCommo mp.Namespace, mp.Name, err) return err } - case factory.IPAMClaimsType: - return nil default: return bsnc.AddResourceCommon(objType, obj) @@ -160,8 +155,6 @@ func (bsnc *BaseUserDefinedNetworkController) UpdateUserDefinedNetworkResourceCo return err } } - case factory.IPAMClaimsType: - return nil default: return fmt.Errorf("object type %s not supported", objType) @@ -204,25 +197,6 @@ func (bsnc *BaseUserDefinedNetworkController) DeleteUserDefinedNetworkResourceCo return err } - case factory.IPAMClaimsType: - ipamClaim, ok := obj.(*ipamclaimsapi.IPAMClaim) - if !ok { - return fmt.Errorf("could not cast obj of type %T to *ipamclaimsapi.IPAMClaim", obj) - } - - switchName, err := bsnc.getExpectedSwitchName(dummyPod()) - if err != nil { - return err - } - ipAllocator := bsnc.lsManager.ForSwitch(switchName) - err = bsnc.ipamClaimsReconciler.Reconcile(ipamClaim, nil, ipAllocator) - if err != nil && !errors.Is(err, persistentips.ErrIgnoredIPAMClaim) { - return fmt.Errorf("error deleting IPAMClaim: %w", err) - } else if errors.Is(err, persistentips.ErrIgnoredIPAMClaim) { - return nil // let's avoid the log below, since nothing was released. - } - klog.Infof("Released IPs %q for network %q", ipamClaim.Status.IPs, ipamClaim.Spec.Network) - default: return bsnc.DeleteResourceCommon(objType, obj) } @@ -897,19 +871,6 @@ func cleanupPolicyLogicalEntities(nbClient libovsdbclient.Client, ops []ovsdb.Op return ops, nil } -// WatchIPAMClaims starts the watching of IPAMClaim resources and calls -// back the appropriate handler logic -func (bsnc *BaseUserDefinedNetworkController) WatchIPAMClaims() error { - if bsnc.ipamClaimsHandler != nil { - return nil - } - handler, err := bsnc.retryIPAMClaims.WatchResource() - if err != nil { - bsnc.ipamClaimsHandler = handler - } - return err -} - func (oc *BaseUserDefinedNetworkController) allowPersistentIPs() bool { return config.OVNKubernetesFeature.EnablePersistentIPs && util.DoesNetworkRequireIPAM(oc.GetNetInfo()) && diff --git a/go-controller/pkg/ovn/base_secondary_layer2_network_controller.go b/go-controller/pkg/ovn/base_secondary_layer2_network_controller.go index 2ec50db6ce..07afdf9bd5 100644 --- a/go-controller/pkg/ovn/base_secondary_layer2_network_controller.go +++ b/go-controller/pkg/ovn/base_secondary_layer2_network_controller.go @@ -42,9 +42,6 @@ func (oc *BaseLayer2UserDefinedNetworkController) stop() { oc.cancelableCtx.Cancel() oc.wg.Wait() - if oc.ipamClaimsHandler != nil { - oc.watchFactory.RemoveIPAMClaimsHandler(oc.ipamClaimsHandler) - } if oc.netPolicyHandler != nil { oc.watchFactory.RemovePolicyHandler(oc.netPolicyHandler) } @@ -120,17 +117,6 @@ func (oc *BaseLayer2UserDefinedNetworkController) run() error { return err } - // when on IC, it will be the NetworkController that returns the IPAMClaims - // IPs back to the pool - if oc.allocatesPodAnnotation() && oc.allowPersistentIPs() { - // WatchIPAMClaims should be started before WatchPods to prevent OVN-K - // master assigning IPs to pods without taking into account the persistent - // IPs set aside for the IPAMClaims - if err := oc.WatchIPAMClaims(); err != nil { - return err - } - } - if err := oc.WatchPods(); err != nil { return err } @@ -346,18 +332,6 @@ func (oc *BaseLayer2UserDefinedNetworkController) syncNodes(nodes []interface{}) return nil } -func (oc *BaseLayer2UserDefinedNetworkController) syncIPAMClaims(ipamClaims []interface{}) error { - switchName, err := oc.getExpectedSwitchName(dummyPod()) - if err != nil { - return err - } - return oc.ipamClaimsReconciler.Sync(ipamClaims, oc.lsManager.ForSwitch(switchName)) -} - -func dummyPod() *corev1.Pod { - return &corev1.Pod{Spec: corev1.PodSpec{NodeName: ""}} -} - // getDenyARPAndNSOnMACVRF provides ACLs to drop ARP and NS from pods to the // gateway IP on the MACVRF port. Even though these requests are unicast, OVN is // flooding them for historic reasons. We don't want these request to be flooded diff --git a/go-controller/pkg/ovn/layer2_user_defined_network_controller.go b/go-controller/pkg/ovn/layer2_user_defined_network_controller.go index c3e20ea040..4fdd6ac316 100644 --- a/go-controller/pkg/ovn/layer2_user_defined_network_controller.go +++ b/go-controller/pkg/ovn/layer2_user_defined_network_controller.go @@ -167,9 +167,6 @@ func (h *layer2UserDefinedNetworkControllerEventHandler) SyncFunc(objs []interfa case factory.MultiNetworkPolicyType: syncFunc = h.oc.syncMultiNetworkPolicies - case factory.IPAMClaimsType: - syncFunc = h.oc.syncIPAMClaims - default: return fmt.Errorf("no sync function for object type %s", h.objType) } @@ -635,9 +632,6 @@ func (oc *Layer2UserDefinedNetworkController) SyncNodes(nodes []*corev1.Node) er func (oc *Layer2UserDefinedNetworkController) initRetryFramework() { oc.retryPods = oc.newRetryFramework(factory.PodType) - if oc.allocatesPodAnnotation() && oc.AllowsPersistentIPs() { - oc.retryIPAMClaims = oc.newRetryFramework(factory.IPAMClaimsType) - } // When a user-defined network is enabled as a primary network for namespace, // then watch for namespace and network policy events. diff --git a/go-controller/pkg/ovn/localnet_user_defined_network_controller.go b/go-controller/pkg/ovn/localnet_user_defined_network_controller.go index a9ace32e45..39b5eb5c9f 100644 --- a/go-controller/pkg/ovn/localnet_user_defined_network_controller.go +++ b/go-controller/pkg/ovn/localnet_user_defined_network_controller.go @@ -138,9 +138,6 @@ func (h *LocalnetUserDefinedNetworkControllerEventHandler) SyncFunc(objs []inter case factory.MultiNetworkPolicyType: syncFunc = h.oc.syncMultiNetworkPolicies - case factory.IPAMClaimsType: - syncFunc = h.oc.syncIPAMClaims - default: return fmt.Errorf("no sync function for object type %s", h.objType) } @@ -330,9 +327,6 @@ func (oc *LocalnetUserDefinedNetworkController) SyncNodes(nodes []*corev1.Node) func (oc *LocalnetUserDefinedNetworkController) initRetryFramework() { oc.retryPods = oc.newRetryFramework(factory.PodType) - if oc.allocatesPodAnnotation() && oc.AllowsPersistentIPs() { - oc.retryIPAMClaims = oc.newRetryFramework(factory.IPAMClaimsType) - } // For secondary networks, we don't have to watch namespace events if // multi-network policy support is not enabled. We don't support From ca608635284281b1832d02e33d637adf26ffa1a1 Mon Sep 17 00:00:00 2001 From: Yun Zhou Date: Tue, 19 May 2026 13:21:49 -0700 Subject: [PATCH 28/51] ovn: drop unreachable IPAMClaim reconciler after non-IC removal With non-IC gone, the controller-side IPAMClaim reconciler and pod-deletion guard are also unreachable for L2/Localnet UDNs. Cluster-manager handles allocation and release in IC mode. Signed-off-by: Yun Zhou --- .../pkg/ovn/base_network_controller.go | 3 - .../base_network_controller_user_defined.go | 86 ------------------- .../layer2_user_defined_network_controller.go | 13 +-- ...ocalnet_user_defined_network_controller.go | 13 +-- 4 files changed, 2 insertions(+), 113 deletions(-) diff --git a/go-controller/pkg/ovn/base_network_controller.go b/go-controller/pkg/ovn/base_network_controller.go index 123995e0da..0e652f24ef 100644 --- a/go-controller/pkg/ovn/base_network_controller.go +++ b/go-controller/pkg/ovn/base_network_controller.go @@ -46,7 +46,6 @@ import ( lsm "github.com/ovn-kubernetes/ovn-kubernetes/go-controller/pkg/ovn/logical_switch_manager" "github.com/ovn-kubernetes/ovn-kubernetes/go-controller/pkg/ovn/routeimport" zoneic "github.com/ovn-kubernetes/ovn-kubernetes/go-controller/pkg/ovn/zone_interconnect" - "github.com/ovn-kubernetes/ovn-kubernetes/go-controller/pkg/persistentips" ovnretry "github.com/ovn-kubernetes/ovn-kubernetes/go-controller/pkg/retry" "github.com/ovn-kubernetes/ovn-kubernetes/go-controller/pkg/syncmap" "github.com/ovn-kubernetes/ovn-kubernetes/go-controller/pkg/types" @@ -119,8 +118,6 @@ type BaseNetworkController struct { // An utility to allocate the PodAnnotation to pods podAnnotationAllocator *pod.PodAnnotationAllocator - ipamClaimsReconciler *persistentips.IPAMClaimReconciler - // A cache of all logical ports known to the controller logicalPortCache *PortCache // optional callback for consumers that need to react when a pod's logical diff --git a/go-controller/pkg/ovn/base_network_controller_user_defined.go b/go-controller/pkg/ovn/base_network_controller_user_defined.go index 8825b4ed1e..0e1e7bd900 100644 --- a/go-controller/pkg/ovn/base_network_controller_user_defined.go +++ b/go-controller/pkg/ovn/base_network_controller_user_defined.go @@ -14,7 +14,6 @@ import ( nadapi "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" corev1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/klog/v2" utilnet "k8s.io/utils/net" "k8s.io/utils/ptr" @@ -489,18 +488,6 @@ func (bsnc *BaseUserDefinedNetworkController) removePodForUserDefinedNetwork(pod continue } - // if we allow for persistent IPs, then we need to check if this pod has an IPAM Claim - if bsnc.allowPersistentIPs() { - hasIPAMClaim, err := bsnc.hasIPAMClaim(pod, nadKey) - if err != nil { - return fmt.Errorf("unable to determine if pod %s has IPAM Claim: %w", podDesc, err) - } - // if there is an IPAM claim, don't release the pod IPs - if hasIPAMClaim { - continue - } - } - // Releasing IPs needs to happen last so that we can deterministically know that if delete failed that // the IP of the pod needs to be released. Otherwise we could have a completed pod failed to be removed // and we dont know if the IP was released or not, and subsequently could accidentally release the IP @@ -517,73 +504,6 @@ func (bsnc *BaseUserDefinedNetworkController) removePodForUserDefinedNetwork(pod return nil } -// hasIPAMClaim determines whether a pod's IPAM is being handled by IPAMClaim CR. -// pod passed should already be validated as having a network connection to nadKey -func (bsnc *BaseUserDefinedNetworkController) hasIPAMClaim(pod *corev1.Pod, nadKey string) (bool, error) { - if !bsnc.AllowsPersistentIPs() { - return false, nil - } - - var ipamClaimName string - var wasPersistentIPRequested bool - if bsnc.IsPrimaryNetwork() { - // 'k8s.ovn.org/primary-udn-ipamclaim' annotation has been deprecated. Maintain backward compatibility by - // using it as a fallback; when defaultNSE.IPAMClaimReference is set, it takes precedence. - if desiredClaimName, isIPAMClaimRequested := pod.Annotations[util.DeprecatedOvnUDNIPAMClaimName]; isIPAMClaimRequested && desiredClaimName != "" { - wasPersistentIPRequested = true - ipamClaimName = desiredClaimName - } - defaultNSE, err := util.GetK8sPodDefaultNetworkSelection(pod) - if err != nil { - return false, err - } - if defaultNSE != nil && defaultNSE.IPAMClaimReference != "" { - wasPersistentIPRequested = true - ipamClaimName = defaultNSE.IPAMClaimReference - } - } else { - // secondary network the IPAM claim reference is on the network selection element - on, networkMap, err := util.GetUDNPodNADToNetworkMapping( - pod, - bsnc.GetNetInfo(), - bsnc.networkManager.GetNetworkNameForNADKey, - ) - if err != nil { - return false, fmt.Errorf("failed to get network mapping for pod %s/%s on network %s: %v", - pod.Namespace, pod.Name, bsnc.GetNetworkName(), err) - } - if !on { - klog.Warningf("Pod %s/%s is not scheduled on network %s", pod.Namespace, pod.Name, bsnc.GetNetworkName()) - return false, nil - } - for key, network := range networkMap { - if key == nadKey { - if len(network.IPAMClaimReference) > 0 { - ipamClaimName = network.IPAMClaimReference - wasPersistentIPRequested = true - } - break - } - } - } - - if !wasPersistentIPRequested || len(ipamClaimName) == 0 { - return false, nil - } - - ipamClaim, err := bsnc.ipamClaimsReconciler.FindIPAMClaim(ipamClaimName, pod.Namespace) - if apierrors.IsNotFound(err) { - klog.Errorf("IPAMClaim %q for namespace: %q not found...will release IPs: %v", - ipamClaimName, pod.Namespace, err) - return false, nil - } else if err != nil { - return false, fmt.Errorf("failed to get IPAMClaim %s/%s: %w", pod.Namespace, ipamClaimName, err) - } - - hasIPAMClaim := ipamClaim != nil && len(ipamClaim.Status.IPs) > 0 - return hasIPAMClaim, nil -} - func (bsnc *BaseUserDefinedNetworkController) syncPodsForUserDefinedNetwork(pods []interface{}) error { annotatedLocalPods := map[*corev1.Pod]map[string]*util.PodAnnotation{} // get the list of logical switch ports (equivalent to pods). Reserve all existing Pod IPs to @@ -871,12 +791,6 @@ func cleanupPolicyLogicalEntities(nbClient libovsdbclient.Client, ops []ovsdb.Op return ops, nil } -func (oc *BaseUserDefinedNetworkController) allowPersistentIPs() bool { - return config.OVNKubernetesFeature.EnablePersistentIPs && - util.DoesNetworkRequireIPAM(oc.GetNetInfo()) && - util.AllowsPersistentIPs(oc.GetNetInfo()) -} - // buildUDNEgressSNAT is used to build the conditional SNAT required on L3 and L2 UDNs to // steer traffic correctly via mp0 when leaving OVN to the host func (bsnc *BaseUserDefinedNetworkController) buildUDNEgressSNAT(localPodSubnets []*net.IPNet, outputPort string, isUDNAdvertised bool) ([]*nbdb.NAT, error) { diff --git a/go-controller/pkg/ovn/layer2_user_defined_network_controller.go b/go-controller/pkg/ovn/layer2_user_defined_network_controller.go index 4fdd6ac316..0ccc7525ad 100644 --- a/go-controller/pkg/ovn/layer2_user_defined_network_controller.go +++ b/go-controller/pkg/ovn/layer2_user_defined_network_controller.go @@ -38,7 +38,6 @@ import ( "github.com/ovn-kubernetes/ovn-kubernetes/go-controller/pkg/ovn/routeimport" "github.com/ovn-kubernetes/ovn-kubernetes/go-controller/pkg/ovn/topology" zoneinterconnect "github.com/ovn-kubernetes/ovn-kubernetes/go-controller/pkg/ovn/zone_interconnect" - "github.com/ovn-kubernetes/ovn-kubernetes/go-controller/pkg/persistentips" "github.com/ovn-kubernetes/ovn-kubernetes/go-controller/pkg/retry" "github.com/ovn-kubernetes/ovn-kubernetes/go-controller/pkg/syncmap" "github.com/ovn-kubernetes/ovn-kubernetes/go-controller/pkg/types" @@ -318,21 +317,11 @@ func NewLayer2UserDefinedNetworkController( } if oc.allocatesPodAnnotation() { - var claimsReconciler persistentips.PersistentAllocations - if oc.allowPersistentIPs() { - ipamClaimsReconciler := persistentips.NewIPAMClaimReconciler( - oc.kube, - oc.GetNetInfo(), - oc.watchFactory.IPAMClaimsInformer().Lister(), - ) - oc.ipamClaimsReconciler = ipamClaimsReconciler - claimsReconciler = ipamClaimsReconciler - } oc.podAnnotationAllocator = pod.NewPodAnnotationAllocator( oc.GetNetInfo(), cnci.watchFactory.PodCoreInformer().Lister(), cnci.kube, - claimsReconciler) + nil) } // enable multicast support for UDN only for primaries + multicast enabled diff --git a/go-controller/pkg/ovn/localnet_user_defined_network_controller.go b/go-controller/pkg/ovn/localnet_user_defined_network_controller.go index 39b5eb5c9f..a71eb77f7e 100644 --- a/go-controller/pkg/ovn/localnet_user_defined_network_controller.go +++ b/go-controller/pkg/ovn/localnet_user_defined_network_controller.go @@ -25,7 +25,6 @@ import ( addressset "github.com/ovn-kubernetes/ovn-kubernetes/go-controller/pkg/ovn/address_set" "github.com/ovn-kubernetes/ovn-kubernetes/go-controller/pkg/ovn/addresssetmanager" lsm "github.com/ovn-kubernetes/ovn-kubernetes/go-controller/pkg/ovn/logical_switch_manager" - "github.com/ovn-kubernetes/ovn-kubernetes/go-controller/pkg/persistentips" "github.com/ovn-kubernetes/ovn-kubernetes/go-controller/pkg/retry" "github.com/ovn-kubernetes/ovn-kubernetes/go-controller/pkg/syncmap" "github.com/ovn-kubernetes/ovn-kubernetes/go-controller/pkg/types" @@ -205,21 +204,11 @@ func NewLocalnetUserDefinedNetworkController( } if oc.allocatesPodAnnotation() { - var claimsReconciler persistentips.PersistentAllocations - if oc.allowPersistentIPs() { - ipamClaimsReconciler := persistentips.NewIPAMClaimReconciler( - oc.kube, - oc.GetNetInfo(), - oc.watchFactory.IPAMClaimsInformer().Lister(), - ) - oc.ipamClaimsReconciler = ipamClaimsReconciler - claimsReconciler = ipamClaimsReconciler - } oc.podAnnotationAllocator = pod.NewPodAnnotationAllocator( netInfo, cnci.watchFactory.PodCoreInformer().Lister(), cnci.kube, - claimsReconciler) + nil) } // disable multicast support for UDNs From d7ac03e4f32a87ce04225d37ecfcb824167f6e18 Mon Sep 17 00:00:00 2001 From: Tim Rozet Date: Mon, 18 May 2026 14:33:07 -0400 Subject: [PATCH 29/51] Support manual DPU simulator Helm installs Teach kind-helm.sh about DPU simulator host and DPU modes, external values files, and existing kubeconfigs so OVN-Kubernetes can be installed manually after dpu-simulator generates DPU-specific values. Also wire the DPU-side FRR-K8S flow to use host-cluster credentials, keep unmanaged BGP resources split between the DPU and host APIs, and recover dpu-simulator-managed system add-ons after OVN-Kubernetes is ready. Signed-off-by: Tim Rozet Assisted-by: Codex --- contrib/kind-common.sh | 147 +++++++++++++++++++++++------- contrib/kind-dpu-sim-lib.sh | 174 ++++++++++++++++++++++++++++++++++++ contrib/kind-helm.sh | 162 +++++++++++++++++++++++++++------ 3 files changed, 426 insertions(+), 57 deletions(-) create mode 100644 contrib/kind-dpu-sim-lib.sh diff --git a/contrib/kind-common.sh b/contrib/kind-common.sh index 3f590eb750..1c577092fe 100644 --- a/contrib/kind-common.sh +++ b/contrib/kind-common.sh @@ -333,8 +333,10 @@ build_ovn_image() { } run_kubectl() { - # Skip kubeconfig export in container mode - it overwrites container IPs with localhost. - if [ "$RUN_IN_CONTAINER" != true ]; then + # In deploy mode, KUBECONFIG points at an existing cluster and must not be + # merged with `kind export kubeconfig`; repeated merges can duplicate entries. + # In container mode, export would also overwrite reachable container IPs with localhost. + if [ "$RUN_IN_CONTAINER" != true ] && [ "${KIND_CREATE:-true}" == true ]; then kind export kubeconfig --name ${KIND_CLUSTER_NAME} fi @@ -735,9 +737,40 @@ delete_metallb_dir() { rm -rf "${METALLB_DIR}" } +wait_for_ovn_daemonset() { + local ds=$1 + local required=${2:-true} + local endtime=$3 + + if ! kubectl -n ovn-kubernetes get daemonset "${ds}" >/dev/null 2>&1; then + if [ "${required}" = true ]; then + echo "DaemonSet ${ds} was not found in namespace ovn-kubernetes" >&2 + return 1 + fi + return 0 + fi + + local timeout + timeout=$(calculate_timeout ${endtime}) + echo "Waiting for k8s to launch all ${ds} pods (timeout ${timeout})..." + # `kubectl rollout status` errors on DaemonSets with updateStrategy=OnDelete + # (upgrade-ovn.sh sets ovs-node to OnDelete so helm upgrade doesn't roll + # OVS out from under still-running ovnkube-node pods). For OnDelete DSes + # we can't observe rollout progress; just wait for pods to be Ready. + local strategy + strategy=$(kubectl -n ovn-kubernetes get daemonset ${ds} \ + -o=jsonpath='{.spec.updateStrategy.type}' 2>/dev/null) + if [ "${strategy}" = "OnDelete" ]; then + kubectl wait -n ovn-kubernetes --for=condition=ready pods \ + -l app=${ds} --timeout=${timeout}s + else + kubectl rollout status daemonset -n ovn-kubernetes ${ds} --timeout ${timeout}s + fi +} + # kubectl_wait_pods will set a total timeout of 300s for IPv4 and 480s for IPv6. It will first wait for all # DaemonSets to complete with kubectl rollout. This command will block until all pods of the DS are actually up. -# Next, it waits for ovnkube-control-plane pods to post "Ready". +# Next, it waits for ovnkube-control-plane pods to post "Ready" when that deployment is part of the mode. # Last, it will do the same with all pods in the kube-system namespace. kubectl_wait_pods() { # IPv6 cluster seems to take a little longer to come up, so extend the wait time. @@ -749,28 +782,31 @@ kubectl_wait_pods() { # We will make sure that we timeout all commands at current seconds + the desired timeout. endtime=$(( SECONDS + OVN_TIMEOUT )) - for ds in ovnkube-node ovs-node; do - timeout=$(calculate_timeout ${endtime}) - echo "Waiting for k8s to launch all ${ds} pods (timeout ${timeout})..." - # `kubectl rollout status` errors on DaemonSets with updateStrategy=OnDelete - # (upgrade-ovn.sh sets ovs-node to OnDelete so helm upgrade doesn't roll - # OVS out from under still-running ovnkube-node pods). For OnDelete DSes - # we can't observe rollout progress; just wait for pods to be Ready. - strategy=$(kubectl -n ovn-kubernetes get daemonset ${ds} \ - -o=jsonpath='{.spec.updateStrategy.type}' 2>/dev/null) - if [ "${strategy}" = "OnDelete" ]; then - kubectl wait -n ovn-kubernetes --for=condition=ready pods \ - -l app=${ds} --timeout=${timeout}s - else - kubectl rollout status daemonset -n ovn-kubernetes ${ds} --timeout ${timeout}s - fi - done - - for name in ovnkube-control-plane; do + case "${DPU_MODE:-none}" in + dpu) + wait_for_ovn_daemonset ovnkube-node-dpu true ${endtime} + ;; + host) + wait_for_ovn_daemonset ovnkube-node-dpu-host true ${endtime} + wait_for_ovn_daemonset ovnkube-node false ${endtime} + wait_for_ovn_daemonset ovs-node false ${endtime} + ;; + *) + wait_for_ovn_daemonset ovnkube-node true ${endtime} + wait_for_ovn_daemonset ovs-node true ${endtime} + ;; + esac + + if [ "${DPU_MODE:-none}" != "dpu" ]; then + local name + name=ovnkube-control-plane timeout=$(calculate_timeout ${endtime}) echo "Waiting for k8s to create ${name} pods (timeout ${timeout})..." kubectl wait pods -n ovn-kubernetes -l name=${name} --for condition=Ready --timeout=${timeout}s - done + fi + + restart_dpu_sim_multus_after_ovnk + restart_dpu_sim_system_deployments_after_ovnk timeout=$(calculate_timeout ${endtime}) if ! kubectl wait -n kube-system --for=condition=ready pods --all --timeout=${timeout}s ; then @@ -916,7 +952,11 @@ docker_create_second_disconnected_interface() { } enable_multi_net() { - install_multus + if [ "${DPU_MODE:-none}" == "none" ]; then + install_multus + else + echo "Skipping multus-cni installation; DPU simulator mode expects Multus to be installed by dpu-simulator" + fi install_mpolicy_crd install_ipamclaim_crd docker_create_second_disconnected_interface "underlay" # localnet scenarios require an extra interface @@ -1247,7 +1287,18 @@ deploy_frr_external_container() { # Add route-reflector-client for IPv6 neighbors sed -i '/neighbor fc00.*remote-as 64512/a \ neighbor {{ . }} route-reflector-client' frr/frr.conf.tmpl fi - ./demo.sh + if [ "${OCI_BIN}" == "podman" ]; then + # frr-k8s' demo script prefers docker when both docker and podman are + # installed. Keep it on the same runtime as kind-helm.sh so later podman + # operations can find the external frr container. + local docker_wrapper_dir + docker_wrapper_dir=$(mktemp -d) + ln -s "$(command -v podman)" "${docker_wrapper_dir}/docker" + PATH="${docker_wrapper_dir}:${PATH}" ./demo.sh + rm -rf "${docker_wrapper_dir}" + else + ./demo.sh + fi popd || exit 1 if [ "$PLATFORM_IPV6_SUPPORT" == true ]; then # Enable IPv6 forwarding in FRR @@ -1379,6 +1430,13 @@ destroy_bgp() { fi } +install_frr_k8s_crds() { + echo "Installing frr-k8s CRDs ..." + clone_frr + kubectl apply -f "${FRR_TMP_DIR}"/frr-k8s/config/crd/bases/frrk8s.metallb.io_frrconfigurations.yaml + kubectl apply -f "${FRR_TMP_DIR}"/frr-k8s/config/crd/bases/frrk8s.metallb.io_frrnodestates.yaml +} + install_frr_k8s() { local bgp_port=${1:-0} echo "Installing frr-k8s ..." @@ -1415,15 +1473,27 @@ install_frr_k8s() { exit 1 } fi + install_frr_k8s_host_api_crds kubectl apply -f "${FRR_TMP_DIR}"/frr-k8s/config/all-in-one/frr-k8s.yaml + create_frr_k8s_remote_kubeconfig_secret + configure_frr_k8s_remote_daemonsets } wait_for_frr_k8s() { - kubectl wait -n frr-k8s-system deployment frr-k8s-statuscleaner --for condition=Available --timeout 2m - kubectl rollout status -n frr-k8s-system daemonset frr-k8s-daemon --timeout 2m + if kubectl -n frr-k8s-system get deployment frr-k8s-statuscleaner >/dev/null 2>&1; then + kubectl wait -n frr-k8s-system deployment frr-k8s-statuscleaner --for condition=Available --timeout 2m + fi + if kubectl -n frr-k8s-system get daemonset frr-k8s-daemon >/dev/null 2>&1; then + kubectl rollout status -n frr-k8s-system daemonset frr-k8s-daemon --timeout 2m + return + fi + local ds + for ds in $(kubectl -n frr-k8s-system get daemonset -l dpu-sim.ovn.org/frr-remote=true -o name); do + kubectl rollout status -n frr-k8s-system "${ds}" --timeout 2m + done } -configure_frr_k8s() { +apply_frr_k8s_receive_config() { # apply a BGP peer configration with the external gateway that does not # exchange routes pushd "${FRR_TMP_DIR}"/frr-k8s/hack/demo/configs || exit 1 @@ -1443,10 +1513,12 @@ configure_frr_k8s() { # frr-k8s webhook is declaring readiness before its endpoint is serving. # Let's do our own probing. Also will print logs in case of failure so we get - # insights on why this is hapenning + # insights on why this is hapenning. In remote mode the host API does not use + # the DPU-cluster webhook service, so skip this probe. local r r=0 - timeout 60s bash -x </dev/null 2>&1; then + return + fi + + echo "Restarting dpu-simulator Multus after OVN-Kubernetes is ready..." + kubectl -n kube-system rollout restart daemonset/kube-multus-ds + kubectl -n kube-system rollout status daemonset/kube-multus-ds --timeout 2m +} + +resume_dpu_sim_system_deployment() { + local namespace=$1 + local name=$2 + local kubeconfig=${3:-} + local kubectl_cmd=(kubectl) + + if [ -n "${kubeconfig}" ]; then + kubectl_cmd=(kubectl --kubeconfig "${kubeconfig}") + fi + + if ! "${kubectl_cmd[@]}" -n "${namespace}" get deployment "${name}" >/dev/null 2>&1; then + return + fi + + local replicas + replicas=$("${kubectl_cmd[@]}" -n "${namespace}" get deployment "${name}" -o jsonpath='{.metadata.annotations.dpu-sim\.io/suspend-replicas}') + if [ -n "${replicas}" ]; then + echo "Restoring dpu-simulator deployment ${namespace}/${name} to ${replicas} replicas..." + "${kubectl_cmd[@]}" -n "${namespace}" patch deployment "${name}" --type=json -p="[{\"op\":\"replace\",\"path\":\"/spec/replicas\",\"value\":${replicas}},{\"op\":\"remove\",\"path\":\"/metadata/annotations/dpu-sim.io~1suspend-replicas\"}]" + fi + + "${kubectl_cmd[@]}" -n "${namespace}" rollout restart deployment/"${name}" + "${kubectl_cmd[@]}" -n "${namespace}" rollout status deployment/"${name}" --timeout 2m +} + +restart_dpu_sim_system_deployments_after_ovnk() { + if [ "${DPU_MODE:-none}" == "none" ]; then + return + fi + + if [ "${DPU_MODE}" == "host" ]; then + echo "Leaving dpu-simulator host system deployments suspended until DPU OVN-Kubernetes is installed" + return + fi + + resume_dpu_sim_system_deployment kube-system coredns + resume_dpu_sim_system_deployment local-path-storage local-path-provisioner + + if [ "${DPU_MODE}" == "dpu" ] && [ -n "${FRR_K8S_HOST_KUBECONFIG:-}" ]; then + echo "Restoring dpu-simulator host system deployments after DPU OVN-Kubernetes is ready..." + resume_dpu_sim_system_deployment kube-system coredns "${FRR_K8S_HOST_KUBECONFIG}" + resume_dpu_sim_system_deployment local-path-storage local-path-provisioner "${FRR_K8S_HOST_KUBECONFIG}" + fi +} + +frr_k8s_remote_enabled() { + [[ -n "${FRR_K8S_REMOTE_KUBECONFIG:-}" || -n "${FRR_K8S_REMOTE_NODE_MAP:-}" ]] +} + +validate_frr_k8s_remote() { + if ! frr_k8s_remote_enabled; then + return + fi + if [[ -z "${FRR_K8S_REMOTE_KUBECONFIG:-}" || -z "${FRR_K8S_REMOTE_NODE_MAP:-}" ]]; then + echo "FRR-K8S remote mode requires both FRR_K8S_REMOTE_KUBECONFIG and FRR_K8S_REMOTE_NODE_MAP" >&2 + exit 1 + fi + if [[ ! -f "${FRR_K8S_REMOTE_KUBECONFIG}" ]]; then + echo "FRR-K8S remote kubeconfig does not exist: ${FRR_K8S_REMOTE_KUBECONFIG}" >&2 + exit 1 + fi + if [[ -n "${FRR_K8S_HOST_KUBECONFIG:-}" && ! -f "${FRR_K8S_HOST_KUBECONFIG}" ]]; then + echo "FRR-K8S host kubeconfig does not exist: ${FRR_K8S_HOST_KUBECONFIG}" >&2 + exit 1 + fi +} + +frr_k8s_host_kubeconfig() { + printf '%s' "${FRR_K8S_HOST_KUBECONFIG:-${FRR_K8S_REMOTE_KUBECONFIG}}" +} + +ensure_frr_k8s_namespace() { + local kubeconfig=${1:-} + local kubectl_cmd=(kubectl) + if [ -n "${kubeconfig}" ]; then + kubectl_cmd=(kubectl --kubeconfig "${kubeconfig}") + fi + + "${kubectl_cmd[@]}" create namespace frr-k8s-system --dry-run=client -o yaml | \ + "${kubectl_cmd[@]}" apply -f - +} + +install_frr_k8s_host_api_crds() { + validate_frr_k8s_remote + if ! frr_k8s_remote_enabled; then + return + fi + + echo "Installing frr-k8s CRDs into the remote host API ..." + local host_kubeconfig + host_kubeconfig=$(frr_k8s_host_kubeconfig) + ensure_frr_k8s_namespace "${host_kubeconfig}" + kubectl --kubeconfig "${host_kubeconfig}" apply \ + -f "${FRR_TMP_DIR}"/frr-k8s/config/crd/bases/frrk8s.metallb.io_frrconfigurations.yaml + kubectl --kubeconfig "${host_kubeconfig}" apply \ + -f "${FRR_TMP_DIR}"/frr-k8s/config/crd/bases/frrk8s.metallb.io_frrnodestates.yaml +} + +create_frr_k8s_remote_kubeconfig_secret() { + validate_frr_k8s_remote + if ! frr_k8s_remote_enabled; then + return + fi + + kubectl -n frr-k8s-system create secret generic frr-k8s-host-kubeconfig \ + --from-file=kubeconfig="${FRR_K8S_REMOTE_KUBECONFIG}" \ + --dry-run=client -o yaml | kubectl apply -f - +} + +configure_frr_k8s_remote_daemonsets() { + validate_frr_k8s_remote + if ! frr_k8s_remote_enabled; then + return + fi + + local source_json="${FRR_TMP_DIR}/frr-k8s-daemon.json" + kubectl -n frr-k8s-system get daemonset frr-k8s-daemon -o json > "${source_json}" + kubectl -n frr-k8s-system delete daemonset -l dpu-sim.ovn.org/frr-remote=true --ignore-not-found + + local pair host_node dpu_node safe_name ds_name + IFS=',' read -ra pairs <<< "${FRR_K8S_REMOTE_NODE_MAP}" + for pair in "${pairs[@]}"; do + host_node="${pair%%=*}" + dpu_node="${pair#*=}" + if [[ -z "${host_node}" || -z "${dpu_node}" || "${pair}" != *"="* ]]; then + echo "Invalid FRR_K8S_REMOTE_NODE_MAP entry: ${pair}" >&2 + exit 1 + fi + safe_name=$(printf '%s' "${host_node}" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9-]/-/g' | sed 's/^-*//;s/-*$//') + if [[ -z "${safe_name}" ]]; then + echo "Invalid host node name for FRR_K8S_REMOTE_NODE_MAP entry: ${pair}" >&2 + exit 1 + fi + ds_name="frr-k8s-daemon-${safe_name:0:45}" + echo "Creating remote frr-k8s daemonset ${ds_name}: host node ${host_node}, DPU node ${dpu_node}" + jq \ + --arg name "${ds_name}" \ + --arg host_node "${host_node}" \ + --arg dpu_node "${dpu_node}" \ + 'del(.metadata.uid, .metadata.resourceVersion, .metadata.generation, .metadata.creationTimestamp, .metadata.managedFields, .status) + | .metadata.name = $name + | .metadata.labels["dpu-sim.ovn.org/frr-remote"] = "true" + | .metadata.labels["dpu-sim.ovn.org/frr-host-node"] = $host_node + | .spec.selector.matchLabels["dpu-sim.ovn.org/frr-host-node"] = $host_node + | .spec.template.metadata.labels["dpu-sim.ovn.org/frr-remote"] = "true" + | .spec.template.metadata.labels["dpu-sim.ovn.org/frr-host-node"] = $host_node + | .spec.template.spec.nodeSelector = {"kubernetes.io/hostname": $dpu_node} + | (.spec.template.spec.containers[] | select(.name == "frr-k8s").args) |= ((. // []) | map(if startswith("--node-name=") then "--node-name=" + $host_node else . end)) + | (.spec.template.spec.containers[] | select(.name == "frr-k8s").env) |= ((. // []) + [{"name":"KUBECONFIG","value":"/var/run/host-kubeconfig/kubeconfig"}]) + | (.spec.template.spec.containers[] | select(.name == "frr-k8s").volumeMounts) |= ((. // []) + [{"name":"host-kubeconfig","mountPath":"/var/run/host-kubeconfig","readOnly":true}]) + | .spec.template.spec.volumes = ((.spec.template.spec.volumes // []) + [{"name":"host-kubeconfig","secret":{"secretName":"frr-k8s-host-kubeconfig"}}])' \ + "${source_json}" | kubectl apply -f - + done + + kubectl -n frr-k8s-system delete daemonset frr-k8s-daemon --ignore-not-found +} diff --git a/contrib/kind-helm.sh b/contrib/kind-helm.sh index 2304b044cc..6f6a65e7b3 100755 --- a/contrib/kind-helm.sh +++ b/contrib/kind-helm.sh @@ -10,12 +10,24 @@ export DIR="$( cd -- "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd ) # Source the kind-common.sh file from the same directory where this script is located source "${DIR}/kind-common.sh" +source "${DIR}/kind-dpu-sim-lib.sh" + +OVN_HELM_EXTRA_VALUES=() set_default_params() { set_common_default_params check_ipv6 set_cluster_cidr_ip_families - OVN_ENABLE_OVNKUBE_IDENTITY=${OVN_ENABLE_OVNKUBE_IDENTITY:-true} + DPU_MODE=${DPU_MODE:-none} + if [[ "${DPU_MODE}" != "none" && "${DPU_MODE}" != "host" && "${DPU_MODE}" != "dpu" ]]; then + echo "Invalid DPU_MODE: ${DPU_MODE}. Expected one of: none, host, dpu" + exit 1 + fi + local ovnkube_identity_default=true + if [ "${DPU_MODE}" != "none" ]; then + ovnkube_identity_default=false + fi + OVN_ENABLE_OVNKUBE_IDENTITY=${OVN_ENABLE_OVNKUBE_IDENTITY:-${ovnkube_identity_default}} } usage() { @@ -51,6 +63,11 @@ usage() { echo " [ -cn | --cluster-name ]" echo " [ -mip | --metrics-ip ]" echo " [ -mtu ]" + echo " [ --dpu-mode ]" + echo " [ -f | --extra-values ]" + echo " [ --frr-k8s-remote-kubeconfig ]" + echo " [ --frr-k8s-host-kubeconfig ]" + echo " [ --frr-k8s-remote-node-map ]" echo " [ --enable-coredumps ]" echo " [ -h ]" echo "" @@ -129,6 +146,11 @@ usage() { echo "-sw | --allow-system-writes Allow the script to write to /etc/hosts and other system files when needed." echo "-ric | --run-in-container Run the script from inside a docker container (adapts kubeconfig API URL)." echo "-kc | --kubeconfig Output kubeconfig path. DEFAULT: \$HOME/\$KIND_CLUSTER_NAME.conf" + echo "--dpu-mode Deploy OVN-Kubernetes for DPU simulator mode: none, host, or dpu. DEFAULT: none" + echo "-f | --extra-values Extra Helm values file appended after the base values file. May be repeated." + echo "--frr-k8s-remote-kubeconfig Kubeconfig used by DPU-cluster FRR-K8S pods to watch the host cluster API." + echo "--frr-k8s-host-kubeconfig Kubeconfig used by this script to write FRR-K8S resources to the host cluster API." + echo "--frr-k8s-remote-node-map Comma-separated host-node=dpu-node pairs for remote FRR-K8S node-name mapping." echo "-nokvipam | --opt-out-kv-ipam Skip installing the KubeVirt IPAM controller (requires --install-kubevirt)." echo "" @@ -252,8 +274,6 @@ parse_args() { ;; -cn | --cluster-name ) shift KIND_CLUSTER_NAME=$1 - # Setup KUBECONFIG - set_default_params ;; -dns | --enable-dnsnameresolver ) OVN_ENABLE_DNSNAMERESOLVER=true ;; @@ -362,6 +382,48 @@ parse_args() { -kc | --kubeconfig ) shift KUBECONFIG=$1 ;; + --dpu-mode ) shift + DPU_MODE=$1 + if [[ "${DPU_MODE}" != "none" && "${DPU_MODE}" != "host" && "${DPU_MODE}" != "dpu" ]]; then + echo "Invalid --dpu-mode: ${DPU_MODE}. Expected one of: none, host, dpu" + exit 1 + fi + if [[ "${DPU_MODE}" != "none" && -z "${OVN_ENABLE_OVNKUBE_IDENTITY:-}" ]]; then + OVN_ENABLE_OVNKUBE_IDENTITY=false + fi + ;; + -f | --extra-values ) shift + if [[ -z "${1:-}" || "${1:-}" == -* ]]; then + echo "Missing value for --extra-values" >&2 + usage + exit 1 + fi + OVN_HELM_EXTRA_VALUES+=("$1") + ;; + --frr-k8s-remote-kubeconfig ) shift + if [[ -z "${1:-}" || "${1:-}" == -* ]]; then + echo "Missing value for --frr-k8s-remote-kubeconfig" >&2 + usage + exit 1 + fi + FRR_K8S_REMOTE_KUBECONFIG=$1 + ;; + --frr-k8s-host-kubeconfig ) shift + if [[ -z "${1:-}" || "${1:-}" == -* ]]; then + echo "Missing value for --frr-k8s-host-kubeconfig" >&2 + usage + exit 1 + fi + FRR_K8S_HOST_KUBECONFIG=$1 + ;; + --frr-k8s-remote-node-map ) shift + if [[ -z "${1:-}" || "${1:-}" == -* ]]; then + echo "Missing value for --frr-k8s-remote-node-map" >&2 + usage + exit 1 + fi + FRR_K8S_REMOTE_NODE_MAP=$1 + ;; -nokvipam | --opt-out-kv-ipam ) KIND_OPT_OUT_KUBEVIRT_IPAM=true ;; * ) usage @@ -439,6 +501,11 @@ print_params() { echo "KIND_INSTALL_PROMETHEUS = $KIND_INSTALL_PROMETHEUS" echo "KIND_ALLOW_SYSTEM_WRITES = $KIND_ALLOW_SYSTEM_WRITES" echo "RUN_IN_CONTAINER = $RUN_IN_CONTAINER" + echo "DPU_MODE = $DPU_MODE" + echo "OVN_HELM_EXTRA_VALUES = ${OVN_HELM_EXTRA_VALUES[*]}" + echo "FRR_K8S_REMOTE_KUBECONFIG = ${FRR_K8S_REMOTE_KUBECONFIG:-}" + echo "FRR_K8S_HOST_KUBECONFIG = ${FRR_K8S_HOST_KUBECONFIG:-}" + echo "FRR_K8S_REMOTE_NODE_MAP = ${FRR_K8S_REMOTE_NODE_MAP:-}" echo "MASTER_LOG_LEVEL = $MASTER_LOG_LEVEL" echo "NODE_LOG_LEVEL = $NODE_LOG_LEVEL" echo "OVN_LOG_LEVEL_NB = $OVN_LOG_LEVEL_NB" @@ -463,25 +530,56 @@ helm_prereqs() { sudo sysctl fs.inotify.max_user_instances=512 } +helm_extra_values_args() { + local args="" + local values_file + for values_file in "${OVN_HELM_EXTRA_VALUES[@]}"; do + args+=" -f $(printf '%q' "${values_file}")" + done + printf '%s' "${args}" +} + +skip_ovn_image_build_load() { + [ "${DPU_MODE}" != "none" ] && [ "${#OVN_HELM_EXTRA_VALUES[@]}" -gt 0 ] && [ "${OVN_IMAGE}" == "local" ] +} + create_ovn_kubernetes() { cd ${DIR}/../helm/ovn-kubernetes label_ovn_single_node_zones value_file="values-single-node-zone.yaml" + if [ "${DPU_MODE}" == "dpu" ]; then + value_file="values-single-node-zone-dpu.yaml" + fi echo "value_file=${value_file}" + local extra_values_args helm_network_args helm_image_args helm_log_args + extra_values_args="$(helm_extra_values_args)" # For multi-pod-subnet case, NET_CIDR_IPV4 is a list of CIDRs separated by comma. # When Helm encounters a comma within a string value in a --set argument, it attempts to parse the comma as a separator # for multiple values (like a list or a map), not as part of a single string value. set -x ESCAPED_NET_CIDR="${NET_CIDR//,/\\,}" ESCAPED_SVC_CIDR="${SVC_CIDR//,/\\,}" + if [ "${DPU_MODE}" == "dpu" ]; then + helm_network_args="--set global.mtu=${OVN_MTU}" + if skip_ovn_image_build_load; then + helm_image_args="" + else + helm_image_args="--set global.dpuImage.repository=${OVN_IMAGE%:*} --set global.dpuImage.tag=${OVN_IMAGE##*:}" + fi + helm_log_args="--set ovnkube-single-node-zone-dpu.ovnkubeNodeLogLevel=${NODE_LOG_LEVEL} --set-string ovnkube-single-node-zone-dpu.nbLogLevel=\"${OVN_LOG_LEVEL_NB}\" --set-string ovnkube-single-node-zone-dpu.sbLogLevel=\"${OVN_LOG_LEVEL_SB}\" --set-string ovnkube-single-node-zone-dpu.northdLogLevel=\"${OVN_LOG_LEVEL_NORTHD}\" --set-string ovnkube-single-node-zone-dpu.ovnControllerLogLevel=\"${OVN_LOG_LEVEL_CONTROLLER}\"" + else + helm_network_args="--set k8sAPIServer=${API_URL} --set podNetwork=\"${ESCAPED_NET_CIDR}\" --set serviceNetwork=\"${ESCAPED_SVC_CIDR}\" --set mtu=${OVN_MTU}" + if skip_ovn_image_build_load; then + helm_image_args="" + else + helm_image_args="--set global.image.repository=${OVN_IMAGE%:*} --set global.image.tag=${OVN_IMAGE##*:}" + fi + helm_log_args="--set ovnkube-control-plane.logLevel=${MASTER_LOG_LEVEL} --set ovnkube-node.logLevel=${NODE_LOG_LEVEL} --set ovnkube-single-node-zone.ovnkubeNodeLogLevel=${NODE_LOG_LEVEL} --set-string ovnkube-single-node-zone.nbLogLevel=\"${OVN_LOG_LEVEL_NB}\" --set-string ovnkube-single-node-zone.sbLogLevel=\"${OVN_LOG_LEVEL_SB}\" --set-string ovnkube-single-node-zone.northdLogLevel=\"${OVN_LOG_LEVEL_NORTHD}\" --set-string ovnkube-node.ovnControllerLogLevel=\"${OVN_LOG_LEVEL_CONTROLLER}\" --set-string ovnkube-single-node-zone.ovnControllerLogLevel=\"${OVN_LOG_LEVEL_CONTROLLER}\"" + fi cmd=$(cat <&2 + exit 1 + fi + validate_frr_k8s_remote +fi print_params helm_prereqs @@ -597,13 +695,21 @@ fi if [ "$RUN_IN_CONTAINER" == true ]; then run_script_in_container fi -# when using a non-default cluster name, fix up the context/cluster/user names in kubeconfig -if [ "$KIND_CLUSTER_NAME" != "ovn" ]; then +# when using a non-default cluster name created by this script, fix up the +# context/cluster/user names in kubeconfig. In deploy mode the kubeconfig is +# supplied by the caller and should be used as-is. +if [ "$KIND_CREATE" == true ] && [ "$KIND_CLUSTER_NAME" != "ovn" ]; then fixup_kubeconfig_names fi -build_ovn_image +if skip_ovn_image_build_load; then + echo "Skipping OVN image build/load; DPU extra values are expected to provide the image" +else + build_ovn_image +fi detect_apiserver_url -install_ovn_image +if ! skip_ovn_image_build_load; then + install_ovn_image +fi if [ "$OVN_ENABLE_DNSNAMERESOLVER" == true ]; then build_dnsnameresolver_images install_dnsnameresolver_images @@ -617,12 +723,16 @@ if [ "$ENABLE_ROUTE_ADVERTISEMENTS" == true ]; then if [ "$ENABLE_NO_OVERLAY_MANAGED_ROUTING" == true ]; then # Enable bgp port listening on node, required for managed mode. FRR will listen on port 179 to receive BGP updates from other nodes. frr_port=179 - else - # external FRR is required for unmanaged mode + elif [ "${DPU_MODE}" != "host" ]; then + # external FRR is required for unmanaged mode where the FRR-K8S speakers run. deploy_frr_external_container deploy_bgp_external_server fi - install_frr_k8s $frr_port + if [ "${DPU_MODE}" == "host" ]; then + install_frr_k8s_crds + else + install_frr_k8s $frr_port + fi fi if [ "$KIND_REMOVE_TAINT" == true ]; then remove_no_schedule_taint @@ -666,7 +776,7 @@ if [ "$KIND_INSTALL_KUBEVIRT" == true ]; then fi fi -if [ "$ENABLE_ROUTE_ADVERTISEMENTS" == true ]; then +if [ "$ENABLE_ROUTE_ADVERTISEMENTS" == true ] && [ "${DPU_MODE}" != "host" ]; then # wait for frr-k8s to be ready wait_for_frr_k8s if [ "$ENABLE_NO_OVERLAY_MANAGED_ROUTING" != true ]; then From 7fc0f4b19752409cd9f7df0b3e185568d6647af6 Mon Sep 17 00:00:00 2001 From: Tim Rozet Date: Tue, 2 Jun 2026 17:24:08 -0400 Subject: [PATCH 30/51] Add DPU simulator no-overlay CI lane Add a second DPU simulator workflow job that exercises the values-only manual Helm install path for no-overlay mode. Place the deployment helper in contrib so developers can use the same flow locally. The helper expects a built dpu-simulator tree, discovers common checkout locations using GOPATH, selects a consistent Kind provider, and installs OVNK on the host and DPU clusters through contrib/kind-helm.sh. Prefer Docker for the local helper when no provider is set, because rootless Podman Kind nodes cannot load the host Open vSwitch kernel module during DPU simulator setup. Explicit KIND_EXPERIMENTAL_PROVIDER continues to override the default. Allow kind-helm.sh OVN pod waits to be extended through KIND_HELM_OVN_TIMEOUT, and use a longer wait for the DPU simulator helper because the DPU host daemonset can take longer than the normal Kind default to settle. Use a DPU-specific external BGP test subnet so the helper does not collide with the standard kind-helm.sh bgpnet subnet left behind by another local container provider. When running with podman, patch the frr-k8s demo to use podman and the Kind network directly. This avoids creating the external FRR container in host networking, which podman cannot later attach to bgpnet. The workflow checks out dpu-simulator main, builds it, deploys the Kind DPU environment without embedded OVN-Kubernetes, and then runs the dpu-simulator traffic flow tests. This covers the manual flow that depends on generated Helm values and remote FRR-K8S host access artifacts. Signed-off-by: Tim Rozet Assisted-by: Codex --- .github/workflows/kind-dpu-offload.yml | 111 +++++++++++++ contrib/kind-common.sh | 46 ++++-- contrib/kind-dpu-sim-no-overlay.sh | 211 +++++++++++++++++++++++++ contrib/kind-helm.sh | 2 + 4 files changed, 353 insertions(+), 17 deletions(-) create mode 100755 contrib/kind-dpu-sim-no-overlay.sh diff --git a/.github/workflows/kind-dpu-offload.yml b/.github/workflows/kind-dpu-offload.yml index 1a4e0bd7ff..b48fba2dfb 100644 --- a/.github/workflows/kind-dpu-offload.yml +++ b/.github/workflows/kind-dpu-offload.yml @@ -35,6 +35,7 @@ jobs: uses: actions/checkout@v4 with: path: ovn-kubernetes + persist-credentials: false - name: Checkout dpu-simulator uses: actions/checkout@v4 @@ -42,6 +43,7 @@ jobs: repository: ovn-kubernetes/dpu-simulator ref: ${{ env.DPU_SIM_REF }} path: dpu-simulator + persist-credentials: false - name: Set up Go from dpu-simulator uses: actions/setup-go@v5 @@ -131,3 +133,112 @@ jobs: if: always() working-directory: dpu-simulator run: ./bin/dpu-sim --config config-kind-ovnk-offload.yaml --ovn-kubernetes-path "$OVN_KUBERNETES_PATH" --cleanup || true + + kind-dpu-no-overlay: + runs-on: ubuntu-latest + timeout-minutes: 180 + env: + DPU_SIM_PATH: ${{ github.workspace }}/dpu-simulator + DPU_SIM_CONFIG: config-kind-ovnk-offload.yaml + KIND_EXPERIMENTAL_PROVIDER: docker + steps: + - name: Checkout OVN-Kubernetes (PR/push) + uses: actions/checkout@v4 + with: + path: ovn-kubernetes + persist-credentials: false + + - name: Checkout dpu-simulator + uses: actions/checkout@v4 + with: + repository: ovn-kubernetes/dpu-simulator + ref: ${{ env.DPU_SIM_REF }} + path: dpu-simulator + persist-credentials: false + + - name: Set up Go from dpu-simulator + uses: actions/setup-go@v5 + with: + go-version-file: dpu-simulator/go.mod + cache: true + + - name: Install libvirt development headers + run: sudo apt-get update && sudo apt-get install -y libvirt-dev + + - name: Install Helm + uses: azure/setup-helm@v4 + + - name: Set up Python for traffic flow tests + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install kind + run: | + KIND_VERSION=v0.31.0 + curl -fsSLo /tmp/kind-linux-amd64 "https://kind.sigs.k8s.io/dl/${KIND_VERSION}/kind-linux-amd64" + curl -fsSLo /tmp/kind-linux-amd64.sha256sum "https://kind.sigs.k8s.io/dl/${KIND_VERSION}/kind-linux-amd64.sha256sum" + (cd /tmp && sha256sum -c kind-linux-amd64.sha256sum) + chmod +x /tmp/kind-linux-amd64 + sudo mv /tmp/kind-linux-amd64 /usr/local/bin/kind + + - name: Build dpu-simulator + working-directory: dpu-simulator + run: make build + + - name: Disable containerd image store + # Workaround for https://github.com/kubernetes-sigs/kind/issues/3795 + run: | + sudo mkdir -p /etc/docker + [ -s "/etc/docker/daemon.json" ] && { + cat "/etc/docker/daemon.json" | jq '. + {"features":{"containerd-snapshotter": false}}' | sudo tee /etc/docker/daemon.$$ + } || { + echo '{"features":{"containerd-snapshotter": false}}' | sudo tee /etc/docker/daemon.$$ + } + sudo mv -f /etc/docker/daemon.$$ /etc/docker/daemon.json + sudo systemctl restart docker + + - name: Deploy no-overlay DPU simulator with Helm + working-directory: ovn-kubernetes + run: ./contrib/kind-dpu-sim-no-overlay.sh + + - name: TFT Python venv + working-directory: dpu-simulator + run: ./bin/dpu-sim tft venv --config "${DPU_SIM_CONFIG}" + + - name: Run traffic flow tests + working-directory: dpu-simulator + run: ./bin/dpu-sim tft run --config "${DPU_SIM_CONFIG}" + + - name: Export kind logs on failure + if: failure() + run: | + mkdir -p /tmp/kind-dpu-no-overlay-logs + for cluster in dpu-sim-host dpu-sim-host-kind dpu-sim-dpu dpu-sim-dpu-kind; do + kind export logs /tmp/kind-dpu-no-overlay-logs --name "${cluster}" 2>/dev/null || true + done + kubectl --kubeconfig dpu-simulator/kubeconfig/dpu-sim-host.kubeconfig get pods -A -o wide \ + > /tmp/kind-dpu-no-overlay-logs/dpu-sim-host-pods.txt 2>&1 || true + kubectl --kubeconfig dpu-simulator/kubeconfig/dpu-sim-dpu.kubeconfig get pods -A -o wide \ + > /tmp/kind-dpu-no-overlay-logs/dpu-sim-dpu-pods.txt 2>&1 || true + + - name: Upload kind logs on failure + if: failure() + uses: actions/upload-artifact@v4 + with: + name: kind-dpu-no-overlay-logs + path: /tmp/kind-dpu-no-overlay-logs + if-no-files-found: ignore + + - name: Upload TFT results on failure + if: failure() + uses: actions/upload-artifact@v4 + with: + name: dpu-no-overlay-tft-results + path: dpu-simulator/kubernetes-traffic-flow-tests/ft-logs + if-no-files-found: ignore + + - name: Cleanup + if: always() + working-directory: dpu-simulator + run: ./bin/dpu-sim --config "${DPU_SIM_CONFIG}" --ovn-kubernetes-path "$OVN_KUBERNETES_PATH" --cleanup || true diff --git a/contrib/kind-common.sh b/contrib/kind-common.sh index 1c577092fe..923d9fecd7 100644 --- a/contrib/kind-common.sh +++ b/contrib/kind-common.sh @@ -768,15 +768,21 @@ wait_for_ovn_daemonset() { fi } -# kubectl_wait_pods will set a total timeout of 300s for IPv4 and 480s for IPv6. It will first wait for all -# DaemonSets to complete with kubectl rollout. This command will block until all pods of the DS are actually up. -# Next, it waits for ovnkube-control-plane pods to post "Ready" when that deployment is part of the mode. -# Last, it will do the same with all pods in the kube-system namespace. +# kubectl_wait_pods will set a total timeout of 300s for IPv4 and 480s for IPv6, +# unless KIND_HELM_OVN_TIMEOUT is set. It will first wait for all DaemonSets to +# complete with kubectl rollout. This command will block until all pods of the +# DS are actually up. Next, it waits for ovnkube-control-plane pods to post +# "Ready" when that deployment is part of the mode. Last, it will do the same +# with all pods in the kube-system namespace. kubectl_wait_pods() { # IPv6 cluster seems to take a little longer to come up, so extend the wait time. - OVN_TIMEOUT=300 + OVN_TIMEOUT=${KIND_HELM_OVN_TIMEOUT:-300} + if ! [[ "${OVN_TIMEOUT}" =~ ^[0-9]+$ ]]; then + echo "Invalid KIND_HELM_OVN_TIMEOUT: ${OVN_TIMEOUT}" + exit 1 + fi if [ "$PLATFORM_IPV6_SUPPORT" == true ]; then - OVN_TIMEOUT=480 + OVN_TIMEOUT=${KIND_HELM_OVN_TIMEOUT:-480} fi # We will make sure that we timeout all commands at current seconds + the desired timeout. @@ -806,7 +812,6 @@ kubectl_wait_pods() { fi restart_dpu_sim_multus_after_ovnk - restart_dpu_sim_system_deployments_after_ovnk timeout=$(calculate_timeout ${endtime}) if ! kubectl wait -n kube-system --for=condition=ready pods --all --timeout=${timeout}s ; then @@ -1289,16 +1294,23 @@ deploy_frr_external_container() { fi if [ "${OCI_BIN}" == "podman" ]; then # frr-k8s' demo script prefers docker when both docker and podman are - # installed. Keep it on the same runtime as kind-helm.sh so later podman - # operations can find the external frr container. - local docker_wrapper_dir - docker_wrapper_dir=$(mktemp -d) - ln -s "$(command -v podman)" "${docker_wrapper_dir}/docker" - PATH="${docker_wrapper_dir}:${PATH}" ./demo.sh - rm -rf "${docker_wrapper_dir}" - else - ./demo.sh - fi + # installed. Force its podman path, and avoid its host-network fallback + # because podman cannot later attach a host-network container to bgpnet. + replace_in_file_or_exit \ + ./demo.sh \ + 'CLI=docker' \ + 'CLI=podman' + replace_in_file_or_exit \ + ./demo.sh \ + 'CLI_BR_NET_BY_SUBNET_FN="docker_get_br_net_by_subnet"' \ + 'CLI_BR_NET_BY_SUBNET_FN="podman_get_br_net_by_subnet"' + sed -i '/^pushd \.\/frr\/ && {/i\ +if [ "$CLI" = "podman" ]; then\ + NETWORK=${FRR_K8S_DEMO_NETWORK:-kind}\ +fi\ +' ./demo.sh + fi + ./demo.sh popd || exit 1 if [ "$PLATFORM_IPV6_SUPPORT" == true ]; then # Enable IPv6 forwarding in FRR diff --git a/contrib/kind-dpu-sim-no-overlay.sh b/contrib/kind-dpu-sim-no-overlay.sh new file mode 100755 index 0000000000..8fd9b7ffc2 --- /dev/null +++ b/contrib/kind-dpu-sim-no-overlay.sh @@ -0,0 +1,211 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright The OVN-Kubernetes Contributors +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" +OVN_KUBERNETES_PATH=${OVN_KUBERNETES_PATH:-$(cd "${SCRIPT_DIR}/.." && pwd)} + +resolve_dpu_sim_path() { + if [ -n "${DPU_SIM_PATH:-}" ]; then + if [ ! -d "${DPU_SIM_PATH}" ]; then + echo "error: DPU_SIM_PATH does not exist: ${DPU_SIM_PATH}" >&2 + exit 1 + fi + cd "${DPU_SIM_PATH}" && pwd + return + fi + + local candidates=( + "${OVN_KUBERNETES_PATH}/../dpu-simulator" + "${OVN_KUBERNETES_PATH}/../../ovn-kubernetes/dpu-simulator" + ) + + local gopath=${GOPATH:-} + if [ -z "${gopath}" ] && command -v go >/dev/null 2>&1; then + gopath=$(go env GOPATH 2>/dev/null || true) + fi + if [ -n "${gopath}" ]; then + local entry + local -a gopath_entries + IFS=: read -ra gopath_entries <<< "${gopath}" + for entry in "${gopath_entries[@]}"; do + candidates+=("${entry}/src/github.com/ovn-kubernetes/dpu-simulator") + done + fi + + local path + for path in "${candidates[@]}"; do + if [ -d "${path}" ]; then + cd "${path}" && pwd + return + fi + done + + echo "error: could not locate dpu-simulator checkout" >&2 + echo "set DPU_SIM_PATH to the dpu-simulator repository path" >&2 + exit 1 +} + +resolve_kind_provider() { + if [ -n "${KIND_EXPERIMENTAL_PROVIDER:-}" ]; then + echo "${KIND_EXPERIMENTAL_PROVIDER}" + return + fi + + if command -v docker >/dev/null 2>&1; then + echo "docker" + return + fi + + if command -v podman >/dev/null 2>&1; then + echo "podman" + return + fi + + echo "error: could not locate podman or docker" >&2 + echo "set KIND_EXPERIMENTAL_PROVIDER to the Kind provider to use" >&2 + exit 1 +} + +DPU_SIM_PATH=$(resolve_dpu_sim_path) +KIND_EXPERIMENTAL_PROVIDER=$(resolve_kind_provider) +export KIND_EXPERIMENTAL_PROVIDER +KIND_HELM_OVN_TIMEOUT=${KIND_HELM_OVN_TIMEOUT:-900} +export KIND_HELM_OVN_TIMEOUT +BGP_SERVER_NET_SUBNET_IPV4=${BGP_SERVER_NET_SUBNET_IPV4:-172.27.0.0/16} +BGP_SERVER_NET_SUBNET_IPV6=${BGP_SERVER_NET_SUBNET_IPV6:-fc00:f853:ccd:e797::/64} +export BGP_SERVER_NET_SUBNET_IPV4 +export BGP_SERVER_NET_SUBNET_IPV6 +DPU_SIM_CONFIG=${DPU_SIM_CONFIG:-config-kind-ovnk-offload.yaml} +HOST_CLUSTER=${HOST_CLUSTER:-dpu-sim-host} +DPU_CLUSTER=${DPU_CLUSTER:-dpu-sim-dpu} +HOST_KUBECONFIG="${DPU_SIM_PATH}/kubeconfig/${HOST_CLUSTER}.kubeconfig" +DPU_KUBECONFIG="${DPU_SIM_PATH}/kubeconfig/${DPU_CLUSTER}.kubeconfig" +HOST_VALUES="${DPU_SIM_PATH}/kubeconfig/helm-values/${HOST_CLUSTER}-ovn-kubernetes-dpu-host-values.yaml" +DPU_VALUES="${DPU_SIM_PATH}/kubeconfig/helm-values/${DPU_CLUSTER}-ovn-kubernetes-dpu-values.yaml" +FRR_ENV="${DPU_SIM_PATH}/kubeconfig/helm-values/${DPU_CLUSTER}-frr-k8s.env" + +kubectl_host() { + kubectl --kubeconfig "${HOST_KUBECONFIG}" "$@" +} + +kubectl_dpu() { + kubectl --kubeconfig "${DPU_KUBECONFIG}" "$@" +} + +cluster_command_arg() { + local kubeconfig=$1 + local selector=$2 + local arg_name=$3 + local command + + command=$(kubectl --kubeconfig "${kubeconfig}" -n kube-system get pod \ + -l "${selector}" -o jsonpath='{.items[0].spec.containers[0].command}') + printf '%s\n' "${command}" | tr '",' '\n' | awk -v prefix="--${arg_name}=" ' + index($0, prefix) == 1 { + print substr($0, length(prefix) + 1) + exit + } + ' +} + +host_cluster_cidrs() { + local net_cidr svc_cidr + + net_cidr=$(cluster_command_arg "${HOST_KUBECONFIG}" "component=kube-controller-manager" "cluster-cidr") + svc_cidr=$(cluster_command_arg "${HOST_KUBECONFIG}" "component=kube-apiserver" "service-cluster-ip-range") + + if [ -z "${net_cidr}" ] || [ -z "${svc_cidr}" ]; then + echo "error: could not determine host cluster pod/service CIDRs" >&2 + exit 1 + fi + + echo "${net_cidr} ${svc_cidr}" +} + +install_ovnk_host() { + local cidrs host_net_cidr host_svc_cidr + + cidrs=$(host_cluster_cidrs) + read -r host_net_cidr host_svc_cidr <<< "${cidrs}" + echo "Using host cluster pod CIDR ${host_net_cidr}" + echo "Using host cluster service CIDR ${host_svc_cidr}" + + pushd "${OVN_KUBERNETES_PATH}" + NET_CIDR_IPV4="${host_net_cidr}" \ + SVC_CIDR_IPV4="${host_svc_cidr}" \ + ./contrib/kind-helm.sh \ + --deploy \ + --cluster-name "${HOST_CLUSTER}" \ + --kubeconfig "${HOST_KUBECONFIG}" \ + --dpu-mode host \ + --network-segmentation-enable \ + --multi-network-enable \ + --route-advertisements-enable \ + --no-overlay-enable \ + --advertise-default-network \ + --extra-values "${HOST_VALUES}" + popd +} + +install_ovnk_dpu() { + # shellcheck disable=SC1090 + source "${FRR_ENV}" + + pushd "${OVN_KUBERNETES_PATH}" + ./contrib/kind-helm.sh \ + --deploy \ + --cluster-name "${DPU_CLUSTER}" \ + --kubeconfig "${DPU_KUBECONFIG}" \ + --dpu-mode dpu \ + --multi-network-enable \ + --network-segmentation-enable \ + --route-advertisements-enable \ + --no-overlay-enable \ + --advertise-default-network \ + --extra-values "${DPU_VALUES}" \ + --frr-k8s-host-kubeconfig "${FRR_K8S_HOST_KUBECONFIG}" \ + --frr-k8s-remote-kubeconfig "${FRR_K8S_REMOTE_KUBECONFIG}" \ + --frr-k8s-remote-node-map "${FRR_K8S_REMOTE_NODE_MAP}" + popd +} + +wait_for_ovn() { + kubectl_host wait --for=condition=Ready nodes --all --timeout=25m + kubectl_dpu wait --for=condition=Ready nodes --all --timeout=25m + kubectl_host -n ovn-kubernetes wait --for=condition=Ready pods --all --timeout=10m + kubectl_dpu -n ovn-kubernetes wait --for=condition=Ready pods --all --timeout=10m + kubectl_dpu -n frr-k8s-system wait --for=condition=Ready pods --all --timeout=10m +} + +if [ ! -x "${DPU_SIM_PATH}/bin/dpu-sim" ]; then + echo "error: ${DPU_SIM_PATH}/bin/dpu-sim does not exist or is not executable" >&2 + echo "run 'make build' in the dpu-simulator repository first" >&2 + exit 1 +fi + +echo "Using KIND_EXPERIMENTAL_PROVIDER=${KIND_EXPERIMENTAL_PROVIDER}" +echo "Using KIND_HELM_OVN_TIMEOUT=${KIND_HELM_OVN_TIMEOUT}" +echo "Using BGP_SERVER_NET_SUBNET_IPV4=${BGP_SERVER_NET_SUBNET_IPV4}" + +pushd "${DPU_SIM_PATH}" +./bin/dpu-sim \ + --config "${DPU_SIM_CONFIG}" \ + --ovn-kubernetes-path "${OVN_KUBERNETES_PATH}" \ + --ovnk-mode values-only + +install_ovnk_host + +./bin/dpu-sim ovnk host-access \ + --config "${DPU_SIM_CONFIG}" \ + --cluster "${HOST_CLUSTER}" +./bin/dpu-sim ovnk values \ + --config "${DPU_SIM_CONFIG}" \ + --cluster "${DPU_CLUSTER}" \ + --require-host-credentials + +install_ovnk_dpu +wait_for_ovn +popd diff --git a/contrib/kind-helm.sh b/contrib/kind-helm.sh index 6f6a65e7b3..155ae82f32 100755 --- a/contrib/kind-helm.sh +++ b/contrib/kind-helm.sh @@ -784,6 +784,8 @@ if [ "$ENABLE_ROUTE_ADVERTISEMENTS" == true ] && [ "${DPU_MODE}" != "host" ]; th fi fi +restart_dpu_sim_system_deployments_after_ovnk + # IPsec pods need the signer-ca ConfigMap and signed CSRs before they can roll out. # The ovn-ipsec DaemonSet is created by the helm chart (tags.ovn-ipsec=true), install_ipsec # handles the CA creation and CSR signing (manifest apply is skipped when helm owns the DS). From 107c5d5b650c2abdd7143436da5d3ab7ed039f44 Mon Sep 17 00:00:00 2001 From: Tim Rozet Date: Thu, 4 Jun 2026 16:41:47 -0400 Subject: [PATCH 31/51] Handle renamed FRR-K8S controller in DPU sim Newer FRR-K8S manifests name the main controller container controller instead of frr-k8s. The DPU remote daemonset rewrite must still inject the host kubeconfig and replace --node-name with the host node name. Keep frr-status on the DPU API so it can read its own daemon pod, while the controller talks to the host API. Install the full host CRD set and reconcile host-side RBAC because the newer controller watches FRRK8sConfiguration in addition to the older FRR-K8S resources. Signed-off-by: Tim Rozet Assisted-by: Codex --- contrib/kind-common.sh | 3 +-- contrib/kind-dpu-sim-lib.sh | 41 +++++++++++++++++++++++++++++++------ 2 files changed, 36 insertions(+), 8 deletions(-) diff --git a/contrib/kind-common.sh b/contrib/kind-common.sh index 923d9fecd7..d5f3fd7970 100644 --- a/contrib/kind-common.sh +++ b/contrib/kind-common.sh @@ -1445,8 +1445,7 @@ destroy_bgp() { install_frr_k8s_crds() { echo "Installing frr-k8s CRDs ..." clone_frr - kubectl apply -f "${FRR_TMP_DIR}"/frr-k8s/config/crd/bases/frrk8s.metallb.io_frrconfigurations.yaml - kubectl apply -f "${FRR_TMP_DIR}"/frr-k8s/config/crd/bases/frrk8s.metallb.io_frrnodestates.yaml + kubectl apply -f "${FRR_TMP_DIR}"/frr-k8s/config/crd/bases/ } install_frr_k8s() { diff --git a/contrib/kind-dpu-sim-lib.sh b/contrib/kind-dpu-sim-lib.sh index cda6192ffe..20bc0961b1 100644 --- a/contrib/kind-dpu-sim-lib.sh +++ b/contrib/kind-dpu-sim-lib.sh @@ -109,9 +109,33 @@ install_frr_k8s_host_api_crds() { host_kubeconfig=$(frr_k8s_host_kubeconfig) ensure_frr_k8s_namespace "${host_kubeconfig}" kubectl --kubeconfig "${host_kubeconfig}" apply \ - -f "${FRR_TMP_DIR}"/frr-k8s/config/crd/bases/frrk8s.metallb.io_frrconfigurations.yaml - kubectl --kubeconfig "${host_kubeconfig}" apply \ - -f "${FRR_TMP_DIR}"/frr-k8s/config/crd/bases/frrk8s.metallb.io_frrnodestates.yaml + -f "${FRR_TMP_DIR}"/frr-k8s/config/crd/bases/ + ensure_frr_k8s_host_api_rbac "${host_kubeconfig}" +} + +ensure_frr_k8s_host_api_rbac() { + local host_kubeconfig=$1 + + local frr_rbac_dir="${FRR_TMP_DIR}/frr-k8s/config/rbac" + local host_rbac_dir="${FRR_TMP_DIR}/frr-k8s/config/host-api-rbac" + mkdir -p "${host_rbac_dir}" + cp \ + "${frr_rbac_dir}/service_account.yaml" \ + "${frr_rbac_dir}/role.yaml" \ + "${frr_rbac_dir}/secrets_role.yaml" \ + "${frr_rbac_dir}/role_binding.yaml" \ + "${host_rbac_dir}/" + cat > "${host_rbac_dir}/kustomization.yaml" <<'EOF' +namespace: frr-k8s-system +namePrefix: frr-k8s- +resources: +- service_account.yaml +- role.yaml +- secrets_role.yaml +- role_binding.yaml +EOF + + kubectl --kubeconfig "${host_kubeconfig}" apply -k "${host_rbac_dir}" } create_frr_k8s_remote_kubeconfig_secret() { @@ -151,6 +175,10 @@ configure_frr_k8s_remote_daemonsets() { fi ds_name="frr-k8s-daemon-${safe_name:0:45}" echo "Creating remote frr-k8s daemonset ${ds_name}: host node ${host_node}, DPU node ${dpu_node}" + # These DaemonSets run in the DPU cluster. Only the FRR-K8S + # controller container uses the host API so it watches host + # FRRConfiguration objects with the host node name. Keep frr-status + # on the DPU API because it reports the local daemon pod status. jq \ --arg name "${ds_name}" \ --arg host_node "${host_node}" \ @@ -162,10 +190,11 @@ configure_frr_k8s_remote_daemonsets() { | .spec.selector.matchLabels["dpu-sim.ovn.org/frr-host-node"] = $host_node | .spec.template.metadata.labels["dpu-sim.ovn.org/frr-remote"] = "true" | .spec.template.metadata.labels["dpu-sim.ovn.org/frr-host-node"] = $host_node + | .spec.template.metadata.annotations["kubectl.kubernetes.io/default-container"] = "controller" | .spec.template.spec.nodeSelector = {"kubernetes.io/hostname": $dpu_node} - | (.spec.template.spec.containers[] | select(.name == "frr-k8s").args) |= ((. // []) | map(if startswith("--node-name=") then "--node-name=" + $host_node else . end)) - | (.spec.template.spec.containers[] | select(.name == "frr-k8s").env) |= ((. // []) + [{"name":"KUBECONFIG","value":"/var/run/host-kubeconfig/kubeconfig"}]) - | (.spec.template.spec.containers[] | select(.name == "frr-k8s").volumeMounts) |= ((. // []) + [{"name":"host-kubeconfig","mountPath":"/var/run/host-kubeconfig","readOnly":true}]) + | (.spec.template.spec.containers[] | select(.name == "frr-k8s" or .name == "controller").args) |= ((. // []) | map(if startswith("--node-name=") then "--node-name=" + $host_node else . end)) + | (.spec.template.spec.containers[] | select(.name == "frr-k8s" or .name == "controller").env) |= ((. // []) + [{"name":"KUBECONFIG","value":"/var/run/host-kubeconfig/kubeconfig"}]) + | (.spec.template.spec.containers[] | select(.name == "frr-k8s" or .name == "controller").volumeMounts) |= ((. // []) + [{"name":"host-kubeconfig","mountPath":"/var/run/host-kubeconfig","readOnly":true}]) | .spec.template.spec.volumes = ((.spec.template.spec.volumes // []) + [{"name":"host-kubeconfig","secret":{"secretName":"frr-k8s-host-kubeconfig"}}])' \ "${source_json}" | kubectl apply -f - done From 95380fd4c1eae9614dc22844a65fc678c642fa80 Mon Sep 17 00:00:00 2001 From: Tim Rozet Date: Fri, 5 Jun 2026 11:53:33 -0400 Subject: [PATCH 32/51] Peer DPU sim FRR over gateway network In DPU simulator remote mode the FRR-K8S daemon runs on the DPU cluster but watches FRRConfiguration objects from the host API. The generic kind BGP helper still created the seed receive configuration with the external FRR address from the kind network, so frr-k8s peered over 172.18.0.0/16 and imported routes with off-link next hops for the OVN gateway router. When the simulator exports a DPU gateway network, connect the external FRR container to it, add BGP peers for the DPU nodes on that network, and rewrite the receive config to use the external FRR gateway-network address. Drop the stale IPv6 kind-network neighbor because the DPU gateway network is IPv4-only. Export the gateway network variables from the no-overlay helper so kind-helm can enable the DPU-specific FRR setup. Clean stale external FRR artifacts before invoking dpu-sim so the simulator can remove its gateway network during redeploy. Signed-off-by: Tim Rozet Assisted-by: Codex --- contrib/kind-common.sh | 13 +++- contrib/kind-dpu-sim-lib.sh | 111 +++++++++++++++++++++++++++++ contrib/kind-dpu-sim-no-overlay.sh | 22 ++++++ contrib/kind-helm.sh | 4 +- 4 files changed, 147 insertions(+), 3 deletions(-) diff --git a/contrib/kind-common.sh b/contrib/kind-common.sh index d5f3fd7970..354550ff58 100644 --- a/contrib/kind-common.sh +++ b/contrib/kind-common.sh @@ -1312,6 +1312,9 @@ fi\ fi ./demo.sh popd || exit 1 + if frr_k8s_remote_enabled && [ -n "${DPU_SIM_GATEWAY_NETWORK:-}" ]; then + configure_dpu_sim_frr_gateway_peers + fi if [ "$PLATFORM_IPV6_SUPPORT" == true ]; then # Enable IPv6 forwarding in FRR $OCI_BIN exec frr sysctl -w net.ipv6.conf.all.forwarding=1 @@ -1498,10 +1501,15 @@ wait_for_frr_k8s() { kubectl rollout status -n frr-k8s-system daemonset frr-k8s-daemon --timeout 2m return fi - local ds + local ds found=false for ds in $(kubectl -n frr-k8s-system get daemonset -l dpu-sim.ovn.org/frr-remote=true -o name); do + found=true kubectl rollout status -n frr-k8s-system "${ds}" --timeout 2m done + if [ "${found}" != true ]; then + echo "No local or remote FRR-K8S daemonsets were found in namespace frr-k8s-system" >&2 + return 1 + fi } apply_frr_k8s_receive_config() { @@ -1509,6 +1517,9 @@ apply_frr_k8s_receive_config() { # exchange routes pushd "${FRR_TMP_DIR}"/frr-k8s/hack/demo/configs || exit 1 sed 's/mode: all/mode: filtered/g' receive_all.yaml > receive_filtered.yaml + if frr_k8s_remote_enabled && [ -n "${DPU_SIM_GATEWAY_NETWORK:-}" ]; then + configure_dpu_sim_frr_receive_config receive_filtered.yaml + fi # Allow receiving the bgp external server's prefix sed -i '/mode: filtered/a\ prefixes:\n - prefix: '"${BGP_SERVER_NET_SUBNET_IPV4}"'' receive_filtered.yaml # If IPv6 is enabled, add the IPv6 prefix as well diff --git a/contrib/kind-dpu-sim-lib.sh b/contrib/kind-dpu-sim-lib.sh index 20bc0961b1..8a0173ed44 100644 --- a/contrib/kind-dpu-sim-lib.sh +++ b/contrib/kind-dpu-sim-lib.sh @@ -87,6 +87,117 @@ frr_k8s_host_kubeconfig() { printf '%s' "${FRR_K8S_HOST_KUBECONFIG:-${FRR_K8S_REMOTE_KUBECONFIG}}" } +dpu_sim_container_network_ipv4() { + local container=$1 + local network=$2 + + $OCI_BIN inspect "${container}" | jq -r --arg network "${network}" \ + '.[0].NetworkSettings.Networks[$network].IPAddress // empty' +} + +configure_dpu_sim_frr_gateway_peers() { + if ! frr_k8s_remote_enabled; then + return + fi + if [ -z "${DPU_SIM_GATEWAY_NETWORK:-}" ]; then + return + fi + + echo "Connecting external FRR to DPU gateway network ${DPU_SIM_GATEWAY_NETWORK}" + if ! $OCI_BIN network inspect "${DPU_SIM_GATEWAY_NETWORK}" >/dev/null 2>&1; then + echo "DPU simulator gateway network does not exist: ${DPU_SIM_GATEWAY_NETWORK}" >&2 + exit 1 + fi + if [ -z "$(dpu_sim_container_network_ipv4 frr "${DPU_SIM_GATEWAY_NETWORK}")" ]; then + $OCI_BIN network connect "${DPU_SIM_GATEWAY_NETWORK}" frr + fi + + DPU_SIM_FRR_IPV4=$(dpu_sim_container_network_ipv4 frr "${DPU_SIM_GATEWAY_NETWORK}") + if [ -z "${DPU_SIM_FRR_IPV4}" ]; then + echo "Failed to determine external FRR IP on ${DPU_SIM_GATEWAY_NETWORK}" >&2 + exit 1 + fi + echo "External FRR DPU gateway network IPv4: ${DPU_SIM_FRR_IPV4}" + + local -a node_ips=() + local -a pairs=() + local pair dpu_node node_ip + IFS=',' read -ra pairs <<< "${FRR_K8S_REMOTE_NODE_MAP}" + for pair in "${pairs[@]}"; do + dpu_node="${pair#*=}" + node_ip=$(dpu_sim_container_network_ipv4 "${dpu_node}" "${DPU_SIM_GATEWAY_NETWORK}") + if [ -z "${node_ip}" ]; then + echo "Failed to determine ${dpu_node} IP on ${DPU_SIM_GATEWAY_NETWORK}" >&2 + exit 1 + fi + node_ips+=("${node_ip}") + done + + local attempts=0 daemon_status + while ! daemon_status=$($OCI_BIN exec frr vtysh -c "show daemons" 2>&1); do + if (( ++attempts > 30 )); then + echo "error: FRR daemons did not become ready after 30 attempts" + echo "last daemon status: $daemon_status" + exit 1 + fi + sleep 1 + done + + local vtysh_cmds=(-c "configure terminal" -c "router bgp 64512") + for node_ip in "${node_ips[@]}"; do + vtysh_cmds+=(-c "neighbor ${node_ip} remote-as 64512") + done + vtysh_cmds+=(-c "address-family ipv4 unicast") + for node_ip in "${node_ips[@]}"; do + vtysh_cmds+=(-c "neighbor ${node_ip} activate") + vtysh_cmds+=(-c "neighbor ${node_ip} route-reflector-client") + done + vtysh_cmds+=(-c "exit-address-family" -c "end" -c "write memory") + $OCI_BIN exec frr vtysh "${vtysh_cmds[@]}" +} + +configure_dpu_sim_frr_receive_config() { + local receive_config=$1 + + if ! frr_k8s_remote_enabled; then + return + fi + if [ -z "${DPU_SIM_GATEWAY_NETWORK:-}" ]; then + return + fi + + DPU_SIM_FRR_IPV4=${DPU_SIM_FRR_IPV4:-$(dpu_sim_container_network_ipv4 frr "${DPU_SIM_GATEWAY_NETWORK}")} + if [ -z "${DPU_SIM_FRR_IPV4}" ]; then + echo "Failed to determine external FRR IP on ${DPU_SIM_GATEWAY_NETWORK}" >&2 + exit 1 + fi + sed -i -E "s/(address: )[0-9.]+/\\1${DPU_SIM_FRR_IPV4}/g" "${receive_config}" + + local filtered + filtered=$(mktemp) + awk ' + function indentation(line) { + match(line, /[^ ]/) + return RSTART ? RSTART - 1 : length(line) + } + skip && indentation($0) <= skip_indent && $0 !~ /^[[:space:]]*$/ { + skip = 0 + } + !skip && $0 ~ /^[[:space:]]*- address: / { + addr = $0 + sub(/^[[:space:]]*- address: /, "", addr) + gsub(/"/, "", addr) + if (addr ~ /:/) { + skip = 1 + skip_indent = indentation($0) + next + } + } + !skip { print } + ' "${receive_config}" > "${filtered}" + mv "${filtered}" "${receive_config}" +} + ensure_frr_k8s_namespace() { local kubeconfig=${1:-} local kubectl_cmd=(kubectl) diff --git a/contrib/kind-dpu-sim-no-overlay.sh b/contrib/kind-dpu-sim-no-overlay.sh index 8fd9b7ffc2..d187d45a9d 100755 --- a/contrib/kind-dpu-sim-no-overlay.sh +++ b/contrib/kind-dpu-sim-no-overlay.sh @@ -69,6 +69,24 @@ resolve_kind_provider() { exit 1 } +cleanup_bgp_artifacts() { + local provider=$1 + + if ! command -v "${provider}" >/dev/null 2>&1; then + return + fi + + if "${provider}" ps -a --format '{{.Names}}' | grep -Eq '^bgpserver$'; then + "${provider}" rm -f bgpserver + fi + if "${provider}" ps -a --format '{{.Names}}' | grep -Eq '^frr$'; then + "${provider}" rm -f frr + fi + if "${provider}" network ls --format '{{.Name}}' | grep -Eq '^bgpnet$'; then + "${provider}" network rm bgpnet || true + fi +} + DPU_SIM_PATH=$(resolve_dpu_sim_path) KIND_EXPERIMENTAL_PROVIDER=$(resolve_kind_provider) export KIND_EXPERIMENTAL_PROVIDER @@ -153,6 +171,8 @@ install_ovnk_host() { install_ovnk_dpu() { # shellcheck disable=SC1090 source "${FRR_ENV}" + export DPU_SIM_GATEWAY_NETWORK + export DPU_SIM_GATEWAY_SUBNET pushd "${OVN_KUBERNETES_PATH}" ./contrib/kind-helm.sh \ @@ -190,6 +210,8 @@ echo "Using KIND_EXPERIMENTAL_PROVIDER=${KIND_EXPERIMENTAL_PROVIDER}" echo "Using KIND_HELM_OVN_TIMEOUT=${KIND_HELM_OVN_TIMEOUT}" echo "Using BGP_SERVER_NET_SUBNET_IPV4=${BGP_SERVER_NET_SUBNET_IPV4}" +cleanup_bgp_artifacts "${KIND_EXPERIMENTAL_PROVIDER}" + pushd "${DPU_SIM_PATH}" ./bin/dpu-sim \ --config "${DPU_SIM_CONFIG}" \ diff --git a/contrib/kind-helm.sh b/contrib/kind-helm.sh index 155ae82f32..cfac2c0480 100755 --- a/contrib/kind-helm.sh +++ b/contrib/kind-helm.sh @@ -653,8 +653,8 @@ check_dependencies parse_args "$@" set_default_params if [ "${DPU_MODE}" == "dpu" ] && [ "${ENABLE_ROUTE_ADVERTISEMENTS}" == true ]; then - if [[ -z "${FRR_K8S_REMOTE_KUBECONFIG:-}" || -z "${FRR_K8S_HOST_KUBECONFIG:-}" || -z "${FRR_K8S_REMOTE_NODE_MAP:-}" ]]; then - echo "DPU mode with route advertisements requires --frr-k8s-remote-kubeconfig, --frr-k8s-host-kubeconfig, and --frr-k8s-remote-node-map" >&2 + if [[ -z "${FRR_K8S_REMOTE_KUBECONFIG:-}" || -z "${FRR_K8S_REMOTE_NODE_MAP:-}" ]]; then + echo "DPU mode with route advertisements requires --frr-k8s-remote-kubeconfig and --frr-k8s-remote-node-map" >&2 exit 1 fi validate_frr_k8s_remote From f8f3c2b56025013a28757cfe4e4fc3569e12cbd8 Mon Sep 17 00:00:00 2001 From: Tim Rozet Date: Sat, 6 Jun 2026 11:15:48 -0400 Subject: [PATCH 33/51] Route DPU host no-overlay traffic through OVN In DPU shared gateway no-overlay mode, host-network pods on the DPU host need remote default-network pod traffic to use routes that exist on the DPU. Stop programming broad default cluster CIDR routes through the management port. Instead route those CIDRs through the shared gateway dummy next hop, SNAT host traffic to the host masquerade IP in nftables, and add bridge flows that steer the traffic through OVN before SNATing it to the node underlay IP on egress. Document the datapath so the Linux route, nftables, OVS, and OVN pieces can be understood together. Signed-off-by: Tim Rozet Assisted-by: Codex --- docs/design/dpu-host-no-overlay-routing.md | 89 +++++++++++++++++ .../pkg/node/bridgeconfig/bridgeflows.go | 97 +++++++++++++++++++ .../pkg/node/bridgeconfig/bridgeflows_test.go | 70 +++++++++++++ go-controller/pkg/node/gateway_init.go | 34 +++++++ .../pkg/node/gateway_init_linux_test.go | 46 +++++++++ .../pkg/node/gateway_nftables_test.go | 33 +++++++ go-controller/pkg/node/gateway_shared_intf.go | 91 +++++++++++++++++ .../pkg/node/managementport/port_config.go | 14 ++- .../node/managementport/port_linux_test.go | 19 ++++ mkdocs.yml | 1 + 10 files changed, 491 insertions(+), 3 deletions(-) create mode 100644 docs/design/dpu-host-no-overlay-routing.md diff --git a/docs/design/dpu-host-no-overlay-routing.md b/docs/design/dpu-host-no-overlay-routing.md new file mode 100644 index 0000000000..ed1c4632b6 --- /dev/null +++ b/docs/design/dpu-host-no-overlay-routing.md @@ -0,0 +1,89 @@ +# DPU Host No-Overlay Routing + +This document describes the default-network host-to-pod routing path used +when OVN-Kubernetes runs in DPU-host mode with shared gateway mode and +`no-overlay` transport. + +## Problem + +In no-overlay mode, pod traffic between nodes is routed on the underlay. +On a DPU deployment, host-networked pods run on the DPU host, while the +OVN gateway router and BGP learned routes exist on the DPU. + +The DPU host cannot route remote pod CIDRs through `ovn-k8s-mp0`. The +management port only reaches local node subnets. If the host keeps a broad +cluster CIDR route through the management port, host-networked pods send +remote pod traffic to the wrong place. + +At the same time, the DPU host should not learn every BGP route from the +DPU. The routing decision should stay on the DPU, where OVN already has the +BGP routes. + +## Datapath + +The datapath uses four pieces: + +1. The DPU host does not install broad default cluster CIDR routes through + `ovn-k8s-mp0`. +2. The DPU host installs broad default cluster CIDR routes through the + shared gateway interface using the dummy next-hop address. +3. Host nftables SNATs host-network traffic for default cluster CIDRs to + the host masquerade IP before the packet enters OVS. +4. The DPU shared gateway bridge steers that already-SNATed traffic into + OVN. After OVN routes it back out the default-network gateway patch, a + higher priority bridge flow SNATs the host masquerade IP to the node + underlay IP and sends it out the physical interface. + +The first SNAT is intentionally done in host nftables, not in OpenFlow. The +bridge only matches the already-SNATed host masquerade IP when steering the +packet into OVN. This avoids double-SNAT in OpenFlow while still letting the +DPU use OVN and BGP routes for the egress decision. + +## Forward Path + +For host-networked pod traffic to a remote default-network pod: + +1. Linux routing on the DPU host selects the shared gateway interface for + the default cluster CIDR. +2. The host nftables postrouting rule SNATs the source to the host + masquerade IP, for example `169.254.0.2`. +3. The DPU bridge matches: + + ```text + in_port=, ip_src=, + ip_dst= + ``` + + and sends the packet to the default-network OVN patch port. +4. OVN routes the packet using its logical router and BGP learned routes. +5. When the packet returns to the DPU bridge from the default-network patch + port, the bridge matches: + + ```text + in_port=, ip_src=, + ip_dst= + ``` + + and commits SNAT in the default conntrack zone to the node underlay IP. +6. The packet leaves the physical interface toward the remote node or next + hop selected by OVN routing. + +## Return Path + +Return traffic enters the DPU bridge from the physical interface and follows +the existing default conntrack-zone path. The bridge sends established +traffic marked for OVN back to the default-network patch port, and OVN +returns it toward the DPU host. The bridge then sends traffic destined to the +host masquerade IP through the normal OVN-to-host dispatch path, where host +conntrack reverses the nftables SNAT. + +## Scope + +This path is only for the default network in: + +- DPU-host mode +- shared gateway mode +- no-overlay transport + +It does not add host-network support for UDNs. Host-networked workloads use +the host default network, so UDN remote pod routing is outside this path. diff --git a/go-controller/pkg/node/bridgeconfig/bridgeflows.go b/go-controller/pkg/node/bridgeconfig/bridgeflows.go index 491dba13d3..55063dcf92 100644 --- a/go-controller/pkg/node/bridgeconfig/bridgeflows.go +++ b/go-controller/pkg/node/bridgeconfig/bridgeflows.go @@ -625,6 +625,85 @@ func isLocalGatewayNoOverlayUDN(netConfig *BridgeUDNConfiguration) bool { config.Gateway.Mode == config.GatewayModeLocal } +func isDPUSharedNoOverlay() bool { + return (config.IsModeDPU() || config.IsModeDPUHost()) && + config.Gateway.Mode == config.GatewayModeShared && + config.Default.Transport == types.NetworkTransportNoOverlay +} + +func dpuHostNoOverlayPodCIDRFlows(defaultNetConfig *BridgeUDNConfiguration, ofPortHost string, isIPv6 bool) []string { + if defaultNetConfig == nil { + return nil + } + + protoPrefix := protoPrefixV4 + masqIP := config.Gateway.MasqueradeIPs.V4HostMasqueradeIP + if isIPv6 { + protoPrefix = protoPrefixV6 + masqIP = config.Gateway.MasqueradeIPs.V6HostMasqueradeIP + } + + var subnets []*net.IPNet + for _, clusterEntry := range defaultNetConfig.Subnets { + subnets = append(subnets, clusterEntry.CIDR) + } + + var flows []string + for _, subnet := range matchIPNetFamily(isIPv6, subnets) { + // Host-network traffic to no-overlay pod CIDRs is SNATed by host + // nftables before it enters OVS. Match the host masquerade IP so only + // that already-SNATed traffic is steered into OVN, avoiding double NAT + // in OpenFlow and preserving the offloadable path. + flows = append(flows, + fmt.Sprintf("cookie=%s, priority=510, in_port=%s, %s, %s_src=%s, %s_dst=%s, "+ + "actions=goto_table:2", + nodetypes.DefaultOpenFlowCookie, ofPortHost, protoPrefix, protoPrefix, + masqIP, protoPrefix, subnet.String())) + // Return traffic comes back from OVN to the host masquerade IP. Send it + // through the normal OVN->host dispatch path; host conntrack reverses + // the nftables SNAT instead of using OpenFlow NAT here. + flows = append(flows, + fmt.Sprintf("cookie=%s, priority=510, in_port=%s, %s, %s_src=%s, %s_dst=%s,"+ + "actions=goto_table:3", + nodetypes.DefaultOpenFlowCookie, defaultNetConfig.OfPortPatch, protoPrefix, + protoPrefix, subnet.String(), protoPrefix, masqIP)) + } + return flows +} + +func dpuHostNoOverlayHostSNATEgressFlows(defaultNetConfig *BridgeUDNConfiguration, ofPortPhys string, bridgeMacAddress string, physicalIP *net.IPNet, isIPv6 bool) []string { + if defaultNetConfig == nil { + return nil + } + + protoPrefix := protoPrefixV4 + masqIP := config.Gateway.MasqueradeIPs.V4HostMasqueradeIP + if isIPv6 { + protoPrefix = protoPrefixV6 + masqIP = config.Gateway.MasqueradeIPs.V6HostMasqueradeIP + } + + var subnets []*net.IPNet + for _, clusterEntry := range defaultNetConfig.Subnets { + subnets = append(subnets, clusterEntry.CIDR) + } + + var flows []string + for _, subnet := range matchIPNetFamily(isIPv6, subnets) { + // After OVN routes host-network traffic back out the default-network + // gateway patch, SNAT the host masquerade IP to this node's underlay IP + // in the default conntrack zone. The reverse direction then follows the + // existing table 1 ct_mark=OVN path back to OVN. + flows = append(flows, + fmt.Sprintf("cookie=%s, priority=510, in_port=%s, dl_src=%s, %s, %s_src=%s, %s_dst=%s, "+ + "actions=ct(commit, zone=%d, nat(src=%s), exec(set_field:%s->ct_mark)), output:%s", + nodetypes.DefaultOpenFlowCookie, defaultNetConfig.OfPortPatch, bridgeMacAddress, + protoPrefix, protoPrefix, masqIP, protoPrefix, subnet.String(), + config.Default.ConntrackZone, physicalIP.IP, defaultNetConfig.MasqCTMark, ofPortPhys)) + } + return flows +} + // getMaxFrameLength returns the maximum frame size (ignoring VLAN header) that a gateway can handle func getMaxFrameLength() int { return config.Default.MTU + 14 @@ -752,6 +831,10 @@ func (b *BridgeConfiguration) commonFlows(hostSubnets []*net.IPNet) ([]string, e physicalIP.IP, netConfig.MasqCTMark, ofPortPhys)) } + if netConfig.IsDefaultNetwork() && isDPUSharedNoOverlay() { + dftFlows = append(dftFlows, + dpuHostNoOverlayHostSNATEgressFlows(netConfig, ofPortPhys, bridgeMacAddress, physicalIP, false)...) + } // table0, packets coming from egressIP pods that have mark 1008 on them // will be SNAT-ed a final time into nodeIP to maintain consistency in traffic even if the GR @@ -811,6 +894,11 @@ func (b *BridgeConfiguration) commonFlows(hostSubnets []*net.IPNet) ([]string, e "actions=ct(commit, zone=%d, exec(set_field:%s->ct_mark)), %soutput:%s", nodetypes.DefaultOpenFlowCookie, ofPortHost, protoPrefixV4, config.Default.ConntrackZone, nodetypes.CtMarkHost, modVLANID, ofPortPhys)) + if isDPUSharedNoOverlay() { + defaultNetConfig := b.netConfig[types.DefaultNetworkName] + dftFlows = append(dftFlows, + dpuHostNoOverlayPodCIDRFlows(defaultNetConfig, ofPortHost, false)...) + } } if config.Gateway.Mode == config.GatewayModeLocal { for _, netConfig := range b.patchedNetConfigs() { @@ -869,6 +957,10 @@ func (b *BridgeConfiguration) commonFlows(hostSubnets []*net.IPNet) ([]string, e physicalIP.IP, netConfig.MasqCTMark, ofPortPhys)) } + if netConfig.IsDefaultNetwork() && isDPUSharedNoOverlay() { + dftFlows = append(dftFlows, + dpuHostNoOverlayHostSNATEgressFlows(netConfig, ofPortPhys, bridgeMacAddress, physicalIP, true)...) + } // table0, packets coming from egressIP pods that have mark 1008 on them // will be DNAT-ed a final time into nodeIP to maintain consistency in traffic even if the GR @@ -928,6 +1020,11 @@ func (b *BridgeConfiguration) commonFlows(hostSubnets []*net.IPNet) ([]string, e "actions=ct(commit, zone=%d, exec(set_field:%s->ct_mark)), %soutput:%s", nodetypes.DefaultOpenFlowCookie, ofPortHost, protoPrefixV6, config.Default.ConntrackZone, nodetypes.CtMarkHost, modVLANID, ofPortPhys)) + if isDPUSharedNoOverlay() { + defaultNetConfig := b.netConfig[types.DefaultNetworkName] + dftFlows = append(dftFlows, + dpuHostNoOverlayPodCIDRFlows(defaultNetConfig, ofPortHost, true)...) + } } if config.Gateway.Mode == config.GatewayModeLocal { diff --git a/go-controller/pkg/node/bridgeconfig/bridgeflows_test.go b/go-controller/pkg/node/bridgeconfig/bridgeflows_test.go index bbf5878fe3..c723c6a022 100644 --- a/go-controller/pkg/node/bridgeconfig/bridgeflows_test.go +++ b/go-controller/pkg/node/bridgeconfig/bridgeflows_test.go @@ -76,6 +76,76 @@ func TestSharedNoOverlayNodeIPFlowUsesNATInDefaultConntrackZone(t *testing.T) { expectFlow(t, flows, expectedIPv6) } +func TestDPUHostNoOverlayPodCIDRFlowsUseHostMasqueradeIP(t *testing.T) { + if err := config.PrepareTestConfig(); err != nil { + t.Fatalf("failed to prepare test config: %v", err) + } + t.Cleanup(func() { + _ = config.PrepareTestConfig() + }) + config.IPv4Mode = true + config.IPv6Mode = true + config.Gateway.Mode = config.GatewayModeShared + config.Default.Transport = types.NetworkTransportNoOverlay + config.OvnKubeNode.Mode = types.NodeModeDPU + + bridgeMAC := mustParseMAC(t, "62:41:d0:54:3d:64") + v4NodeIP := mustParseIPNet(t, "172.18.0.3/24") + v6NodeIP := mustParseIPNet(t, "fd00::3/64") + + bridge := &BridgeConfiguration{ + ofPortPhys: "eth0", + ofPortHost: nodetypes.OvsLocalPort, + ips: []*net.IPNet{v4NodeIP, v6NodeIP}, + macAddress: bridgeMAC, + netConfig: map[string]*BridgeUDNConfiguration{ + types.DefaultNetworkName: { + OfPortPatch: "patch-breth0_ov", + MasqCTMark: nodetypes.CtMarkOVN, + Subnets: []config.CIDRNetworkEntry{ + {CIDR: mustParseIPNet(t, "10.244.0.0/16")}, + {CIDR: mustParseIPNet(t, "fd10:244::/64")}, + }, + }, + }, + } + + flows, err := bridge.commonFlows(nil) + if err != nil { + t.Fatalf("failed to render bridge flows: %v", err) + } + + expectedIPv4HostToOVN := fmt.Sprintf("cookie=%s, priority=510, in_port=LOCAL, ip, ip_src=%s, "+ + "ip_dst=10.244.0.0/16, actions=goto_table:2", + nodetypes.DefaultOpenFlowCookie, config.Gateway.MasqueradeIPs.V4HostMasqueradeIP) + expectedIPv4OVNToHost := fmt.Sprintf("cookie=%s, priority=510, in_port=patch-breth0_ov, ip, "+ + "ip_src=10.244.0.0/16, ip_dst=%s,actions=goto_table:3", + nodetypes.DefaultOpenFlowCookie, config.Gateway.MasqueradeIPs.V4HostMasqueradeIP) + expectedIPv4OVNToPhysical := fmt.Sprintf("cookie=%s, priority=510, in_port=patch-breth0_ov, "+ + "dl_src=%s, ip, ip_src=%s, ip_dst=10.244.0.0/16, "+ + "actions=ct(commit, zone=%d, nat(src=172.18.0.3), exec(set_field:%s->ct_mark)), output:eth0", + nodetypes.DefaultOpenFlowCookie, bridgeMAC, config.Gateway.MasqueradeIPs.V4HostMasqueradeIP, + config.Default.ConntrackZone, nodetypes.CtMarkOVN) + expectedIPv6HostToOVN := fmt.Sprintf("cookie=%s, priority=510, in_port=LOCAL, ipv6, ipv6_src=%s, "+ + "ipv6_dst=fd10:244::/64, actions=goto_table:2", + nodetypes.DefaultOpenFlowCookie, config.Gateway.MasqueradeIPs.V6HostMasqueradeIP) + expectedIPv6OVNToHost := fmt.Sprintf("cookie=%s, priority=510, in_port=patch-breth0_ov, ipv6, "+ + "ipv6_src=fd10:244::/64, ipv6_dst=%s,actions=goto_table:3", + nodetypes.DefaultOpenFlowCookie, config.Gateway.MasqueradeIPs.V6HostMasqueradeIP) + expectedIPv6OVNToPhysical := fmt.Sprintf("cookie=%s, priority=510, in_port=patch-breth0_ov, "+ + "dl_src=%s, ipv6, ipv6_src=%s, ipv6_dst=fd10:244::/64, "+ + "actions=ct(commit, zone=%d, nat(src=fd00::3), exec(set_field:%s->ct_mark)), output:eth0", + nodetypes.DefaultOpenFlowCookie, bridgeMAC, config.Gateway.MasqueradeIPs.V6HostMasqueradeIP, + config.Default.ConntrackZone, nodetypes.CtMarkOVN) + + expectFlow(t, flows, expectedIPv4HostToOVN) + expectFlow(t, flows, expectedIPv4OVNToHost) + expectFlow(t, flows, expectedIPv4OVNToPhysical) + expectFlow(t, flows, expectedIPv6HostToOVN) + expectFlow(t, flows, expectedIPv6OVNToHost) + expectFlow(t, flows, expectedIPv6OVNToPhysical) +} + func TestLocalNoOverlayServiceHairpinUsesUDNGatewayMasqueradeIP(t *testing.T) { if err := config.PrepareTestConfig(); err != nil { t.Fatalf("failed to prepare test config: %v", err) diff --git a/go-controller/pkg/node/gateway_init.go b/go-controller/pkg/node/gateway_init.go index 5d55b8aa72..b66bc456ff 100644 --- a/go-controller/pkg/node/gateway_init.go +++ b/go-controller/pkg/node/gateway_init.go @@ -194,6 +194,40 @@ func configureSvcRouteViaInterface(routeManager *routemanager.Controller, iface return nil } +func shouldConfigureDPUHostNoOverlayPodCIDRRoute() bool { + return config.IsModeDPUHost() && + config.Gateway.Mode == config.GatewayModeShared && + config.Default.Transport == types.NetworkTransportNoOverlay +} + +func configureDPUHostNoOverlayPodCIDRRoute(routeManager *routemanager.Controller, iface string, gwIPs []net.IP) error { + link, err := util.LinkSetUp(iface) + if err != nil { + return fmt.Errorf("unable to get link for %s, error: %v", iface, err) + } + + mtu := config.Default.MTU + if config.Default.RoutableMTU != 0 { + mtu = config.Default.RoutableMTU + } + + for _, clusterSubnet := range config.Default.ClusterSubnets { + subnet := clusterSubnet.CIDR + isV6 := utilnet.IsIPv6CIDR(subnet) + gwIP, err := util.MatchIPFamily(isV6, gwIPs) + if err != nil { + return fmt.Errorf("unable to find gateway IP for subnet: %v, found IPs: %v", subnet, gwIPs) + } + subnetCopy := *subnet + gwIPCopy := gwIP[0] + err = routeManager.Add(netlink.Route{LinkIndex: link.Attrs().Index, Gw: gwIPCopy, Dst: &subnetCopy, MTU: mtu}) + if err != nil { + return fmt.Errorf("unable to add gateway IP route for subnet: %v, %v", subnet, err) + } + } + return nil +} + // getNodePrimaryIfAddrs returns the appropriate interface addresses based on the node mode func getNodePrimaryIfAddrs(watchFactory factory.NodeWatchFactory, nodeName string, gatewayIntf string) ([]*net.IPNet, error) { switch config.OvnKubeNode.Mode { diff --git a/go-controller/pkg/node/gateway_init_linux_test.go b/go-controller/pkg/node/gateway_init_linux_test.go index 7662d6d087..b0840628c9 100644 --- a/go-controller/pkg/node/gateway_init_linux_test.go +++ b/go-controller/pkg/node/gateway_init_linux_test.go @@ -1920,6 +1920,52 @@ var _ = Describe("Gateway unit tests", func() { }) }) + Context("configureDPUHostNoOverlayPodCIDRRoute", func() { + It("configures pod CIDR routes on interface without a route source", func() { + _, ipnet, err := net.ParseCIDR("10.244.0.0/16") + Expect(err).ToNot(HaveOccurred()) + config.Default.ClusterSubnets = []config.CIDRNetworkEntry{{CIDR: ipnet, HostSubnetLength: 24}} + gwIPs := []net.IP{net.ParseIP("169.254.0.4")} + + lnk := &linkMock.Link{} + lnkAttr := &netlink.LinkAttrs{ + Name: "ens1f0", + Index: 5, + } + expectedRoute := &netlink.Route{ + Dst: ipnet, + LinkIndex: 5, + Scope: netlink.SCOPE_UNIVERSE, + Gw: gwIPs[0], + MTU: config.Default.MTU, + Table: syscall.RT_TABLE_MAIN, + } + + lnk.On("Attrs").Return(lnkAttr) + netlinkMock.On("LinkByName", lnkAttr.Name).Return(lnk, nil) + netlinkMock.On("LinkByIndex", lnkAttr.Index).Return(lnk, nil) + netlinkMock.On("LinkSetUp", mock.Anything).Return(nil) + netlinkMock.On("RouteReplace", expectedRoute).Return(nil) + + wg := &sync.WaitGroup{} + rm := routemanager.NewController() + util.SetNetLinkOpMockInst(netlinkMock) + stopCh := make(chan struct{}) + wg.Add(1) + go func() { + rm.Run(stopCh, 10*time.Second) + wg.Done() + }() + defer func() { + close(stopCh) + wg.Wait() + }() + + err = configureDPUHostNoOverlayPodCIDRRoute(rm, "ens1f0", gwIPs) + Expect(err).ToNot(HaveOccurred()) + }) + }) + Context("getGatewayNextHops", func() { It("Finds correct gateway interface and nexthops without configuration", func() { diff --git a/go-controller/pkg/node/gateway_nftables_test.go b/go-controller/pkg/node/gateway_nftables_test.go index 986a37a2d1..5b85b20802 100644 --- a/go-controller/pkg/node/gateway_nftables_test.go +++ b/go-controller/pkg/node/gateway_nftables_test.go @@ -10,6 +10,7 @@ import ( "sigs.k8s.io/knftables" "github.com/ovn-kubernetes/ovn-kubernetes/go-controller/pkg/config" + nodenft "github.com/ovn-kubernetes/ovn-kubernetes/go-controller/pkg/node/nftables" ovntest "github.com/ovn-kubernetes/ovn-kubernetes/go-controller/pkg/testing" "github.com/ovn-kubernetes/ovn-kubernetes/go-controller/pkg/types" @@ -268,4 +269,36 @@ var _ = Describe("Gateway NFTables", func() { }) }) }) + + Describe("setupDPUHostNoOverlaySNAT", func() { + It("SNATs default cluster CIDR traffic to the host masquerade IP", func() { + nft := nodenft.SetFakeNFTablesHelper() + config.Default.ClusterSubnets = []config.CIDRNetworkEntry{ + {CIDR: ovntest.MustParseIPNet("10.244.0.0/16")}, + {CIDR: ovntest.MustParseIPNet("fd00:10:244::/48")}, + } + + Expect(setupDPUHostNoOverlaySNAT("breth0")).To(Succeed()) + dump := nft.Dump() + + Expect(dump).To(ContainSubstring("add chain inet ovn-kubernetes dpu-host-no-overlay-snat")) + Expect(dump).To(ContainSubstring("add rule inet ovn-kubernetes dpu-host-no-overlay-snat oifname != breth0 return")) + Expect(dump).To(ContainSubstring("ip daddr 10.244.0.0/16 ip saddr != 169.254.169.2 snat ip to 169.254.169.2")) + Expect(dump).To(ContainSubstring("ip6 daddr fd00:10:244::/48 ip6 saddr != fd69::2 snat ip6 to fd69::2")) + }) + + It("removes DPU host no-overlay SNAT rules", func() { + nft := nodenft.SetFakeNFTablesHelper() + config.Default.ClusterSubnets = []config.CIDRNetworkEntry{ + {CIDR: ovntest.MustParseIPNet("10.244.0.0/16")}, + } + + Expect(setupDPUHostNoOverlaySNAT("breth0")).To(Succeed()) + Expect(nft.Dump()).To(ContainSubstring("dpu-host-no-overlay-snat")) + + Expect(teardownDPUHostNoOverlaySNAT()).To(Succeed()) + Expect(nft.Dump()).NotTo(ContainSubstring("dpu-host-no-overlay-snat")) + Expect(teardownDPUHostNoOverlaySNAT()).To(Succeed()) + }) + }) }) diff --git a/go-controller/pkg/node/gateway_shared_intf.go b/go-controller/pkg/node/gateway_shared_intf.go index b01a343b3f..0cb3b8afaa 100644 --- a/go-controller/pkg/node/gateway_shared_intf.go +++ b/go-controller/pkg/node/gateway_shared_intf.go @@ -75,6 +75,10 @@ const ( // from accessing any of the advertised UDN networks nftablesUDNBGPOutputChain = "udn-bgp-drop" + // nftablesDPUHostNoOverlaySNATChain SNATs DPU-host host-network traffic + // to the host masquerade IP before handing it to OVN for no-overlay pod CIDRs. + nftablesDPUHostNoOverlaySNATChain = "dpu-host-no-overlay-snat" + // nftablesAdvertisedUDNsSetV[4|6] is a set containing advertised UDN subnets nftablesAdvertisedUDNsSetV4 = "advertised-udn-subnets-v4" nftablesAdvertisedUDNsSetV6 = "advertised-udn-subnets-v6" @@ -2231,6 +2235,18 @@ func (r *masqueradeReconciler) ensure() error { if err := configureSvcRouteViaInterface(r.routeManager, gwIface, DummyNextHopIPs()); err != nil { return fmt.Errorf("failed to configure service route: %w", err) } + if shouldConfigureDPUHostNoOverlayPodCIDRRoute() { + if err := configureDPUHostNoOverlayPodCIDRRoute(r.routeManager, gwIface, DummyNextHopIPs()); err != nil { + return fmt.Errorf("failed to configure DPU host no-overlay pod CIDR route: %w", err) + } + if err := setupDPUHostNoOverlaySNAT(gwIface); err != nil { + return fmt.Errorf("failed to configure DPU host no-overlay SNAT: %w", err) + } + } else { + if err := teardownDPUHostNoOverlaySNAT(); err != nil { + return fmt.Errorf("failed to remove DPU host no-overlay SNAT: %w", err) + } + } // 3. ARP/ND entries for masquerade IPs. if err := addHostMACBindings(gwIface); err != nil { return fmt.Errorf("failed to add MAC bindings: %w", err) @@ -2241,6 +2257,81 @@ func (r *masqueradeReconciler) ensure() error { return nil } +func setupDPUHostNoOverlaySNAT(gwIface string) error { + nft, err := nodenft.GetNFTablesHelper() + if err != nil { + return err + } + tx := nft.NewTransaction() + + tx.Add(&knftables.Chain{ + Name: nftablesDPUHostNoOverlaySNATChain, + Comment: knftables.PtrTo("OVN DPU host no-overlay SNAT"), + Type: knftables.PtrTo(knftables.NATType), + Hook: knftables.PtrTo(knftables.PostroutingHook), + Priority: knftables.PtrTo(knftables.SNATPriority), + }) + tx.Flush(&knftables.Chain{Name: nftablesDPUHostNoOverlaySNATChain}) + tx.Add(&knftables.Rule{ + Chain: nftablesDPUHostNoOverlaySNATChain, + Rule: knftables.Concat( + "oifname", "!=", gwIface, + "return", + ), + }) + + for _, clusterSubnet := range config.Default.ClusterSubnets { + subnet := clusterSubnet.CIDR + if utilnet.IsIPv6CIDR(subnet) { + tx.Add(&knftables.Rule{ + Chain: nftablesDPUHostNoOverlaySNATChain, + Rule: knftables.Concat( + "ip6 daddr", subnet, + "ip6 saddr", "!=", config.Gateway.MasqueradeIPs.V6HostMasqueradeIP, + "snat ip6 to", config.Gateway.MasqueradeIPs.V6HostMasqueradeIP, + ), + }) + continue + } + tx.Add(&knftables.Rule{ + Chain: nftablesDPUHostNoOverlaySNATChain, + Rule: knftables.Concat( + "ip daddr", subnet, + "ip saddr", "!=", config.Gateway.MasqueradeIPs.V4HostMasqueradeIP, + "snat ip to", config.Gateway.MasqueradeIPs.V4HostMasqueradeIP, + ), + }) + } + + if err := nft.Run(context.TODO(), tx); err != nil { + return fmt.Errorf("could not update nftables rule for DPU host no-overlay SNAT: %w", err) + } + return nil +} + +func teardownDPUHostNoOverlaySNAT() error { + nft, err := nodenft.GetNFTablesHelper() + if err != nil { + return err + } + tx := nft.NewTransaction() + + chain := &knftables.Chain{ + Name: nftablesDPUHostNoOverlaySNATChain, + Type: knftables.PtrTo(knftables.NATType), + Hook: knftables.PtrTo(knftables.PostroutingHook), + Priority: knftables.PtrTo(knftables.SNATPriority), + } + tx.Add(chain) + tx.Flush(chain) + tx.Delete(chain) + + if err := nft.Run(context.TODO(), tx); err != nil && !knftables.IsNotFound(err) { + return fmt.Errorf("could not remove nftables rule for DPU host no-overlay SNAT: %w", err) + } + return nil +} + func addHostMACBindings(bridgeName string) error { // Add a neighbour entry on the K8s node to map dummy next-hop masquerade // addresses with MACs. This is required because these addresses do not diff --git a/go-controller/pkg/node/managementport/port_config.go b/go-controller/pkg/node/managementport/port_config.go index c3286ec860..3ff6f48eb0 100644 --- a/go-controller/pkg/node/managementport/port_config.go +++ b/go-controller/pkg/node/managementport/port_config.go @@ -131,9 +131,11 @@ func newManagementPortIPFamilyConfig(hostSubnet *net.IPNet, isIPv6 bool, netInfo } // capture all the subnets for which we need to add routes through management port - for _, subnet := range config.Default.ClusterSubnets { - if utilnet.IsIPv6CIDR(subnet.CIDR) == isIPv6 { - cfg.clusterSubnets = append(cfg.clusterSubnets, subnet.CIDR) + if managementPortRoutesDefaultClusterSubnets() { + for _, subnet := range config.Default.ClusterSubnets { + if utilnet.IsIPv6CIDR(subnet.CIDR) == isIPv6 { + cfg.clusterSubnets = append(cfg.clusterSubnets, subnet.CIDR) + } } } // add the .3 masqueradeIP to add the route via mp0 for ETP=local case @@ -154,3 +156,9 @@ func newManagementPortIPFamilyConfig(hostSubnet *net.IPNet, isIPv6 bool, netInfo return cfg, nil } + +func managementPortRoutesDefaultClusterSubnets() bool { + return !(config.IsModeDPUHost() && + config.Gateway.Mode == config.GatewayModeShared && + config.Default.Transport == types.NetworkTransportNoOverlay) +} diff --git a/go-controller/pkg/node/managementport/port_linux_test.go b/go-controller/pkg/node/managementport/port_linux_test.go index ae83fe99e4..4cfe9400c8 100644 --- a/go-controller/pkg/node/managementport/port_linux_test.go +++ b/go-controller/pkg/node/managementport/port_linux_test.go @@ -1250,6 +1250,25 @@ var _ = Describe("Management Port tests", func() { netInfo.On("GetPodNetworkAdvertisedOnNodeVRFs", "worker-node").Return(nil) netInfo.On("GetNodeGatewayIP", hostSubnets[0]).Return(util.GetNodeGatewayIfAddr(hostSubnets[0])) netInfo.On("GetNodeManagementIP", hostSubnets[0]).Return(util.GetNodeManagementIfAddr(hostSubnets[0])) + It("does not add default cluster subnet routes through the management port for DPU host no-overlay", func() { + config.OvnKubeNode.Mode = types.NodeModeDPUHost + config.Gateway.Mode = config.GatewayModeShared + config.Default.Transport = types.NetworkTransportNoOverlay + config.Default.ClusterSubnets = []config.CIDRNetworkEntry{{ + CIDR: ovntest.MustParseIPNet("10.1.0.0/16"), + HostSubnetLength: 24, + }} + + cfg, err := newManagementPortIPFamilyConfig(hostSubnets[0], false, netInfo) + Expect(err).NotTo(HaveOccurred()) + + var routeSubnets []string + for _, subnet := range cfg.clusterSubnets { + routeSubnets = append(routeSubnets, subnet.String()) + } + Expect(routeSubnets).NotTo(ContainElement("10.1.0.0/16")) + Expect(routeSubnets).To(ConsistOf(fmt.Sprintf("%s/32", config.Gateway.MasqueradeIPs.V4HostETPLocalMasqueradeIP.String()))) + }) It("Creates managementPort by default", func() { mgmtPort, err := NewManagementPortController(nil, node, hostSubnets, netdevName, rep, nil, netInfo) Expect(err).NotTo(HaveOccurred()) diff --git a/mkdocs.yml b/mkdocs.yml index d9d0e421fc..a64e328b7b 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -84,6 +84,7 @@ nav: - Service Traffic Policy: design/service-traffic-policy.md - Host To NodePort Hairpin: design/host-to-node-port-hairpin-trafficflow.md - ExternalIPs/LoadBalancerIngress: design/external-ip-and-loadbalancer-ingress.md + - DPU Host No-Overlay Routing: design/dpu-host-no-overlay-routing.md - Internal Subnets: design/ovn-kubernetes-subnets.md - Kubevirt VM Live Migration: features/live-migration.md - Getting Started: From 1160194c7f54af62d1e3f7214d7b7c5214505bf8 Mon Sep 17 00:00:00 2001 From: Tim Rozet Date: Sat, 6 Jun 2026 11:23:11 -0400 Subject: [PATCH 34/51] Make TFT report failure for DPU CI jobs TFT runs were not enabling check mode, so traffic flow test failures did not cause the CI step to fail. Run dpu-sim TFT with --check so the wrapper passes --check through to tft.py and returns the test validation status. Signed-off-by: Tim Rozet Assisted-by: Codex --- .github/workflows/kind-dpu-offload.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/kind-dpu-offload.yml b/.github/workflows/kind-dpu-offload.yml index b48fba2dfb..2f3a593c73 100644 --- a/.github/workflows/kind-dpu-offload.yml +++ b/.github/workflows/kind-dpu-offload.yml @@ -105,7 +105,7 @@ jobs: - name: Run traffic flow tests working-directory: dpu-simulator - run: ./bin/dpu-sim tft run --config config-kind-ovnk-offload.yaml + run: ./bin/dpu-sim tft run --check --config config-kind-ovnk-offload.yaml - name: Export kind logs on failure if: failure() @@ -208,7 +208,7 @@ jobs: - name: Run traffic flow tests working-directory: dpu-simulator - run: ./bin/dpu-sim tft run --config "${DPU_SIM_CONFIG}" + run: ./bin/dpu-sim tft run --check --config "${DPU_SIM_CONFIG}" - name: Export kind logs on failure if: failure() From 2d545a0bf4b58407dfe061263bb77f72393c89d7 Mon Sep 17 00:00:00 2001 From: Mykola Yurchenko Date: Wed, 13 May 2026 15:00:35 -0700 Subject: [PATCH 35/51] contrib: refresh OVN pods on kind-helm.sh --deploy This was wrongfully dropped in 80be988d0, recover the funcionality so that pods are deleted for same tag redeploy. ovs-node is deliberately ommited. Signed-off-by: Mykola Yurchenko --- contrib/kind-common.sh | 11 +++++++++++ contrib/kind-helm.sh | 5 +++++ 2 files changed, 16 insertions(+) diff --git a/contrib/kind-common.sh b/contrib/kind-common.sh index 3f590eb750..071e4a3ffe 100644 --- a/contrib/kind-common.sh +++ b/contrib/kind-common.sh @@ -93,6 +93,9 @@ set_common_default_params() { OVN_IMAGE=${OVN_IMAGE:-local} OVN_REPO=${OVN_REPO:-""} OVN_GITREF=${OVN_GITREF:-""} + # Pods to force-delete on --deploy so they respawn with the kind-loaded image. + # ovs-node excluded: restarting it under live ovnkube-node pods breaks the cluster. + OVN_DEPLOY_PODS=${OVN_DEPLOY_PODS:-"ovnkube-identity ovnkube-control-plane ovnkube-node"} # Subnet params # Input not currently validated. Modify outside script at your own risk. @@ -1029,6 +1032,14 @@ install_ovn_image() { install_image "${OVN_IMAGE}" } +# Force-respawn OVN pods so their controllers pick up the kind-loaded image +# on --deploy, where helm sees no spec diff (OVN_IMAGE tag is fixed). +refresh_ovn_pods() { + for pod in ${OVN_DEPLOY_PODS}; do + kubectl delete pod -n ovn-kubernetes -l name="${pod}" --ignore-not-found + done +} + # install_image accepts the image name along with the tag as an argument and installs it. install_image() { # If local registry is being used push image there for consumption by kind cluster diff --git a/contrib/kind-helm.sh b/contrib/kind-helm.sh index 2304b044cc..68d912d8d4 100755 --- a/contrib/kind-helm.sh +++ b/contrib/kind-helm.sh @@ -629,6 +629,11 @@ if [ "$KIND_REMOVE_TAINT" == true ]; then fi create_ovn_kubernetes +# --deploy: helm sees no spec diff (same OVN_IMAGE tag), so refresh pods manually. +if [ "$KIND_CREATE" == false ] && [ "$KIND_LOCAL_REGISTRY" == false ]; then + refresh_ovn_pods +fi + install_online_ovn_kubernetes_crds if [ "$KIND_INSTALL_INGRESS" == true ]; then install_ingress From a11c389db8fdee5efd90087910ece9f9ee09529f Mon Sep 17 00:00:00 2001 From: Surya Seetharaman Date: Mon, 1 Jun 2026 19:42:44 +0200 Subject: [PATCH 36/51] area-maintainers: introduce kubevirt area with merge bot Introduce the Area Maintainer concept for ovn-kubernetes, starting with the KubeVirt / Live Migration area as the first area. Area Maintainers can merge PRs that exclusively touch files within their area by commenting /area-approve, which triggers a GitHub Actions merge bot that verifies file scope, CI status, and authorization before merging. - Add KubeVirt / Live Migration section to CODEOWNERS with @maiqueb as Area Maintainer and @qinqon, @ormergi as reviewers - Add .github/workflows/area-merge.yml merge bot triggered by /area-approve and check_suite events, with SHA-bound approvals, pagination, self-merge prevention, and pinned action versions - Add Area Maintainers governance section to GOVERNANCE.md with roles, responsibilities, and appointment/removal process - Establish Maintainer purview over all areas and Area Maintainers - Update REVIEWING.md with area maintainer merge process, no-self-merge rule, and exception to maintainer-approval rule Signed-off-by: Surya Seetharaman Co-authored-by: Cursor --- .github/scripts/area-merge.js | 338 +++++++++++++++++++++++++++++++ .github/workflows/area-merge.yml | 38 ++++ CODEOWNERS | 50 +++++ GOVERNANCE.md | 84 ++++++++ REVIEWING.md | 9 +- docs/developer-guide/areas.md | 55 +++++ mkdocs.yml | 1 + 7 files changed, 572 insertions(+), 3 deletions(-) create mode 100644 .github/scripts/area-merge.js create mode 100644 .github/workflows/area-merge.yml create mode 100644 docs/developer-guide/areas.md diff --git a/.github/scripts/area-merge.js b/.github/scripts/area-merge.js new file mode 100644 index 0000000000..abf8f25746 --- /dev/null +++ b/.github/scripts/area-merge.js @@ -0,0 +1,338 @@ +// Area Maintainer Merge Bot +// +// Parses CODEOWNERS to determine area maintainer authorization, then verifies +// file scope and CI status before merging PRs via /area-maintainer-approved. +// +// Called from .github/workflows/area-merge.yml via actions/github-script. + +const fs = require('fs'); + +module.exports = async ({ github, context, core }) => { + // --- Determine which PRs to process and who triggered --- + let pullNumbers = []; + let commenter = null; + + if (context.eventName === 'issue_comment') { + pullNumbers.push(context.payload.issue.number); + commenter = context.payload.comment.user.login.toLowerCase(); + } else if (context.eventName === 'check_suite') { + const suite = context.payload.check_suite; + for (const pr of (suite.pull_requests || [])) { + pullNumbers.push(pr.number); + } + } + + if (pullNumbers.length === 0) { + core.info('No pull requests to process'); + return; + } + + // --- Parse CODEOWNERS --- + // Extracts two things: + // 1. Pattern rules: which users are listed on each file pattern line (for review assignment) + // 2. Area maintainers: parsed from section header comments matching + // "Area Maintainer: @user1 @user2" — only these users can trigger merges + const codeownersContent = fs.readFileSync('CODEOWNERS', 'utf8'); + const rules = []; + let currentAreaMaintainers = []; + + for (const line of codeownersContent.split('\n')) { + const trimmed = line.trim(); + + // Parse area maintainer(s) from section header comments + // Format: # ... (Area Maintainer: @user1 @user2) + const maintainerMatch = trimmed.match(/Area Maintainer[s]?:\s*((?:@[\w-]+[\s,]*)+)/i); + if (maintainerMatch) { + currentAreaMaintainers = maintainerMatch[1] + .match(/@[\w-]+/g) + .map(u => u.replace('@', '').toLowerCase()); + continue; + } + + if (!trimmed || trimmed.startsWith('#')) continue; + const parts = trimmed.split(/\s+/); + if (parts.length < 2) continue; + const pattern = parts[0]; + if (pattern === '*') continue; + const owners = parts.slice(1).map(o => o.replace('@', '').toLowerCase()); + rules.push({ pattern, owners, areaMaintainers: [...currentAreaMaintainers] }); + } + + function matchPattern(pattern, filePath) { + const normalizedFile = '/' + filePath; + if (pattern.endsWith('/')) { + return normalizedFile.startsWith(pattern) || + normalizedFile === pattern.slice(0, -1); + } + return normalizedFile === pattern; + } + + // Last matching rule wins, mirroring CODEOWNERS precedence. + function findAreaMaintainers(filePath) { + let maintainers = null; + for (const rule of rules) { + if (matchPattern(rule.pattern, filePath)) { + maintainers = rule.areaMaintainers; + } + } + return maintainers; + } + + async function fetchAllComments(prNumber) { + const allComments = []; + let page = 1; + while (true) { + const { data: comments } = await github.rest.issues.listComments({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: prNumber, + per_page: 100, + page: page, + }); + if (comments.length === 0) break; + allComments.push(...comments); + if (comments.length < 100) break; + page++; + } + return allComments; + } + + // --- Process each PR --- + for (const prNumber of pullNumbers) { + core.info(`Processing PR #${prNumber}`); + + const { data: pr } = await github.rest.pulls.get({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: prNumber, + }); + + if (pr.state !== 'open') { + core.info(`PR #${prNumber} is ${pr.state}, skipping`); + continue; + } + + // Prevent PR authors from merging their own PRs + const prAuthor = pr.user.login.toLowerCase(); + if (commenter && commenter === prAuthor) { + core.info(`PR #${prNumber} author ${prAuthor} cannot approve their own PR`); + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: prNumber, + body: `@${commenter} you cannot use \`/area-maintainer-approved\` on your own PR. ` + + `Another area maintainer or a repo maintainer must approve and merge it.`, + }); + continue; + } + + // For check_suite retries, find who previously ran /area-maintainer-approved + let requester = commenter; + let allComments = null; + if (!requester) { + allComments = await fetchAllComments(prNumber); + const mergeComment = [...allComments] + .reverse() + .find(c => c.body.includes('/area-maintainer-approved') && + c.user.login !== 'github-actions[bot]' && + c.user.login.toLowerCase() !== prAuthor); + if (mergeComment) { + requester = mergeComment.user.login.toLowerCase(); + } + } + + if (!requester) { + core.info(`No /area-maintainer-approved request found for PR #${prNumber}, skipping`); + continue; + } + + // For check_suite, verify the bot posted a "waiting" comment for the current head SHA. + // If new commits were pushed after approval, the SHA won't match and we skip + // to prevent merging code the area maintainer never reviewed. + if (context.eventName === 'check_suite') { + if (!allComments) allComments = await fetchAllComments(prNumber); + const waitingMarker = ``; + const isWaiting = allComments.some(c => + c.user.login === 'github-actions[bot]' && + c.body.includes(waitingMarker) + ); + if (!isWaiting) { + core.info(`PR #${prNumber} has no waiting marker for current SHA ${pr.head.sha}, skipping`); + continue; + } + } + + core.info(`/area-maintainer-approved requested by: ${requester}`); + + // Get changed files (paginate for large PRs) + const changedFiles = []; + let page = 1; + while (true) { + const { data: files } = await github.rest.pulls.listFiles({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: prNumber, + per_page: 100, + page: page, + }); + if (files.length === 0) break; + changedFiles.push(...files.map(f => f.filename)); + if (files.length < 100) break; + page++; + } + + core.info(`Changed files (${changedFiles.length}): ${changedFiles.join(', ')}`); + + // Check authorization: requester must be an area maintainer for ALL changed files + const unauthorizedFiles = []; + for (const file of changedFiles) { + const maintainers = findAreaMaintainers(file); + if (!maintainers || !maintainers.includes(requester)) { + unauthorizedFiles.push(file); + } + } + + if (unauthorizedFiles.length > 0) { + core.info(`Unauthorized files: ${unauthorizedFiles.join(', ')}`); + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: prNumber, + body: `@${requester} cannot merge this PR via \`/area-maintainer-approved\`.\n\n` + + `You are not listed as an Area Maintainer for the following files in \`CODEOWNERS\`:\n` + + unauthorizedFiles.map(f => `- \`${f}\``).join('\n') + + `\n\nOnly area maintainers (listed in the \`Area Maintainer:\` section header in CODEOWNERS) can trigger merges. ` + + `A repo maintainer from \`ovn-kubernetes-committers\` is required to merge this PR.`, + }); + continue; + } + + // Check CI status + const ref = pr.head.sha; + + const allCheckRuns = []; + let checkPage = 1; + while (true) { + const { data } = await github.rest.checks.listForRef({ + owner: context.repo.owner, + repo: context.repo.repo, + ref: ref, + per_page: 100, + page: checkPage, + }); + allCheckRuns.push(...data.check_runs); + if (allCheckRuns.length >= data.total_count) break; + checkPage++; + } + + const { data: combinedStatus } = await github.rest.repos.getCombinedStatusForRef({ + owner: context.repo.owner, + repo: context.repo.repo, + ref: ref, + }); + + const pendingChecks = []; + const failedChecks = []; + + for (const run of allCheckRuns) { + if (run.name === 'area-merge') continue; + if (run.status !== 'completed') { + pendingChecks.push(run.name); + } else if (run.conclusion !== 'success' && run.conclusion !== 'neutral' && run.conclusion !== 'skipped') { + failedChecks.push(run.name); + } + } + + // Gate on the aggregate state first — it covers ALL commit statuses + // regardless of pagination, so we never miss a failure or pending status. + // Then iterate the (possibly partial) statuses array for detailed names. + if (combinedStatus.state === 'failure') { + for (const status of combinedStatus.statuses) { + if (status.state === 'failure' || status.state === 'error') { + failedChecks.push(status.context); + } + } + if (failedChecks.length === 0) { + failedChecks.push('(commit status failure — details may be on a later page)'); + } + } else if (combinedStatus.state === 'pending') { + for (const status of combinedStatus.statuses) { + if (status.state === 'pending') { + pendingChecks.push(status.context); + } else if (status.state === 'failure' || status.state === 'error') { + failedChecks.push(status.context); + } + } + if (pendingChecks.length === 0 && failedChecks.length === 0) { + pendingChecks.push('(commit status pending — details may be on a later page)'); + } + } + + if (failedChecks.length > 0) { + core.info(`Failed checks: ${failedChecks.join(', ')}`); + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: prNumber, + body: `Cannot merge via \`/area-maintainer-approved\`: the following CI checks have failed:\n` + + failedChecks.map(c => `- \`${c}\``).join('\n') + + `\n\nPlease fix the failures and run \`/area-maintainer-approved\` again.`, + }); + continue; + } + + if (pendingChecks.length > 0) { + core.info(`Pending checks: ${pendingChecks.join(', ')}`); + + const shaMarker = ``; + if (!allComments) allComments = await fetchAllComments(prNumber); + const alreadyWaiting = allComments.some(c => + c.user.login === 'github-actions[bot]' && + c.body.includes(shaMarker) + ); + if (!alreadyWaiting) { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: prNumber, + body: `${shaMarker}\nArea maintainer @${requester} has approved this PR via \`/area-maintainer-approved\`. ` + + `Waiting on CI checks to complete — will merge automatically when all checks pass.\n\n` + + `Pending checks:\n` + + pendingChecks.map(c => `- \`${c}\``).join('\n'), + }); + } + continue; + } + + // All checks passed, authorized — merge! + core.info(`All checks passed. Merging PR #${prNumber}`); + try { + await github.rest.pulls.merge({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: prNumber, + sha: ref, + merge_method: 'merge', + commit_title: `Merge pull request #${prNumber} (/area-maintainer-approved by @${requester})`, + }); + + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: prNumber, + body: `PR merged by area maintainer @${requester} via \`/area-maintainer-approved\`.`, + }); + + core.info(`Successfully merged PR #${prNumber}`); + } catch (e) { + core.setFailed(`Failed to merge PR #${prNumber}: ${e.message}`); + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: prNumber, + body: `Failed to merge this PR: \`${e.message}\`\n\n` + + `A repo maintainer may need to merge manually.`, + }); + } + } +}; diff --git a/.github/workflows/area-merge.yml b/.github/workflows/area-merge.yml new file mode 100644 index 0000000000..b99c923f1c --- /dev/null +++ b/.github/workflows/area-merge.yml @@ -0,0 +1,38 @@ +name: "Area Maintainer Merge" + +on: + issue_comment: + types: [created] + check_suite: + types: [completed] + +jobs: + area-merge: + if: > + (github.event_name == 'issue_comment' && + github.event.issue.pull_request && + contains(github.event.comment.body, '/area-maintainer-approved')) || + github.event_name == 'check_suite' + permissions: + contents: write + pull-requests: write + checks: read + statuses: read + runs-on: ubuntu-latest + steps: + - name: Checkout base branch (for CODEOWNERS and scripts) + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + ref: ${{ github.event.repository.default_branch }} + sparse-checkout: | + CODEOWNERS + .github/scripts/area-merge.js + persist-credentials: false + + - name: Area maintainer merge check + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const script = require('./.github/scripts/area-merge.js'); + await script({ github, context, core }); diff --git a/CODEOWNERS b/CODEOWNERS index fcd3dbc423..8d0eaf71a9 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -1 +1,51 @@ +# CODEOWNERS - Area Maintainers for ovn-kubernetes +# +# Each line is a file pattern followed by one or more owners (GitHub usernames +# or team slugs). When a PR modifies files matching a pattern, GitHub will +# automatically request reviews from the listed owners. +# +# Order matters: the LAST matching pattern takes precedence. +# See https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners +# +# To add a new area: +# 1. Add a section below with the file patterns and area maintainer(s) +# 2. Optionally create a GitHub team in the ovn-kubernetes org +# 3. Use individual @usernames or @org/team-slug + +# ============================================================================ +# Default: catch-all for anything not matched below +# ============================================================================ * @ovn-kubernetes/ovn-kubernetes-members + +# ============================================================================ +# Virtualization (Area Maintainer: @maiqueb) +# ============================================================================ +# E2E tests +/test/e2e/kubevirt.go @maiqueb @qinqon @ormergi +/test/e2e/kubevirt/ @maiqueb @qinqon @ormergi +/test/e2e/multihoming.go @maiqueb @qinqon @ormergi +/test/e2e/multihoming_utils.go @maiqueb @qinqon @ormergi +/test/e2e/multihoming_external_router_utils.go @maiqueb @qinqon @ormergi +/test/e2e/network_segmentation_localnet.go @maiqueb @qinqon @ormergi +/test/e2e/localnet-underlay.go @maiqueb @qinqon @ormergi +/test/e2e/network_segmentation_preconfigured_layer2.go @maiqueb @qinqon @ormergi +/test/e2e/testscenario/cudn/valid-scenarios-localnet.go @maiqueb @qinqon @ormergi +/test/e2e/testscenario/cudn/invalid-scenarios-localnet-vlan.go @maiqueb @qinqon @ormergi +/test/e2e/testscenario/cudn/invalid-scenarios-localnet-subnets.go @maiqueb @qinqon @ormergi +/test/e2e/testscenario/cudn/invalid-scenarios-localnet-role.go @maiqueb @qinqon @ormergi +/test/e2e/testscenario/cudn/invalid-scenarios-localnet-phynetname.go @maiqueb @qinqon @ormergi +/test/e2e/testscenario/cudn/invalid-scenarios-localnet-mtu.go @maiqueb @qinqon @ormergi +# Unit tests +/go-controller/pkg/ovn/kubevirt_test.go @maiqueb @qinqon @ormergi +/go-controller/pkg/ovn/multihoming_test.go @maiqueb @qinqon @ormergi +/go-controller/pkg/ovn/multipolicy_test.go @maiqueb @qinqon @ormergi +/go-controller/pkg/ovn/layer2_user_defined_network_controller_test.go @maiqueb @qinqon @ormergi +/go-controller/pkg/util/multi_network_test.go @maiqueb @qinqon @ormergi +# Production code +/go-controller/pkg/kubevirt/ @maiqueb @qinqon @ormergi +/go-controller/pkg/util/arp.go @maiqueb @qinqon @ormergi +/go-controller/pkg/util/ndp/ @maiqueb @qinqon @ormergi +# Docs +/docs/features/live-migration.md @maiqueb @qinqon @ormergi +/docs/features/multiple-networks/multi-homing.md @maiqueb @qinqon @ormergi +/docs/features/multiple-networks/multi-network-policies.md @maiqueb @qinqon @ormergi diff --git a/GOVERNANCE.md b/GOVERNANCE.md index 4e9ea2fbdf..0d61463686 100644 --- a/GOVERNANCE.md +++ b/GOVERNANCE.md @@ -11,6 +11,9 @@ This governance explains how the project is run. - [Members](#members) - [Becoming a Member](#becoming-a-member) - [Removing a Member](#removing-a-member) +- [Area Maintainers](#area-maintainers) + - [Becoming an Area Maintainer](#becoming-an-area-maintainer) + - [Removing an Area Maintainer](#removing-an-area-maintainer) - [Meetings](#meetings) - [Code of Conduct](#code-of-conduct) - [Security Response Team](#security-response-team) @@ -59,6 +62,13 @@ the project succeed. The collective team of all Maintainers is known as the Maintainer Council, which is the governing body for the project. +Maintainers have purview over all areas and Area Maintainers. They create and +dissolve areas, appoint and remove Area Maintainers, approve changes to an +area's file scope in `CODEOWNERS`, and may override or merge any PR regardless +of area boundaries. Area Maintainers operate under the authority delegated by +the Maintainers and are expected to escalate cross-area concerns or contentious +decisions to them. + ### Becoming a Maintainer To become a Maintainer you need to demonstrate the following: @@ -137,6 +147,80 @@ Members who are consistently unresponsive to assigned PR reviews may be contacted by Maintainers to discuss their availability and commitment. If the pattern of non-responsiveness continues, the Member may be removed. +## Area Maintainers + +Area Maintainers are trusted contributors who own a specific area of the +codebase (e.g. KubeVirt, Egress IP, Services). They have the authority to review, +approve, and merge pull requests that **exclusively** touch files within their +area, as defined in `CODEOWNERS`. Area Maintainers are not full Maintainers — +they cannot merge PRs that touch files outside their designated area. + +Area Maintainers are automatically requested as reviewers by GitHub when a PR +modifies files matching their `CODEOWNERS` patterns. They can merge qualifying +PRs by commenting `/area-maintainer-approved` on the PR, which triggers the +merge bot (`.github/workflows/area-merge.yml`) to verify file scope and CI +status before merging. + +### Area Maintainer Responsibilities + +- **Area health and ownership:** Take ownership of the overall health of the + area. Maintain high code quality standards, ensure technical debt is managed, + foster innovation without rushing changes that compromise stability, and + promptly bring up stuck PRs for review at community meetings. +- **PR approval:** Review and approve pull requests within their area. + Ensure contributions meet quality standards and are well-tested. Area + Maintainers must not approve or merge their own pull requests — another + area maintainer, reviewer, or repo maintainer must review and approve them. +- **Design proposals:** Own, review, and drive OKEPs (OVN-Kubernetes Enhancement + Proposals) related to their area. +- **Documentation:** Ensure documentation for the area is accurate and + up to date. +- **CI health:** Monitor CI for their area and address failures and flakes + promptly without needing to be pinged. +- **Upstream engagement:** Attend OVN-Kubernetes upstream meetings and represent + the area's interests, especially for cross-area collaborations. +- **Community support:** Help other contributors working in the area with + reviews, guidance, and mentoring. +- **Communication:** Keep the Maintainers informed of important changes, + design decisions, and roadmap items happening in the area. +- **Scope management:** If the area's file list in `CODEOWNERS` needs + expansion or contraction, file a request to the Maintainers, who have the + final say on the area's scope. + +### Becoming an Area Maintainer + +Area Maintainers are typically individuals who are already fulfilling the +responsibilities listed above — the role formalizes what they are already doing. +To become an Area Maintainer you need to demonstrate the following: + +- commitment to the specific area: + - participate in discussions, contributions, code and documentation reviews + related to the area for 3 months or more, + - perform reviews for 5 non-trivial pull requests in the area, + - contribute 10 non-trivial pull requests to the area and have them merged, +- demonstrated ownership of area health: maintaining code quality, addressing + CI failures, keeping documentation current, and driving improvements without + compromising stability, +- deep understanding of the area's code, design, and interactions with the + rest of the project, +- ability to write quality code and/or documentation, +- ability to collaborate with the team. + +A new Area Maintainer must be proposed by an existing Maintainer by sending a +message to the [developer mailing list](https://groups.google.com/g/ovn-kubernetes). +The appointment requires approval from a simple majority of the Maintainers. +Once approved, the new Area Maintainer's GitHub username is added to the +relevant entries in `CODEOWNERS`. + +### Removing an Area Maintainer + +Area Maintainers may resign at any time. + +Area Maintainers may also be removed after being inactive in their area for a +period of 6 months or more, for failure to fulfill their responsibilities, or +for violating the Code of Conduct. An Area Maintainer may be removed at any +time by a simple majority vote of the Maintainers. + ## Meetings Time zones permitting, Maintainers are expected to participate in the public diff --git a/REVIEWING.md b/REVIEWING.md index 9358104bdb..0032a59185 100644 --- a/REVIEWING.md +++ b/REVIEWING.md @@ -35,10 +35,13 @@ Be trustworthy. During a review, your actions both build and help maintain the t ## Process -* Reviewers are automatically assigned via the load-balancing algorithm using contributors from the ovn-kubernetes/ovn-kubernetes-members team. -* Reviewers may opt out of reviewing of any PR or the reviewing process altogether by contacting committers or setting their github profile status as "busy" and removing themselves from any currently assigned PR. +* **Reviewers for area-owned files** are automatically assigned when a PR touches files listed in `CODEOWNERS`. GitHub requests reviews from the owners listed on the matching pattern lines (area maintainers and reviewers alike). Merge authority is determined separately by the merge bot based on the `Area Maintainer:` header in `CODEOWNERS`. +* If no area-specific pattern matches, reviewers are assigned via the load-balancing algorithm using contributors from the ovn-kubernetes/ovn-kubernetes-members team (configured in `CODEOWNERS`). +* Area maintainers are appointed by the repo Maintainers (see [Area Maintainers](./GOVERNANCE.md#area-maintainers) in the governance docs). +* **Area maintainer merge:** Area maintainers can merge PRs that **only** touch files within their area by commenting `/area-maintainer-approved` on the PR. The merge bot (`.github/workflows/area-merge.yml`) verifies that all changed files are within the commenter's area in `CODEOWNERS` and that all CI checks pass before merging. If CI is still running, the bot waits and merges automatically when checks go green. Area maintainers cannot use `/area-maintainer-approved` on their own PRs — the bot will reject the attempt; another area maintainer or repo maintainer must approve and merge instead. PRs touching files outside the area maintainer's scope require a committer from `ovn-kubernetes-committers` to merge. +* Reviewers may opt out of reviewing of any PR or the reviewing process altogether by contacting committers or setting their GitHub profile status as "busy" and removing themselves from any currently assigned PR. * Reviewers should wait for automated checks to pass before reviewing -* At least 1 approved review is required from a maintainer before a pull request can be merged +* At least 1 approved review is required from a maintainer before a pull request can be merged. **Exception:** PRs that exclusively touch files within a single area (as defined in `CODEOWNERS`) may be merged by the designated area maintainer via `/area-maintainer-approved` without a full maintainer approval. * All CI checks must pass * If a PR is stuck for some reason it is down to the reviewer to determine the best course of action: * PRs may be closed if they are no longer relevant diff --git a/docs/developer-guide/areas.md b/docs/developer-guide/areas.md new file mode 100644 index 0000000000..9437d4cb5f --- /dev/null +++ b/docs/developer-guide/areas.md @@ -0,0 +1,55 @@ +# Project Areas + +ovn-kubernetes organises its codebase into **areas** — focused domains that are +owned by designated **Area Maintainers**. Each area has a clear scope of files +defined in [`CODEOWNERS`](https://github.com/ovn-kubernetes/ovn-kubernetes/blob/master/CODEOWNERS) +and a set of maintainers and reviewers responsible for its health. + +For the full governance details — roles, responsibilities, appointment and +removal process — see [Area Maintainers](../governance/GOVERNANCE.md#area-maintainers) +in the governance docs. + +## How Areas Work + +* Every area is declared as a section in `CODEOWNERS` with a header comment + identifying the Area Maintainer(s): + ``` + # Virtualization (Area Maintainer: @user) + ``` +* GitHub automatically assigns reviewers from the listed owners when a PR + touches files matching the area's patterns. +* Area Maintainers can merge PRs that **exclusively** touch files within their + area by commenting `/area-maintainer-approved` on the PR. The merge bot + (`.github/workflows/area-merge.yml`) verifies file scope, CI status, and + authorization before merging. +* PRs that touch files across multiple areas require a repo Maintainer to merge. + +## Current Areas + +### Virtualization + +| | | +|---|---| +| **Scope** | KubeVirt integration, live migration, multi-homing, localnet | +| **Area Maintainer** | [@maiqueb](https://github.com/maiqueb) | +| **Reviewers** | [@qinqon](https://github.com/qinqon), [@ormergi](https://github.com/ormergi) | + +**Files:** + +| Category | Paths | +|---|---| +| E2E tests | `/test/e2e/kubevirt.go`, `/test/e2e/kubevirt/`, `/test/e2e/multihoming.go`, `/test/e2e/multihoming_utils.go`, `/test/e2e/multihoming_external_router_utils.go`, `/test/e2e/network_segmentation_localnet.go`, `/test/e2e/localnet-underlay.go`, `/test/e2e/network_segmentation_preconfigured_layer2.go`, `/test/e2e/testscenario/cudn/valid-scenarios-localnet.go`, `/test/e2e/testscenario/cudn/invalid-scenarios-localnet-*.go` | +| Unit tests | `/go-controller/pkg/ovn/kubevirt_test.go`, `/go-controller/pkg/ovn/multihoming_test.go`, `/go-controller/pkg/ovn/multipolicy_test.go`, `/go-controller/pkg/ovn/layer2_user_defined_network_controller_test.go`, `/go-controller/pkg/util/multi_network_test.go` | +| Production code | `/go-controller/pkg/kubevirt/`, `/go-controller/pkg/util/arp.go`, `/go-controller/pkg/util/ndp/` | +| Docs | `/docs/features/live-migration.md`, `/docs/features/multiple-networks/multi-homing.md`, `/docs/features/multiple-networks/multi-network-policies.md` | + +## Adding a New Area + +1. Open a PR proposing the new area — the PR must be approved by the repo + Maintainers (see [Governance](../governance/GOVERNANCE.md#area-maintainers)). +2. Add a new section to `CODEOWNERS` with the file patterns and the proposed + Area Maintainer in the section header comment. +3. Add an entry to this document describing the area's scope, maintainer(s), + and reviewers. +4. Once merged, the Area Maintainer can begin using `/area-maintainer-approved` + to merge qualifying PRs. diff --git a/mkdocs.yml b/mkdocs.yml index a64e328b7b..7f5205536e 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -102,6 +102,7 @@ nav: - Contributing Guide: governance/CONTRIBUTING.md - Governance: governance/GOVERNANCE.md - Reviewing Guide: governance/REVIEWING.md + - Project Areas: developer-guide/areas.md - Coding Guide: developer-guide/developer.md - OVN-Kubernetes Container Images: developer-guide/image-build.md - Documentation Guide: developer-guide/documentation.md From 9e8bb31cdff6389f76c23e3f0dbf65df877b542f Mon Sep 17 00:00:00 2001 From: Enrique Llorente Date: Tue, 26 May 2026 11:50:46 +0200 Subject: [PATCH 37/51] fix(evpn): gate neighbor programming on migration domain readiness Refactor reconcilePod to use shouldDeleteNeighbors/shouldEnsureNeighbors helpers that check migration status before programming PERMANENT neighbors. This avoids the race where neighbors are programmed before the source has withdrawn, allowing zebra's extern_learn to overwrite them. Signed-off-by: Enrique Llorente --- .../controllers/evpn/evpn_pod_controller.go | 56 ++++++++++--------- 1 file changed, 31 insertions(+), 25 deletions(-) diff --git a/go-controller/pkg/node/controllers/evpn/evpn_pod_controller.go b/go-controller/pkg/node/controllers/evpn/evpn_pod_controller.go index e7c400989a..532bc4864f 100644 --- a/go-controller/pkg/node/controllers/evpn/evpn_pod_controller.go +++ b/go-controller/pkg/node/controllers/evpn/evpn_pod_controller.go @@ -64,28 +64,32 @@ func (c *Controller) podNeedsUpdate(oldObj, newObj *corev1.Pod) bool { return oldAnnot != newAnnot } -// handleLiveMigrationTargetReady is called when a non-local kubevirt migration target pod -// becomes ready. It finds the local source pod and removes its neighbor/FDB entries so -// FRR withdraws the Type-2 routes from this node. -func (c *Controller) handleLiveMigrationTargetReady(targetPod *corev1.Pod) error { - migrationStatus, err := kubevirt.DiscoverLiveMigrationStatus(c.podLister, targetPod) - if err != nil { - return fmt.Errorf("failed to discover live migration status: %w", err) +// shouldDeleteNeighbors returns true when a pod's neighbor/FDB entries should be removed. +// This happens when the pod has completed, or when this node is the migration source +// and the target domain is ready (so FRR withdraws the Type-2 routes from this node). +func (c *Controller) shouldDeleteNeighbors(migrationStatus *kubevirt.LiveMigrationStatus, pod *corev1.Pod) bool { + if util.PodCompleted(pod) { + return true } if migrationStatus == nil || !migrationStatus.IsTargetDomainReady() { - return nil - } - if migrationStatus.SourcePod.Spec.NodeName != c.nodeName { - return nil + return false } + return migrationStatus.SourcePod.Spec.NodeName == c.nodeName +} - key, err := cache.MetaNamespaceKeyFunc(migrationStatus.SourcePod) - if err != nil { - return err +// shouldEnsureNeighbors returns true when a pod's PERMANENT neighbor entries should be +// programmed. For non-kubevirt pods (migrationStatus==nil) it always returns true. +// During live migration, it returns true only on the target node after the target +// domain is ready, ensuring neighbors are programmed after the source has withdrawn +// its routes and zebra's extern_learn entries are gone. +func (c *Controller) shouldEnsureNeighbors(migrationStatus *kubevirt.LiveMigrationStatus) bool { + if migrationStatus == nil { + return true + } + if !migrationStatus.IsTargetDomainReady() { + return false } - klog.Infof("Live migration target %s/%s ready, removing entries for local source pod %s", - targetPod.Namespace, targetPod.Name, key) - return c.deletePodNeighbors(key) + return migrationStatus.TargetPod.Spec.NodeName == c.nodeName } func (c *Controller) reconcilePod(key string) error { @@ -102,13 +106,12 @@ func (c *Controller) reconcilePod(key string) error { return err } - if pod.Spec.NodeName != c.nodeName { - // Non-local pod: this is a kubevirt migration target whose ready timestamp changed. - // Find the local source pod and remove its entries so FRR withdraws the Type-2 routes. - return c.handleLiveMigrationTargetReady(pod) + migrationStatus, err := kubevirt.DiscoverLiveMigrationStatus(c.podLister, pod) + if err != nil { + return fmt.Errorf("failed to discover live migration status: %w", err) } - if util.PodCompleted(pod) { + if c.shouldDeleteNeighbors(migrationStatus, pod) { return c.deletePodNeighbors(key) } @@ -119,7 +122,7 @@ func (c *Controller) reconcilePod(key string) error { c.podNeighLock.Lock() existing, hasExisting := c.podNeighbors[key] c.podNeighLock.Unlock() - if hasExisting && existing.uid == pod.UID { + if hasExisting && existing.uid == pod.UID && c.shouldEnsureNeighbors(migrationStatus) { return c.ensurePodNeighbors(existing) } @@ -155,8 +158,11 @@ func (c *Controller) reconcilePod(key string) error { for _, ipNet := range podAnnotation.IPs { entries.ips = append(entries.ips, ipNet.IP) } - if err := c.ensurePodNeighbors(entries); err != nil { - return err + + if c.shouldEnsureNeighbors(migrationStatus) { + if err := c.ensurePodNeighbors(entries); err != nil { + return err + } } c.podNeighLock.Lock() From ff6114335b717d36be9c1b17a15e5dd20db99f69 Mon Sep 17 00:00:00 2001 From: Enrique Llorente Date: Tue, 26 May 2026 11:54:00 +0200 Subject: [PATCH 38/51] refactor: rename LinkNeighAdd to LinkNeighSet The function now uses NeighSet internally, name should reflect that. Signed-off-by: Enrique Llorente --- .../pkg/node/controllers/evpn/evpn_pod_controller.go | 2 +- go-controller/pkg/node/gateway_shared_intf.go | 2 +- go-controller/pkg/node/managementport/port_linux.go | 2 +- go-controller/pkg/util/net_linux.go | 8 ++++---- go-controller/pkg/util/net_linux_unit_test.go | 4 ++-- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/go-controller/pkg/node/controllers/evpn/evpn_pod_controller.go b/go-controller/pkg/node/controllers/evpn/evpn_pod_controller.go index 532bc4864f..2a90bba439 100644 --- a/go-controller/pkg/node/controllers/evpn/evpn_pod_controller.go +++ b/go-controller/pkg/node/controllers/evpn/evpn_pod_controller.go @@ -191,7 +191,7 @@ func (c *Controller) ensurePodNeighbors(entries *neighEntries) error { } klog.V(5).Infof("Configured FDB %s vlan %d on %s", entries.mac, entries.macvrfVID, entries.ovsPortName) for _, ip := range entries.ips { - if err := util.LinkNeighAdd(svi, ip, entries.mac); err != nil { + if err := util.LinkNeighSet(svi, ip, entries.mac); err != nil { return fmt.Errorf("failed to add neighbor %s on %s: %w", ip, entries.sviName, err) } klog.V(5).Infof("Configured neighbor %s lladdr %s on %s", ip, entries.mac, entries.sviName) diff --git a/go-controller/pkg/node/gateway_shared_intf.go b/go-controller/pkg/node/gateway_shared_intf.go index 0cb3b8afaa..2b6499d39a 100644 --- a/go-controller/pkg/node/gateway_shared_intf.go +++ b/go-controller/pkg/node/gateway_shared_intf.go @@ -2361,7 +2361,7 @@ func addHostMACBindings(bridgeName string) error { klog.Warningf("Failed to remove IP neighbor entry for ip %s, on iface %s: %v", ip, bridgeName, err) } - if err = util.LinkNeighAdd(link, net.ParseIP(ip), dummyNextHopMAC); err != nil { + if err = util.LinkNeighSet(link, net.ParseIP(ip), dummyNextHopMAC); err != nil { return fmt.Errorf("failed to configure neighbor: %s, on iface %s: %v", ip, bridgeName, err) } diff --git a/go-controller/pkg/node/managementport/port_linux.go b/go-controller/pkg/node/managementport/port_linux.go index 7240f235c6..a51dc4ab89 100644 --- a/go-controller/pkg/node/managementport/port_linux.go +++ b/go-controller/pkg/node/managementport/port_linux.go @@ -302,7 +302,7 @@ func setupManagementPortIPFamilyConfig(link netlink.Link, mpcfg *managementPortC klog.Warningf("Could not remove remove stale IP neighbor entry for IP %s, on iface %s: %v", cfg.gwIP.String(), types.K8sMgmtIntfName, err) } } - err = util.LinkNeighAdd(link, cfg.gwIP, mpcfg.gwMAC) + err = util.LinkNeighSet(link, cfg.gwIP, mpcfg.gwMAC) } if err != nil { return err diff --git a/go-controller/pkg/util/net_linux.go b/go-controller/pkg/util/net_linux.go index c2ebbe15fb..8a30b0bd22 100644 --- a/go-controller/pkg/util/net_linux.go +++ b/go-controller/pkg/util/net_linux.go @@ -663,10 +663,10 @@ func LinkNeighDel(link netlink.Link, neighIP net.IP) error { return nil } -// LinkNeighAdd adds or replaces MAC/IP bindings for the given link. -// It uses NeighSet (NLM_F_CREATE|NLM_F_REPLACE) so that existing entries -// (e.g. extern_learn from EVPN) are replaced with the desired state. -func LinkNeighAdd(link netlink.Link, neighIP net.IP, neighMAC net.HardwareAddr) error { +// LinkNeighSet adds or replaces MAC/IP bindings for the given link. +// It uses NeighSet (NLM_F_CREATE|NLM_F_REPLACE) so it succeeds whether +// the entry is new or already exists (e.g. zebra's extern_learn). +func LinkNeighSet(link netlink.Link, neighIP net.IP, neighMAC net.HardwareAddr) error { neigh := &netlink.Neigh{ LinkIndex: link.Attrs().Index, Family: getFamily(neighIP), diff --git a/go-controller/pkg/util/net_linux_unit_test.go b/go-controller/pkg/util/net_linux_unit_test.go index e4c0821c19..9f9bba1c27 100644 --- a/go-controller/pkg/util/net_linux_unit_test.go +++ b/go-controller/pkg/util/net_linux_unit_test.go @@ -818,7 +818,7 @@ func TestLinkRouteExists(t *testing.T) { } } -func TestLinkNeighAdd(t *testing.T) { +func TestLinkNeighSet(t *testing.T) { mockNetLinkOps := new(mocks.NetLinkOps) mockLink := new(netlink_mocks.Link) // below is defined in net_linux.go @@ -861,7 +861,7 @@ func TestLinkNeighAdd(t *testing.T) { ovntest.ProcessMockFnList(&mockNetLinkOps.Mock, tc.onRetArgsNetLinkLibOpers) ovntest.ProcessMockFnList(&mockLink.Mock, tc.onRetArgsLinkIfaceOpers) - err := LinkNeighAdd(tc.inputLink, tc.inputNeigIP, tc.inputMacAddr) + err := LinkNeighSet(tc.inputLink, tc.inputNeigIP, tc.inputMacAddr) t.Log(err) if tc.errExp { require.Error(t, err) From 8fd2cad4ef06afb74ecc1a985bf90dd04af87337 Mon Sep 17 00:00:00 2001 From: Enrique Llorente Date: Tue, 26 May 2026 11:56:14 +0200 Subject: [PATCH 39/51] refactor: rename LinkFDBAdd to LinkFDBSet, use NeighSet Same EEXIST issue as LinkNeighAdd: NeighAdd fails silently when an FDB entry already exists. Switch to NeighSet for consistency. Assisted-By: Claude Opus 4.6 Signed-off-by: Enrique Llorente --- .../pkg/node/controllers/evpn/evpn_pod_controller.go | 2 +- .../node/controllers/evpn/evpn_pod_controller_test.go | 3 +-- go-controller/pkg/util/net_linux.go | 10 ++++++---- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/go-controller/pkg/node/controllers/evpn/evpn_pod_controller.go b/go-controller/pkg/node/controllers/evpn/evpn_pod_controller.go index 2a90bba439..12b9c42d19 100644 --- a/go-controller/pkg/node/controllers/evpn/evpn_pod_controller.go +++ b/go-controller/pkg/node/controllers/evpn/evpn_pod_controller.go @@ -184,7 +184,7 @@ func (c *Controller) ensurePodNeighbors(entries *neighEntries) error { if err != nil { return fmt.Errorf("failed to get OVS port %s: %w", entries.ovsPortName, err) } - if err := util.LinkFDBAdd(ovsPort, entries.mac, entries.macvrfVID); err != nil { + if err := util.LinkFDBSet(ovsPort, entries.mac, entries.macvrfVID); err != nil { if !errors.Is(err, syscall.EEXIST) { return fmt.Errorf("failed to add FDB entry for %s on %s: %w", entries.mac, entries.ovsPortName, err) } diff --git a/go-controller/pkg/node/controllers/evpn/evpn_pod_controller_test.go b/go-controller/pkg/node/controllers/evpn/evpn_pod_controller_test.go index 583c33ee1d..bd444df154 100644 --- a/go-controller/pkg/node/controllers/evpn/evpn_pod_controller_test.go +++ b/go-controller/pkg/node/controllers/evpn/evpn_pod_controller_test.go @@ -82,7 +82,6 @@ var _ = Describe("EVPN pod controller", func() { nlMock.On("LinkByName", sviName).Return(sviLink, nil) nlMock.On("LinkByName", ovsPortName).Return(ovsPortLink, nil) - nlMock.On("NeighAdd", mock.Anything).Return(nil) nlMock.On("NeighSet", mock.Anything).Return(nil) mac, _ := net.ParseMAC("0a:58:0a:00:00:05") @@ -100,7 +99,7 @@ var _ = Describe("EVPN pod controller", func() { Expect(ctrl.reconcilePod("test-ns/test-pod")).To(Succeed()) By("verifying FDB entry was added on OVS port") - nlMock.AssertCalled(GinkgoT(), "NeighAdd", mock.MatchedBy(func(n *netlink.Neigh) bool { + nlMock.AssertCalled(GinkgoT(), "NeighSet", mock.MatchedBy(func(n *netlink.Neigh) bool { return n.LinkIndex == 20 && n.HardwareAddr.String() == mac.String() && n.Vlan == 100 })) diff --git a/go-controller/pkg/util/net_linux.go b/go-controller/pkg/util/net_linux.go index 8a30b0bd22..251216b4c1 100644 --- a/go-controller/pkg/util/net_linux.go +++ b/go-controller/pkg/util/net_linux.go @@ -618,8 +618,10 @@ func LinkRouteGetByDstAndGw(link netlink.Link, gwIP net.IP, subnet *net.IPNet) ( return route, err } -// LinkFDBAdd adds a static FDB entry on the bridge that owns the given port. -func LinkFDBAdd(port netlink.Link, mac net.HardwareAddr, vlan int) error { +// LinkFDBSet adds or replaces a static FDB entry on the bridge that owns the given port. +// It uses NeighSet (NLM_F_CREATE|NLM_F_REPLACE) so it succeeds whether +// the entry is new or already exists. +func LinkFDBSet(port netlink.Link, mac net.HardwareAddr, vlan int) error { neigh := &netlink.Neigh{ LinkIndex: port.Attrs().Index, Family: syscall.AF_BRIDGE, @@ -628,8 +630,8 @@ func LinkFDBAdd(port netlink.Link, mac net.HardwareAddr, vlan int) error { Vlan: vlan, HardwareAddr: mac, } - if err := netLinkOps.NeighAdd(neigh); err != nil { - return fmt.Errorf("failed to add FDB entry %s vlan %d on %s: %w", mac, vlan, port.Attrs().Name, err) + if err := netLinkOps.NeighSet(neigh); err != nil { + return fmt.Errorf("failed to set FDB entry %s vlan %d on %s: %w", mac, vlan, port.Attrs().Name, err) } return nil } From 94cf737b7779a2607add484ca06d62556f204190 Mon Sep 17 00:00:00 2001 From: Enrique Llorente Date: Tue, 26 May 2026 12:03:39 +0200 Subject: [PATCH 40/51] fix(evpn): remove dead EEXIST check from LinkFDBSet caller NeighSet (NLM_F_CREATE|NLM_F_REPLACE) never returns EEXIST, so the guard was dead code after the NeighAdd to NeighSet switch. Assisted-By: Claude Opus 4.6 Signed-off-by: Enrique Llorente --- .../pkg/node/controllers/evpn/evpn_pod_controller.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/go-controller/pkg/node/controllers/evpn/evpn_pod_controller.go b/go-controller/pkg/node/controllers/evpn/evpn_pod_controller.go index 12b9c42d19..434afc63bb 100644 --- a/go-controller/pkg/node/controllers/evpn/evpn_pod_controller.go +++ b/go-controller/pkg/node/controllers/evpn/evpn_pod_controller.go @@ -185,9 +185,7 @@ func (c *Controller) ensurePodNeighbors(entries *neighEntries) error { return fmt.Errorf("failed to get OVS port %s: %w", entries.ovsPortName, err) } if err := util.LinkFDBSet(ovsPort, entries.mac, entries.macvrfVID); err != nil { - if !errors.Is(err, syscall.EEXIST) { - return fmt.Errorf("failed to add FDB entry for %s on %s: %w", entries.mac, entries.ovsPortName, err) - } + return fmt.Errorf("failed to set FDB entry for %s on %s: %w", entries.mac, entries.ovsPortName, err) } klog.V(5).Infof("Configured FDB %s vlan %d on %s", entries.mac, entries.macvrfVID, entries.ovsPortName) for _, ip := range entries.ips { From 08dee44fd5d0becd19e80484aabea5478e99a1a8 Mon Sep 17 00:00:00 2001 From: Enrique Llorente Date: Wed, 27 May 2026 12:11:54 +0200 Subject: [PATCH 41/51] test(evpn): add unit tests for neighbor timing during live migration Verify that the EVPN pod controller programs and deletes neighbor entries at the correct migration phases: - Target pod created but domain not ready: no NeighSet calls - Target domain ready on target node: NeighSet + FDB programmed - Target domain ready on source node: NeighDel cleans up source These tests guard the fix in commit 77fd91110 that gates neighbor programming on IsTargetDomainReady() to prevent zebra from overwriting PERMANENT entries with extern_learn during MAC mobility. Assisted-By: Claude Opus 4.6 Signed-off-by: Enrique Llorente --- .../evpn/evpn_pod_controller_test.go | 145 ++++++++++++++++++ 1 file changed, 145 insertions(+) diff --git a/go-controller/pkg/node/controllers/evpn/evpn_pod_controller_test.go b/go-controller/pkg/node/controllers/evpn/evpn_pod_controller_test.go index bd444df154..9294180dd1 100644 --- a/go-controller/pkg/node/controllers/evpn/evpn_pod_controller_test.go +++ b/go-controller/pkg/node/controllers/evpn/evpn_pod_controller_test.go @@ -7,9 +7,11 @@ import ( "fmt" "net" "syscall" + "time" "github.com/stretchr/testify/mock" "github.com/vishvananda/netlink" + kubevirtv1 "kubevirt.io/api/core/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -344,5 +346,148 @@ var _ = Describe("EVPN pod controller", func() { Expect(ctrl.reconcilePod("test-ns/remote-pod")).To(Succeed()) nlMock.AssertNotCalled(GinkgoT(), "NeighSet", mock.Anything) }) + + Context("during live migration", func() { + const ( + vmName = "test-vm" + sourceNode = "source-node" + podAnnotation = `{"test-ns/test-nad":{"ip_addresses":["10.0.0.5/24"],"mac_address":"0a:58:0a:00:00:05","ip_address":"10.0.0.5/24"}}` + ) + + var ( + netInfo *multinetworkmocks.NetInfo + sviLink netlink.Link + ovsPortLink netlink.Link + ) + + // newVirtLauncherPod builds a kubevirt virt-launcher pod with the + // labels and annotations needed to be recognized by + // DiscoverLiveMigrationStatus and the EVPN pod controller. + newVirtLauncherPod := func(name, nodeName string, creationOffset time.Duration, targetReady bool) *corev1.Pod { + annotations := map[string]string{ + kubevirtv1.DomainAnnotation: vmName, + types.OvnPodAnnotationName: podAnnotation, + } + if targetReady { + annotations[kubevirtv1.MigrationTargetReadyTimestamp] = "2024-01-01T00:00:00Z" + } + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: "test-ns", + UID: k8stypes.UID(name + "-uid"), + Annotations: annotations, + Labels: map[string]string{kubevirtv1.AppLabel: "virt-launcher", kubevirtv1.VirtualMachineNameLabel: vmName}, + CreationTimestamp: metav1.Time{Time: time.Now().Add(creationOffset)}, + }, + Spec: corev1.PodSpec{NodeName: nodeName}, + Status: corev1.PodStatus{Phase: corev1.PodRunning}, + } + } + + BeforeEach(func() { + netInfo = &multinetworkmocks.NetInfo{} + netInfo.On("EVPNVTEPName").Return("vtep1") + netInfo.On("EVPNMACVRFVID").Return(100) + netInfo.On("EVPNMACVRFVNI").Return(int32(10100)) + netInfo.On("GetNetworkName").Return("mynet") + netInfo.On("GetNetworkID").Return(5) + netInfo.On("IsPrimaryNetwork").Return(true) + const nadKey = "test-ns/test-nad" + fakeNM.NADNetworks = map[string]util.NetInfo{nadKey: netInfo} + fakeNM.PrimaryNetworks = map[string]util.NetInfo{"test-ns": netInfo} + + sviName := GetEVPNL2SVIName(netInfo) + ovsPortName := GetEVPNOVSPortName(netInfo) + sviLink = &netlink.Dummy{LinkAttrs: netlink.LinkAttrs{Name: sviName, Index: 10}} + ovsPortLink = &netlink.Dummy{LinkAttrs: netlink.LinkAttrs{Name: ovsPortName, Index: 20}} + + nlMock.On("LinkByName", sviName).Return(sviLink, nil) + nlMock.On("LinkByName", ovsPortName).Return(ovsPortLink, nil) + }) + + It("does NOT program neighbors on target node before domain is ready", func() { + sourcePod := newVirtLauncherPod("virt-launcher-source", sourceNode, -time.Minute, false) + targetPod := newVirtLauncherPod("virt-launcher-target", nodeName, 0, false) + ctrl.podLister = newFakePodLister(sourcePod, targetPod) + + Expect(ctrl.reconcilePod("test-ns/virt-launcher-target")).To(Succeed()) + + nlMock.AssertNotCalled(GinkgoT(), "NeighSet", mock.Anything) + + By("verifying cache entry was created but neighbors were not programmed") + ctrl.podNeighLock.Lock() + _, exists := ctrl.podNeighbors["test-ns/virt-launcher-target"] + ctrl.podNeighLock.Unlock() + Expect(exists).To(BeTrue(), "cache entry should exist even though neighbors were deferred") + }) + + It("programs neighbors on target node after domain is ready", func() { + sourcePod := newVirtLauncherPod("virt-launcher-source", sourceNode, -time.Minute, false) + targetPod := newVirtLauncherPod("virt-launcher-target", nodeName, 0, true) + ctrl.podLister = newFakePodLister(sourcePod, targetPod) + + nlMock.On("NeighSet", mock.Anything).Return(nil) + + Expect(ctrl.reconcilePod("test-ns/virt-launcher-target")).To(Succeed()) + + mac, _ := net.ParseMAC("0a:58:0a:00:00:05") + + By("verifying FDB entry was added on OVS port") + nlMock.AssertCalled(GinkgoT(), "NeighSet", mock.MatchedBy(func(n *netlink.Neigh) bool { + return n.LinkIndex == ovsPortLink.Attrs().Index && + n.HardwareAddr.String() == mac.String() && + n.Vlan == 100 + })) + + By("verifying neighbor entry was added on SVI") + nlMock.AssertCalled(GinkgoT(), "NeighSet", mock.MatchedBy(func(n *netlink.Neigh) bool { + return n.LinkIndex == sviLink.Attrs().Index && + n.IP.Equal(net.ParseIP("10.0.0.5")) + })) + }) + + It("deletes neighbors on source node after target domain is ready", func() { + mac, _ := net.ParseMAC("0a:58:0a:00:00:05") + sourceKey := "test-ns/virt-launcher-source" + + // Controller runs on source node for this test. + ctrl.nodeName = sourceNode + + // Pre-seed cache as if source pod was previously reconciled. + ctrl.podNeighbors[sourceKey] = &neighEntries{ + uid: "virt-launcher-source-uid", + sviName: GetEVPNL2SVIName(netInfo), + ovsPortName: GetEVPNOVSPortName(netInfo), + macvrfVID: 100, + ips: []net.IP{net.ParseIP("10.0.0.5")}, + mac: mac, + } + + sourcePod := newVirtLauncherPod("virt-launcher-source", sourceNode, -time.Minute, false) + targetPod := newVirtLauncherPod("virt-launcher-target", nodeName, 0, true) + ctrl.podLister = newFakePodLister(sourcePod, targetPod) + + nlMock.On("NeighDel", mock.Anything).Return(nil) + + Expect(ctrl.reconcilePod(sourceKey)).To(Succeed()) + + By("verifying FDB entry was deleted from OVS port") + nlMock.AssertCalled(GinkgoT(), "NeighDel", mock.MatchedBy(func(n *netlink.Neigh) bool { + return n.LinkIndex == ovsPortLink.Attrs().Index && + n.HardwareAddr.String() == mac.String() + })) + + By("verifying neighbor entry was deleted from SVI") + nlMock.AssertCalled(GinkgoT(), "NeighDel", mock.MatchedBy(func(n *netlink.Neigh) bool { + return n.LinkIndex == sviLink.Attrs().Index && + n.IP.Equal(net.ParseIP("10.0.0.5")) + })) + + By("verifying cache was cleared") + _, exists := ctrl.podNeighbors[sourceKey] + Expect(exists).To(BeFalse()) + }) + }) }) }) From 859795070fecc3b0c70f76077162e501bb3f655d Mon Sep 17 00:00:00 2001 From: Enrique Llorente Date: Wed, 27 May 2026 12:28:39 +0200 Subject: [PATCH 42/51] e2e(virt): Add evpn failed migration test Signed-off-by: Enrique Llorente --- test/e2e/kubevirt.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/test/e2e/kubevirt.go b/test/e2e/kubevirt.go index 95a5ec84ba..f824b3d3a8 100644 --- a/test/e2e/kubevirt.go +++ b/test/e2e/kubevirt.go @@ -2331,6 +2331,17 @@ ip route add %[3]s via %[4]s test: liveMigrateFailed, topology: udnv1.NetworkTopologyLocalnet, }), + Entry(nil, testData{ + resource: virtualMachineWithUDN, + test: liveMigrateFailed, + topology: udnv1.NetworkTopologyLayer2, + role: udnv1.NetworkRolePrimary, + ingress: "routed", + evpn: &udnv1.EVPNConfig{ + MACVRF: &udnv1.VRFConfig{}, + IPVRF: &udnv1.VRFConfig{}, + }, + }), ) }) Context("with kubevirt VM using layer2 UDPN", Ordered, func() { From d59e62b07a618c963ca4647e462f63f35d572faf Mon Sep 17 00:00:00 2001 From: Igal Tsoiref Date: Wed, 27 May 2026 22:11:48 -0400 Subject: [PATCH 43/51] cni: gracefully handle missing device info file for primary UDN MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit LoadDeviceInfoFromDP returns an error when the device-info file does not exist under /var/run/k8s.cni.cncf.io/devinfo/dp/. Per the device-info spec the file is optional — the device plugin only writes it when NAD discovery is configured. In deployments where the device plugin does not write devinfo (e.g. DPF/DPU-host environments using the NodeSRIOVDevicePlugin controller), the CNI ADD for primary UDN pods failed unconditionally. Handle os.IsNotExist gracefully: if the file is absent, proceed with an empty DeviceInfo and let GetNetdevNameFromDeviceId resolve the netdev from sysfs. If the file exists but is malformed, still return an error. Signed-off-by: Igal Tsoiref Co-authored-by: Cursor --- go-controller/pkg/cni/cniserver.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/go-controller/pkg/cni/cniserver.go b/go-controller/pkg/cni/cniserver.go index a9f6d78ee6..4611c06794 100644 --- a/go-controller/pkg/cni/cniserver.go +++ b/go-controller/pkg/cni/cniserver.go @@ -11,6 +11,7 @@ import ( "fmt" "io" "net/http" + "os" "strings" "time" @@ -419,12 +420,17 @@ func (s *Server) getPrimaryUDNPodRequest(originalPodRequest *PodRequest) (*PodRe pod.Namespace, pod.Name, resourceName, err) } primaryPodRequest.CNIConf.DeviceID = deviceID + // Load device info if the device plugin wrote it. The file may not + // exist (e.g. when the device plugin has no NAD discovery configured), + // in which case GetNetdevNameFromDeviceId falls back to sysfs. deviceInfo, err := nadutils.LoadDeviceInfoFromDP(resourceName, deviceID) - if err != nil { + if err != nil && !os.IsNotExist(err) { return nil, fmt.Errorf("failed to load primary UDN's device info for pod %s/%s resource %s deviceID %s: %w", pod.Namespace, pod.Name, resourceName, deviceID, err) } - primaryPodRequest.deviceInfo = *deviceInfo + if deviceInfo != nil { + primaryPodRequest.deviceInfo = *deviceInfo + } } if err = updateDeviceInfo(primaryPodRequest); err != nil { return nil, err From 268848f5b668991790e84cfdd011f3f8d804bca5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Caama=C3=B1o=20Ruiz?= Date: Tue, 9 Jun 2026 10:01:58 +0000 Subject: [PATCH 44/51] e2e: fix UDN subnet overlap with downstream service CIDR MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PR #5433 introduced hardcoded UDN subnets that overlap with the downstream service CIDR. Replace these with subnets sourced from the allocators package which takes care to prevent such overlaps. Assisted-By: Claude Opus 4.6 Signed-off-by: Jaime Caamaño Ruiz --- test/e2e/allocators/subnet_spec.go | 1 + test/e2e/allocators/udn.go | 38 ++++++++++++++++++++++++------ test/e2e/egress_firewall.go | 15 +++++++----- test/e2e/multihoming_utils.go | 33 +++++++++++++++++++------- test/e2e/network_segmentation.go | 26 ++++++++++---------- test/e2e/util.go | 14 +++++++++++ 6 files changed, 92 insertions(+), 35 deletions(-) diff --git a/test/e2e/allocators/subnet_spec.go b/test/e2e/allocators/subnet_spec.go index 1a7f80e246..3aa1074d7d 100644 --- a/test/e2e/allocators/subnet_spec.go +++ b/test/e2e/allocators/subnet_spec.go @@ -131,6 +131,7 @@ func (s subnetSpec) usable() int { return s.total() - s.excluded.Len() } +// nthFree returns the index of the n-th non-excluded subnet (1-indexed). func (s subnetSpec) nthFree(n int) int { count := 0 for i := range s.total() { diff --git a/test/e2e/allocators/udn.go b/test/e2e/allocators/udn.go index b481b635e9..857bff5203 100644 --- a/test/e2e/allocators/udn.go +++ b/test/e2e/allocators/udn.go @@ -12,16 +12,40 @@ var ( udnV4, udnV6 subnetSpec ) -// AllocateUDNSubnets always allocates the same fixed UDN IPv4 and IPv6 subnet -// within the dedicated UDN subnet broader range. Used when overlaps across UDNs -// are not a concern but still prevents overlaps with other subnets. -func AllocateUDNSubnets() (ipv4, ipv6 string) { +func initSubnetSpecs() { udnOnce.Do(func() { udnV4 = newSubnetSpec(udnSubnets, nil) udnV6 = newSubnetSpec(udnSubnets6, nil) }) +} + +// GetFirstUDNSubnets always allocates the first UDN IPv4 and IPv6 subnet +// within the dedicated UDN subnet broader range. Used when overlaps across UDNs +// are not a concern but still prevents overlaps with other subnets. +func GetFirstUDNSubnets() (ipv4, ipv6 string) { + subnets4, subnets6 := GetNthFirstUDNSubnets(1) + return subnets4[0], subnets6[0] +} + +// GetNthFirstUDNSubnets returns the first n UDN IPv4 and IPv6 subnets within +// the dedicated UDN subnet broader range. Used when overlaps across UDNs are +// not a concern but still prevents overlaps with other subnets. +func GetNthFirstUDNSubnets(n int) (ipv4, ipv6 []string) { + if n < 1 { + panic("GetNthFirstUDNSubnets: n must be >= 1") + } + initSubnetSpecs() + if n > udnV4.usable() || n > udnV6.usable() { + panic("GetNthFirstUDNSubnets: not enough free subnets available") + } - udnV4Idx := udnV4.nthFree(1) - udnV6Idx := udnV6.nthFree(1) - return udnV4.cidr(udnV4Idx), udnV6.cidr(udnV6Idx) + ipv4 = make([]string, 0, n) + ipv6 = make([]string, 0, n) + for i := 1; i < n+1; i++ { + udnV4Idx := udnV4.nthFree(i) + udnV6Idx := udnV6.nthFree(i) + ipv4 = append(ipv4, udnV4.cidr(udnV4Idx)) + ipv6 = append(ipv6, udnV6.cidr(udnV6Idx)) + } + return ipv4, ipv6 } diff --git a/test/e2e/egress_firewall.go b/test/e2e/egress_firewall.go index 965ea07409..df847b32c6 100644 --- a/test/e2e/egress_firewall.go +++ b/test/e2e/egress_firewall.go @@ -14,6 +14,7 @@ import ( "strings" "time" + "github.com/ovn-kubernetes/ovn-kubernetes/test/e2e/allocators" "github.com/ovn-kubernetes/ovn-kubernetes/test/e2e/deploymentconfig" "github.com/ovn-kubernetes/ovn-kubernetes/test/e2e/images" "github.com/ovn-kubernetes/ovn-kubernetes/test/e2e/infraprovider" @@ -152,18 +153,20 @@ func egressFirewallPolicyValidationTests(useUDN bool, udnTopology string) { f.Namespace = namespace framework.ExpectNoError(err) - userDefinedNetworkIPv4Subnets := []string{"172.31.0.0/16"} - userDefinedNetworkIPv6Subnets := []string{"2014:100:200::0/60"} + userDefinedNetworkIPv4Subnets, userDefinedNetworkIPv6Subnets := allocators.GetNthFirstUDNSubnets(1) + cidr := joinStrings(joinStrings(userDefinedNetworkIPv4Subnets...), joinStrings(userDefinedNetworkIPv6Subnets...)) if udnTopology == "layer3" { - userDefinedNetworkIPv4Subnets = []string{"172.31.0.0/23/24", "172.30.0.0/16/24"} - userDefinedNetworkIPv6Subnets = []string{"2014:100:200::0/63/64", "2014:100:100::0/48/64"} + // For Layer 3, use multiple CIDRs per IP family. The first + // CIDR is just big enough to allocate hostSubnets for the + // first two nodes, remaining nodes will be allocated + // hostSubnets from the second CIDR + cidr = primaryLayer3MultiCIDRs() } - userDefinedNetworkSubnets := append(append([]string{}, userDefinedNetworkIPv4Subnets...), userDefinedNetworkIPv6Subnets...) nadCfg := networkAttachmentConfigParams{ name: "tenant-red", topology: udnTopology, - cidr: joinStrings(userDefinedNetworkSubnets...), + cidr: cidr, role: "primary", } diff --git a/test/e2e/multihoming_utils.go b/test/e2e/multihoming_utils.go index f56ef85dd7..86093287e8 100644 --- a/test/e2e/multihoming_utils.go +++ b/test/e2e/multihoming_utils.go @@ -29,6 +29,7 @@ import ( utilnet "k8s.io/utils/net" "github.com/ovn-kubernetes/ovn-kubernetes/go-controller/pkg/generator/ip" + "github.com/ovn-kubernetes/ovn-kubernetes/test/e2e/allocators" ) func netCIDR(netCIDR string, netPrefixLengthPerNode int) string { @@ -43,18 +44,34 @@ func primaryLayer3MultiCIDRs() string { return joinStrings(primaryLayer3MultiIPv4CIDRs(), primaryLayer3MultiIPv6CIDRs()) } +// primaryLayer3IPv4CIDRs returns two IPv4 CIDRs for a primary network in layer3 +// topology. The first CIDR is resized to a /23 to force an overflow to the +// second CIDR for a third and subsequent nodes hostSubnet. +func primaryLayer3IPv4CIDRs() (string, string) { + subnets4, _ := allocators.GetNthFirstUDNSubnets(2) + subnets4[0] = firstSubnetOf(subnets4[0], 23) + "/24" + subnets4[1] = subnets4[1] + "/24" + return subnets4[0], subnets4[1] +} + +// primaryLayer3IPv6CIDRs returns two IPv6 CIDRs for a primary network in layer3 +// topology. The first CIDR is resized to a /63 to force an overflow to the +// second CIDR for a third and subsequent nodes hostSubnet. +func primaryLayer3IPv6CIDRs() (string, string) { + _, subnets6 := allocators.GetNthFirstUDNSubnets(2) + subnets6[0] = firstSubnetOf(subnets6[0], 63) + "/64" + subnets6[1] = subnets6[1] + "/64" + return subnets6[0], subnets6[1] +} + func primaryLayer3MultiIPv4CIDRs() string { - return joinStrings( - "172.31.0.0/23/24", - "172.30.0.0/16/24", - ) + subnet1, subnet2 := primaryLayer3IPv4CIDRs() + return joinStrings(subnet1, subnet2) } func primaryLayer3MultiIPv6CIDRs() string { - return joinStrings( - "2014:100:200::0/63/64", - "2014:100:100::0/48/64", - ) + subnet1, subnet2 := primaryLayer3IPv6CIDRs() + return joinStrings(subnet1, subnet2) } func filterCIDRsAndJoin(cs clientset.Interface, cidrs string) string { diff --git a/test/e2e/network_segmentation.go b/test/e2e/network_segmentation.go index 65d1a773ea..4dec7ef64a 100644 --- a/test/e2e/network_segmentation.go +++ b/test/e2e/network_segmentation.go @@ -79,13 +79,6 @@ var _ = Describe("Network Segmentation", feature.NetworkSegmentation, func() { customL2IPv6InfraCIDR = "2014:100:200::/122" userDefinedNetworkName = "hogwarts" nadName = "gryffindor" - - // The first subnet can support 2 nodes; when the cluster has more than 2 nodes, - // both subnets will be utilized. - userDefinedNetworkIPv4Subnet1 = "172.31.0.0/23/24" - userDefinedNetworkIPv4Subnet2 = "172.30.0.0/16/24" - userDefinedNetworkIPv6Subnet1 = "2014:100:200::0/63/64" - userDefinedNetworkIPv6Subnet2 = "2014:100:100::0/48/64" ) BeforeEach(func() { @@ -1103,12 +1096,11 @@ var _ = Describe("Network Segmentation", feature.NetworkSegmentation, func() { &networkAttachmentConfigParams{ name: nadName, topology: "layer3", - cidr: joinStrings( - // the 1st set of CIDR can only allocate subnet for 2 nodes - userDefinedNetworkIPv4Subnet1, userDefinedNetworkIPv4Subnet2, - // the 2nd set of CIDR can allocate subnet for remaining nodes - userDefinedNetworkIPv6Subnet1, userDefinedNetworkIPv6Subnet2, - ), + // Use multiple CIDRs per IP family. The first CIDR + // is just big enough to allocate hostSubnets for + // the first two nodes, remaining nodes will be + // allocated hostSubnets from the second CIDR + cidr: primaryLayer3MultiCIDRs(), role: "primary", }, ), @@ -1151,7 +1143,13 @@ var _ = Describe("Network Segmentation", feature.NetworkSegmentation, func() { e2eskipper.Skipf("need at least 3 ready schedulable nodes to run this test") } - By("creating the intial network with one CIDR") + // Use multiple CIDRs per IP family. The first CIDR is just + // big enough to allocate hostSubnets for the first two + // nodes, remaining nodes will be allocated hostSubnets from + // the second CIDR + userDefinedNetworkIPv4Subnet1, userDefinedNetworkIPv4Subnet2 := primaryLayer3IPv4CIDRs() + userDefinedNetworkIPv6Subnet1, userDefinedNetworkIPv6Subnet2 := primaryLayer3IPv6CIDRs() + By("creating the initial network with one CIDR") netConfig := &networkAttachmentConfigParams{ name: randomNetworkMetaName(), namespace: f.Namespace.Name, diff --git a/test/e2e/util.go b/test/e2e/util.go index 1275718436..c7c9644586 100644 --- a/test/e2e/util.go +++ b/test/e2e/util.go @@ -2016,3 +2016,17 @@ func waitForNodeReadyState(f *framework.Framework, nodeName string, timeout time return false }, timeout, 10*time.Second).Should(gomega.BeTrue(), expectationMessage) } + +// firstSubnetOf returns the first subnet of a given size within the provided +// subnet +func firstSubnetOf(subnet string, subnetSize int) string { + _, ipNet, err := net.ParseCIDR(subnet) + if err != nil { + panic(fmt.Sprintf("firstSubnetOf: invalid subnet %q: %v", subnet, err)) + } + ones, bits := ipNet.Mask.Size() + if subnetSize < ones || subnetSize > bits { + panic(fmt.Sprintf("firstSubnetOf: requested size /%d is not between min /%d and max /%d for %s", subnetSize, ones, bits, subnet)) + } + return fmt.Sprintf("%s/%d", ipNet.IP, subnetSize) +} From 9ebc3afa1ebb7ec76b22c6a4c2339e4d79ba3de7 Mon Sep 17 00:00:00 2001 From: jtalerico Date: Fri, 5 Jun 2026 15:54:25 -0400 Subject: [PATCH 45/51] Adding missing labels for UDN workloads We are missing the proper ns labels for the UDN workloads Signed-off-by: jtalerico --- contrib/perf/workloads/udn-density-l2-pods-netpol.yml | 8 +++++++- contrib/perf/workloads/udn-density-l3-pods-netpol.yml | 8 +++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/contrib/perf/workloads/udn-density-l2-pods-netpol.yml b/contrib/perf/workloads/udn-density-l2-pods-netpol.yml index ef0693cf34..74eb800075 100644 --- a/contrib/perf/workloads/udn-density-l2-pods-netpol.yml +++ b/contrib/perf/workloads/udn-density-l2-pods-netpol.yml @@ -36,7 +36,13 @@ jobs: churnConfig: percent: 10 cycles: 10 - mode: objects + mode: namespaces + namespaceLabels: + security.openshift.io/scc.podSecurityLabelSync: false + pod-security.kubernetes.io/enforce: privileged + pod-security.kubernetes.io/audit: privileged + pod-security.kubernetes.io/warn: privileged + k8s.ovn.org/primary-user-defined-network: "" objects: - objectTemplate: workloads/templates/udn-density/udn_l2.yml diff --git a/contrib/perf/workloads/udn-density-l3-pods-netpol.yml b/contrib/perf/workloads/udn-density-l3-pods-netpol.yml index 7389ced47f..9b563033c2 100644 --- a/contrib/perf/workloads/udn-density-l3-pods-netpol.yml +++ b/contrib/perf/workloads/udn-density-l3-pods-netpol.yml @@ -36,7 +36,13 @@ jobs: churnConfig: percent: 10 cycles: 10 - mode: objects + mode: namespaces + namespaceLabels: + security.openshift.io/scc.podSecurityLabelSync: false + pod-security.kubernetes.io/enforce: privileged + pod-security.kubernetes.io/audit: privileged + pod-security.kubernetes.io/warn: privileged + k8s.ovn.org/primary-user-defined-network: "" objects: - objectTemplate: workloads/templates/udn-density/udn_l3.yml From 75597a672135f4b182762ac16131609255030cce Mon Sep 17 00:00:00 2001 From: Matteo Dallaglio Date: Fri, 17 Apr 2026 13:27:05 +0200 Subject: [PATCH 46/51] Add transport label to CUDN count metric Add a "transport" label to the ovnkube_clustermanager_cluster_user_defined_networks gauge to distinguish CUDNs by transport type (Geneve, EVPN, NoOverlay). Empty transport (default OVN overlay) is mapped to "Geneve". Add seedCUDNCountsMetric to establish correct baselines at startup so the metric survives leader restarts where Inc/Dec from finalizer guards would not re-fire. Add dedicated CUDN metrics test context covering increment on create (Geneve and EVPN transports) and decrement on delete. Signed-off-by: Matteo Dallaglio --- docs/observability/metrics.md | 1 + .../userdefinednetwork/controller.go | 22 +- .../userdefinednetwork/controller_metrics.go | 108 ++++++++++ .../userdefinednetwork/controller_test.go | 193 ++++++++++++++++-- go-controller/pkg/metrics/cluster_manager.go | 19 +- 5 files changed, 312 insertions(+), 31 deletions(-) create mode 100644 go-controller/pkg/clustermanager/userdefinednetwork/controller_metrics.go diff --git a/docs/observability/metrics.md b/docs/observability/metrics.md index 441520fce8..e5c3315560 100644 --- a/docs/observability/metrics.md +++ b/docs/observability/metrics.md @@ -17,6 +17,7 @@ Measurement accuracy can be impacted by other parallel processing that might be ## Change log This list is to help notify if there are additions, changes or removals to metrics. Latest changes are at the top of this list. +- Add `transport` label to `ovnkube_clustermanager_cluster_user_defined_networks` to distinguish CUDNs by transport type (Geneve, EVPN, NoOverlay, Localnet) - Add metrics to track logfile size for ovnkube processes - ovnkube_node_logfile_size_bytes and ovnkube_controller_logfile_size_bytes - Remove ovnkube_controller_ovn_cli_latency_seconds metrics since we have moved most of the OVN DB operations to libovsdb. - Effect of OVN IC architecture: diff --git a/go-controller/pkg/clustermanager/userdefinednetwork/controller.go b/go-controller/pkg/clustermanager/userdefinednetwork/controller.go index 99389f3cec..ad2e34aeba 100644 --- a/go-controller/pkg/clustermanager/userdefinednetwork/controller.go +++ b/go-controller/pkg/clustermanager/userdefinednetwork/controller.go @@ -138,6 +138,10 @@ type Controller struct { // Keys are CR name, value is affected namespace names slice. namespaceTracker map[string]sets.Set[string] namespaceTrackerLock sync.RWMutex + // cudnMetricTracker tracks which CUDNs are counted in the CUDN gauge metric. + // Maps CUDN name to its (role, topology, transport) labels. + // Only mutated from syncClusterUDN and initializeController (single-threaded). + cudnMetricTracker map[string]cudnMetricKey // renderNadFn render NAD manifest from given object, enable replacing in tests. renderNadFn RenderNetAttachDefManifest // createNetworkLock lock should be held when NAD is created to avoid having two components @@ -206,6 +210,7 @@ func New( namespaceInformer: namespaceInformer, networkManager: networkManager, namespaceTracker: map[string]sets.Set[string]{}, + cudnMetricTracker: map[string]cudnMetricKey{}, vidAllocator: vidAllocator, reservedVNIs: map[vniKey]string{}, eventRecorder: eventRecorder, @@ -299,6 +304,8 @@ func (c *Controller) initializeController() error { } c.initializeNamespaceTracker(cudnNADs) + c.seedCUDNCountMetrics(cudnNADs) + if util.IsEVPNEnabled() { // Recover VID allocations from existing EVPN CUDNs. // Recovery failures are logged and the affected CUDNs are enqueued for reconciliation, @@ -933,16 +940,6 @@ func (c *Controller) syncClusterUDN(cudn *userdefinednetworkv1.ClusterUserDefine cudnName := cudn.Name affectedNamespaces := c.namespaceTracker[cudnName] - var role, topology string - if cudn.Spec.Network.Layer2 != nil { - role = string(cudn.Spec.Network.Layer2.Role) - } else if cudn.Spec.Network.Layer3 != nil { - role = string(cudn.Spec.Network.Layer3.Role) - } else if cudn.Spec.Network.Localnet != nil { - role = string(cudn.Spec.Network.Localnet.Role) - } - topology = string(cudn.Spec.Network.Topology) - if !cudn.DeletionTimestamp.IsZero() { if controllerutil.ContainsFinalizer(cudn, template.FinalizerUserDefinedNetwork) { var errs []error @@ -974,7 +971,7 @@ func (c *Controller) syncClusterUDN(cudn *userdefinednetworkv1.ClusterUserDefine } klog.Infof("Finalizer removed from ClusterUserDefinedNetwork %q", cudn.Name) delete(c.namespaceTracker, cudnName) - metrics.DecrementCUDNCount(role, topology) + c.cudnMetricUncounted(cudnName) metrics.DeleteDynamicUDNNodeCount(util.GenerateCUDNNetworkName(cudn.Name)) c.releaseEVPNIDsForNetwork(cudnName) } @@ -983,7 +980,6 @@ func (c *Controller) syncClusterUDN(cudn *userdefinednetworkv1.ClusterUserDefine } if _, exist := c.namespaceTracker[cudnName]; !exist { - // start tracking CR c.namespaceTracker[cudnName] = sets.Set[string]{} } @@ -994,7 +990,7 @@ func (c *Controller) syncClusterUDN(cudn *userdefinednetworkv1.ClusterUserDefine return nil, fmt.Errorf("failed to add finalizer to ClusterUserDefinedNetwork %q: %w", cudnName, err) } klog.Infof("Added Finalizer to ClusterUserDefinedNetwork %q", cudnName) - metrics.IncrementCUDNCount(role, topology) + c.cudnMetricCounted(cudnName, &cudn.Spec.Network) } if evpnErr != nil { diff --git a/go-controller/pkg/clustermanager/userdefinednetwork/controller_metrics.go b/go-controller/pkg/clustermanager/userdefinednetwork/controller_metrics.go new file mode 100644 index 0000000000..5c2b7d0c2b --- /dev/null +++ b/go-controller/pkg/clustermanager/userdefinednetwork/controller_metrics.go @@ -0,0 +1,108 @@ +// SPDX-FileCopyrightText: Copyright The OVN-Kubernetes Contributors +// SPDX-License-Identifier: Apache-2.0 + +package userdefinednetwork + +import ( + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + "github.com/ovn-kubernetes/ovn-kubernetes/go-controller/pkg/clustermanager/userdefinednetwork/template" + userdefinednetworkv1 "github.com/ovn-kubernetes/ovn-kubernetes/go-controller/pkg/crd/userdefinednetwork/v1" + "github.com/ovn-kubernetes/ovn-kubernetes/go-controller/pkg/metrics" +) + +// cudnMetricKey holds the label values for a single CUDN in the count gauge. +type cudnMetricKey struct { + role, topology, transport string +} + +// seedCUDNCountMetrics populates the metric tracker from existing CUDNs at +// startup, then sets the gauge once. +// +// Only finalized CUDNs are seeded. This preserves the tracker invariant: every +// entry in cudnMetricTracker has a finalizer that guarantees cudnMetricUncounted +// will run during deletion. Unfinalized CUDNs are safe to skip — they will be +// counted by cudnMetricCounted when the reconcile loop adds the finalizer. +// +// Without this guard a tracker leak is possible: if an unfinalized CUDN is +// deleted before the controller reconciles it, Kubernetes garbage-collects it +// immediately (no finalizer), reconcileCUDN sees NotFound, syncClusterUDN(nil) +// short-circuits, and cudnMetricUncounted is never called — leaving a phantom +// entry in the tracker. +func (c *Controller) seedCUDNCountMetrics(cudnNADs cudnToNADs) { + counts := map[cudnMetricKey]float64{} + for _, entry := range cudnNADs { + if !controllerutil.ContainsFinalizer(entry.cudn, template.FinalizerUserDefinedNetwork) { + continue + } + role, topology, transport := cudnMetricLabels(&entry.cudn.Spec.Network) + key := cudnMetricKey{role, topology, transport} + c.cudnMetricTracker[entry.cudn.Name] = key + counts[key]++ + } + for k, count := range counts { + metrics.SetCUDNCount(k.role, k.topology, k.transport, count) + } +} + +// cudnMetricCounted records that a CUDN with the given spec is now counted in +// the gauge metric. The caller must have already persisted the finalizer — this +// is what guarantees a matching cudnMetricUncounted call during deletion. +func (c *Controller) cudnMetricCounted(name string, spec *userdefinednetworkv1.NetworkSpec) { + role, topology, transport := cudnMetricLabels(spec) + key := cudnMetricKey{role, topology, transport} + c.cudnMetricTracker[name] = key + metrics.SetCUDNCount(key.role, key.topology, key.transport, c.countCUDNMetricKey(key)) +} + +// cudnMetricUncounted records that a CUDN is no longer counted in the gauge +// metric. The caller must have already removed the finalizer. +func (c *Controller) cudnMetricUncounted(name string) { + key, existed := c.cudnMetricTracker[name] + if !existed { + return + } + delete(c.cudnMetricTracker, name) + remaining := c.countCUDNMetricKey(key) + if remaining == 0 { + metrics.DeleteCUDNCount(key.role, key.topology, key.transport) + } else { + metrics.SetCUDNCount(key.role, key.topology, key.transport, remaining) + } +} + +// countCUDNMetricKey counts how many CUDNs in the tracker share the given label combination. +func (c *Controller) countCUDNMetricKey(target cudnMetricKey) float64 { + var count float64 + for _, k := range c.cudnMetricTracker { + if k == target { + count++ + } + } + return count +} + +// cudnMetricLabels extracts the role, topology, and transport label values from +// a CUDN network spec for use in Prometheus metrics. An empty transport (the +// default OVN overlay) is mapped to "Geneve" for overlay topologies. Localnet +// topology uses direct provider-network attachment (no overlay encapsulation), +// so its transport is labeled "Localnet". +func cudnMetricLabels(spec *userdefinednetworkv1.NetworkSpec) (role, topology, transport string) { + if spec.Layer2 != nil { + role = string(spec.Layer2.Role) + } else if spec.Layer3 != nil { + role = string(spec.Layer3.Role) + } else if spec.Localnet != nil { + role = string(spec.Localnet.Role) + } + topology = string(spec.Topology) + transport = string(spec.Transport) + if transport == "" { + if spec.Topology == userdefinednetworkv1.NetworkTopologyLocalnet { + transport = "Localnet" + } else { + transport = "Geneve" + } + } + return +} diff --git a/go-controller/pkg/clustermanager/userdefinednetwork/controller_test.go b/go-controller/pkg/clustermanager/userdefinednetwork/controller_test.go index 71c7074af4..e886297cf8 100644 --- a/go-controller/pkg/clustermanager/userdefinednetwork/controller_test.go +++ b/go-controller/pkg/clustermanager/userdefinednetwork/controller_test.go @@ -14,6 +14,8 @@ import ( netv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" netv1clientset "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/clientset/versioned" netv1fakeclientset "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/clientset/versioned/fake" + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -36,6 +38,7 @@ import ( vtepv1 "github.com/ovn-kubernetes/ovn-kubernetes/go-controller/pkg/crd/vtep/v1" vtepinformer "github.com/ovn-kubernetes/ovn-kubernetes/go-controller/pkg/crd/vtep/v1/apis/informers/externalversions/vtep/v1" "github.com/ovn-kubernetes/ovn-kubernetes/go-controller/pkg/factory" + "github.com/ovn-kubernetes/ovn-kubernetes/go-controller/pkg/metrics" "github.com/ovn-kubernetes/ovn-kubernetes/go-controller/pkg/networkmanager" ovntypes "github.com/ovn-kubernetes/ovn-kubernetes/go-controller/pkg/types" "github.com/ovn-kubernetes/ovn-kubernetes/go-controller/pkg/util" @@ -59,6 +62,7 @@ var _ = Describe("User Defined Network Controller", func() { // Enable EVPN for EVPN-related tests config.OVNKubernetesFeature.EnableRouteAdvertisements = true config.OVNKubernetesFeature.EnableEVPN = true + metrics.RegisterClusterManagerFunctional() // satisfy EVPN LGW restriction, otherwise no effect config.Gateway.Mode = config.GatewayModeLocal }) @@ -128,6 +132,17 @@ var _ = Describe("User Defined Network Controller", func() { }, 2*time.Second, 100*time.Millisecond).Should(BeTrue()) } + // markCUDNForDeletion marks a CUDN for deletion by setting its DeletionTimestamp to the current time. + markCUDNForDeletion := func(name string) { + GinkgoHelper() + cudnObj, err := cs.UserDefinedNetworkClient.K8sV1().ClusterUserDefinedNetworks().Get(context.Background(), name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + now := metav1.Now() + cudnObj.DeletionTimestamp = &now + _, err = cs.UserDefinedNetworkClient.K8sV1().ClusterUserDefinedNetworks().Update(context.Background(), cudnObj, metav1.UpdateOptions{}) + Expect(err).NotTo(HaveOccurred()) + } + Context("manager", func() { var c *Controller AfterEach(func() { @@ -693,12 +708,7 @@ var _ = Describe("User Defined Network Controller", func() { Expect(c.vidAllocator.GetID("evpn-delete-cudn/macvrf")).To(BeNumerically(">=", 0), "VID should be allocated") // Trigger deletion by setting DeletionTimestamp and processing - now := metav1.Now() - cudn, err := cs.UserDefinedNetworkClient.K8sV1().ClusterUserDefinedNetworks().Get(context.Background(), cudn.Name, metav1.GetOptions{}) - Expect(err).NotTo(HaveOccurred()) - cudn.DeletionTimestamp = &now - _, err = cs.UserDefinedNetworkClient.K8sV1().ClusterUserDefinedNetworks().Update(context.Background(), cudn, metav1.UpdateOptions{}) - Expect(err).NotTo(HaveOccurred()) + markCUDNForDeletion(cudn.Name) // Wait for finalizer to be removed (indicating deletion was processed) Eventually(func(g Gomega) { @@ -732,12 +742,7 @@ var _ = Describe("User Defined Network Controller", func() { Expect(c.vidAllocator.GetID("evpn-irb-delete/ipvrf")).To(Equal(3), "IP-VRF VID should be allocated") // Trigger deletion - now := metav1.Now() - cudn, err := cs.UserDefinedNetworkClient.K8sV1().ClusterUserDefinedNetworks().Get(context.Background(), cudn.Name, metav1.GetOptions{}) - Expect(err).NotTo(HaveOccurred()) - cudn.DeletionTimestamp = &now - _, err = cs.UserDefinedNetworkClient.K8sV1().ClusterUserDefinedNetworks().Update(context.Background(), cudn, metav1.UpdateOptions{}) - Expect(err).NotTo(HaveOccurred()) + markCUDNForDeletion(cudn.Name) // Wait for finalizer to be removed and verify both VIDs are released Eventually(func(g Gomega) { @@ -2573,6 +2578,104 @@ var _ = Describe("User Defined Network Controller", func() { expectTransportCondition("test-cudn", metav1.ConditionTrue, ReasonEVPNTransportAccepted, "Transport has been configured as 'EVPN'.") }) }) + + Context("CUDN metrics", func() { + var c *Controller + + BeforeEach(func() { + metrics.ResetCUDNCount() + }) + + AfterEach(func() { + if c != nil { + c.Shutdown() + } + }) + + It("should increment CUDN count on create with Geneve transport", func() { + baseVal, _ := getCUDNCountMetricValue("Primary", "Layer3", "Geneve") + + testNs := testNamespace("metrics-ns") + cudn := testLayer3PrimaryClusterUDN("metrics-cudn", testNs.Name) + cudn.Finalizers = nil + c = newTestController(template.RenderNetAttachDefManifest, cudn, testNs) + Expect(c.Run()).To(Succeed()) + + Eventually(func() []metav1.Condition { + cudn, err := cs.UserDefinedNetworkClient.K8sV1().ClusterUserDefinedNetworks().Get(context.Background(), cudn.Name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + return normalizeConditions(filterTransportConditions(cudn.Status.Conditions)) + }).Should(Equal([]metav1.Condition{{ + Type: "NetworkCreated", + Status: "True", + Reason: "NetworkAttachmentDefinitionCreated", + Message: "NetworkAttachmentDefinition has been created in following namespaces: [metrics-ns]", + }})) + + val, found := getCUDNCountMetricValue("Primary", "Layer3", "Geneve") + Expect(found).To(BeTrue(), "CUDN count metric should exist for default Geneve transport") + Expect(val).To(Equal(baseVal + 1)) + }) + + It("should increment CUDN count on create with EVPN transport", func() { + baseVal, _ := getCUDNCountMetricValue("Primary", "Layer2", "EVPN") + + testNs := testNamespace("metrics-evpn-ns") + vtep := testVTEP("vtep-metrics") + cudn := testEVPNClusterUDN("metrics-evpn-cudn", vtep.Name, testNs.Name) + cudn.Finalizers = nil + cudn.Spec.Network.Layer2.Role = udnv1.NetworkRolePrimary + + c = newTestControllerWithNetworkManager(template.RenderNetAttachDefManifest, cudn, testNs, vtep) + Expect(c.Run()).To(Succeed()) + + Eventually(func() []metav1.Condition { + cudn, err := cs.UserDefinedNetworkClient.K8sV1().ClusterUserDefinedNetworks().Get(context.Background(), cudn.Name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + return normalizeConditions(filterTransportConditions(cudn.Status.Conditions)) + }).Should(Equal([]metav1.Condition{{ + Type: "NetworkCreated", + Status: "True", + Reason: "NetworkAttachmentDefinitionCreated", + Message: "NetworkAttachmentDefinition has been created in following namespaces: [metrics-evpn-ns]", + }})) + + val, found := getCUDNCountMetricValue("Primary", "Layer2", "EVPN") + Expect(found).To(BeTrue(), "CUDN count metric should exist for EVPN transport") + Expect(val).To(Equal(baseVal + 1)) + }) + + It("should decrement CUDN count on delete", func() { + testNs := testNamespace("metrics-del-ns") + cudn := testLayer2SecondaryClusterUDN("metrics-del-cudn", testNs.Name) + cudn.Finalizers = nil + c = newTestController(template.RenderNetAttachDefManifest, cudn, testNs) + Expect(c.Run()).To(Succeed()) + + Eventually(func() []metav1.Condition { + cudn, err := cs.UserDefinedNetworkClient.K8sV1().ClusterUserDefinedNetworks().Get(context.Background(), cudn.Name, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + return normalizeConditions(filterTransportConditions(cudn.Status.Conditions)) + }).Should(Equal([]metav1.Condition{{ + Type: "NetworkCreated", + Status: "True", + Reason: "NetworkAttachmentDefinitionCreated", + Message: "NetworkAttachmentDefinition has been created in following namespaces: [metrics-del-ns]", + }})) + + beforeVal, found := getCUDNCountMetricValue("Secondary", "Layer2", "Geneve") + Expect(found).To(BeTrue()) + + markCUDNForDeletion(cudn.Name) + + expected := beforeVal - 1 + Eventually(func() float64 { + val, _ := getCUDNCountMetricValue("Secondary", "Layer2", "Geneve") + return val + }).Should(Equal(expected), "CUDN count metric should decrement by exactly one after deletion") + }) + + }) }) // assertConditionReportNetworkInUse checks conditions reflect network consumers. @@ -2786,6 +2889,32 @@ func testClusterUDN(name string, targetNamespaces ...string) *udnv1.ClusterUserD } } +// testLayer3PrimaryClusterUDN returns a CUDN with Layer3 Primary topology and default Geneve transport. +func testLayer3PrimaryClusterUDN(name string, targetNamespaces ...string) *udnv1.ClusterUserDefinedNetwork { + cudn := testClusterUDN(name, targetNamespaces...) + cudn.Spec.Network = udnv1.NetworkSpec{ + Topology: udnv1.NetworkTopologyLayer3, + Layer3: &udnv1.Layer3Config{ + Role: udnv1.NetworkRolePrimary, + Subnets: []udnv1.Layer3Subnet{{CIDR: "10.100.0.0/16"}}, + }, + } + return cudn +} + +// testLayer2SecondaryClusterUDN returns a CUDN with Layer2 Secondary topology and default Geneve transport. +func testLayer2SecondaryClusterUDN(name string, targetNamespaces ...string) *udnv1.ClusterUserDefinedNetwork { + cudn := testClusterUDN(name, targetNamespaces...) + cudn.Spec.Network = udnv1.NetworkSpec{ + Topology: udnv1.NetworkTopologyLayer2, + Layer2: &udnv1.Layer2Config{ + Role: udnv1.NetworkRoleSecondary, + Subnets: udnv1.DualStackCIDRs{"10.20.0.0/16"}, + }, + } + return cudn +} + func testClusterUdnNAD(name, namespace string) *netv1.NetworkAttachmentDefinition { return &netv1.NetworkAttachmentDefinition{ ObjectMeta: metav1.ObjectMeta{ @@ -2975,3 +3104,43 @@ func setNADEVPNVIDs(nad *netv1.NetworkAttachmentDefinition, macVID, ipVID int) e nad.Spec.Config = string(configBytes) return nil } + +// getCUDNCountMetricValue returns the current gauge value for the CUDN count metric with the given labels. +func getCUDNCountMetricValue(role, topology, transport string) (float64, bool) { + metricName := prometheus.BuildFQName(ovntypes.MetricOvnkubeNamespace, ovntypes.MetricOvnkubeSubsystemClusterManager, "cluster_user_defined_networks") + mf := findMetricFamily(metricName) + if mf == nil { + return 0, false + } + for _, metric := range mf.GetMetric() { + if metricLabelValue(metric.GetLabel(), "role") == role && + metricLabelValue(metric.GetLabel(), "topology") == topology && + metricLabelValue(metric.GetLabel(), "transport") == transport { + return metric.GetGauge().GetValue(), true + } + } + return 0, false +} + +// findMetricFamily returns the MetricFamily with the given name from the default Prometheus gatherer. +func findMetricFamily(name string) *dto.MetricFamily { + GinkgoHelper() + mfs, err := prometheus.DefaultGatherer.Gather() + Expect(err).NotTo(HaveOccurred(), "failed to gather metrics") + for _, mf := range mfs { + if mf.GetName() == name { + return mf + } + } + return nil +} + +// metricLabelValue returns the value of the label with the given name, or empty string if not found. +func metricLabelValue(labels []*dto.LabelPair, name string) string { + for _, label := range labels { + if label.GetName() == name { + return label.GetValue() + } + } + return "" +} diff --git a/go-controller/pkg/metrics/cluster_manager.go b/go-controller/pkg/metrics/cluster_manager.go index 233fe0bc36..b16ec8fb43 100644 --- a/go-controller/pkg/metrics/cluster_manager.go +++ b/go-controller/pkg/metrics/cluster_manager.go @@ -115,6 +115,7 @@ var metricCUDNCount = prometheus.NewGaugeVec(prometheus.GaugeOpts{ []string{ "role", "topology", + "transport", }, ) @@ -223,14 +224,20 @@ func DecrementUDNCount(role, topology string) { metricUDNCount.WithLabelValues(role, topology).Dec() } -// IncrementCUDNCount increments the number of ClusterUserDefinedNetworks of the given type -func IncrementCUDNCount(role, topology string) { - metricCUDNCount.WithLabelValues(role, topology).Inc() +// SetCUDNCount sets the CUDN count for a specific label combination. +func SetCUDNCount(role, topology, transport string, count float64) { + metricCUDNCount.WithLabelValues(role, topology, transport).Set(count) } -// DecrementCUDNCount decrements the number of ClusterUserDefinedNetworks of the given type -func DecrementCUDNCount(role, topology string) { - metricCUDNCount.WithLabelValues(role, topology).Dec() +// DeleteCUDNCount removes a label combination from the CUDN count gauge. +func DeleteCUDNCount(role, topology, transport string) { + metricCUDNCount.DeleteLabelValues(role, topology, transport) +} + +// ResetCUDNCount resets the CUDN count gauge, removing all label combinations. +// Intended for use in tests to ensure metric isolation between test cases. +func ResetCUDNCount() { + metricCUDNCount.Reset() } // SetDynamicUDNNodeCount sets the number of nodes currently active with a CUDN/UDN. From 8512a3bf6fb2caac7a924197b700391780799106 Mon Sep 17 00:00:00 2001 From: Matteo Dallaglio Date: Wed, 10 Jun 2026 13:23:56 +0200 Subject: [PATCH 47/51] Address review feedback on CUDN transport metric - Use "Default" instead of "Geneve"/"Localnet" for the transport label when spec.Transport is empty. This avoids assuming a specific encapsulation protocol (OVN may use Geneve or VXLAN) and removes the conflation of Localnet topology with transport. - Invert cudnMetricTracker from map[name]key to map[key]sets.Set[name], giving O(1) count via set length and removing the O(n) countCUDNMetricKey scan. - Extract trackCUDN helper to deduplicate nil-check + insert logic between seedCUDNCountMetrics and cudnMetricCounted. Signed-off-by: Matteo Dallaglio --- docs/observability/metrics.md | 2 +- .../userdefinednetwork/controller.go | 9 +-- .../userdefinednetwork/controller_metrics.go | 68 +++++++++---------- .../userdefinednetwork/controller_test.go | 20 +++--- 4 files changed, 47 insertions(+), 52 deletions(-) diff --git a/docs/observability/metrics.md b/docs/observability/metrics.md index e5c3315560..14f9c89bc1 100644 --- a/docs/observability/metrics.md +++ b/docs/observability/metrics.md @@ -17,7 +17,7 @@ Measurement accuracy can be impacted by other parallel processing that might be ## Change log This list is to help notify if there are additions, changes or removals to metrics. Latest changes are at the top of this list. -- Add `transport` label to `ovnkube_clustermanager_cluster_user_defined_networks` to distinguish CUDNs by transport type (Geneve, EVPN, NoOverlay, Localnet) +- Add `transport` label to `ovnkube_clustermanager_cluster_user_defined_networks` to distinguish CUDNs by transport type (Default, EVPN, NoOverlay) - Add metrics to track logfile size for ovnkube processes - ovnkube_node_logfile_size_bytes and ovnkube_controller_logfile_size_bytes - Remove ovnkube_controller_ovn_cli_latency_seconds metrics since we have moved most of the OVN DB operations to libovsdb. - Effect of OVN IC architecture: diff --git a/go-controller/pkg/clustermanager/userdefinednetwork/controller.go b/go-controller/pkg/clustermanager/userdefinednetwork/controller.go index ad2e34aeba..35f01b9fa8 100644 --- a/go-controller/pkg/clustermanager/userdefinednetwork/controller.go +++ b/go-controller/pkg/clustermanager/userdefinednetwork/controller.go @@ -139,9 +139,10 @@ type Controller struct { namespaceTracker map[string]sets.Set[string] namespaceTrackerLock sync.RWMutex // cudnMetricTracker tracks which CUDNs are counted in the CUDN gauge metric. - // Maps CUDN name to its (role, topology, transport) labels. + // Maps (role, topology, transport) labels to the set of CUDN names sharing + // that combination. // Only mutated from syncClusterUDN and initializeController (single-threaded). - cudnMetricTracker map[string]cudnMetricKey + cudnMetricTracker map[cudnMetricKey]sets.Set[string] // renderNadFn render NAD manifest from given object, enable replacing in tests. renderNadFn RenderNetAttachDefManifest // createNetworkLock lock should be held when NAD is created to avoid having two components @@ -210,7 +211,7 @@ func New( namespaceInformer: namespaceInformer, networkManager: networkManager, namespaceTracker: map[string]sets.Set[string]{}, - cudnMetricTracker: map[string]cudnMetricKey{}, + cudnMetricTracker: map[cudnMetricKey]sets.Set[string]{}, vidAllocator: vidAllocator, reservedVNIs: map[vniKey]string{}, eventRecorder: eventRecorder, @@ -971,7 +972,7 @@ func (c *Controller) syncClusterUDN(cudn *userdefinednetworkv1.ClusterUserDefine } klog.Infof("Finalizer removed from ClusterUserDefinedNetwork %q", cudn.Name) delete(c.namespaceTracker, cudnName) - c.cudnMetricUncounted(cudnName) + c.cudnMetricUncounted(cudnName, &cudn.Spec.Network) metrics.DeleteDynamicUDNNodeCount(util.GenerateCUDNNetworkName(cudn.Name)) c.releaseEVPNIDsForNetwork(cudnName) } diff --git a/go-controller/pkg/clustermanager/userdefinednetwork/controller_metrics.go b/go-controller/pkg/clustermanager/userdefinednetwork/controller_metrics.go index 5c2b7d0c2b..149e917a16 100644 --- a/go-controller/pkg/clustermanager/userdefinednetwork/controller_metrics.go +++ b/go-controller/pkg/clustermanager/userdefinednetwork/controller_metrics.go @@ -4,6 +4,7 @@ package userdefinednetwork import ( + "k8s.io/apimachinery/pkg/util/sets" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "github.com/ovn-kubernetes/ovn-kubernetes/go-controller/pkg/clustermanager/userdefinednetwork/template" @@ -11,7 +12,7 @@ import ( "github.com/ovn-kubernetes/ovn-kubernetes/go-controller/pkg/metrics" ) -// cudnMetricKey holds the label values for a single CUDN in the count gauge. +// cudnMetricKey holds the label values for a CUDN metric label combination. type cudnMetricKey struct { role, topology, transport string } @@ -30,18 +31,14 @@ type cudnMetricKey struct { // short-circuits, and cudnMetricUncounted is never called — leaving a phantom // entry in the tracker. func (c *Controller) seedCUDNCountMetrics(cudnNADs cudnToNADs) { - counts := map[cudnMetricKey]float64{} for _, entry := range cudnNADs { if !controllerutil.ContainsFinalizer(entry.cudn, template.FinalizerUserDefinedNetwork) { continue } - role, topology, transport := cudnMetricLabels(&entry.cudn.Spec.Network) - key := cudnMetricKey{role, topology, transport} - c.cudnMetricTracker[entry.cudn.Name] = key - counts[key]++ + c.trackCUDN(entry.cudn.Name, &entry.cudn.Spec.Network) } - for k, count := range counts { - metrics.SetCUDNCount(k.role, k.topology, k.transport, count) + for key, names := range c.cudnMetricTracker { + metrics.SetCUDNCount(key.role, key.topology, key.transport, float64(names.Len())) } } @@ -49,44 +46,45 @@ func (c *Controller) seedCUDNCountMetrics(cudnNADs cudnToNADs) { // the gauge metric. The caller must have already persisted the finalizer — this // is what guarantees a matching cudnMetricUncounted call during deletion. func (c *Controller) cudnMetricCounted(name string, spec *userdefinednetworkv1.NetworkSpec) { + key, names := c.trackCUDN(name, spec) + metrics.SetCUDNCount(key.role, key.topology, key.transport, float64(names.Len())) +} + +// trackCUDN inserts a CUDN name into the metric tracker bucket for its label +// combination, initializing the set if needed. Returns the bucket key and set. +func (c *Controller) trackCUDN(name string, spec *userdefinednetworkv1.NetworkSpec) (cudnMetricKey, sets.Set[string]) { role, topology, transport := cudnMetricLabels(spec) key := cudnMetricKey{role, topology, transport} - c.cudnMetricTracker[name] = key - metrics.SetCUDNCount(key.role, key.topology, key.transport, c.countCUDNMetricKey(key)) + if c.cudnMetricTracker[key] == nil { + c.cudnMetricTracker[key] = sets.New[string]() + } + c.cudnMetricTracker[key].Insert(name) + return key, c.cudnMetricTracker[key] } // cudnMetricUncounted records that a CUDN is no longer counted in the gauge // metric. The caller must have already removed the finalizer. -func (c *Controller) cudnMetricUncounted(name string) { - key, existed := c.cudnMetricTracker[name] - if !existed { +func (c *Controller) cudnMetricUncounted(name string, spec *userdefinednetworkv1.NetworkSpec) { + role, topology, transport := cudnMetricLabels(spec) + key := cudnMetricKey{role, topology, transport} + names := c.cudnMetricTracker[key] + if names == nil || !names.Has(name) { return } - delete(c.cudnMetricTracker, name) - remaining := c.countCUDNMetricKey(key) - if remaining == 0 { + names.Delete(name) + if names.Len() == 0 { + delete(c.cudnMetricTracker, key) metrics.DeleteCUDNCount(key.role, key.topology, key.transport) } else { - metrics.SetCUDNCount(key.role, key.topology, key.transport, remaining) + metrics.SetCUDNCount(key.role, key.topology, key.transport, float64(names.Len())) } } -// countCUDNMetricKey counts how many CUDNs in the tracker share the given label combination. -func (c *Controller) countCUDNMetricKey(target cudnMetricKey) float64 { - var count float64 - for _, k := range c.cudnMetricTracker { - if k == target { - count++ - } - } - return count -} - // cudnMetricLabels extracts the role, topology, and transport label values from -// a CUDN network spec for use in Prometheus metrics. An empty transport (the -// default OVN overlay) is mapped to "Geneve" for overlay topologies. Localnet -// topology uses direct provider-network attachment (no overlay encapsulation), -// so its transport is labeled "Localnet". +// a CUDN network spec for use in Prometheus metrics. When spec.Transport is +// empty (the user did not configure an explicit transport), the label is set to +// "Default" — meaning the standard OVN overlay (Geneve or VXLAN, depending on +// ovn-encap-type). This avoids assuming a specific encapsulation protocol. func cudnMetricLabels(spec *userdefinednetworkv1.NetworkSpec) (role, topology, transport string) { if spec.Layer2 != nil { role = string(spec.Layer2.Role) @@ -98,11 +96,7 @@ func cudnMetricLabels(spec *userdefinednetworkv1.NetworkSpec) (role, topology, t topology = string(spec.Topology) transport = string(spec.Transport) if transport == "" { - if spec.Topology == userdefinednetworkv1.NetworkTopologyLocalnet { - transport = "Localnet" - } else { - transport = "Geneve" - } + transport = "Default" } return } diff --git a/go-controller/pkg/clustermanager/userdefinednetwork/controller_test.go b/go-controller/pkg/clustermanager/userdefinednetwork/controller_test.go index e886297cf8..f357acb3fb 100644 --- a/go-controller/pkg/clustermanager/userdefinednetwork/controller_test.go +++ b/go-controller/pkg/clustermanager/userdefinednetwork/controller_test.go @@ -2475,7 +2475,7 @@ var _ = Describe("User Defined Network Controller", func() { } }) - It("should not update status for empty transport (defaults to Geneve)", func() { + It("should not update status for empty transport (default OVN overlay)", func() { cudn := newCUDNWithTransport("test-cudn", map[string]string{"app": "test"}, "") c = newTestController(template.RenderNetAttachDefManifest, cudn) Expect(c.Run()).To(Succeed()) @@ -2592,8 +2592,8 @@ var _ = Describe("User Defined Network Controller", func() { } }) - It("should increment CUDN count on create with Geneve transport", func() { - baseVal, _ := getCUDNCountMetricValue("Primary", "Layer3", "Geneve") + It("should increment CUDN count on create with default transport", func() { + baseVal, _ := getCUDNCountMetricValue("Primary", "Layer3", "Default") testNs := testNamespace("metrics-ns") cudn := testLayer3PrimaryClusterUDN("metrics-cudn", testNs.Name) @@ -2612,8 +2612,8 @@ var _ = Describe("User Defined Network Controller", func() { Message: "NetworkAttachmentDefinition has been created in following namespaces: [metrics-ns]", }})) - val, found := getCUDNCountMetricValue("Primary", "Layer3", "Geneve") - Expect(found).To(BeTrue(), "CUDN count metric should exist for default Geneve transport") + val, found := getCUDNCountMetricValue("Primary", "Layer3", "Default") + Expect(found).To(BeTrue(), "CUDN count metric should exist for default transport") Expect(val).To(Equal(baseVal + 1)) }) @@ -2622,7 +2622,7 @@ var _ = Describe("User Defined Network Controller", func() { testNs := testNamespace("metrics-evpn-ns") vtep := testVTEP("vtep-metrics") - cudn := testEVPNClusterUDN("metrics-evpn-cudn", vtep.Name, testNs.Name) + cudn := testEVPNClusterUDN("metrics-evpn-cudn", &udnv1.EVPNConfig{VTEP: vtep.Name, MACVRF: &udnv1.VRFConfig{VNI: 100}}, testNs.Name) cudn.Finalizers = nil cudn.Spec.Network.Layer2.Role = udnv1.NetworkRolePrimary @@ -2663,14 +2663,14 @@ var _ = Describe("User Defined Network Controller", func() { Message: "NetworkAttachmentDefinition has been created in following namespaces: [metrics-del-ns]", }})) - beforeVal, found := getCUDNCountMetricValue("Secondary", "Layer2", "Geneve") + beforeVal, found := getCUDNCountMetricValue("Secondary", "Layer2", "Default") Expect(found).To(BeTrue()) markCUDNForDeletion(cudn.Name) expected := beforeVal - 1 Eventually(func() float64 { - val, _ := getCUDNCountMetricValue("Secondary", "Layer2", "Geneve") + val, _ := getCUDNCountMetricValue("Secondary", "Layer2", "Default") return val }).Should(Equal(expected), "CUDN count metric should decrement by exactly one after deletion") }) @@ -2889,7 +2889,7 @@ func testClusterUDN(name string, targetNamespaces ...string) *udnv1.ClusterUserD } } -// testLayer3PrimaryClusterUDN returns a CUDN with Layer3 Primary topology and default Geneve transport. +// testLayer3PrimaryClusterUDN returns a CUDN with Layer3 Primary topology and default transport. func testLayer3PrimaryClusterUDN(name string, targetNamespaces ...string) *udnv1.ClusterUserDefinedNetwork { cudn := testClusterUDN(name, targetNamespaces...) cudn.Spec.Network = udnv1.NetworkSpec{ @@ -2902,7 +2902,7 @@ func testLayer3PrimaryClusterUDN(name string, targetNamespaces ...string) *udnv1 return cudn } -// testLayer2SecondaryClusterUDN returns a CUDN with Layer2 Secondary topology and default Geneve transport. +// testLayer2SecondaryClusterUDN returns a CUDN with Layer2 Secondary topology and default transport. func testLayer2SecondaryClusterUDN(name string, targetNamespaces ...string) *udnv1.ClusterUserDefinedNetwork { cudn := testClusterUDN(name, targetNamespaces...) cudn.Spec.Network = udnv1.NetworkSpec{ From 857ce9a902e4d01531505a4c0a7e46def23607ac Mon Sep 17 00:00:00 2001 From: Tom Pantelis Date: Wed, 27 May 2026 10:32:41 -0400 Subject: [PATCH 48/51] Add CLI flags for configuring TLS to ovnkube-identity Added optional CLI flags `--tls-min-version` and `--tls-cipher-suites` that will be used to configure TLS parameters for the webhook. Also added a `tls.ApplyConfigOptionsFor` function that will be used to parse the CLI flags and apply to a `tls.Config`. Signed-off-by: Tom Pantelis --- .../cmd/ovnkube-identity/ovnkubeidentity.go | 12 +++ go-controller/go.mod | 2 +- go-controller/pkg/tls/tls.go | 29 +++++++ go-controller/pkg/tls/tls_suite_test.go | 16 ++++ go-controller/pkg/tls/tls_test.go | 75 +++++++++++++++++++ 5 files changed, 133 insertions(+), 1 deletion(-) create mode 100644 go-controller/pkg/tls/tls.go create mode 100644 go-controller/pkg/tls/tls_suite_test.go create mode 100644 go-controller/pkg/tls/tls_test.go diff --git a/go-controller/cmd/ovnkube-identity/ovnkubeidentity.go b/go-controller/cmd/ovnkube-identity/ovnkubeidentity.go index 5524445482..106554a56d 100644 --- a/go-controller/cmd/ovnkube-identity/ovnkubeidentity.go +++ b/go-controller/cmd/ovnkube-identity/ovnkubeidentity.go @@ -56,6 +56,8 @@ type config struct { csrAcceptanceConditions []csrapprover.CSRAcceptanceCondition podAdmissionConditionFile string podAdmissionConditions []ovnwebhook.PodAdmissionConditionOption + minTLSVersion string + tlsCipherSuites cli.StringSlice } var cliCfg config @@ -261,6 +263,16 @@ func main() { Usage: "Configure additional pod validate admission conditions", Destination: &cliCfg.podAdmissionConditionFile, }, + &cli.StringFlag{ + Name: "tls-min-version", + Usage: "Minimum TLS version supported by the webhook server", + Destination: &cliCfg.minTLSVersion, + }, + &cli.StringSliceFlag{ + Name: "tls-cipher-suites", + Usage: "Comma-separated list of cipher suites for the webhook server", + Destination: &cliCfg.tlsCipherSuites, + }, } ctx := context.Background() diff --git a/go-controller/go.mod b/go-controller/go.mod index 4cafa209d1..ff48447b56 100644 --- a/go-controller/go.mod +++ b/go-controller/go.mod @@ -61,6 +61,7 @@ require ( k8s.io/apimachinery v0.35.1 k8s.io/apiserver v0.35.1 k8s.io/client-go v0.35.1 + k8s.io/component-base v0.35.1 k8s.io/component-helpers v0.35.1 k8s.io/klog/v2 v2.130.1 k8s.io/kubelet v0.35.1 @@ -152,7 +153,6 @@ require ( gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/apiextensions-apiserver v0.35.1 // indirect - k8s.io/component-base v0.35.1 // indirect k8s.io/controller-manager v0.35.1 // indirect k8s.io/kube-openapi v0.0.0-20260304202019-5b3e3fdb0acf // indirect kubevirt.io/containerized-data-importer-api v1.55.0 // indirect diff --git a/go-controller/pkg/tls/tls.go b/go-controller/pkg/tls/tls.go new file mode 100644 index 0000000000..7403a25eaa --- /dev/null +++ b/go-controller/pkg/tls/tls.go @@ -0,0 +1,29 @@ +// SPDX-FileCopyrightText: Copyright The OVN-Kubernetes Contributors +// SPDX-License-Identifier: Apache-2.0 + +package tls + +import ( + "crypto/tls" + + cliflag "k8s.io/component-base/cli/flag" +) + +type ApplyConfigOptions func(*tls.Config) + +func NewApplyConfigOptions(minVersion string, cipherSuites []string) (ApplyConfigOptions, error) { + minVersionID, err := cliflag.TLSVersion(minVersion) + if err != nil { + return nil, err + } + + cipherSuiteIDs, err := cliflag.TLSCipherSuites(cipherSuites) + if err != nil { + return nil, err + } + + return func(cfg *tls.Config) { + cfg.CipherSuites = cipherSuiteIDs + cfg.MinVersion = minVersionID + }, nil +} diff --git a/go-controller/pkg/tls/tls_suite_test.go b/go-controller/pkg/tls/tls_suite_test.go new file mode 100644 index 0000000000..97fe332925 --- /dev/null +++ b/go-controller/pkg/tls/tls_suite_test.go @@ -0,0 +1,16 @@ +// SPDX-FileCopyrightText: Copyright The OVN-Kubernetes Contributors +// SPDX-License-Identifier: Apache-2.0 + +package tls_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestTLS(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "TLS Suite") +} diff --git a/go-controller/pkg/tls/tls_test.go b/go-controller/pkg/tls/tls_test.go new file mode 100644 index 0000000000..bfe81ecd91 --- /dev/null +++ b/go-controller/pkg/tls/tls_test.go @@ -0,0 +1,75 @@ +// SPDX-FileCopyrightText: Copyright The OVN-Kubernetes Contributors +// SPDX-License-Identifier: Apache-2.0 + +package tls_test + +import ( + "crypto/tls" + + ovntls "github.com/ovn-kubernetes/ovn-kubernetes/go-controller/pkg/tls" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("NewApplyConfigOptions", func() { + assertApplySuccess := func(minVersion string, ciperSuites []string) *tls.Config { + applyOpts, err := ovntls.NewApplyConfigOptions(minVersion, ciperSuites) + Expect(err).ToNot(HaveOccurred()) + Expect(applyOpts).ToNot(BeNil()) + + cfg := &tls.Config{} + applyOpts(cfg) + + return cfg + } + + Context("with valid inputs", func() { + It("should correctly apply the settings", func() { + cfg := assertApplySuccess("VersionTLS12", []string{ + "TLS_AES_128_GCM_SHA256", + "TLS_AES_256_GCM_SHA384", + }) + Expect(cfg.MinVersion).To(Equal(uint16(tls.VersionTLS12))) + Expect(cfg.CipherSuites).To(ConsistOf(tls.TLS_AES_128_GCM_SHA256, tls.TLS_AES_256_GCM_SHA384)) + }) + }) + + Context("with empty cipher suites list", func() { + It("should apply an empty cipher suites list", func() { + cfg := assertApplySuccess("VersionTLS13", []string{}) + Expect(cfg.MinVersion).To(Equal(uint16(tls.VersionTLS13))) + Expect(cfg.CipherSuites).To(BeEmpty()) + }) + }) + + Context("with nil cipher suites list", func() { + It("should apply an empty cipher suites list", func() { + cfg := assertApplySuccess("VersionTLS11", nil) + Expect(cfg.MinVersion).To(Equal(uint16(tls.VersionTLS11))) + Expect(cfg.CipherSuites).To(BeEmpty()) + }) + }) + + Context("with empty min version", func() { + It("should apply the default min version", func() { + cfg := assertApplySuccess("", []string{}) + Expect(cfg.MinVersion).To(Equal(uint16(tls.VersionTLS12))) + }) + }) + + DescribeTableSubtree("with invalid", + func(minVersion string, ciperSuites []string) { + It("should return an error", func() { + applyOpts, err := ovntls.NewApplyConfigOptions(minVersion, ciperSuites) + Expect(err).To(HaveOccurred()) + Expect(applyOpts).To(BeNil()) + }) + }, + Entry("TLS version string", "InvalidTLSVersion", []string{}), + Entry("cipher suite name", "VersionTLS12", []string{ + "TLS_AES_128_GCM_SHA256", + "InvalidCipherSuite", + "TLS_AES_256_GCM_SHA384"}), + ) +}) From 3ac5bf9516a458142b18470ee2f3143418f15c82 Mon Sep 17 00:00:00 2001 From: Tom Pantelis Date: Thu, 28 May 2026 12:29:07 -0400 Subject: [PATCH 49/51] Use TLS config params in the ovnkube-identity webhook Adds an `ApplyTLSOptions` field to `webhook.Config` that allows callers to customize TLS configuration (MinVersion, CipherSuites, etc.) for the webhook server. Includes unit tests to verify the server applies the custom TLS options during handshake. Signed-off-by: Tom Pantelis --- .../cmd/ovnkube-identity/ovnkubeidentity.go | 7 +++ .../cmd/ovnkube-identity/webhook/webhook.go | 15 +++-- .../ovnkube-identity/webhook/webhook_test.go | 61 ++++++++++++++++++- 3 files changed, 74 insertions(+), 9 deletions(-) diff --git a/go-controller/cmd/ovnkube-identity/ovnkubeidentity.go b/go-controller/cmd/ovnkube-identity/ovnkubeidentity.go index 106554a56d..b672382496 100644 --- a/go-controller/cmd/ovnkube-identity/ovnkubeidentity.go +++ b/go-controller/cmd/ovnkube-identity/ovnkubeidentity.go @@ -34,6 +34,7 @@ import ( identitywebhook "github.com/ovn-kubernetes/ovn-kubernetes/go-controller/cmd/ovnkube-identity/webhook" "github.com/ovn-kubernetes/ovn-kubernetes/go-controller/pkg/csrapprover" "github.com/ovn-kubernetes/ovn-kubernetes/go-controller/pkg/ovnwebhook" + "github.com/ovn-kubernetes/ovn-kubernetes/go-controller/pkg/tls" utilerrors "github.com/ovn-kubernetes/ovn-kubernetes/go-controller/pkg/util/errors" ) @@ -308,6 +309,11 @@ func main() { } func runWebhook(ctx context.Context, restCfg *rest.Config) error { + applyTLSOptions, err := tls.NewApplyConfigOptions(cliCfg.minTLSVersion, cliCfg.tlsCipherSuites.Value()) + if err != nil { + return err + } + return identitywebhook.Run(ctx, restCfg, identitywebhook.Config{ EnableHybridOverlay: cliCfg.enableHybridOverlay, ExtraAllowedUsers: cliCfg.extraAllowedUsers.Value(), @@ -316,6 +322,7 @@ func runWebhook(ctx context.Context, restCfg *rest.Config) error { CertDir: cliCfg.certDir, Host: cliCfg.host, Port: cliCfg.port, + ApplyTLSOptions: applyTLSOptions, }) } diff --git a/go-controller/cmd/ovnkube-identity/webhook/webhook.go b/go-controller/cmd/ovnkube-identity/webhook/webhook.go index bc9aab89b1..d0c1f59454 100644 --- a/go-controller/cmd/ovnkube-identity/webhook/webhook.go +++ b/go-controller/cmd/ovnkube-identity/webhook/webhook.go @@ -29,6 +29,7 @@ import ( "github.com/ovn-kubernetes/ovn-kubernetes/go-controller/pkg/crd/egressip/v1/apis/clientset/versioned/scheme" "github.com/ovn-kubernetes/ovn-kubernetes/go-controller/pkg/csrapprover" "github.com/ovn-kubernetes/ovn-kubernetes/go-controller/pkg/ovnwebhook" + ovntls "github.com/ovn-kubernetes/ovn-kubernetes/go-controller/pkg/tls" ) var logger = klog.NewKlogr() @@ -42,6 +43,7 @@ type Config struct { CertDir string Host string Port int + ApplyTLSOptions ovntls.ApplyConfigOptions // This field is a constructor used for creating clients and is intended for testing. NewKubernetesClient func(*rest.Config) (kubernetes.Interface, error) @@ -111,18 +113,19 @@ func Run(ctx context.Context, restCfg *rest.Config, config Config) error { } webhookMux.Handle("/pod", podHandler) - cfg := &tls.Config{ - NextProtos: []string{"h2"}, - MinVersion: tls.VersionTLS12, - } - certPath := filepath.Join(config.CertDir, "tls.crt") keyPath := filepath.Join(config.CertDir, "tls.key") certWatcher, err := certwatcher.New(certPath, keyPath) if err != nil { return fmt.Errorf("failed to setup certwatcher: %v", err) } - cfg.GetCertificate = certWatcher.GetCertificate + + cfg := &tls.Config{ + NextProtos: []string{"h2"}, + GetCertificate: certWatcher.GetCertificate, + } + + config.ApplyTLSOptions(cfg) go func() { if err := certWatcher.Start(ctx); err != nil { diff --git a/go-controller/cmd/ovnkube-identity/webhook/webhook_test.go b/go-controller/cmd/ovnkube-identity/webhook/webhook_test.go index 08d4df5426..f13a04f432 100644 --- a/go-controller/cmd/ovnkube-identity/webhook/webhook_test.go +++ b/go-controller/cmd/ovnkube-identity/webhook/webhook_test.go @@ -138,6 +138,60 @@ var _ = Describe("Run", func() { "Server should be using new certificate after rotation") }).Within(3 * time.Second).ProbeEvery(200 * time.Millisecond).Should(Succeed()) }) + + Context("with applied TLS config options", func() { + BeforeEach(func() { + config.ApplyTLSOptions = func(tlsConfig *tls.Config) { + tlsConfig.MinVersion = tls.VersionTLS12 + tlsConfig.CipherSuites = []uint16{ + tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, + } + } + }) + + Specify("the HTTP server should offer the applied TLS options", func() { + Eventually(func(g Gomega) { + conn, err := tls.Dial("tcp", fmt.Sprintf("%s:%d", config.Host, config.Port), &tls.Config{ + InsecureSkipVerify: true, + MaxVersion: tls.VersionTLS12, // Force TLS 1.2 to test minimum version + NextProtos: []string{"h2"}, // Request HTTP/2 via ALPN + }) + + g.Expect(err).NotTo(HaveOccurred()) + defer conn.Close() + + err = conn.Handshake() + g.Expect(err).NotTo(HaveOccurred()) + + state := conn.ConnectionState() + + // For Intermediate profile with MaxVersion TLS 1.2, should negotiate exactly TLS 1.2 + g.Expect(int(state.Version)).To(Equal(tls.VersionTLS12)) + g.Expect(state.CipherSuite).To(Equal(tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256)) + g.Expect(state.NegotiatedProtocol).To(Equal("h2")) + }).Within(5 * time.Second).ProbeEvery(100 * time.Millisecond).Should(Succeed()) + }) + + Specify("the HTTP server should reject TLS versions below the minimum", func() { + Eventually(func(g Gomega) { + // Attempt to connect with TLS 1.1 (below the server's minimum of TLS 1.2) + conn, err := tls.Dial("tcp", fmt.Sprintf("%s:%d", config.Host, config.Port), &tls.Config{ + InsecureSkipVerify: true, + MinVersion: tls.VersionTLS10, + MaxVersion: tls.VersionTLS11, // Restrict client to TLS 1.1 or below + }) + + if err == nil { + err = conn.Handshake() + conn.Close() + } + + // The connection or handshake should fail because the server requires TLS 1.2+ + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring("protocol version")) + }).Within(5 * time.Second).ProbeEvery(100 * time.Millisecond).Should(Succeed()) + }) + }) }) func newWebhookConfig(port int) webhook.Config { @@ -160,9 +214,10 @@ func newWebhookConfig(port int) webhook.Config { Expect(os.WriteFile(keyPath, keyPEM, 0600)).To(Succeed()) return webhook.Config{ - CertDir: tempDir, - Host: "localhost", - Port: port, + CertDir: tempDir, + Host: "localhost", + Port: port, + ApplyTLSOptions: func(*tls.Config) {}, NewKubernetesClient: func(_ *rest.Config) (kubernetes.Interface, error) { return k8sfake.NewClientset(), nil }, From dfcb2426d7e0d157f74b4f7b74687c053ec7e62d Mon Sep 17 00:00:00 2001 From: Jamo Luhrsen Date: Tue, 9 Jun 2026 11:55:40 -0700 Subject: [PATCH 50/51] Fix UDN controller startup timeout during pod sync During controller initialization, syncPodsForUserDefinedNetwork() calls GetActiveNetworkForNamespace() which can fail when a namespace is deleted (NotFound) or has the primary UDN label but no NAD yet (InvalidPrimaryNetworkError). Without error handling, these failures cause the factory retry loop to spin for 60 seconds until context deadline expires, blocking controller startup and preventing pods from getting their logical ports created. Add error handling to skip these pods during initial sync. Normal event-driven processing will handle them once their namespaces are ready. Signed-Off-By: Jamo Luhrsen Co-Authored-By: Claude Sonnet 4.5 --- .../base_network_controller_user_defined.go | 16 +++++++ ...se_network_controller_user_defined_test.go | 48 +++++++++++++++++++ 2 files changed, 64 insertions(+) diff --git a/go-controller/pkg/ovn/base_network_controller_user_defined.go b/go-controller/pkg/ovn/base_network_controller_user_defined.go index 0e1e7bd900..7b0eae2719 100644 --- a/go-controller/pkg/ovn/base_network_controller_user_defined.go +++ b/go-controller/pkg/ovn/base_network_controller_user_defined.go @@ -14,6 +14,7 @@ import ( nadapi "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/klog/v2" utilnet "k8s.io/utils/net" "k8s.io/utils/ptr" @@ -520,6 +521,21 @@ func (bsnc *BaseUserDefinedNetworkController) syncPodsForUserDefinedNetwork(pods if bsnc.IsPrimaryNetwork() { activeNetwork, err = bsnc.networkManager.GetActiveNetworkForNamespace(pod.Namespace) if err != nil { + if apierrors.IsNotFound(err) { + // namespace deleted after pod listing - safe to skip + klog.Infof("%s network controller pod sync: pod %s/%s namespace deleted, skipping", + bsnc.GetNetworkName(), pod.Namespace, pod.Name) + continue + } + if util.IsInvalidPrimaryNetworkError(err) { + // If network manager isn't aware of the primary network for this pod's namespace, + // it can't possibly be already wired to our network unless someone is directly messing + // with the NADs owned by a CUDN, as network manager syncs all NADs at startup. + // Skip during initial sync to avoid blocking startup. + klog.V(5).Infof("%s network controller pod sync: pod %s/%s namespace network not ready, skipping", + bsnc.GetNetworkName(), pod.Namespace, pod.Name) + continue + } return fmt.Errorf("failed to find the active network for pod %s/%s: %w", pod.Namespace, pod.Name, err) } if activeNetwork == nil || activeNetwork.IsDefault() { diff --git a/go-controller/pkg/ovn/base_network_controller_user_defined_test.go b/go-controller/pkg/ovn/base_network_controller_user_defined_test.go index 961df255c1..d150d8e684 100644 --- a/go-controller/pkg/ovn/base_network_controller_user_defined_test.go +++ b/go-controller/pkg/ovn/base_network_controller_user_defined_test.go @@ -415,6 +415,54 @@ var _ = Describe("BaseUserDefinedNetworkController", func() { Expect(err).NotTo(HaveOccurred()) }) + It("should not fail to sync pods if namespace has primary UDN label but NAD not ready", func() { + config.OVNKubernetesFeature.EnableNetworkSegmentation = true + config.OVNKubernetesFeature.EnableMultiNetwork = true + fakeOVN := NewFakeOVN(false) + // Create namespace with primary UDN label but no NAD + namespace := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-namespace", + Labels: map[string]string{ + types.RequiredUDNNamespaceLabel: "", + }, + }, + } + fakeOVN.start( + &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "worker1", + Annotations: map[string]string{ + "k8s.ovn.org/network-ids": `{"other": "3"}`, + }, + }, + }, + namespace, + ) + Expect(fakeOVN.NewUserDefinedNetworkController(nad)).To(Succeed()) + controller, ok := fakeOVN.userDefinedNetworkControllers["bluenet"] + Expect(ok).To(BeTrue()) + // inject a real networkManager so GetActiveNetworkForNamespace will get called + nadController, err := networkmanager.NewForZone("dummyZone", nil, fakeOVN.watcher) + Expect(err).NotTo(HaveOccurred()) + controller.bnc.networkManager = nadController.Interface() + + // Pod in namespace with primary UDN label but no NAD causes InvalidPrimaryNetworkError + podInLabeledNamespace := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "test-namespace", + Name: "test-pod", + }, + } + + var initialPodList []interface{} + initialPodList = append(initialPodList, podInLabeledNamespace) + + // Should skip pod without error when GetActiveNetworkForNamespace returns InvalidPrimaryNetworkError + err = controller.bnc.syncPodsForUserDefinedNetwork(initialPodList) + Expect(err).NotTo(HaveOccurred()) + }) + }) func TestAdvertisedSharedGatewaySNATUsesLiveAllowedExtIPSets(t *gotesting.T) { From 8e2a679f7cc1864992439e2a12b919a533025ba0 Mon Sep 17 00:00:00 2001 From: origin-release-container Date: Mon, 15 Jun 2026 15:05:50 +0000 Subject: [PATCH 51/51] sync test annotations with upstream changes - go mod vendor - ./openshift/hack/update-tests-annotation.sh Automated sync after downstream merge to keep test annotations in sync with upstream test modifications and rules.go changes. --- .../generated/zz_generated.annotations.go | 132 ++---------------- 1 file changed, 10 insertions(+), 122 deletions(-) diff --git a/openshift/test/generated/zz_generated.annotations.go b/openshift/test/generated/zz_generated.annotations.go index c7a876f63b..298a58edc9 100644 --- a/openshift/test/generated/zz_generated.annotations.go +++ b/openshift/test/generated/zz_generated.annotations.go @@ -1087,32 +1087,6 @@ var AppendedAnnotations = map[string]string{ "EgressService [LGW] Should validate pods' egress uses node's IP when setting Network without SNAT ipv6 pods": "[Disabled:Unimplemented]", - "External Gateway When migrating from Annotations to Admin Policy Based External Route CRs e2e multiple external gateway stale conntrack entry deletion validation ExternalGWPod annotation: Should validate conntrack entry remains unchanged when deleting the annotation in the pods while the CR dynamic hop still references the same pods with the pod selector IPV4 tcp": "[Disabled:Unimplemented]", - - "External Gateway When migrating from Annotations to Admin Policy Based External Route CRs e2e multiple external gateway stale conntrack entry deletion validation ExternalGWPod annotation: Should validate conntrack entry remains unchanged when deleting the annotation in the pods while the CR dynamic hop still references the same pods with the pod selector IPV4 udp": "[Disabled:Unimplemented]", - - "External Gateway When migrating from Annotations to Admin Policy Based External Route CRs e2e multiple external gateway stale conntrack entry deletion validation ExternalGWPod annotation: Should validate conntrack entry remains unchanged when deleting the annotation in the pods while the CR dynamic hop still references the same pods with the pod selector IPV6 tcp": "[Disabled:Unimplemented]", - - "External Gateway When migrating from Annotations to Admin Policy Based External Route CRs e2e multiple external gateway stale conntrack entry deletion validation ExternalGWPod annotation: Should validate conntrack entry remains unchanged when deleting the annotation in the pods while the CR dynamic hop still references the same pods with the pod selector IPV6 udp": "[Disabled:Unimplemented]", - - "External Gateway When migrating from Annotations to Admin Policy Based External Route CRs e2e multiple external gateway stale conntrack entry deletion validation Namespace annotation: Should validate conntrack entry remains unchanged when deleting the annotation in the namespace while the CR static hop still references the same namespace in the policy IPV4 tcp": "[Disabled:Unimplemented]", - - "External Gateway When migrating from Annotations to Admin Policy Based External Route CRs e2e multiple external gateway stale conntrack entry deletion validation Namespace annotation: Should validate conntrack entry remains unchanged when deleting the annotation in the namespace while the CR static hop still references the same namespace in the policy IPV4 udp": "[Disabled:Unimplemented]", - - "External Gateway When migrating from Annotations to Admin Policy Based External Route CRs e2e multiple external gateway stale conntrack entry deletion validation Namespace annotation: Should validate conntrack entry remains unchanged when deleting the annotation in the namespace while the CR static hop still references the same namespace in the policy IPV6 tcp": "[Disabled:Unimplemented]", - - "External Gateway When migrating from Annotations to Admin Policy Based External Route CRs e2e multiple external gateway stale conntrack entry deletion validation Namespace annotation: Should validate conntrack entry remains unchanged when deleting the annotation in the namespace while the CR static hop still references the same namespace in the policy IPV6 udp": "[Disabled:Unimplemented]", - - "External Gateway When migrating from Annotations to Admin Policy Based External Route CRs e2e non-vxlan external gateway through a gateway pod Should validate ICMP connectivity to an external gateway's loopback address via a pod with external gateway annotations and a policy CR and after the annotations are removed ipv4": "[Disabled:Unimplemented]", - - "External Gateway When migrating from Annotations to Admin Policy Based External Route CRs e2e non-vxlan external gateway through a gateway pod Should validate TCP/UDP connectivity to an external gateway's loopback address via a pod when deleting the annotation and supported by a CR with the same gateway IPs TCP ipv4": "[Disabled:Unimplemented]", - - "External Gateway When migrating from Annotations to Admin Policy Based External Route CRs e2e non-vxlan external gateway through a gateway pod Should validate TCP/UDP connectivity to an external gateway's loopback address via a pod when deleting the annotation and supported by a CR with the same gateway IPs TCP ipv6": "[Disabled:Unimplemented]", - - "External Gateway When migrating from Annotations to Admin Policy Based External Route CRs e2e non-vxlan external gateway through a gateway pod Should validate TCP/UDP connectivity to an external gateway's loopback address via a pod when deleting the annotation and supported by a CR with the same gateway IPs UDP ipv4": "[Disabled:Unimplemented]", - - "External Gateway When migrating from Annotations to Admin Policy Based External Route CRs e2e non-vxlan external gateway through a gateway pod Should validate TCP/UDP connectivity to an external gateway's loopback address via a pod when deleting the annotation and supported by a CR with the same gateway IPs UDP ipv6": "[Disabled:Unimplemented]", - "External Gateway When validating the Admin Policy Based External Route status Should update the status of a successful and failed CRs": "[Disabled:Unimplemented]", "External Gateway With Admin Policy Based External Route CRs BFD e2e multiple external gateway validation Should validate ICMP connectivity to multiple external gateways for an ECMP scenario IPV4": "[Disabled:Unimplemented]", @@ -1139,28 +1113,28 @@ var AppendedAnnotations = map[string]string{ "External Gateway With Admin Policy Based External Route CRs BFD e2e non-vxlan external gateway through a dynamic hop Should validate TCP/UDP connectivity to an external gateway's loopback address via a pod with a dynamic hop UDP ipv6": "[Disabled:Unimplemented]", - "External Gateway With Admin Policy Based External Route CRs e2e multiple external gateway stale conntrack entry deletion validation Dynamic Hop: Should validate conntrack entry deletion for TCP/UDP traffic via multiple external gateways a.k.a ECMP routes IPV4 tcp + pod annotation update": "[Disabled:Unimplemented]", - "External Gateway With Admin Policy Based External Route CRs e2e multiple external gateway stale conntrack entry deletion validation Dynamic Hop: Should validate conntrack entry deletion for TCP/UDP traffic via multiple external gateways a.k.a ECMP routes IPV4 tcp + pod deletion timestamp": "[Disabled:Unimplemented]", - "External Gateway With Admin Policy Based External Route CRs e2e multiple external gateway stale conntrack entry deletion validation Dynamic Hop: Should validate conntrack entry deletion for TCP/UDP traffic via multiple external gateways a.k.a ECMP routes IPV4 tcp + pod not ready": "[Disabled:Unimplemented]", + "External Gateway With Admin Policy Based External Route CRs e2e multiple external gateway stale conntrack entry deletion validation Dynamic Hop: Should validate conntrack entry deletion for TCP/UDP traffic via multiple external gateways a.k.a ECMP routes IPV4 tcp + pod label update": "[Disabled:Unimplemented]", - "External Gateway With Admin Policy Based External Route CRs e2e multiple external gateway stale conntrack entry deletion validation Dynamic Hop: Should validate conntrack entry deletion for TCP/UDP traffic via multiple external gateways a.k.a ECMP routes IPV4 udp + pod annotation update": "[Disabled:Unimplemented]", + "External Gateway With Admin Policy Based External Route CRs e2e multiple external gateway stale conntrack entry deletion validation Dynamic Hop: Should validate conntrack entry deletion for TCP/UDP traffic via multiple external gateways a.k.a ECMP routes IPV4 tcp + pod not ready": "[Disabled:Unimplemented]", "External Gateway With Admin Policy Based External Route CRs e2e multiple external gateway stale conntrack entry deletion validation Dynamic Hop: Should validate conntrack entry deletion for TCP/UDP traffic via multiple external gateways a.k.a ECMP routes IPV4 udp + pod deletion timestamp": "[Disabled:Unimplemented]", - "External Gateway With Admin Policy Based External Route CRs e2e multiple external gateway stale conntrack entry deletion validation Dynamic Hop: Should validate conntrack entry deletion for TCP/UDP traffic via multiple external gateways a.k.a ECMP routes IPV4 udp + pod not ready": "[Disabled:Unimplemented]", + "External Gateway With Admin Policy Based External Route CRs e2e multiple external gateway stale conntrack entry deletion validation Dynamic Hop: Should validate conntrack entry deletion for TCP/UDP traffic via multiple external gateways a.k.a ECMP routes IPV4 udp + pod label update": "[Disabled:Unimplemented]", - "External Gateway With Admin Policy Based External Route CRs e2e multiple external gateway stale conntrack entry deletion validation Dynamic Hop: Should validate conntrack entry deletion for TCP/UDP traffic via multiple external gateways a.k.a ECMP routes IPV6 tcp + pod annotation update": "[Disabled:Unimplemented]", + "External Gateway With Admin Policy Based External Route CRs e2e multiple external gateway stale conntrack entry deletion validation Dynamic Hop: Should validate conntrack entry deletion for TCP/UDP traffic via multiple external gateways a.k.a ECMP routes IPV4 udp + pod not ready": "[Disabled:Unimplemented]", "External Gateway With Admin Policy Based External Route CRs e2e multiple external gateway stale conntrack entry deletion validation Dynamic Hop: Should validate conntrack entry deletion for TCP/UDP traffic via multiple external gateways a.k.a ECMP routes IPV6 tcp + pod deletion timestamp": "[Disabled:Unimplemented]", - "External Gateway With Admin Policy Based External Route CRs e2e multiple external gateway stale conntrack entry deletion validation Dynamic Hop: Should validate conntrack entry deletion for TCP/UDP traffic via multiple external gateways a.k.a ECMP routes IPV6 tcp + pod not ready": "[Disabled:Unimplemented]", + "External Gateway With Admin Policy Based External Route CRs e2e multiple external gateway stale conntrack entry deletion validation Dynamic Hop: Should validate conntrack entry deletion for TCP/UDP traffic via multiple external gateways a.k.a ECMP routes IPV6 tcp + pod label update": "[Disabled:Unimplemented]", - "External Gateway With Admin Policy Based External Route CRs e2e multiple external gateway stale conntrack entry deletion validation Dynamic Hop: Should validate conntrack entry deletion for TCP/UDP traffic via multiple external gateways a.k.a ECMP routes IPV6 udp + pod annotation update": "[Disabled:Unimplemented]", + "External Gateway With Admin Policy Based External Route CRs e2e multiple external gateway stale conntrack entry deletion validation Dynamic Hop: Should validate conntrack entry deletion for TCP/UDP traffic via multiple external gateways a.k.a ECMP routes IPV6 tcp + pod not ready": "[Disabled:Unimplemented]", "External Gateway With Admin Policy Based External Route CRs e2e multiple external gateway stale conntrack entry deletion validation Dynamic Hop: Should validate conntrack entry deletion for TCP/UDP traffic via multiple external gateways a.k.a ECMP routes IPV6 udp + pod deletion timestamp": "[Disabled:Unimplemented]", + "External Gateway With Admin Policy Based External Route CRs e2e multiple external gateway stale conntrack entry deletion validation Dynamic Hop: Should validate conntrack entry deletion for TCP/UDP traffic via multiple external gateways a.k.a ECMP routes IPV6 udp + pod label update": "[Disabled:Unimplemented]", + "External Gateway With Admin Policy Based External Route CRs e2e multiple external gateway stale conntrack entry deletion validation Dynamic Hop: Should validate conntrack entry deletion for TCP/UDP traffic via multiple external gateways a.k.a ECMP routes IPV6 udp + pod not ready": "[Disabled:Unimplemented]", "External Gateway With Admin Policy Based External Route CRs e2e multiple external gateway stale conntrack entry deletion validation Static Hop: Should validate conntrack entry deletion for TCP/UDP traffic via multiple external gateways a.k.a ECMP routes IPV4 tcp": "[Disabled:Unimplemented]", @@ -1203,94 +1177,6 @@ var AppendedAnnotations = map[string]string{ "External Gateway With Admin Policy Based External Route CRs e2e non-vxlan external gateway through a gateway pod Should validate TCP/UDP connectivity to an external gateway's loopback address via a gateway pod UDP ipv6": "[Disabled:Unimplemented]", - "External Gateway With annotations BFD e2e multiple external gateway validation Should validate ICMP connectivity to multiple external gateways for an ECMP scenario IPV4": "[Disabled:Unimplemented]", - - "External Gateway With annotations BFD e2e multiple external gateway validation Should validate ICMP connectivity to multiple external gateways for an ECMP scenario IPV6": "[Disabled:Unimplemented]", - - "External Gateway With annotations BFD e2e multiple external gateway validation Should validate TCP/UDP connectivity to multiple external gateways for a UDP / TCP scenario IPV4 tcp": "[Disabled:Unimplemented]", - - "External Gateway With annotations BFD e2e multiple external gateway validation Should validate TCP/UDP connectivity to multiple external gateways for a UDP / TCP scenario IPV4 udp": "[Disabled:Unimplemented]", - - "External Gateway With annotations BFD e2e multiple external gateway validation Should validate TCP/UDP connectivity to multiple external gateways for a UDP / TCP scenario IPV6 tcp": "[Disabled:Unimplemented]", - - "External Gateway With annotations BFD e2e multiple external gateway validation Should validate TCP/UDP connectivity to multiple external gateways for a UDP / TCP scenario IPV6 udp": "[Disabled:Unimplemented]", - - "External Gateway With annotations BFD e2e non-vxlan external gateway through an annotated gateway pod Should validate ICMP connectivity to an external gateway's loopback address via a pod with external gateway annotations enabled ipv4": "[Disabled:Unimplemented]", - - "External Gateway With annotations BFD e2e non-vxlan external gateway through an annotated gateway pod Should validate ICMP connectivity to an external gateway's loopback address via a pod with external gateway annotations enabled ipv6": "[Disabled:Unimplemented]", - - "External Gateway With annotations BFD e2e non-vxlan external gateway through an annotated gateway pod Should validate TCP/UDP connectivity to an external gateway's loopback address via a pod with external gateway annotations enabled TCP ipv4": "[Disabled:Unimplemented]", - - "External Gateway With annotations BFD e2e non-vxlan external gateway through an annotated gateway pod Should validate TCP/UDP connectivity to an external gateway's loopback address via a pod with external gateway annotations enabled TCP ipv6": "[Disabled:Unimplemented]", - - "External Gateway With annotations BFD e2e non-vxlan external gateway through an annotated gateway pod Should validate TCP/UDP connectivity to an external gateway's loopback address via a pod with external gateway annotations enabled UDP ipv4": "[Disabled:Unimplemented]", - - "External Gateway With annotations BFD e2e non-vxlan external gateway through an annotated gateway pod Should validate TCP/UDP connectivity to an external gateway's loopback address via a pod with external gateway annotations enabled UDP ipv6": "[Disabled:Unimplemented]", - - "External Gateway With annotations e2e multiple external gateway stale conntrack entry deletion validation ExternalGWPod annotation: Should validate conntrack entry deletion for TCP/UDP traffic via multiple external gateways a.k.a ECMP routes IPV4 tcp + pod annotation update": "[Disabled:Unimplemented]", - - "External Gateway With annotations e2e multiple external gateway stale conntrack entry deletion validation ExternalGWPod annotation: Should validate conntrack entry deletion for TCP/UDP traffic via multiple external gateways a.k.a ECMP routes IPV4 tcp + pod deletion timestamp": "[Disabled:Unimplemented]", - - "External Gateway With annotations e2e multiple external gateway stale conntrack entry deletion validation ExternalGWPod annotation: Should validate conntrack entry deletion for TCP/UDP traffic via multiple external gateways a.k.a ECMP routes IPV4 tcp + pod not ready": "[Disabled:Unimplemented]", - - "External Gateway With annotations e2e multiple external gateway stale conntrack entry deletion validation ExternalGWPod annotation: Should validate conntrack entry deletion for TCP/UDP traffic via multiple external gateways a.k.a ECMP routes IPV4 udp + pod annotation update": "[Disabled:Unimplemented]", - - "External Gateway With annotations e2e multiple external gateway stale conntrack entry deletion validation ExternalGWPod annotation: Should validate conntrack entry deletion for TCP/UDP traffic via multiple external gateways a.k.a ECMP routes IPV4 udp + pod delete": "[Disabled:Unimplemented]", - - "External Gateway With annotations e2e multiple external gateway stale conntrack entry deletion validation ExternalGWPod annotation: Should validate conntrack entry deletion for TCP/UDP traffic via multiple external gateways a.k.a ECMP routes IPV4 udp + pod deletion timestamp": "[Disabled:Unimplemented]", - - "External Gateway With annotations e2e multiple external gateway stale conntrack entry deletion validation ExternalGWPod annotation: Should validate conntrack entry deletion for TCP/UDP traffic via multiple external gateways a.k.a ECMP routes IPV4 udp + pod not ready": "[Disabled:Unimplemented]", - - "External Gateway With annotations e2e multiple external gateway stale conntrack entry deletion validation ExternalGWPod annotation: Should validate conntrack entry deletion for TCP/UDP traffic via multiple external gateways a.k.a ECMP routes IPV6 tcp + pod annotation update": "[Disabled:Unimplemented]", - - "External Gateway With annotations e2e multiple external gateway stale conntrack entry deletion validation ExternalGWPod annotation: Should validate conntrack entry deletion for TCP/UDP traffic via multiple external gateways a.k.a ECMP routes IPV6 tcp + pod delete": "[Disabled:Unimplemented]", - - "External Gateway With annotations e2e multiple external gateway stale conntrack entry deletion validation ExternalGWPod annotation: Should validate conntrack entry deletion for TCP/UDP traffic via multiple external gateways a.k.a ECMP routes IPV6 tcp + pod deletion timestamp": "[Disabled:Unimplemented]", - - "External Gateway With annotations e2e multiple external gateway stale conntrack entry deletion validation ExternalGWPod annotation: Should validate conntrack entry deletion for TCP/UDP traffic via multiple external gateways a.k.a ECMP routes IPV6 tcp + pod not ready": "[Disabled:Unimplemented]", - - "External Gateway With annotations e2e multiple external gateway stale conntrack entry deletion validation ExternalGWPod annotation: Should validate conntrack entry deletion for TCP/UDP traffic via multiple external gateways a.k.a ECMP routes IPV6 udp + pod annotation update": "[Disabled:Unimplemented]", - - "External Gateway With annotations e2e multiple external gateway stale conntrack entry deletion validation ExternalGWPod annotation: Should validate conntrack entry deletion for TCP/UDP traffic via multiple external gateways a.k.a ECMP routes IPV6 udp + pod deletion timestamp": "[Disabled:Unimplemented]", - - "External Gateway With annotations e2e multiple external gateway stale conntrack entry deletion validation ExternalGWPod annotation: Should validate conntrack entry deletion for TCP/UDP traffic via multiple external gateways a.k.a ECMP routes IPV6 udp + pod not ready": "[Disabled:Unimplemented]", - - "External Gateway With annotations e2e multiple external gateway stale conntrack entry deletion validation Namespace annotation: Should validate conntrack entry deletion for TCP/UDP traffic via multiple external gateways a.k.a ECMP routes IPV4 tcp": "[Disabled:Unimplemented]", - - "External Gateway With annotations e2e multiple external gateway stale conntrack entry deletion validation Namespace annotation: Should validate conntrack entry deletion for TCP/UDP traffic via multiple external gateways a.k.a ECMP routes IPV4 udp": "[Disabled:Unimplemented]", - - "External Gateway With annotations e2e multiple external gateway stale conntrack entry deletion validation Namespace annotation: Should validate conntrack entry deletion for TCP/UDP traffic via multiple external gateways a.k.a ECMP routes IPV6 tcp": "[Disabled:Unimplemented]", - - "External Gateway With annotations e2e multiple external gateway stale conntrack entry deletion validation Namespace annotation: Should validate conntrack entry deletion for TCP/UDP traffic via multiple external gateways a.k.a ECMP routes IPV6 udp": "[Disabled:Unimplemented]", - - "External Gateway With annotations e2e multiple external gateway validation Should validate ICMP connectivity to multiple external gateways for an ECMP scenario IPV4": "[Disabled:Unimplemented]", - - "External Gateway With annotations e2e multiple external gateway validation Should validate ICMP connectivity to multiple external gateways for an ECMP scenario IPV6": "[Disabled:Unimplemented]", - - "External Gateway With annotations e2e multiple external gateway validation Should validate TCP/UDP connectivity to multiple external gateways for a UDP / TCP scenario IPV4 tcp": "[Disabled:Unimplemented]", - - "External Gateway With annotations e2e multiple external gateway validation Should validate TCP/UDP connectivity to multiple external gateways for a UDP / TCP scenario IPV4 udp": "[Disabled:Unimplemented]", - - "External Gateway With annotations e2e multiple external gateway validation Should validate TCP/UDP connectivity to multiple external gateways for a UDP / TCP scenario IPV6 tcp": "[Disabled:Unimplemented]", - - "External Gateway With annotations e2e multiple external gateway validation Should validate TCP/UDP connectivity to multiple external gateways for a UDP / TCP scenario IPV6 udp": "[Disabled:Unimplemented]", - - "External Gateway With annotations e2e non-vxlan external gateway through a gateway pod Should validate ICMP connectivity to an external gateway's loopback address via a pod with external gateway CR ipv4": "[Disabled:Unimplemented]", - - "External Gateway With annotations e2e non-vxlan external gateway through a gateway pod Should validate ICMP connectivity to an external gateway's loopback address via a pod with external gateway CR ipv6": "[Disabled:Unimplemented]", - - "External Gateway With annotations e2e non-vxlan external gateway through a gateway pod Should validate TCP/UDP connectivity to an external gateway's loopback address via a pod with external gateway annotations enabled TCP ipv4": "[Disabled:Unimplemented]", - - "External Gateway With annotations e2e non-vxlan external gateway through a gateway pod Should validate TCP/UDP connectivity to an external gateway's loopback address via a pod with external gateway annotations enabled TCP ipv6": "[Disabled:Unimplemented]", - - "External Gateway With annotations e2e non-vxlan external gateway through a gateway pod Should validate TCP/UDP connectivity to an external gateway's loopback address via a pod with external gateway annotations enabled UDP ipv4": "[Disabled:Unimplemented]", - - "External Gateway With annotations e2e non-vxlan external gateway through a gateway pod Should validate TCP/UDP connectivity to an external gateway's loopback address via a pod with external gateway annotations enabled UDP ipv6": "[Disabled:Unimplemented]", - - "External Gateway e2e ingress gateway traffic validation Should validate ingress connectivity from an external gateway": "[Disabled:Unimplemented]", - - "External Gateway e2e non-vxlan external gateway and update validation Should validate connectivity without vxlan before and after updating the namespace annotation to a new external gateway": "[Disabled:Unimplemented]", - "Kubevirt Virtual Machines IP family validation for layer2 primary networks should fail when dual-stack network requests only IPv4": "[Disabled:Unimplemented]", "Kubevirt Virtual Machines IP family validation for layer2 primary networks should fail when dual-stack network requests only IPv6": "[Disabled:Unimplemented]", @@ -1315,6 +1201,8 @@ var AppendedAnnotations = map[string]string{ "Kubevirt Virtual Machines with kubevirt VM using layer2 UDPN should configure IPv4 and IPv6 using DHCP and NDP": "[Disabled:Unimplemented]", + "Kubevirt Virtual Machines with user defined networks and persistent ips configured should keep ip after live migration failed of VirtualMachine with interface binding for UDN with Primary/Layer2 ingress routed over evpn": "[Disabled:Unimplemented]", + "Kubevirt Virtual Machines with user defined networks and persistent ips configured should keep ip after live migration failed of VirtualMachineInstance with Secondary/Localnet ingress snat": "[Disabled:Unimplemented]", "Kubevirt Virtual Machines with user defined networks and persistent ips configured should keep ip after live migration failed of VirtualMachineInstance with interface binding for UDN with Primary/Layer2 ingress snat": "[Disabled:Unimplemented]",