From 9e1722dc5264255bbd513bb4fb23f33a9de85927 Mon Sep 17 00:00:00 2001 From: Varun Ramesh Date: Wed, 21 Oct 2026 12:37:04 +0530 Subject: [PATCH 1/8] feat(nvsentinel): add opt-in log collector mixin Signed-off-by: Varun Ramesh --- .../kwok-nvsentinel-remediation.yaml | 5 + .../nvsentinel-log-collector-e2e.yaml | 152 ++++++ ...nvsentinel-observability-render-check.yaml | 4 +- .github/workflows/vuln-scan-images.yaml | 32 +- Makefile | 6 + docs/contributor/recipe.md | 2 +- docs/integrator/recipe-development.md | 2 + docs/user/component-catalog.md | 47 +- docs/user/container-images.md | 1 + kwok/scripts/validate-scheduling.sh | 9 + pkg/bundler/validations/checks.go | 1 + .../validations/nvsentinel_log_collector.go | 281 ++++++++++ .../nvsentinel_log_collector_render_test.go | 200 ++++++++ .../nvsentinel_log_collector_test.go | 430 ++++++++++++++++ pkg/chainsaw/nvsentinel_check_states_test.go | 1 + .../nvsentinel_log_collector_mixin_test.go | 229 +++++++++ .../testdata/catalog_parity_golden.yaml | 140 ++--- recipes/checks/nvsentinel/health-check.yaml | 42 +- recipes/mixins/nvsentinel-log-collector.yaml | 76 +++ recipes/registry.yaml | 25 +- tests/e2e/nvsentinel-log-collector/run.sh | 483 ++++++++++++++++++ tools/mirror-e2e | 30 +- 22 files changed, 2106 insertions(+), 92 deletions(-) create mode 100644 .github/workflows/nvsentinel-log-collector-e2e.yaml create mode 100644 pkg/bundler/validations/nvsentinel_log_collector.go create mode 100644 pkg/bundler/validations/nvsentinel_log_collector_render_test.go create mode 100644 pkg/bundler/validations/nvsentinel_log_collector_test.go create mode 100644 pkg/recipe/nvsentinel_log_collector_mixin_test.go create mode 100644 recipes/mixins/nvsentinel-log-collector.yaml create mode 100755 tests/e2e/nvsentinel-log-collector/run.sh diff --git a/.github/workflows/kwok-nvsentinel-remediation.yaml b/.github/workflows/kwok-nvsentinel-remediation.yaml index 893f0aeca0..703509942b 100644 --- a/.github/workflows/kwok-nvsentinel-remediation.yaml +++ b/.github/workflows/kwok-nvsentinel-remediation.yaml @@ -116,6 +116,11 @@ jobs: # The event exporter (#2621), which reads the datastore a step adds. - mixin: nvsentinel-observe,nvsentinel-event-exporter deployer: helm + # The log collector's file server runs on emptyDir under KWOK + # (kwok/scripts/validate-scheduling.sh); EKS's GPU Operator installs + # the driver, so CheckNVSentinelLogCollectorCoherent accepts it. + - mixin: 'nvsentinel-remediation,nvsentinel-health-events-analyzer,nvsentinel-log-collector' + deployer: helm steps: - name: Checkout Code uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 diff --git a/.github/workflows/nvsentinel-log-collector-e2e.yaml b/.github/workflows/nvsentinel-log-collector-e2e.yaml new file mode 100644 index 0000000000..25e0cd8414 --- /dev/null +++ b/.github/workflows/nvsentinel-log-collector-e2e.yaml @@ -0,0 +1,152 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Runtime Kind test for the nvsentinel-log-collector mixin. +# +# Installs the remediation step with the log collector, and the health events +# analyzer that live remediation requires, via the bundle's own install.sh, +# sends one synthetic GPU fault into platform-connectors, and checks +# that fault-remediation runs the collector Job on the node, that the bug report +# it uploads is listed by the in-cluster file server, that the file server's +# volume took the --storage-class value, and that the GPU reset is requested +# only after collection. The collector runs in its MOCK_MODE: a Kind node has no +# GPU driver. Not wired into `make qualify`: it needs a Kind cluster. +# +# Path filter kept narrow for the same reason as nvsentinel-preflight-e2e.yaml. + +name: NVSentinel Log Collector E2E + +on: + push: + branches: [main] + paths: &paths + - 'recipes/mixins/nvsentinel-remediation.yaml' + - 'recipes/mixins/nvsentinel-health-events-analyzer.yaml' + - 'recipes/mixins/nvsentinel-log-collector.yaml' + - 'pkg/bundler/validations/nvsentinel_log_collector.go' + - 'pkg/bundler/validations/checks.go' + - 'recipes/registry.yaml' + - 'recipes/components/nvsentinel/**' + - 'recipes/components/nvsentinel-mongodb/**' + - 'recipes/components/psmdb-operator/**' + # The tested overlay inherits kind.yaml, which inherits base.yaml, and + # base.yaml is where nvsentinel enters the chain at all. + - 'recipes/overlays/h100-kind-training.yaml' + - 'recipes/overlays/kind.yaml' + - 'recipes/overlays/base.yaml' + # The run installs cert-manager, for the Percona cluster's certificates, + # and prometheus-operator-crds from the bundle. + - 'recipes/components/cert-manager/**' + - 'recipes/components/prometheus-operator-crds/**' + - 'tests/e2e/nvsentinel-log-collector/**' + - 'pkg/bundler/deployer/localformat/templates/**' + # run.sh sources tools/common and is invoked through a make target. + - 'tools/common' + - 'Makefile' + - '.github/workflows/nvsentinel-log-collector-e2e.yaml' + - '.github/actions/setup-build-tools/**' + - '.github/actions/prep-kind-runner/**' + - '.github/actions/load-versions/**' + - '.settings.yaml' + pull_request: + branches: [main] + paths: *paths + workflow_dispatch: {} + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + test: + name: NVSentinel Log Collector E2E + runs-on: ubuntu-24.04 + # The script's bounded waits sum to ~70 minutes worst case (node Ready 2, + # cert-manager 10, Percona datastore 15, nvsentinel rollout 15, event + # delivery 3, collection 15, reset request 5, proto fetch 1, port-forward + # and fetches 4), before kind create, go build and the installs' own helm + # waits. A healthy run takes a fraction of that. + timeout-minutes: 100 + steps: + - name: Checkout Code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Load versions + id: versions + uses: ./.github/actions/load-versions + + - name: Prep Kind runner + uses: ./.github/actions/prep-kind-runner + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version: ${{ steps.versions.outputs.go }} + cache: false + + # run.sh's has_tools gate also needs jq and curl, which the runner image + # already ships; installing only some of the rest fails the lane before + # it exercises anything. + - name: Install Kind, kubectl, Helm and yq + uses: ./.github/actions/setup-build-tools + with: + install_kind: 'true' + kind_version: '${{ steps.versions.outputs.kind }}' + install_kubectl: 'true' + kubectl_version: '${{ steps.versions.outputs.kubectl }}' + install_helm: 'true' + helm_version: '${{ steps.versions.outputs.helm }}' + install_yq: 'true' + yq_version: '${{ steps.versions.outputs.yq }}' + + # KEEP_CLUSTER so the cluster survives for the diagnostics below; the + # script's own EXIT trap would otherwise delete it before they run, + # leaving a failure with nothing to inspect. The always() step after + # diagnostics does the teardown instead. + - name: Run nvsentinel-log-collector e2e + env: + KEEP_CLUSTER: 'true' + run: make nvsentinel-log-collector-e2e + + # run.sh writes its kubeconfig inside a mktemp dir and removes that dir on + # exit, so KEEP_CLUSTER leaves the cluster running but no credentials to + # reach it. Re-export before querying anything. cancelled() because a + # job timeout cancels rather than fails. + - name: Collect diagnostics on failure + if: ${{ failure() || cancelled() }} + run: | + set +e + cluster=aicr-nvsentinel-lc-e2e + export KUBECONFIG="${RUNNER_TEMP}/lc-diag.kubeconfig" + kind export kubeconfig --name "$cluster" --kubeconfig "$KUBECONFIG" || exit 0 + kubectl get pods -A + kubectl get node -o yaml + kubectl -n nvsentinel get pvc,statefulset,deployment,job,perconaservermongodb,certificate + kubectl get gpuresets.janitor.dgxc.nvidia.com -A -o yaml + kubectl -n nvsentinel logs -l app=log-collector --tail=200 + for d in fault-quarantine node-drainer fault-remediation janitor nvsentinel-incluster-file-server; do + kubectl -n nvsentinel logs "deployment/$d" --tail=200 + done + kubectl -n nvsentinel logs daemonset/platform-connectors --tail=200 + kubectl -n nvsentinel logs deployment/psmdb-operator --tail=200 + kubectl -n nvsentinel logs statefulset/nvsentinel-mongodb-rs0 -c mongod --tail=100 + + - name: Delete Kind cluster + if: ${{ always() }} + run: kind delete cluster --name aicr-nvsentinel-lc-e2e || true diff --git a/.github/workflows/nvsentinel-observability-render-check.yaml b/.github/workflows/nvsentinel-observability-render-check.yaml index a58b8c3320..5dd855ead4 100644 --- a/.github/workflows/nvsentinel-observability-render-check.yaml +++ b/.github/workflows/nvsentinel-observability-render-check.yaml @@ -26,7 +26,8 @@ # health-events-analyzer one asserts the workload, its datastore wiring and # that the mixin names every rule the chart ships, the event-exporter one # asserts the workload, its OIDC Secret and datastore wiring and its config, -# and the CRD +# the log-collector one asserts the collector Job, the file server it uploads +# to and that server's volume and images, and the CRD # exclusion one asserts the chart still ships mongodb-store's CRDs where # nvsentinel's ownsCRDsExcludeSubcharts expects them. # All are gated on testing.Short() -- `make @@ -103,6 +104,7 @@ jobs: TestNVSentinelSlinkyDrainerChartRender TestNVSentinelHealthEventsAnalyzerChartRender TestNVSentinelEventExporterChartRender + TestNVSentinelLogCollectorChartRender ) pattern="^($(IFS='|'; echo "${tests[*]}"))$" out=$(go test ./pkg/bundler/validations/... -run "$pattern" -v -count=1 2>&1) && rc=0 || rc=$? diff --git a/.github/workflows/vuln-scan-images.yaml b/.github/workflows/vuln-scan-images.yaml index beaad321d0..5fc048e798 100644 --- a/.github/workflows/vuln-scan-images.yaml +++ b/.github/workflows/vuln-scan-images.yaml @@ -333,8 +333,8 @@ jobs: # opt-in nvsentinel-object-monitor (#2612), nvsentinel-preflight # (#2610), nvsentinel-nic-health-monitor, nvsentinel-slurm-drain-monitor # (#2611), remediation, nvsentinel-slinky-drainer (#2615), - # nvsentinel-health-events-analyzer (#2618) and - # nvsentinel-event-exporter (#2621) mixins, and no other job + # nvsentinel-health-events-analyzer (#2618), nvsentinel-event-exporter + # (#2621) and nvsentinel-log-collector (#2619) mixins, and no other job # scans a mixin-gated image, so nothing else would surface a CVE in # them. # Because they are not built by this workflow they carry their @@ -355,6 +355,14 @@ jobs: - ghcr.io/nvidia/nvsentinel/slinky-drainer - ghcr.io/nvidia/nvsentinel/health-events-analyzer - ghcr.io/nvidia/nvsentinel/event-exporter + - ghcr.io/nvidia/nvsentinel/log-collector + - ghcr.io/nvidia/nvsentinel/file-server-cleanup + # The nvsentinel-log-collector mixin's file server, at the nvsentinel + # chart's own pins. bitnamilegacy is Bitnami's frozen archive and is + # never patched, so expect findings here. + - docker.io/bitnamilegacy/nginx + - public.ecr.aws/docker/library/busybox + - quay.io/martinhelmich/prometheus-nginxlog-exporter # Percona, from the remediation mixins' separately bundled datastore # (psmdb-operator, nvsentinel-mongodb). Pinned by AICR, by digest. - docker.io/percona/percona-server-mongodb-operator @@ -381,9 +389,9 @@ jobs: # TestSlurmDrainMonitorImagePinnedEverywhere, # TestRemediationImagesPinnedEverywhere, # TestSlinkyDrainerImagePinnedEverywhere, - # TestHealthEventsAnalyzerImagePinnedEverywhere and - # TestEventExporterImagePinnedEverywhere fail if these drift - # apart. + # TestHealthEventsAnalyzerImagePinnedEverywhere, + # TestEventExporterImagePinnedEverywhere and + # TestLogCollectorImagesPinnedEverywhere fail if these drift apart. - image: ghcr.io/nvidia/nvsentinel/kubernetes-object-monitor tag: v1.28.0 - image: ghcr.io/nvidia/nvsentinel/preflight @@ -417,6 +425,20 @@ jobs: tag: v1.28.0 - image: ghcr.io/nvidia/nvsentinel/event-exporter tag: v1.28.0 + - image: ghcr.io/nvidia/nvsentinel/log-collector + tag: v1.28.0 + - image: ghcr.io/nvidia/nvsentinel/file-server-cleanup + tag: v1.28.0 + # The nvsentinel chart's own pins for the file server's third-party + # images. The weekly TestNVSentinelLogCollectorChartRender fails if a + # chart bump moves them; TestLogCollectorImagesPinnedEverywhere then + # keeps this list, the mirror lane and the docs on the new tags. + - image: docker.io/bitnamilegacy/nginx + tag: 1.27.2-debian-12-r2 + - image: public.ecr.aws/docker/library/busybox + tag: 1.37.0 + - image: quay.io/martinhelmich/prometheus-nginxlog-exporter + tag: v1.10.0 # AICR's own pins in recipes/components/{psmdb-operator,nvsentinel-mongodb}; # TestRemediationImagesPinnedEverywhere fails if these drift apart. - image: docker.io/percona/percona-server-mongodb-operator diff --git a/Makefile b/Makefile index 0628f4f7a2..af96714d97 100644 --- a/Makefile +++ b/Makefile @@ -456,6 +456,12 @@ nvsentinel-event-exporter-e2e: ## Live Kind test: nvsentinel-event-exporter deli echo "Running nvsentinel-event-exporter mixin e2e test..."; \ tests/e2e/nvsentinel-event-exporter/run.sh +.PHONY: nvsentinel-log-collector-e2e +nvsentinel-log-collector-e2e: ## Live Kind test: nvsentinel-log-collector captures a node's diagnostics into the in-cluster file server before remediation (#2619) + @set -e; \ + echo "Running nvsentinel-log-collector mixin e2e test..."; \ + tests/e2e/nvsentinel-log-collector/run.sh + .PHONY: npd-nvsentinel-object-monitor-e2e npd-nvsentinel-object-monitor-e2e: ## Live Kind test: an NPD Node condition becomes an NVSentinel health event (#2614) @set -e; \ diff --git a/docs/contributor/recipe.md b/docs/contributor/recipe.md index a4039f4399..421afdf53b 100644 --- a/docs/contributor/recipe.md +++ b/docs/contributor/recipe.md @@ -360,7 +360,7 @@ Mixin files currently in the tree: `os-ubuntu`, `os-talos`, `os-amazonlinux`, `o `nvsentinel-object-monitor`, `nvsentinel-nic-health-monitor`, `nvsentinel-preflight`, `nvsentinel-slurm-drain-monitor`, `nvsentinel-observe`, `nvsentinel-quarantine`, `nvsentinel-remediation`, `nvsentinel-slinky-drainer`, -`nvsentinel-health-events-analyzer`, `nvsentinel-event-exporter`, `nvsentinel-datastore-eks`, `npd`. +`nvsentinel-health-events-analyzer`, `nvsentinel-event-exporter`, `nvsentinel-datastore-eks`, `nvsentinel-log-collector`, `npd`. The OS-scoped mixins (`os-ubuntu`, `os-talos`, `os-amazonlinux`, `os-sles`) are mutually exclusive: diff --git a/docs/integrator/recipe-development.md b/docs/integrator/recipe-development.md index e139f0eca2..1de5fc5a6e 100644 --- a/docs/integrator/recipe-development.md +++ b/docs/integrator/recipe-development.md @@ -184,6 +184,8 @@ When authoring a recipe targeting Talos (`criteria.os: talos`), append the `os-t **`nvsentinel-datastore-eks`** declares the storage edge of NVSentinel's datastore on EKS. An EKS leaf that composes a remediation step lists it after the step (`spec.mixins: [nvsentinel-observe, nvsentinel-datastore-eks]`); it adds `aws-ebs-csi-driver` to `nvsentinel-mongodb`'s `dependencyRefs`, so the driver that provisions the datastore's volumes deploys first. It contributes only `dependencyRefs` to an already-chained component, which `mixinComponentRefSafeForMerge` allows; listed before the step it would introduce `nvsentinel-mongodb` itself, and the step's `type` for that component then fails resolution as an identity conflict. On a cluster that uses the EKS-managed EBS addon, `aws-ebs-csi-driver` stays declared with `enabled: false` and the edge counts as satisfied outside the recipe. See [component catalog](../user/component-catalog.md#graded-remediation). +**`nvsentinel-log-collector`** captures a faulty node's diagnostics before NVSentinel repairs it, composed beside the remediation step and the health events analyzer that live remediation requires (`spec.mixins: [nvsentinel-remediation, nvsentinel-health-events-analyzer, nvsentinel-log-collector]`), because only live `fault-remediation` runs the collector. It turns on `fault-remediation.logCollector` and `global.inclusterFileServer` together, pins the file server's name and the upload URL that names it, sets must-gather explicitly off and the volume size and cleanup cycle deliberately, all through `nvsentinel`'s `mixinSafeOverridePaths`; it sets no path a step sets. The file server's claim is in `nvsentinel`'s `storageClassPaths`, so `--storage-class` reaches it. `CheckNVSentinelLogCollectorCoherent` fails the bundle without live remediation, with only one of the two settings on, and on a platform where the collector cannot reach the driver (AKS and OKE at their default `gpuStack`, Kind). See [component catalog](../user/component-catalog.md#diagnostics-capture) for where bundles land, how to retrieve them, and the privileges involved. + **Cross-cutting overlays with wildcard criteria** apply across one criteria dimension without being referenced via `spec.base` or listed in `spec.mixins`. The resolver can return multiple independent maximal-leaf overlays for a single query, so a `service: any` overlay is picked up alongside the service-specific maximal leaf and its inheritance chain: ```yaml diff --git a/docs/user/component-catalog.md b/docs/user/component-catalog.md index 3cd146053b..68d475f8c5 100644 --- a/docs/user/component-catalog.md +++ b/docs/user/component-catalog.md @@ -423,7 +423,7 @@ AICR ships NVSentinel in the upstream chart's **monitoring-only** configuration: Also off: `healthEventsAnalyzer`, `lifecycleManager`, `cspHealthMonitor`, `kubernetesObjectMonitor`, `nvcreCertificationMonitor`, `nicHealthMonitor`, `slurmDrainMonitor`, `preflight`, `eventExporter`, `inclusterFileServer`, `k8sdatastoreCrds`. Verified against chart `v1.28.0`, the version pinned in `recipes/registry.yaml`. -Two entries above are turned back on by AICR's shipped recipes: `nicHealthMonitor` on AKS and OKE (see [NIC and fabric fault detection](#nic-and-fabric-fault-detection)), and `slurmDrainMonitor` on every `platform: slurm` recipe (see [Slurm Drain Monitor](#slurm-drain-monitor)). The remediation components are turned on only by the opt-in [remediation steps](#graded-remediation), which bring their own datastore rather than `mongodbStore`, `healthEventsAnalyzer` only by the opt-in [analyzer mixin](#health-events-analyzer), and `eventExporter` only by the opt-in [event exporter mixin](#health-event-export). +Two entries above are turned back on by AICR's shipped recipes: `nicHealthMonitor` on AKS and OKE (see [NIC and fabric fault detection](#nic-and-fabric-fault-detection)), and `slurmDrainMonitor` on every `platform: slurm` recipe (see [Slurm Drain Monitor](#slurm-drain-monitor)). The remediation components are turned on only by the opt-in [remediation steps](#graded-remediation), which bring their own datastore rather than `mongodbStore`, `healthEventsAnalyzer` only by the opt-in [analyzer mixin](#health-events-analyzer), `eventExporter` only by the opt-in [event exporter mixin](#health-event-export), and `inclusterFileServer` only by the opt-in [log collector mixin](#diagnostics-capture). **The practical effect.** A stock AICR bundle surfaces GPU faults; it does not act on them. A node that needs a reboot is reported, not rebooted, and an operator intervenes. That is deliberate: `janitor` can reboot or terminate nodes, and enabling it without the operator having chosen to is not a safe default. @@ -1061,6 +1061,51 @@ The name is `event-exporter.oidcSecretName` (the mixin sets `event-exporter-oidc **Deployment validation.** The registry's `nvsentinel` check asserts that the `event-exporter` Deployment is fully rolled out wherever it renders, and passes where it does not. To stop exporting, regenerate the bundle without the mixin and redeploy; events already delivered stay at the sink. +### Diagnostics Capture + +When NVSentinel resets a GPU or reboots a node, the evidence of what went wrong goes with it. The opt-in `nvsentinel-log-collector` mixin captures it first: before `fault-remediation` acts on any action other than `NONE`, it runs a Job on the faulty node that collects an `nvidia-bug-report` and uploads it to an in-cluster file server. Compose it beside the [remediation step](#graded-remediation) and the [health events analyzer](#health-events-analyzer), which live remediation requires; no shipped recipe composes it. + +```yaml +# your-leaf-overlay.yaml +spec: + mixins: + - nvsentinel-remediation + - nvsentinel-health-events-analyzer + - nvsentinel-log-collector +``` + +The mixin turns on two settings together, because neither is useful alone: `fault-remediation.logCollector.enabled`, the collector, and `global.inclusterFileServer.enabled`, the nginx file server that stores and serves what it collects. + +**Order and timing.** `fault-remediation` waits for the collection Job before it creates the repair, so each repair is delayed by the collection, up to the mixin's `logCollector.timeout` of 10 minutes. A Job that fails or times out does not block the repair: `fault-remediation` records the failure and remediates, and a timed-out Job is left running. A Job that cannot be created at all does block it, and `fault-remediation` retries the event. The collector is skipped in dry run, so the `nvsentinel-observe` step collects nothing, and `nvsentinel-quarantine` does not run `fault-remediation` at all. It is also skipped once a node reaches the repair cap (`fault-remediation.maxRemediationAttempts: 3`): `fault-remediation` gives up on the event before collecting, so a fault that keeps firing on a capped node does not start a new collection each time. + +**Supported platforms.** The collector runs `nvidia-bug-report.sh` inside the GPU Operator's driver container on the node, which it finds by label in the one namespace `logCollector.gpuOperatorNamespaces` names (`gpu-operator`; despite the name, it takes a single namespace), or, on GKE COS, from the managed driver install at `/home/kubernetes/bin/nvidia`. It therefore works where the GPU Operator installs the driver (EKS, BCM, LKE, generic, most RKE2 and k0s leaves, and AKS and OKE with `gpuStack=operator-managed`). The collector can also read GKE COS's managed install, but the remediation step it needs is not supported on GKE COS (see [Graded Remediation](#graded-remediation)), so the mixin cannot be used there today. Where the node image carries the driver and the GPU Operator runs no driver container (AKS and OKE at their default `gpuStack`, Kind, the VR200 RKE2 and H200 k0s leaves), every collection would fail, so the bundle does. Custom cluster domains are not supported: the upload URL names `svc.cluster.local`, as the datastore's does. + +**Where bundles land and how to retrieve them.** Each collection is stored on the file server's volume under `/upload///`, as `nvidia-bug-report--.log.gz`. The server lists directories, so browse and download through a port-forward: + +```shell +kubectl -n nvsentinel port-forward svc/nvsentinel-incluster-file-server 8081:80 +# in another terminal; replace the node and timestamp +curl -s http://localhost:8081/upload/ +curl -sO http://localhost:8081/upload/gpu-node-1/20261015-101500/nvidia-bug-report-gpu-node-1-20261015-101500.log.gz +``` + +**Retention and sizing.** A cleanup container runs `find -type f -mtime +7 -delete` (`incluster-file-server.logCleanup.retentionDays: 7`) at start and then every hour (`sleepInterval: 3600`, where the chart's default is daily), so a report is deleted within an hour of being at least eight full days old: `find` counts whole days, so `retentionDays: N` keeps a report for N+1. Only files are deleted; the emptied `//` directories stay. Upstream estimates 5–50 MB per collection, so the 50Gi volume (`persistence.size`) holds at least a thousand of the largest within one retention window. The cleanup container exports `fileserver_disk_space_free_bytes` on the file server's `cleanup-metrics` port (9002), refreshed each cycle; alert on it if collections are frequent. + +**Storage.** The file server claims one `ReadWriteOnce` volume. Its `storageClassName` is registered in `nvsentinel`'s `storageClassPaths`, so `--storage-class` sets it; without the flag the claim takes the cluster's default StorageClass, `aicr bundle` warns, and on a cluster with none the claim stays `Pending` and the file server never starts. AICR provisions no storage. + +**What is not collected.** GPU Operator must-gather is set explicitly off (`enableGpuOperatorMustGather: false`): it collects from every node in the cluster, not the faulty one, so on a large cluster one fault becomes a cluster-wide collection that also outlasts the timeout. The GCP and AWS SOS reports stay at the chart default, off. + +**Privileges and exposure.** + +- The collection Job is privileged and mounts the node's root filesystem read-write at `/host`. +- Its service account, `log-collector-job`, is bound to a ClusterRole that can list pods and namespaces and create `pods/exec` in every namespace, which it needs to run the bug report in the driver container. That is a cluster-wide exec grant. +- An `nvidia-bug-report` contains system configuration. The file server speaks plain HTTP inside the cluster and accepts `PUT` and `DELETE` on `/upload/` from any pod that can reach its Service, with no size limit. RBAC does not govern HTTP to a Service, and the chart's own NetworkPolicy excludes the file server, so restricting who can read or delete reports takes a NetworkPolicy you supply. +- The file server image is `docker.io/bitnamilegacy/nginx`, from Bitnami's frozen archive, which receives no patches; it is scanned with the other mixin images (see [Container Images](https://github.com/NVIDIA/aicr/blob/main/docs/user/container-images.md)). + +**Bundle-time gates.** `CheckNVSentinelLogCollectorCoherent` blocks a bundle with only one of the two settings on; with the collector on but `fault-remediation` off, in dry run, or with a `dryRun` that is not a boolean; without the file server's name pinned by `incluster-file-server.fullnameOverride` (the Service name otherwise follows the Helm release name, which Flux sets to `-`), or with an `uploadURL` that is not that Service; and on a platform where the collector cannot reach the driver, as above. It also rejects `--dynamic` on any path it verifies. The collector's `MOCK_MODE` (exactly `true`), which uploads a synthetic report without a driver, skips the platform rule; AICR's Kind e2e uses it. + +**Health check.** The `nvsentinel` health check fails when the `nvsentinel-incluster-file-server` Deployment renders without a complete rollout, and passes where it is absent. It does not prove a collection succeeds; the Kind e2e (`tests/e2e/nvsentinel-log-collector/`) drives one fault through live remediation and checks the uploaded report and that the repair was requested after it. A collection Job that fails leaves its Failed pods in the `nvsentinel` namespace for an hour (`ttlSecondsAfterFinished: 3600`, up to seven pods at the default retry limit), and the health check's failed-pod rule reports them until then. + ## NVSentinel on Provider-Installed-Driver Platforms The recipes configure NVSentinel for you on every platform that needs it. This section explains what they set and why, so the values are recognizable in a generated bundle and the failure signatures are diagnosable if they ever reappear. diff --git a/docs/user/container-images.md b/docs/user/container-images.md index 36491e82f3..dffd8e16b6 100644 --- a/docs/user/container-images.md +++ b/docs/user/container-images.md @@ -472,6 +472,7 @@ The trade-off is intentional. Pinning an image gives reproducibility; deferring - The [`nvsentinel-slurm-drain-monitor` mixin](component-catalog.md#slurm-drain-monitor) sets `global.slurmDrainMonitor.enabled`, turning on the chart's `slurm-drain-monitor` subchart and pulling in `ghcr.io/nvidia/nvsentinel/slurm-drain-monitor:v1.28.0`. Every `platform: slurm` recipe composes it, so those recipes deploy this image and the others do not; it is in AICR's weekly image scan for the same reason as the object monitor's. - The [`nvsentinel-health-events-analyzer` mixin](component-catalog.md#health-events-analyzer) sets `global.healthEventsAnalyzer.enabled`, turning on the chart's `health-events-analyzer` subchart and pulling in `ghcr.io/nvidia/nvsentinel/health-events-analyzer:v1.28.0`, with no init container on the MongoDB datastore AICR bundles. No shipped recipe composes it; it is required beside the `nvsentinel-remediation` step. It is in the weekly image scan and the mirror lane, and `TestNVSentinelHealthEventsAnalyzerChartRender` pins it against the rendered chart. - The [`nvsentinel-event-exporter` mixin](component-catalog.md#health-event-export) sets `global.eventExporter.enabled`, turning on the chart's `event-exporter` subchart and pulling in `ghcr.io/nvidia/nvsentinel/event-exporter:v1.28.0`, with no init container on the MongoDB datastore AICR bundles. No shipped recipe composes it; it is in the weekly image scan and the mirror lane, and `TestNVSentinelEventExporterChartRender` pins it against the rendered chart. +- The [`nvsentinel-log-collector` mixin](component-catalog.md#diagnostics-capture) sets `global.inclusterFileServer.enabled` and `fault-remediation.logCollector.enabled`, adding five images: the collection Job's `ghcr.io/nvidia/nvsentinel/log-collector:v1.28.0`, which is named only inside `fault-remediation`'s ConfigMap, and the file server's `ghcr.io/nvidia/nvsentinel/file-server-cleanup:v1.28.0`, `docker.io/bitnamilegacy/nginx:1.27.2-debian-12-r2`, `public.ecr.aws/docker/library/busybox:1.37.0` (its init container) and `quay.io/martinhelmich/prometheus-nginxlog-exporter:v1.10.0`. The three third-party tags are the chart's own pins, not AICR's; `bitnamilegacy` is Bitnami's frozen archive and is never patched. No shipped recipe composes it; all five are in the weekly image scan and the mirror lane, and `TestNVSentinelLogCollectorChartRender` pins them against the rendered chart. A recipe composing any of these mixins **with `nvsentinel` still enabled** adds that mixin's images to what it deploys and mirrors; `aicr bundle`/`aicr mirror` on such a recipe surfaces them even though this static BOM cannot. A chain that disables `nvsentinel` (the OCP overlay, for example) can compose a mixin and ship none of them. diff --git a/kwok/scripts/validate-scheduling.sh b/kwok/scripts/validate-scheduling.sh index 7fdb2a3af6..725e91be81 100755 --- a/kwok/scripts/validate-scheduling.sh +++ b/kwok/scripts/validate-scheduling.sh @@ -1050,6 +1050,15 @@ generate_bundle() { --set "dynamoplatform:nats.config.jetstream.fileStore.enabled=false" ) fi + # The nvsentinel-log-collector mixin's in-cluster file server claims a + # ReadWriteOnce volume that KWOK's fake nodes cannot bind, so its pod stays + # Pending; run it on emptyDir, as the Slurm and Dynamo stores above do. + if yq eval '.componentRefs[] | select(.name == "nvsentinel") | .overrides.global.inclusterFileServer.enabled' \ + "${WORK_DIR}/recipe.yaml" 2>/dev/null | grep -qx true; then + conditional_sets+=( + --set "nvsentinel:incluster-file-server.persistence.enabled=false" + ) + fi # No NVSentinel --set remedies here. The recipes assign # labeler.assumeDriverInstalled and, on AKS, diff --git a/pkg/bundler/validations/checks.go b/pkg/bundler/validations/checks.go index 5b88ea62de..6ecdf02896 100644 --- a/pkg/bundler/validations/checks.go +++ b/pkg/bundler/validations/checks.go @@ -68,6 +68,7 @@ func init() { registerCheck("CheckNVSentinelRemediationPipelineCoherent", CheckNVSentinelRemediationPipelineCoherent) registerCheck("CheckNVSentinelSlurmCustomDrainCoherent", CheckNVSentinelSlurmCustomDrainCoherent) registerCheck("CheckNVSentinelEventExporterCoherent", CheckNVSentinelEventExporterCoherent) + registerCheck("CheckNVSentinelLogCollectorCoherent", CheckNVSentinelLogCollectorCoherent) registerCheck("CheckNVSentinelMongoDBCoherent", CheckNVSentinelMongoDBCoherent) registerCheck("CheckNVSentinelDatastorePrerequisites", CheckNVSentinelDatastorePrerequisites) } diff --git a/pkg/bundler/validations/nvsentinel_log_collector.go b/pkg/bundler/validations/nvsentinel_log_collector.go new file mode 100644 index 0000000000..330b504313 --- /dev/null +++ b/pkg/bundler/validations/nvsentinel_log_collector.go @@ -0,0 +1,281 @@ +// Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package validations + +import ( + "context" + "fmt" + "log/slog" + "path" + "strings" + + "github.com/NVIDIA/aicr/pkg/bundler/config" + "github.com/NVIDIA/aicr/pkg/component" + aicrerrors "github.com/NVIDIA/aicr/pkg/errors" + "github.com/NVIDIA/aicr/pkg/recipe" +) + +// Value paths CheckNVSentinelLogCollectorCoherent reads, under nvsentinel and +// the enabled GPU Operator, and the names its findings use. +const ( + logCollectorPath = "fault-remediation.logCollector" + logCollectorEnabledPath = "fault-remediation.logCollector.enabled" + logCollectorUploadURLPath = "fault-remediation.logCollector.uploadURL" + logCollectorNamespacesPath = "fault-remediation.logCollector.gpuOperatorNamespaces" + logCollectorMockModePath = "fault-remediation.logCollector.env.MOCK_MODE" + fileServerPath = "global.inclusterFileServer" + fileServerEnabledPath = "global.inclusterFileServer.enabled" + fileServerNamePath = "incluster-file-server.fullnameOverride" + fileServerPortPath = "incluster-file-server.service.port" + logCollectorMixinName = "nvsentinel-log-collector" + gpuOperatorDriverEnabledPath = "driver.enabled" + gpuOperatorDriverInstallDirPath = "hostPaths.driverInstallDir" + logCollectorDefaultOperatorNS = "gpu-operator" +) + +// CheckNVSentinelLogCollectorCoherent blocks a bundle in which NVSentinel's +// log collector cannot capture or upload anything. The collector and the +// in-cluster file server it uploads to are separate subcharts, and neither +// checks the other or what the collector needs from the node, so every case +// below deploys cleanly. It fails when: +// +// - only one of fault-remediation.logCollector.enabled and +// global.inclusterFileServer.enabled is on: the collector has nowhere to +// upload, or the file server and its volume hold nothing; +// - fault-remediation does not run, or runs in dry run or with a dryRun +// that is not a boolean: fault-remediation launches the collector and +// skips it in dry run, so it never captures. A logCollector.enabled that +// is not a boolean is rejected too: the chart also renders it into +// fault-remediation's --enable-log-collector flag, which parses strictly; +// - the file server's name is not pinned by fullnameOverride, so its +// Service follows the Helm release name, which differs by deployer (Flux +// names the release -), or the upload URL is not that +// Service; +// - the collector cannot reach nvidia-bug-report. It runs the script in the +// GPU Operator's driver container, which it looks up in the single +// namespace gpuOperatorNamespaces names, or from GKE's managed driver +// install, so a recipe whose node image carries the driver otherwise (AKS, +// OKE, Kind) fails every collection. +// +// The collector's MOCK_MODE uploads a synthetic report without a driver, +// which the Kind e2e uses, so it skips the last rule. The collector enters it +// only on the exact string "true", so that is all this reads as mock. +// +// It is a no-op unless either setting is on, and rejects --dynamic on any path +// it verifies. +func CheckNVSentinelLogCollectorCoherent(ctx context.Context, componentName string, recipeResult *recipe.RecipeResult, bundlerConfig *config.Config, conditions map[string][]string) ([]string, []error) { + if recipeResult == nil || !checkConditions(recipeResult, conditions) { + return nil, nil + } + sentinelRef := recipeResult.GetComponentRef(componentName) + if sentinelRef == nil { + return nil, nil + } + provider := recipeResult.DataProvider() + sentinelKeys := componentOverrideKeys(componentName, provider) + if componentDisabled(sentinelRef, bundlerConfig, sentinelKeys) { + return nil, nil + } + values, err := effectiveComponentValues(ctx, recipeResult, bundlerConfig, componentName, sentinelKeys, + "NVSentinel log collector") + if err != nil { + return nil, []error{err} + } + + collectorRaw, _ := component.GetValueByPath(values, logCollectorEnabledPath) + collector := helmTruthy(collectorRaw) + fileServer := nvsentinelSubchartRenders(values, "inclusterFileServer") + mockRaw, mockSet := component.GetValueByPath(values, logCollectorMockModePath) + mock := mockSet && fmt.Sprint(mockRaw) == "true" + gpuOp := enabledGPUOperator(recipeResult, bundlerConfig) + + if msgs := logCollectorDynamicViolations(bundlerConfig, componentName, sentinelKeys, collector, fileServer, mock, + gpuOp); len(msgs) > 0 { + for _, msg := range msgs { + slog.Warn(msg, logKeyComponent, componentName) + } + return msgs, nil + } + var errs []error + fail := func(format string, args ...any) { + errs = append(errs, aicrerrors.New(aicrerrors.ErrCodeInvalidRequest, + fmt.Sprintf("component %q: ", componentName)+fmt.Sprintf(format, args...))) + } + if _, isBool := collectorRaw.(bool); collectorRaw != nil && !isBool { + fail("%s is %v, not a boolean, and fault-remediation parses it strictly as --enable-log-collector -- "+ + "set it to true or false", logCollectorEnabledPath, collectorRaw) + return nil, errs + } + if !collector && !fileServer { + return nil, nil + } + + if !collector { + fail("%s is on without %s, so the file server and its volume hold nothing -- compose the %s mixin, "+ + "which turns on both", fileServerEnabledPath, logCollectorEnabledPath, logCollectorMixinName) + return nil, errs + } + if !fileServer { + fail("%s is on without %s, so every collection fails to upload and is lost -- compose the %s mixin, "+ + "which turns on both", logCollectorEnabledPath, fileServerEnabledPath, logCollectorMixinName) + } + + remediation := nvsentinelSubchartRenders(values, "faultRemediation") + dryRun, dryRunWellFormed := nvsentinelDryRun(values) + switch { + case !remediation: + fail("%s is on but fault-remediation, which launches the collector, does not run -- compose %s with "+ + "nvsentinel-remediation", logCollectorEnabledPath, logCollectorMixinName) + case !dryRunWellFormed: + fail("%s is not a boolean, so whether fault-remediation runs the collector cannot be verified -- set it "+ + "to false", nvsentinelDryRunPath) + case dryRun: + fail("%s is on but fault-remediation runs in dry run (%s=true), which skips the collector -- compose %s "+ + "with nvsentinel-remediation, not nvsentinel-observe", logCollectorEnabledPath, nvsentinelDryRunPath, + logCollectorMixinName) + } + + if fileServer { + name, _, _ := resolvedStringValue(values, fileServerNamePath) + if name == "" { + fail("%s is unset, so the file server's Service is named after the Helm release, which differs by "+ + "deployer (Flux names it -), and %s cannot name it -- compose the %s mixin, which "+ + "pins it", fileServerNamePath, logCollectorUploadURLPath, logCollectorMixinName) + } else { + host := fmt.Sprintf("%s.%s.svc.cluster.local", name, componentNamespace(recipeResult, componentName)) + // A null or 80 port is the chart's default, which the URL may omit. + if port, ok := component.GetValueByPath(values, fileServerPortPath); ok && port != nil && fmt.Sprint(port) != "80" { + host += ":" + fmt.Sprint(port) + } + want := "http://" + host + "/upload" + got, _, valid := resolvedStringValue(values, logCollectorUploadURLPath) + if !valid || strings.Replace(strings.TrimSuffix(got, "/"), ".svc.cluster.local:80/", ".svc.cluster.local/", 1) != want { + fail("%s is %q, not the in-cluster file server's Service, so collections upload elsewhere and "+ + "the file server holds nothing -- set it to %q", logCollectorUploadURLPath, got, want) + } + } + } + + if !mock { + errs = append(errs, logCollectorDriverViolations(ctx, recipeResult, bundlerConfig, componentName, values, + gpuOp)...) + } + return nil, errs +} + +// gpuOperatorRef names the first enabled GPU Operator variant in a recipe. +type gpuOperatorRef struct { + name, namespace string + keys []string +} + +// enabledGPUOperator returns the first enabled GPU Operator variant, or nil. +// OCP recipes carry a disabled gpu-operator beside the gpu-operator-ocp they +// deploy, so the first enabled one is the one that runs. +func enabledGPUOperator(recipeResult *recipe.RecipeResult, bundlerConfig *config.Config) *gpuOperatorRef { + view := declaredUnionView(recipeResult) + for _, name := range gpuOperatorComponentNames { + ref := view.GetComponentRef(name) + if ref == nil { + continue + } + keys := componentOverrideKeys(name, recipeResult.DataProvider()) + if !componentDisabled(ref, bundlerConfig, keys) { + return &gpuOperatorRef{name: name, namespace: componentNamespace(view, name), keys: keys} + } + } + return nil +} + +// logCollectorDriverViolations checks that the collector can run +// nvidia-bug-report: in the GPU Operator's driver container, which it finds by +// label in the namespace gpuOperatorNamespaces names (despite the name, the +// collector passes the whole value to kubectl -n), or from GKE's managed +// driver install, which it reads from the host. +func logCollectorDriverViolations(ctx context.Context, recipeResult *recipe.RecipeResult, bundlerConfig *config.Config, componentName string, values map[string]any, gpuOp *gpuOperatorRef) []error { + unsupported := func(why string) []error { + return []error{aicrerrors.New(aicrerrors.ErrCodeInvalidRequest, + fmt.Sprintf("component %q: %s is on but %s, so every collection fails and no bug report is captured. "+ + "The collector runs nvidia-bug-report in the GPU Operator's driver container, or from GKE's "+ + "managed driver install -- let the GPU Operator install the driver (on AKS and OKE, "+ + "--profile gpuStack=operator-managed), or do not compose %s", componentName, + logCollectorEnabledPath, why, logCollectorMixinName))} + } + if gpuOp == nil { + return unsupported("the recipe deploys no GPU Operator") + } + gpuOpValues, err := effectiveComponentValues(ctx, declaredUnionView(recipeResult), bundlerConfig, gpuOp.name, + gpuOp.keys, "NVSentinel log collector") + if err != nil { + return []error{err} + } + driverToggle, problem := ownershipToggle(gpuOpValues, "driver") + if problem != "" { + // CheckDriverOwnershipCoherence rejects the toggle itself. + return nil + } + if driverToggle != nil && !*driverToggle { + installDir := operatorContainerDriverRoot + if dir, ok, _ := resolvedStringValue(gpuOpValues, gpuOperatorDriverInstallDirPath); ok && dir != "" { + installDir = path.Clean(dir) + } + if installDir == gkeManagedDriverRootPath { + return nil + } + return unsupported(fmt.Sprintf("%s leaves the driver to the node image (%s=false) and runs no driver "+ + "container", gpuOp.name, gpuOperatorDriverEnabledPath)) + } + + namespace := logCollectorDefaultOperatorNS + ns, set, valid := resolvedStringValue(values, logCollectorNamespacesPath) + if !valid { + // The chart quotes whatever it is given, so a list renders as the + // literal "[gpu-operator]" and names no namespace. + raw, _ := component.GetValueByPath(values, logCollectorNamespacesPath) + return []error{aicrerrors.New(aicrerrors.ErrCodeInvalidRequest, + fmt.Sprintf("component %q: %s is %v, not a string, so the collector searches a namespace that does "+ + "not exist and every collection fails -- set it to %q", componentName, logCollectorNamespacesPath, + raw, gpuOp.namespace))} + } + if set && ns != "" { + namespace = ns + } + if namespace != gpuOp.namespace { + return []error{aicrerrors.New(aicrerrors.ErrCodeInvalidRequest, + fmt.Sprintf("component %q: %s deploys to namespace %q, but %s is %q, the one namespace the collector "+ + "searches for the driver container, so every collection fails -- set it to %q", + componentName, gpuOp.name, gpuOp.namespace, logCollectorNamespacesPath, namespace, gpuOp.namespace))} + } + return nil +} + +// logCollectorDynamicViolations returns a violation for each --dynamic path a +// rule of CheckNVSentinelLogCollectorCoherent reads. The two toggles always +// matter; the rest only once either is on. +func logCollectorDynamicViolations(bundlerConfig *config.Config, componentName string, sentinelKeys []string, collector, fileServer, mock bool, gpuOp *gpuOperatorRef) []string { + const reason = "decides whether NVSentinel's log collector runs, uploads to the in-cluster file server " + + "and can reach the driver" + guarded := []string{logCollectorEnabledPath, fileServerEnabledPath} + if collector || fileServer { + guarded = append(guarded, logCollectorPath, fileServerPath, fileServerNamePath, fileServerPortPath, + nvsentinelFaultRemediationEnabledPath, nvsentinelDryRunPath) + } + msgs := nvsentinelDynamicGuardViolations(bundlerConfig, componentName, sentinelKeys, guarded, reason) + if collector && !mock && gpuOp != nil { + msgs = append(msgs, nvsentinelDynamicGuardViolations(bundlerConfig, gpuOp.name, gpuOp.keys, + []string{gpuOperatorDriverEnabledPath, gpuOperatorDriverInstallDirPath}, reason)...) + } + return msgs +} diff --git a/pkg/bundler/validations/nvsentinel_log_collector_render_test.go b/pkg/bundler/validations/nvsentinel_log_collector_render_test.go new file mode 100644 index 0000000000..45dfae3840 --- /dev/null +++ b/pkg/bundler/validations/nvsentinel_log_collector_render_test.go @@ -0,0 +1,200 @@ +// Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package validations + +import ( + "context" + "errors" + "io" + "slices" + "strings" + "testing" + + "gopkg.in/yaml.v3" + + "github.com/NVIDIA/aicr/pkg/helm/helmtest" + "github.com/NVIDIA/aicr/pkg/recipe" +) + +const ( + logCollectorMixinFile = "nvsentinel-log-collector" + // The images docs/user/container-images.md discloses for the mixin. The + // static BOM cannot see a mixin-gated subchart, so these assertions are + // what keep that note honest. + logCollectorImage = "ghcr.io/nvidia/nvsentinel/log-collector:v1.28.0" + fileServerCleanupImage = "ghcr.io/nvidia/nvsentinel/file-server-cleanup:v1.28.0" + fileServerNginxImage = "docker.io/bitnamilegacy/nginx:1.27.2-debian-12-r2" + fileServerInitImage = "public.ecr.aws/docker/library/busybox:1.37.0" + fileServerExporterImage = "quay.io/martinhelmich/prometheus-nginxlog-exporter:v1.10.0" + fileServerName = "nvsentinel-incluster-file-server" +) + +// containerEnv returns a container's literal env values by name. +func containerEnv(container map[string]any) map[string]string { + out := map[string]string{} + list, _ := container["env"].([]any) + for _, e := range list { + entry, _ := e.(map[string]any) + name, _ := entry["name"].(string) + if value, ok := entry["value"].(string); ok { + out[name] = value + } + } + return out +} + +// renderedConfigMapKey returns the value of key in the rendered ConfigMap +// that carries it. +func renderedConfigMapKey(t *testing.T, rendered, key string) string { + t.Helper() + dec := yaml.NewDecoder(strings.NewReader(rendered)) + for { + var doc map[string]any + if err := dec.Decode(&doc); err != nil { + if errors.Is(err, io.EOF) { + break + } + t.Fatalf("decode rendered chart: %v", err) + } + if doc["kind"] != "ConfigMap" { + continue + } + data, _ := doc["data"].(map[string]any) + if v, ok := data[key].(string); ok { + return v + } + } + t.Fatalf("no rendered ConfigMap carries %s", key) + return "" +} + +// TestNVSentinelLogCollectorChartRender renders the pinned nvsentinel chart +// with the remediation step and the log collector mixin composed, and asserts +// what #2619 promises reaches the workloads: every image the docs list, the +// file server's Service at the address the collector uploads to under any +// release name (the render helper's differs from every deployer's), its volume +// size and the storage class --storage-class writes, the cleanup cycle, and a +// collection Job with must-gather off. +// +// Pulls the chart over the network by mutable tag, so it is excluded from +// `make test`/`make qualify` (always -short), same gating as +// TestNVSentinelSlurmDrainMonitorChartRender. Run on demand: +// `go test ./pkg/bundler/validations/... -run TestNVSentinelLogCollectorChartRender`. +func TestNVSentinelLogCollectorChartRender(t *testing.T) { + if testing.Short() { + t.Skip("skipping live chart-render integration test in short mode") + } + helmtest.RequireBinary(t) + + store, err := recipe.LoadMetadataStoreFor(context.Background(), nil) + if err != nil { + t.Fatalf("LoadMetadataStoreFor: %v", err) + } + values := nvsentinelRenderedValues(t, store, "nvsentinel-remediation", logCollectorMixinFile) + // The value --storage-class writes through nvsentinel's storageClassPaths; + // the injection itself is covered by the bundle check in the e2e. + mergeValuesForRender(values, map[string]any{ + "incluster-file-server": map[string]any{"persistence": map[string]any{"storageClassName": "aicr-rwo"}}, + }) + out := renderNVSentinel(t, values) + + // The collector uploads to this Service: same name, namespace and port. + collector, _ := values["fault-remediation"].(map[string]any)["logCollector"].(map[string]any) + uploadURL, _ := collector["uploadURL"].(string) + service := renderedObject(t, out, "Service", fileServerName) + svcSpec, _ := service["spec"].(map[string]any) + // helm template omits metadata.namespace, so the namespace is the one + // every deployer installs nvsentinel into. + registry, err := recipe.GetComponentRegistry() + if err != nil { + t.Fatalf("GetComponentRegistry: %v", err) + } + namespace := registry.Get("nvsentinel").Helm.DefaultNamespace + if want := "http://" + fileServerName + "." + namespace + ".svc.cluster.local/upload"; uploadURL != want { + t.Errorf("logCollector.uploadURL = %q, want the rendered Service %q", uploadURL, want) + } + ports, _ := svcSpec["ports"].([]any) + httpPort := false + for _, p := range ports { + port, _ := p.(map[string]any) + httpPort = httpPort || (port["name"] == "http" && port["port"] == 80) + } + if !httpPort { + t.Errorf("file server Service ports = %v, want http on port 80, which the upload URL omits", ports) + } + + claim := renderedObject(t, out, "PersistentVolumeClaim", fileServerName+"-data") + claimSpec, _ := claim["spec"].(map[string]any) + if got := claimSpec["storageClassName"]; got != "aicr-rwo" { + t.Errorf("file server claim storageClassName = %v, want aicr-rwo from storageClassPaths", got) + } + if got := stringList(claimSpec["accessModes"]); !slices.Equal(got, []string{"ReadWriteOnce"}) { + t.Errorf("file server claim accessModes = %v, want [ReadWriteOnce]", got) + } + requests, _ := claimSpec["resources"].(map[string]any)["requests"].(map[string]any) + if requests["storage"] != "50Gi" { + t.Errorf("file server claim size = %v, want 50Gi", requests["storage"]) + } + + server := podContainers(t, renderedObject(t, out, "Deployment", fileServerName)) + images := make([]string, 0, len(server)) + for _, c := range server { + image, _ := c["image"].(string) + images = append(images, image) + } + slices.Sort(images) + want := []string{fileServerNginxImage, fileServerInitImage, fileServerCleanupImage, fileServerExporterImage} + slices.Sort(want) + if !slices.Equal(images, want) { + t.Errorf("file server images = %v, want %v", images, want) + } + cleanup := containerEnv(server["log-cleanup"]) + if cleanup["LOG_RETENTION_DAYS"] != "7" || cleanup["LOG_CLEANUP_SLEEP_INTERVAL"] != "3600" { + t.Errorf("log-cleanup env = %v, want LOG_RETENTION_DAYS=7 and LOG_CLEANUP_SLEEP_INTERVAL=3600", cleanup) + } + + remediation := podContainers(t, renderedObject(t, out, "Deployment", "fault-remediation")) + if got := containerEnv(remediation["fault-remediation"])["LOG_COLLECTOR_TIMEOUT"]; got != "10m" { + t.Errorf("fault-remediation LOG_COLLECTOR_TIMEOUT = %q, want 10m", got) + } + + var job map[string]any + if err := yaml.Unmarshal([]byte(renderedConfigMapKey(t, out, "log-collector-job.yaml")), &job); err != nil { + t.Fatalf("decode log-collector-job.yaml: %v", err) + } + jobContainers := podContainers(t, job) + jobContainer := jobContainers["log-collector"] + if jobContainer["image"] != logCollectorImage { + t.Errorf("log collector Job image = %v, want %s", jobContainer["image"], logCollectorImage) + } + env := containerEnv(jobContainer) + if env["ENABLE_GPU_OPERATOR_MUST_GATHER"] != "false" { + t.Errorf("log collector ENABLE_GPU_OPERATOR_MUST_GATHER = %q, want false", env["ENABLE_GPU_OPERATOR_MUST_GATHER"]) + } + if env["UPLOAD_URL_BASE"] != uploadURL { + t.Errorf("log collector UPLOAD_URL_BASE = %q, want %q", env["UPLOAD_URL_BASE"], uploadURL) + } + if env["GPU_OPERATOR_NAMESPACE"] != "gpu-operator" { + t.Errorf("log collector GPU_OPERATOR_NAMESPACE = %q, want gpu-operator", env["GPU_OPERATOR_NAMESPACE"]) + } + + // Negative control: the remediation step alone pulls in none of it. + base := renderNVSentinel(t, nvsentinelRenderedValues(t, store, "nvsentinel-remediation")) + for _, image := range []string{logCollectorImage, fileServerNginxImage, fileServerCleanupImage} { + if strings.Contains(base, image) { + t.Errorf("%s renders without the %s mixin", image, logCollectorMixinFile) + } + } +} diff --git a/pkg/bundler/validations/nvsentinel_log_collector_test.go b/pkg/bundler/validations/nvsentinel_log_collector_test.go new file mode 100644 index 0000000000..9577d199cb --- /dev/null +++ b/pkg/bundler/validations/nvsentinel_log_collector_test.go @@ -0,0 +1,430 @@ +// Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package validations + +import ( + "strings" + "testing" + + "github.com/NVIDIA/aicr/pkg/bundler/config" + "github.com/NVIDIA/aicr/pkg/recipe" +) + +// TestCheckNVSentinelLogCollectorCoherent covers each rule, the platforms the +// collector can and cannot reach the driver on, mock mode, and the --dynamic +// guard. +func TestCheckNVSentinelLogCollectorCoherent(t *testing.T) { + t.Parallel() + + const fileServerURL = "http://nvsentinel-incluster-file-server.nvsentinel.svc.cluster.local/upload" + on := map[string]any{"enabled": true} + remediation := func() map[string]any { + return map[string]any{"dryRun": false, "faultQuarantine": on, "nodeDrainer": on, "faultRemediation": on} + } + withServer := func(global map[string]any) map[string]any { + global["inclusterFileServer"] = map[string]any{"enabled": true} + return global + } + // named adds the mixin's pinned file server name. + named := func(r *recipe.RecipeResult) *recipe.RecipeResult { + r.ComponentRefs[0].Overrides["incluster-file-server"] = map[string]any{ + "fullnameOverride": "nvsentinel-incluster-file-server", + } + return r + } + // recipeOf builds nvsentinel with global and fault-remediation.logCollector + // overrides, plus the given GPU Operator refs. + recipeOf := func(global, collector map[string]any, operators ...recipe.ComponentRef) *recipe.RecipeResult { + overrides := map[string]any{"global": global} + if collector != nil { + overrides["fault-remediation"] = map[string]any{"logCollector": collector} + } + refs := make([]recipe.ComponentRef, 0, 1+len(operators)) + refs = append(refs, recipe.ComponentRef{Name: nvsentinelComponent, Overrides: overrides}) + r := &recipe.RecipeResult{ComponentRefs: append(refs, operators...)} + if server, _ := global["inclusterFileServer"].(map[string]any); server != nil { + named(r) + } + return r + } + collector := func() map[string]any { return map[string]any{"enabled": true, "uploadURL": fileServerURL} } + with := func(m map[string]any, key string, value any) map[string]any { + m[key] = value + return m + } + operator := recipe.ComponentRef{Name: "gpu-operator"} + hostDriver := recipe.ComponentRef{Name: "gpu-operator", Overrides: map[string]any{ + "driver": map[string]any{"enabled": false}, + }} + gkeDriver := recipe.ComponentRef{Name: "gpu-operator", Overrides: map[string]any{ + "driver": map[string]any{"enabled": false}, + "hostPaths": map[string]any{"driverInstallDir": gkeManagedDriverRootPath + "/"}, + }} + disabledOperator := recipe.ComponentRef{Name: "gpu-operator", Overrides: map[string]any{"enabled": false}} + // mixin is the nvsentinel-log-collector shape on the remediation step. + mixin := func(operators ...recipe.ComponentRef) *recipe.RecipeResult { + return recipeOf(withServer(remediation()), collector(), operators...) + } + mocked := func() map[string]any { + return with(collector(), "env", map[string]any{"MOCK_MODE": "true"}) + } + dynamic := func(paths ...string) *config.Config { + return config.NewConfig(config.WithDynamicValues(map[string][]string{"nv-sentinel": paths})) + } + + const ( + wantNoServer = "every collection fails to upload" + wantNoCollector = "hold nothing -- compose" + wantNoRemedy = "fault-remediation, which launches the collector, does not run" + wantDryRun = "runs in dry run" + wantDryRunType = "dryRun is not a boolean" + wantUploadURL = "not the in-cluster file server's Service" + wantNoOperator = "the recipe deploys no GPU Operator" + wantHostDriver = "leaves the driver to the node image" + wantOperatorNS = "the one namespace the collector searches" + wantEnabledType = "not a boolean, and fault-remediation parses it strictly" + wantOperatorNSType = "not a string, so the collector searches a namespace that does not exist" + wantUnnamed = "incluster-file-server.fullnameOverride is unset" + wantDynamic = "--dynamic" + wantDynamicCause = "decides whether NVSentinel's log collector runs" + ) + + tests := []struct { + name string + recipeResult *recipe.RecipeResult + bundlerConfig *config.Config + want []string + }{ + { + name: "stock nvsentinel", + recipeResult: recipeOf(map[string]any{}, nil, operator), + }, + { + name: "remediation step without the mixin", + recipeResult: recipeOf(remediation(), nil, operator), + }, + { + name: "mixin on the remediation step with a GPU-Operator-managed driver", + recipeResult: mixin(operator), + }, + { + name: "mixin with GKE's managed driver install", + recipeResult: mixin(gkeDriver), + }, + { + name: "mixin on OCP, where gpu-operator-ocp runs beside a disabled gpu-operator", + recipeResult: mixin(disabledOperator, + recipe.ComponentRef{Name: "gpu-operator-ocp"}), + }, + { + name: "upload URL with a trailing slash", + recipeResult: recipeOf(withServer(remediation()), with(collector(), "uploadURL", fileServerURL+"/"), operator), + }, + { + name: "collector without the file server", + recipeResult: recipeOf(remediation(), collector(), operator), + want: []string{wantNoServer}, + }, + { + name: "file server without the collector", + recipeResult: recipeOf(withServer(remediation()), nil, operator), + want: []string{wantNoCollector}, + }, + { + name: "collector switched off by --set", + recipeResult: mixin(operator), + bundlerConfig: config.NewConfig(config.WithValueOverrides(map[string]map[string]string{"nv-sentinel": {"fault-remediation.logCollector.enabled": "false"}})), + want: []string{wantNoCollector}, + }, + { + name: "mixin on the observe step", + recipeResult: recipeOf(withServer(with(remediation(), "dryRun", true)), collector(), operator), + want: []string{wantDryRun}, + }, + { + name: "mixin with a dryRun that is not a boolean", + recipeResult: recipeOf(withServer(with(remediation(), "dryRun", "false")), collector(), operator), + want: []string{wantDryRunType}, + }, + { + name: "mixin on the quarantine step", + recipeResult: recipeOf(withServer(map[string]any{"dryRun": false, "faultQuarantine": on, "nodeDrainer": on}), + collector(), operator), + want: []string{wantNoRemedy}, + }, + { + name: "upload URL pointing elsewhere", + recipeResult: recipeOf(withServer(remediation()), with(collector(), "uploadURL", "http://bucket.example/upload"), operator), + want: []string{wantUploadURL}, + }, + { + name: "upload URL unset", + recipeResult: recipeOf(withServer(remediation()), map[string]any{"enabled": true}, operator), + want: []string{wantUploadURL}, + }, + { + name: "upload URL that is not a string", + recipeResult: recipeOf(withServer(remediation()), with(collector(), "uploadURL", 8080), operator), + want: []string{wantUploadURL}, + }, + { + name: "nvsentinel in another namespace than the upload URL names", + recipeResult: func() *recipe.RecipeResult { + r := mixin(operator) + r.ComponentRefs[0].Namespace = "gpu-health" + return r + }(), + want: []string{wantUploadURL}, + }, + { + name: "file server name left to the Helm release", + recipeResult: func() *recipe.RecipeResult { + r := mixin(operator) + delete(r.ComponentRefs[0].Overrides, "incluster-file-server") + return r + }(), + want: []string{wantUnnamed}, + }, + { + name: "upload URL naming a file server other than the pinned one", + recipeResult: func() *recipe.RecipeResult { + r := mixin(operator) + r.ComponentRefs[0].Overrides["incluster-file-server"] = map[string]any{"fullnameOverride": "diagnostics"} + return r + }(), + want: []string{wantUploadURL}, + }, + { + name: "file server name dynamic with the mixin", + recipeResult: mixin(operator), + bundlerConfig: dynamic("incluster-file-server.fullnameOverride"), + want: []string{wantDynamic}, + }, + { + name: "no GPU Operator", + recipeResult: mixin(), + want: []string{wantNoOperator}, + }, + { + name: "GPU Operator disabled", + recipeResult: mixin(disabledOperator), + want: []string{wantNoOperator}, + }, + { + name: "driver in the node image (AKS, OKE, Kind)", + recipeResult: mixin(hostDriver), + want: []string{wantHostDriver}, + }, + { + name: "GPU Operator in a namespace the collector does not search", + recipeResult: mixin(recipe.ComponentRef{Name: "gpu-operator", Namespace: "nvidia-gpu-operator"}), + want: []string{wantOperatorNS}, + }, + { + // The collector passes the whole value to kubectl -n, so a list + // finds no namespace at all. + name: "GPU Operator namespace listed among several", + recipeResult: recipeOf(withServer(remediation()), + with(collector(), "gpuOperatorNamespaces", "gpu-operator,nvidia-gpu-operator"), + recipe.ComponentRef{Name: "gpu-operator", Namespace: "nvidia-gpu-operator"}), + want: []string{wantOperatorNS}, + }, + { + name: "GPU Operator namespace given as a list", + recipeResult: recipeOf(withServer(remediation()), + with(collector(), "gpuOperatorNamespaces", []any{"gpu-operator"}), operator), + want: []string{wantOperatorNSType}, + }, + { + name: "GPU Operator namespace given as a number", + recipeResult: recipeOf(withServer(remediation()), + with(collector(), "gpuOperatorNamespaces", 7), operator), + want: []string{wantOperatorNSType}, + }, + { + name: "GPU Operator namespace list in mock mode, where no driver is searched", + recipeResult: recipeOf(withServer(remediation()), + with(mocked(), "gpuOperatorNamespaces", []any{"gpu-operator"}), operator), + }, + { + name: "collector pointed at the GPU Operator's own namespace", + recipeResult: recipeOf(withServer(remediation()), + with(collector(), "gpuOperatorNamespaces", "nvidia-gpu-operator"), + recipe.ComponentRef{Name: "gpu-operator", Namespace: "nvidia-gpu-operator"}), + }, + { + name: "MOCK_MODE false is not mock mode", + recipeResult: recipeOf(withServer(remediation()), with(collector(), "env", map[string]any{"MOCK_MODE": "false"}), hostDriver), + want: []string{wantHostDriver}, + }, + { + name: "MOCK_MODE True is not the collector's mock mode", + recipeResult: recipeOf(withServer(remediation()), with(collector(), "env", map[string]any{"MOCK_MODE": "True"}), hostDriver), + want: []string{wantHostDriver}, + }, + { + name: "MOCK_MODE as a boolean", + recipeResult: recipeOf(withServer(remediation()), with(collector(), "env", map[string]any{"MOCK_MODE": true}), hostDriver), + }, + { + name: "collector toggle that is an empty string", + recipeResult: recipeOf(withServer(remediation()), with(collector(), "enabled", ""), operator), + want: []string{wantEnabledType}, + }, + { + name: "upload URL naming the default port", + recipeResult: recipeOf(withServer(remediation()), with(collector(), "uploadURL", "http://nvsentinel-incluster-file-server.nvsentinel.svc.cluster.local:80/upload"), operator), + }, + { + name: "file server port set to null", + recipeResult: func() *recipe.RecipeResult { + r := mixin(operator) + r.ComponentRefs[0].Overrides["incluster-file-server"].(map[string]any)["service"] = map[string]any{"port": nil} + return r + }(), + }, + { + name: "collector toggle that is not a boolean", + recipeResult: recipeOf(withServer(remediation()), with(collector(), "enabled", "yes"), operator), + want: []string{wantEnabledType}, + }, + { + name: "file server on another Service port", + recipeResult: func() *recipe.RecipeResult { + r := recipeOf(withServer(remediation()), + with(collector(), "uploadURL", "http://nvsentinel-incluster-file-server.nvsentinel.svc.cluster.local:8081/upload"), operator) + r.ComponentRefs[0].Overrides["incluster-file-server"].(map[string]any)["service"] = map[string]any{"port": 8081} + return r + }(), + }, + { + name: "file server on another Service port the URL omits", + recipeResult: func() *recipe.RecipeResult { + r := mixin(operator) + r.ComponentRefs[0].Overrides["incluster-file-server"].(map[string]any)["service"] = map[string]any{"port": 8081} + return r + }(), + want: []string{wantUploadURL}, + }, + { + name: "driver toggle another check rejects", + recipeResult: mixin(recipe.ComponentRef{Name: "gpu-operator", Overrides: map[string]any{ + "driver": map[string]any{"enabled": "no"}, + }}), + }, + { + name: "mock mode needs no driver", + recipeResult: recipeOf(withServer(remediation()), mocked()), + }, + { + name: "mock mode with a host driver", + recipeResult: recipeOf(withServer(remediation()), mocked(), hostDriver), + }, + { + name: "mock mode still needs live remediation", + recipeResult: recipeOf(withServer(with(remediation(), "dryRun", true)), mocked()), + want: []string{wantDryRun}, + }, + { + name: "every rule at once", + recipeResult: recipeOf(with(remediation(), "faultRemediation", map[string]any{"enabled": false}), with(collector(), "uploadURL", "x"), hostDriver), + want: []string{wantNoServer, wantNoRemedy, wantHostDriver}, + }, + { + name: "nvsentinel disabled", + recipeResult: func() *recipe.RecipeResult { + r := mixin() + r.ComponentRefs[0].Overrides["enabled"] = false + return r + }(), + }, + { + name: "collector toggle dynamic on a stock recipe", + recipeResult: recipeOf(map[string]any{}, nil, operator), + bundlerConfig: dynamic("fault-remediation.logCollector.enabled"), + want: []string{wantDynamicCause}, + }, + { + name: "file server toggle dynamic on a stock recipe", + recipeResult: recipeOf(map[string]any{}, nil, operator), + bundlerConfig: dynamic("global.inclusterFileServer.enabled"), + want: []string{wantDynamic}, + }, + { + name: "dryRun dynamic without the mixin", + recipeResult: recipeOf(remediation(), nil, operator), + bundlerConfig: dynamic("global.dryRun"), + }, + { + name: "dryRun dynamic with the mixin", + recipeResult: mixin(operator), + bundlerConfig: dynamic("global.dryRun"), + want: []string{wantDynamic}, + }, + { + name: "upload URL dynamic with the mixin", + recipeResult: mixin(operator), + bundlerConfig: dynamic("fault-remediation.logCollector.uploadURL"), + want: []string{wantDynamic}, + }, + { + name: "GPU Operator driver toggle dynamic with the mixin", + recipeResult: mixin(operator), + bundlerConfig: config.NewConfig(config.WithDynamicValues(map[string][]string{"gpuoperator": {"driver.enabled"}})), + want: []string{wantDynamic}, + }, + { + name: "GPU Operator driver toggle dynamic in mock mode", + recipeResult: recipeOf(withServer(remediation()), mocked(), operator), + bundlerConfig: config.NewConfig(config.WithDynamicValues(map[string][]string{"gpuoperator": {"driver.enabled"}})), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + cfg := tt.bundlerConfig + if cfg == nil { + cfg = config.NewConfig() + } + warnings, errs := CheckNVSentinelLogCollectorCoherent(t.Context(), nvsentinelComponent, tt.recipeResult, cfg, nil) + got := append([]string{}, warnings...) + for _, err := range errs { + got = append(got, err.Error()) + } + if len(got) != len(tt.want) { + t.Fatalf("got %d findings %q, want %d matching %q", len(got), got, len(tt.want), tt.want) + } + for i, want := range tt.want { + if !strings.Contains(got[i], want) { + t.Errorf("finding %d = %q, want it to contain %q", i, got[i], want) + } + } + }) + } +} + +// TestCheckNVSentinelLogCollectorCoherentNilRecipe covers the nil and +// condition-filtered entry paths. +func TestCheckNVSentinelLogCollectorCoherentNilRecipe(t *testing.T) { + t.Parallel() + if w, e := CheckNVSentinelLogCollectorCoherent(t.Context(), nvsentinelComponent, nil, config.NewConfig(), nil); w != nil || e != nil { + t.Errorf("nil recipe: got %v %v, want nothing", w, e) + } + r := &recipe.RecipeResult{ComponentRefs: []recipe.ComponentRef{{Name: "gpu-operator"}}} + if w, e := CheckNVSentinelLogCollectorCoherent(t.Context(), nvsentinelComponent, r, config.NewConfig(), nil); w != nil || e != nil { + t.Errorf("no nvsentinel: got %v %v, want nothing", w, e) + } +} diff --git a/pkg/chainsaw/nvsentinel_check_states_test.go b/pkg/chainsaw/nvsentinel_check_states_test.go index b0619336bc..abea794900 100644 --- a/pkg/chainsaw/nvsentinel_check_states_test.go +++ b/pkg/chainsaw/nvsentinel_check_states_test.go @@ -84,6 +84,7 @@ func TestNVSentinelHealthCheckClusterStates(t *testing.T) { {"Deployment", "janitor-provider"}, {"Deployment", "health-events-analyzer"}, {"Deployment", "event-exporter"}, + {"Deployment", "nvsentinel-incluster-file-server"}, } // pipeline returns every remediation workload fully rolled out, except // that the one named faulty gets status[field] = value (nil omits it, diff --git a/pkg/recipe/nvsentinel_log_collector_mixin_test.go b/pkg/recipe/nvsentinel_log_collector_mixin_test.go new file mode 100644 index 0000000000..19d982f255 --- /dev/null +++ b/pkg/recipe/nvsentinel_log_collector_mixin_test.go @@ -0,0 +1,229 @@ +// Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package recipe + +import ( + "fmt" + "maps" + "os" + "slices" + "strings" + "testing" +) + +const logCollectorMixin = "nvsentinel-log-collector" + +// logCollectorChartImages follow nvsentinel's chart version; the file server's +// third-party images are pinned by the chart's own values instead. +var ( + logCollectorChartImages = []string{ + "ghcr.io/nvidia/nvsentinel/log-collector", + "ghcr.io/nvidia/nvsentinel/file-server-cleanup", + } + logCollectorThirdPartyImages = []struct{ image, tag string }{ + {"docker.io/bitnamilegacy/nginx", "1.27.2-debian-12-r2"}, + {"public.ecr.aws/docker/library/busybox", "1.37.0"}, + {"quay.io/martinhelmich/prometheus-nginxlog-exporter", "v1.10.0"}, + } +) + +// TestMixinNVSentinelLogCollector_Composes pins what the mixin sets, alone and +// beside each remediation step: the collector and the file server together, +// the upload URL, must-gather explicitly off, and deliberate volume and +// cleanup values, with nothing else. +func TestMixinNVSentinelLogCollector_Composes(t *testing.T) { + ctx, store := remediationStore(t) + if _, ok := store.Mixins[logCollectorMixin]; !ok { + t.Fatalf("%s mixin not present; check recipes/mixins/%s.yaml", logCollectorMixin, logCollectorMixin) + } + + combos := make([][]string, 0, 1+len(remediationSteps)) + combos = append(combos, []string{logCollectorMixin}) + for _, step := range remediationSteps { + combos = append(combos, []string{step.mixin, logCollectorMixin}) + } + for _, mixins := range combos { + t.Run(strings.Join(mixins, "+"), func(t *testing.T) { + spec := remediationLeaf(mixins...) + if _, err := store.mergeMixins(ctx, &spec); err != nil { + t.Fatalf("mergeMixins: %v", err) + } + nvsentinel, ok := findComponentRefByName(spec.ComponentRefs, "nvsentinel") + if !ok { + t.Fatal("nvsentinel missing from merged spec") + } + global, _ := nvsentinel.Overrides["global"].(map[string]any) + if server, _ := global["inclusterFileServer"].(map[string]any); len(server) != 1 || server["enabled"] != true { + t.Errorf("global.inclusterFileServer = %v, want exactly {enabled: true}", global["inclusterFileServer"]) + } + + fr, _ := nvsentinel.Overrides["fault-remediation"].(map[string]any) + collector, _ := fr["logCollector"].(map[string]any) + wantCollector := map[string]any{ + "enabled": true, + "uploadURL": "http://nvsentinel-incluster-file-server.nvsentinel.svc.cluster.local/upload", + "enableGpuOperatorMustGather": false, + "timeout": "10m", + } + if !maps.Equal(collector, wantCollector) { + t.Errorf("fault-remediation.logCollector = %v, want exactly %v", collector, wantCollector) + } + + server, _ := nvsentinel.Overrides["incluster-file-server"].(map[string]any) + persistence, _ := server["persistence"].(map[string]any) + if !maps.Equal(persistence, map[string]any{"size": "50Gi"}) { + t.Errorf("incluster-file-server.persistence = %v, want exactly {size: 50Gi}", persistence) + } + cleanup, _ := server["logCleanup"].(map[string]any) + if want := map[string]any{"enabled": true, "retentionDays": 7, "sleepInterval": 3600}; !maps.Equal(cleanup, want) { + t.Errorf("incluster-file-server.logCleanup = %v, want exactly %v", cleanup, want) + } + }) + } + + ref, ok := findComponentRefByName(store.Mixins[logCollectorMixin].Spec.ComponentRefs, "nvsentinel") + if !ok { + t.Fatal("mixin has no nvsentinel componentRef") + } + if got := slices.Sorted(maps.Keys(ref.Overrides)); !slices.Equal(got, []string{"fault-remediation", "global", "incluster-file-server"}) { + t.Errorf("mixin top-level overrides = %v, want [fault-remediation global incluster-file-server]", got) + } + global, _ := ref.Overrides["global"].(map[string]any) + if got := slices.Sorted(maps.Keys(global)); !slices.Equal(got, []string{"inclusterFileServer"}) { + t.Errorf("mixin global overrides = %v, want [inclusterFileServer]", got) + } + fr, _ := ref.Overrides["fault-remediation"].(map[string]any) + if got := slices.Sorted(maps.Keys(fr)); !slices.Equal(got, []string{"logCollector"}) { + t.Errorf("mixin fault-remediation overrides = %v, want [logCollector]", got) + } + if n := len(store.Mixins[logCollectorMixin].Spec.ComponentRefs); n != 1 { + t.Errorf("mixin has %d componentRefs, want only nvsentinel: fault-remediation comes from a remediation step", n) + } +} + +// TestLogCollectorFileServerVolumeTakesStorageClass keeps the file server's +// claim on --storage-class: without it the claim takes the cluster default, +// the shape that left a deploy on a Pending PVC in #898. +func TestLogCollectorFileServerVolumeTakesStorageClass(t *testing.T) { + _, store := remediationStore(t) + registry, err := GetComponentRegistryFor(store.provider) + if err != nil { + t.Fatalf("GetComponentRegistryFor: %v", err) + } + nvsentinel := registry.Get("nvsentinel") + if nvsentinel == nil { + t.Fatal("nvsentinel not found in registry") + } + const path = "incluster-file-server.persistence.storageClassName" + if !slices.Contains(nvsentinel.GetStorageClassPaths(), path) { + t.Errorf("nvsentinel storageClassPaths = %v, want %s: the file server's ReadWriteOnce claim", nvsentinel.GetStorageClassPaths(), path) + } + if slices.Contains(nvsentinel.SharedStorageClassPaths, path) { + t.Errorf("%s is in sharedStorageClassPaths; the claim is ReadWriteOnce", path) + } +} + +// TestNoShippedRecipeAdoptsLogCollector keeps log collection opt-in: no +// overlay composes the mixin, and no resolved recipe turns on either setting. +func TestNoShippedRecipeAdoptsLogCollector(t *testing.T) { + ctx, store := remediationStore(t) + + if slices.Contains(store.Base.Spec.Mixins, logCollectorMixin) { + t.Errorf("recipes/overlays/base.yaml adopts %s; log collection stays opt-in", logCollectorMixin) + } + for name, overlay := range store.Overlays { + if slices.Contains(overlay.Spec.Mixins, logCollectorMixin) { + t.Errorf("overlay %q adopts %s; log collection stays opt-in", name, logCollectorMixin) + } + } + + resolved := 0 + for name, overlay := range store.Overlays { + if overlay.Spec.Criteria == nil { + continue + } + result, err := store.BuildRecipeResult(ctx, overlay.Spec.Criteria) + if err != nil { + t.Errorf("resolving %s: %v", name, err) + continue + } + resolved++ + if _, ok := findComponentRefByName(result.ComponentRefs, "nvsentinel"); !ok { + continue + } + values, err := result.GetValuesForComponentWithContext(ctx, "nvsentinel") + if err != nil { + t.Errorf("%s: GetValuesForComponentWithContext: %v", name, err) + continue + } + global, _ := values["global"].(map[string]any) + if server, _ := global["inclusterFileServer"].(map[string]any); server["enabled"] == true { + t.Errorf("%s resolves with global.inclusterFileServer.enabled = true; it must stay off", name) + } + fr, _ := values["fault-remediation"].(map[string]any) + if collector, _ := fr["logCollector"].(map[string]any); collector["enabled"] == true { + t.Errorf("%s resolves with fault-remediation.logCollector.enabled = true; it must stay off", name) + } + } + if resolved == 0 { + t.Fatal("no shipped recipes were resolved -- the walker is broken") + } +} + +// TestLogCollectorImagesPinnedEverywhere keeps every hand-maintained copy of +// the images the mixin adds in step with what the chart deploys: the two +// NVSentinel images follow nvsentinel's registry pin, and the file server's +// third-party images follow the chart's own values. +func TestLogCollectorImagesPinnedEverywhere(t *testing.T) { + _, store := remediationStore(t) + registry, err := GetComponentRegistryFor(store.provider) + if err != nil { + t.Fatalf("GetComponentRegistryFor: %v", err) + } + nvsentinel := registry.Get("nvsentinel") + if nvsentinel == nil { + t.Fatal("nvsentinel not found in registry") + } + version := nvsentinel.Helm.DefaultVersion + if version == "" { + t.Fatal("nvsentinel registry entry has no defaultVersion") + } + + refs := make([]string, 0, len(logCollectorChartImages)+len(logCollectorThirdPartyImages)) + for _, image := range logCollectorChartImages { + assertScanned(t, image, version) + refs = append(refs, fmt.Sprintf("%s:%s", image, version)) + } + for _, p := range logCollectorThirdPartyImages { + assertScanned(t, p.image, p.tag) + refs = append(refs, fmt.Sprintf("%s:%s", p.image, p.tag)) + } + for _, s := range []struct{ path, why string }{ + {"../../tools/mirror-e2e", "the mirror lane asserts every mixin-gated image"}, + {"../../docs/user/container-images.md", "the static BOM cannot see a mixin-gated image, so it is documented by hand"}, + {"../../pkg/bundler/validations/nvsentinel_log_collector_render_test.go", "the render test asserts each image reaches the chart"}, + } { + data, err := os.ReadFile(s.path) + if err != nil { + t.Errorf("reading %s: %v", s.path, err) + continue + } + for _, ref := range refs { + if !strings.Contains(string(data), ref) { + t.Errorf("%s does not pin %s -- %s; bump it alongside the chart", s.path, ref, s.why) + } + } + } +} diff --git a/pkg/recipe/testdata/catalog_parity_golden.yaml b/pkg/recipe/testdata/catalog_parity_golden.yaml index 27f88789d8..cfea9405f3 100644 --- a/pkg/recipe/testdata/catalog_parity_golden.yaml +++ b/pkg/recipe/testdata/catalog_parity_golden.yaml @@ -3,73 +3,73 @@ # # One entry per leaf overlay: sha256 of its deterministically-marshalled # resolved recipe. A moved digest means that recipe's resolved bytes changed. -a100-aks-ubuntu-training-kubeflow: 0c51f648e6a5dff5030f533a35ffddf8c722d57b7e02aea4b7e7c1d8b5c4f594 -a100-any: eb5d7e9661d6953fcbcffab21dd5373151c33a182c7236c59ca04e952760b6c0 -a100-eks-ubuntu-training-kubeflow: 35697974d51c44ab75d3c237fd656f745df515a08431d9a2365b8d919eed6502 -a100-gke-cos-training-kubeflow: b6d0dc83941a528f7a39cefad46205571a71b6f493c7ed1f3e7ee29325878834 -a100-oke-ubuntu-training-kubeflow: ae0745cb4fbe9b1fb1c4b91b72e4e73bfa92b242eace23b736851950b8608de9 -aks-ubuntu: d226fb7c0f20e4e22437d360b00b0bac828a018ee5cb2ea67dc85830427380c3 -b200-any: 41794c1de2e1dc41bf4551504cc61464d3299cf9a485454b3962849e319b420b -b200-gke-cos-inference-dynamo: 24c7c168fda0c016c080b14ceed49f1d09047d63e45e30480f3da77e58ce457b -b200-gke-cos-training-kubeflow: fc5d8b8b5177adec8eb1f0bff4d64de359d291dab2be122bd057178bb90a295d -b300-mk8s-ubuntu-inference-dynamo: 2e919d1bb4a3dc5f9f66eebe7dade38978ac2c6094fe210dc337527d42bda603 -bcm-inference: 63a555254442738ea6f63f95f52827d89b5452a4f6163609ba534a36433dd3dd -eks-ubuntu: 963ef38010e99d0a03258f881e14629cda48273783f791b6df8c065190e4bc03 -gb200-any: bc077aafac8fc519f9e7f053676af2f920d639a3fd3b7ada5042d5d5f49a2819 -gb200-eks-amazonlinux-inference: 7b77feefcf7e6f484366af74a958efbe3a3e370c54e7e75d9968834703b173c3 -gb200-eks-ubuntu-inference-dynamo: ab2d98e29237c9b28fcf8a3cb82d48a9d57093962b6a9c22dccf693ff53ce738 -gb200-eks-ubuntu-training-kubeflow: a46df291f6b1a9ffd8d8f2a18eefcd79c4455c288885a2dc3dda132fc6b4ffef -gb200-eks-ubuntu-training-slurm: 3e5afb5d346e0f49cbf5f55a691e424999db76490f7dca1c187d92cf1dd94640 -gb200-gke-cos-inference-dynamo: 973ae3f3ed286577d7b9f0611f7c2b8c224c816a0b2db90e1a2d542a1d65b60e -gb200-gke-cos-training-kubeflow: 59a3433e29b0af052bc1f4a6a739352844542a6cf7a8f03b7d553904872b3490 -gb200-gke-cos-training-slurm: f00c19c2595d5866a17101245795f4a0b7822733bc0f023b7c5daa790ff62248 -gb200-oke-ubuntu-inference-dynamo: e4e97a066f521b8c728f95aedac638a8d2b8c74c08c7a27f9e0492a417ec11ae -gb200-oke-ubuntu-training-kubeflow: e27a7675203ab121b2d37e7d3b0cc7a4ec2767b89edfc30b9b7050a77d72c766 -gb300-any: afb128dac1a60500e483752097a8254d2d187d96af2aee61fca7e6a35921fcc0 -gb300-eks-amazonlinux-inference: b2b545756361549609a2702a1bb9af09a2613ef940ea1c3cdda185597fe1aea9 -gb300-eks-ubuntu-inference-dynamo: 0c618d586e346793631bb2c2aa92f6df8130006d7266fe5d5012849425bf5b94 -gb300-eks-ubuntu-training-kubeflow: 06c648c8d9dbd495635e307e89fdd5c1841dfc6da76e1079da9068238318121f -gb300-eks-ubuntu-training-slurm: a660f7ec95cd75f3badb4661cdb6067f92ebd15ba3439b58e9623f076b90e6fc -gb300-generic-ubuntu-training: 32464d76ce634d812bb1b1ce5ef7e65ee7f062aecfe2b47b05b571d95bbe97eb -gb300-gke-cos-inference-dynamo: f401feea88bb46a3f66459d95678b248e9aa8386d64cd2930a9a15435a2a686a -gb300-gke-cos-training-kubeflow: 15c171551dd4207c07435ee430d9dc52585f562a3f4be901d5543e1d7628f41d -h100-aks-ubuntu-inference-dynamo: f12276d893bfa3fc5eda18441a14d10b011ee283b25b6d12a3b0af73dddc22a7 -h100-aks-ubuntu-training-kubeflow: 8d46cd66d818f707d6205bc76580578df3bc4180b62e4cc0d96b457994355e62 -h100-aks-ubuntu-training-slurm: 092ed890e00ab4d2fe90f53d86ee360d80049d804d2dd0eac1eee0b5ddbde0e2 -h100-any: 0fbb1fcf566452f8f658a39586ed829493eb8077aeb2611694ae458a75ba3a14 -h100-bcm-ubuntu-training-kubeflow: c99c87468c84ab83922d99a305f97166350e9d0a5be2887111076146e0c97629 -h100-eks-amazonlinux-inference: 3821d23a0baa67e2378d8e387e1c3f5e065cdbcd8954733b505af5ad74e3f69e -h100-eks-ubuntu-inference-dynamo: 4a856ca2333bd64f167e08cf506c66573ddd1a61c3963098458fe971188a7d38 -h100-eks-ubuntu-inference-nim: 1f3ad28976aa546812621a21e58b83848ec101e60cc3e193f423012f88d96d57 -h100-eks-ubuntu-training-kubeflow: da3e468053129206585e957ab46ce20168e6f22ff98b14d1d9d6bdbfe5918a3e -h100-eks-ubuntu-training-kueue: e6308603d2d6c34aa3f299ab838df7ccb428564cd8b2b4e2c211a613d7d66a3e -h100-eks-ubuntu-training-slurm: 6ab2a88c43720842441693d36870fe347a8b0f1da077cd8e96311fa07b4d4ff4 -h100-gke-cos-inference-dynamo: 7ed7dea541b3d1e07ff4e3b7d39862f58b1082c57d1ec3f4d0433e6913fd2d65 -h100-gke-cos-training-kubeflow: 1cf5c5f50ab9471ac4d435c3cad0e87d14b474bb5a1bd1ffdfb03e98ed38c0bf -h100-gke-cos-training-kueue: ee23253c270026082bbf10bbed1c9c343eb99a16afa928f259e7e1c14c232b84 -h100-gke-cos-training-slurm: ac0acf992d9388816c8a39b445214bd756528a989192067114580c8f45f478aa -h100-kind-inference-dynamo: 364551ab4dbc7b8ae4342c18094b68b62ab8dbea76363de5fe744ab4c038ddd0 -h100-kind-training-kubeflow: b7a95fd36a34b94f9dd8ac242e84f0ba490e650f5743dfb9b64224d7ac1c344d -h100-kind-training-kueue: 9abd7c1a4638add34571dd529a4f28e87fa18398468813df3683e78ff892e443 -h100-kind-training-slurm: bb06ddec30b93be8e820394dd0c6bf52b90ea22112977f808e98278b0e7ae947 -h200-any: 5c2a67687fc5da0fc1fe6e634373d749316ed03c08dfd2392054876e51e174cf -h200-eks-inference: fb612aa915dafeabac6ddac1c4a7a023969db0d1811f16b44c0562852caccb10 -h200-eks-training-kubeflow: e9edb955631a88a6a3a5756dd12ffbea748cfb77ae070030b0c674d32540dc19 -h200-k0s-ubuntu-training: 88b6ea10acbe6af59f075682e0fc79654d3f3efc238a5ce897b981ad74eac432 -l40-any: 4c1cd703763d137dca2825c5df06e057f9223d35a2d683d2747bd63ac2b35800 -l40s-any: 8aa35c189c1d5245bb65035c92a03ef544d2c7cf0676914e28c625b018f60abc -l40s-oke-inference: 34ffc7247cd2ef228e492463cdb2c14264529ea159d86b50b81e75aad41f0d08 -l40s-oke-training-kubeflow: 15c744d9f79b74cd29c2591c5da45a006c9f6414921814a5d263136f82f87f7e -monitoring-hpa: 8fd76edc73603362dce9f9939c5245d605ace8991f66fab462fb98b96fe96c45 -ocp-inference-nim: 314c5fbdbe50d8ce1f8a7777254c750bf974b4e74d016b787dad5caab823a7fc -ocp-training: 3a9603a366ea9f62599ddeac7d15ec498eef938769d82482a8b64511734249e6 -rke2-sles: bc31a0fa0f8afdbbfd03456f66eb257277302d45df0bf78e35c6ad19f375bea9 -rtx-pro-6000-any: 300e0621e72ac4a5a3ffca11d11ae69f35b8abecc6895c78f67dcb555b576df3 -rtx-pro-6000-eks-amazonlinux-inference: dd10788c617d4c3cb03b6f8899c540d5ea889195a0cc31c55d15eec26977fce1 -rtx-pro-6000-eks-ubuntu-inference-dynamo: 951c841444a276b1969fb2daf77a31aede2a10a4fe704dad6b51b07ef573ff22 -rtx-pro-6000-eks-ubuntu-inference-nim: cf5f625e83e943dc70a730f4aa6c27fcf58713f9d36ab8418acd6f35e2c53d34 -rtx-pro-6000-eks-ubuntu-training-kubeflow: 14440b7bd6ee552a940d0a2cb877aa7a6ea2823d158c79e956053919505eb6d4 -rtx-pro-6000-lke-ubuntu-inference: 433c954ea7491d4e078ce9305bbb57f38836d0df965abf820cf8d3e5cae03689 -rtx-pro-6000-lke-ubuntu-training-kubeflow: 4484a86200d14888ae01faeee6b2ed3b9ff0cf79d9f5f4600c369d2323a0109d -vr200-rke2-ubuntu-inference-dynamo: 39298769d704d2aa53cbd2fa008f4f49b21ba4fff7520ac52de96e5a3792345a -vr200-rke2-ubuntu-training-kubeflow: 25313116f972e8d8397cd414694deb76c2632b617d12ed2fc73d26d976f6674a +a100-aks-ubuntu-training-kubeflow: 17c8e7c69fe3ee388d89f69d2dfce9a86573a7a1785c2113675dbcbfcc631b5a +a100-any: f32eb5257ba7d9c3b6683d4b14301522851f0f51820785f333c65061d6432c40 +a100-eks-ubuntu-training-kubeflow: 33348c739b0c24b7457535018d7f63b1da8d2a6d1008edb6e87876ca45171aef +a100-gke-cos-training-kubeflow: 81849bd3b581676ad6a8869c9c9128f169e11c139235045c888b7c150d7a119f +a100-oke-ubuntu-training-kubeflow: 9dc2e8ba05b54cd6c880acdad6d821967e78421c98b28b8eee2ce744ac1d240e +aks-ubuntu: bebad113e2cc1ef375289095ea82c3e43c66fcd489681256f110cb3a0e1f8580 +b200-any: 4941146c7edc44d41021c53a482594244f3c25993145ff5aeb5da0a6b164e21e +b200-gke-cos-inference-dynamo: 8d15bf10cec089a7143c6df8e87f82bc41a0cb274d2ddfc677cb28aa29a4b606 +b200-gke-cos-training-kubeflow: 2381973d89ca29f86c02a95bb774e7d9b6313cf662984b7ffc3b51e5845b0b20 +b300-mk8s-ubuntu-inference-dynamo: 8dde34c018bf13ab199999b430712884b3c0276669c6fc8832ef47dbc7a0f14d +bcm-inference: fdc540e09dd9a86175112292a75aa402f0cd06b5e37366b28319aff3d3254a06 +eks-ubuntu: 8f17962605986947de4b9a9fb47c54a2cb6ba8ec29e2c8637efbd17735196e92 +gb200-any: 8d8a36b7a94373f25913410247e10bd62252f5e5c404fb2c407dcea785204331 +gb200-eks-amazonlinux-inference: 9bc2287e85b280efcea73f166dc2f75e9d13127dfeefbca55328419067cc884b +gb200-eks-ubuntu-inference-dynamo: 4430b84cf647ad1be5546ec6dd76ca3adde34448f41a45cbf62c75bf5320cba1 +gb200-eks-ubuntu-training-kubeflow: fbb24488efc20dd208ce467fe66b23fe9d4db7d3a2f366cb1618afa09e96a7d0 +gb200-eks-ubuntu-training-slurm: 349f9a2c806f4657cb25d76f4abf27c731367462097da1d7b5bc84f53bdc4993 +gb200-gke-cos-inference-dynamo: bc236b2b2392b78d11d36d8b017d5fbb5dc42cf64ad79ba5bd66e662f9f0bfdc +gb200-gke-cos-training-kubeflow: 7aeee3cc776b61c195dd2946aa1c5515acea147130ba29369e1d22b86e5bcc44 +gb200-gke-cos-training-slurm: 1bc31c7c666d5dec3dc40be1400559b859c12680f60cbd4a7dfdb1f19ed2dfda +gb200-oke-ubuntu-inference-dynamo: 2f1a7dcb23915681a83006588072a4152d059fce70ff17d7165311f1d23734d8 +gb200-oke-ubuntu-training-kubeflow: 2d1c852128de965abec0602e0e2a6c0a081a38a93a3c6a1518ac79c5a586205a +gb300-any: a9fbe2160b9b7764734ef26f1bcfa57d380e2077d8b8a20f9e7ee743d122ea10 +gb300-eks-amazonlinux-inference: e2165dfcaabe695d74d8b7f9025f86f9c03af63f2f1ac8c86195d89cb4d63d57 +gb300-eks-ubuntu-inference-dynamo: 9ceaa00f5ff6e975d3113abb23b078d3c5c7f0396e2acd1302e9a305f00f7b5e +gb300-eks-ubuntu-training-kubeflow: 585ac71ab9366f522b11afb1b7aa05fb9f62d09958bca8dea199f46230c7cc36 +gb300-eks-ubuntu-training-slurm: 232caf791561834b30788fed4372e88333441ccc451d31d4edc1b88e170a8393 +gb300-generic-ubuntu-training: b2951e46aeffe5bc500af605bd8b980734de841595da5f5edbb6b3cfa87492d0 +gb300-gke-cos-inference-dynamo: 9b221a9ef97ff89575cc69e86feaecb7c3898145a003159ce09fee8e2e400c38 +gb300-gke-cos-training-kubeflow: 48d6a867ebf82474f95aa00367ea5763455d91f9b95498dea10af74cb291b12f +h100-aks-ubuntu-inference-dynamo: b004ffda83d4802ebbf65056a8b11c701da9d43a81a08c18754abbc3f92385e1 +h100-aks-ubuntu-training-kubeflow: 6c227f7ff64175d7d44fddc0a9d0563606fe19897a0679f807998691993e0378 +h100-aks-ubuntu-training-slurm: d49d24573b1d66d943bd1eff87d56a5d05473b3f9d044c8028f7cf13c7fbf4a3 +h100-any: 0701c87b92dd4abc43827dedc065e925be236b86bd4b8f8d8150235e7c18179a +h100-bcm-ubuntu-training-kubeflow: f3685cfc3a32352ebb6da505b6abe9c08d52ba0d402e27a949a4deeb3cfa7d67 +h100-eks-amazonlinux-inference: 6051e297b3a054d8849fe80a9c9636f2fbe880d0e1e914a5b4209a6c6a85c8d7 +h100-eks-ubuntu-inference-dynamo: d48b4dfaf7ca1f517da348a2074dab93371d3a259c41d5f9a9402788bf41a05b +h100-eks-ubuntu-inference-nim: 54927ef5169a3bf43d4e2a48d2cbee71bf1e79e0f15ab5e4c02be4543eb98a08 +h100-eks-ubuntu-training-kubeflow: e8b7b822f5e2ddc8a7bb6081081ff60a5f3db9bb61428c60ddd8e733ffa38e1f +h100-eks-ubuntu-training-kueue: 4e5e91e319473c278b47ee00187e6a358782a367318d1df2e6cf064586b0529f +h100-eks-ubuntu-training-slurm: 0321d7dd61d483e7897e3adb1337b2862545747bd698ee0ea2a112fc16fd2513 +h100-gke-cos-inference-dynamo: 87d3a8b8fba4f9fa472a656769af8d308542ca6da0afd16df57e6516811c05af +h100-gke-cos-training-kubeflow: 8bd97bbf5d96fae6d0a566f29bf64f1fe5b33f411195772f78cbe73b58a67248 +h100-gke-cos-training-kueue: 108404f2753ee3c8162dea0fc4a61154bf0b6ab98cb5b114079873f8f450f587 +h100-gke-cos-training-slurm: ac955624bc5d3e45cfad73ac676ad0db17e5bd741dfb6b8467b1b7247b09aaa6 +h100-kind-inference-dynamo: 82861e3beb26bab95e97faf8ab5ce78bb5adbbc3e5bb14111ba42c5b2377ccae +h100-kind-training-kubeflow: fc30d2a0bff98bffc807263fba1797fd87fb977a653b1c5d4fab5ee496821a3e +h100-kind-training-kueue: 077c041935ef24eb08164aaa9bb1888d2171b99f68f931d7add75cbed0248d1c +h100-kind-training-slurm: 32cef8be43061fa2aa6feb45938492bfc00e6e3c9c4f73f3286cc6bfd440b29c +h200-any: 4e7ca085808da786159e3679d4f8c4c63eb4e0e58a4cd43ef4196d148fb508a7 +h200-eks-inference: e86cbfad3845c729c8c84bf2884a389a3674448680f710d13a4a1eebc4d1464a +h200-eks-training-kubeflow: f40643022f2885a1d03dd5116344c030f9db16ad143d96bdc4890735bc43e03f +h200-k0s-ubuntu-training: 666bcbb05a8f5a8a2a048c3c7c3c80138a12becc62d7e252acc6ce806160d222 +l40-any: 3ca5ea03ea70a2b9e5f46a17eaa6171ca3aac1603c9af563089fe68a93242987 +l40s-any: d7eec5663814d96f6d3cde5e81ce8ca85e5f02eb5a6640c8cf3d4fe12fce0c8d +l40s-oke-inference: 797762a0d9ac80280027fe31edb601ca4669e2564c874bbd859bbf37cbb955a9 +l40s-oke-training-kubeflow: 76477313dc275e5158f7a670f0824d1433921ba320f2fea4ddd0bcc14e1c8265 +monitoring-hpa: 1edd83687e7a4602cdedf8140f82a114b9ef94110e39628db6be0e18fecff2d6 +ocp-inference-nim: de2306c6ec1f737a0a3164bb3494b143a9d0b6a745034b92919ca526b53540c3 +ocp-training: 6b9e63b7e51acd0a86ac9a7464cdf3490a0e587bf3b8a87027e0df269436ba1a +rke2-sles: 85513e75b290255b6301c3d7ebb31471e4a685973efef5e0abe0c1e5304bdbcd +rtx-pro-6000-any: b994432c0909cd065937623f439b2ae45e941ade9ed4486e3d7b4a26badc449d +rtx-pro-6000-eks-amazonlinux-inference: 444dae5e5a14ced9ff9392cd9c8b3dd3159e509216e0b8bfd62c20470b6189d3 +rtx-pro-6000-eks-ubuntu-inference-dynamo: dcf990b0c2a71b5d5c810b288a00c27c460ffe20740ed9867426a7239591a30e +rtx-pro-6000-eks-ubuntu-inference-nim: 5550d4b7e8b3acc62d9c531604b2fd3c16edb3fe7687a4c50651de1833f07735 +rtx-pro-6000-eks-ubuntu-training-kubeflow: c7402aa83cf1aded86e64cf7e44e6b48195fa10b921c9cb1d8a3c2340fdbafd2 +rtx-pro-6000-lke-ubuntu-inference: d916cb9a2a760f62c741ed1f29c6c6c9991e4d7ba53a9119c8de06370d3f69d9 +rtx-pro-6000-lke-ubuntu-training-kubeflow: 495ee0c7bdddee92a2b3e450dca10d6cb7e8ca604b7e51b8a80ccbfddcb69d85 +vr200-rke2-ubuntu-inference-dynamo: 0b7f513e38f4555b542081449010dfade3a71edcfca8bd1762457a898d4cd1d4 +vr200-rke2-ubuntu-training-kubeflow: fa105629371df4577ca7f9076ca9d8e534cf1181b67aee52bb52ffa801e6bc66 diff --git a/recipes/checks/nvsentinel/health-check.yaml b/recipes/checks/nvsentinel/health-check.yaml index 3e2a309260..61d180c848 100644 --- a/recipes/checks/nvsentinel/health-check.yaml +++ b/recipes/checks/nvsentinel/health-check.yaml @@ -234,11 +234,11 @@ spec: try: # The remediation workloads the nvsentinel-observe, -quarantine and # -remediation mixins turn on, the health-events-analyzer the - # nvsentinel-health-events-analyzer mixin adds, and the - # event-exporter the nvsentinel-event-exporter mixin adds; their - # datastore is checked by nvsentinel-mongodb's own health check. - # Negative form, as above: off in every shipped recipe, so each op - # passes on a 404 and + # nvsentinel-health-events-analyzer mixin adds, the event-exporter the + # nvsentinel-event-exporter mixin adds, and the in-cluster file server + # the nvsentinel-log-collector mixin adds; their datastore is checked + # by nvsentinel-mongodb's own health check. Negative form, as above: + # off in every shipped recipe, so each op passes on a 404 and # fails only when the workload renders without a complete rollout. # The status counters are omitempty, hence the `|| 0`. # updatedReplicas catches pods still on the previous revision, and @@ -468,6 +468,38 @@ spec: name: event-exporter namespace: nvsentinel ((status.observedGeneration || `0`) < metadata.generation): true + - error: + resource: + apiVersion: apps/v1 + kind: Deployment + metadata: + name: nvsentinel-incluster-file-server + namespace: nvsentinel + ((status.readyReplicas || `0`) < spec.replicas): true + - error: + resource: + apiVersion: apps/v1 + kind: Deployment + metadata: + name: nvsentinel-incluster-file-server + namespace: nvsentinel + ((status.updatedReplicas || `0`) < spec.replicas): true + - error: + resource: + apiVersion: apps/v1 + kind: Deployment + metadata: + name: nvsentinel-incluster-file-server + namespace: nvsentinel + ((status.replicas || `0`) > spec.replicas): true + - error: + resource: + apiVersion: apps/v1 + kind: Deployment + metadata: + name: nvsentinel-incluster-file-server + namespace: nvsentinel + ((status.observedGeneration || `0`) < metadata.generation): true - name: validate-all-pods-healthy try: # Assert no pods are in unhealthy phases (Pending / Failed / diff --git a/recipes/mixins/nvsentinel-log-collector.yaml b/recipes/mixins/nvsentinel-log-collector.yaml new file mode 100644 index 0000000000..c57c6e2178 --- /dev/null +++ b/recipes/mixins/nvsentinel-log-collector.yaml @@ -0,0 +1,76 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Turns on NVSentinel's log collector and the in-cluster file server it uploads +# to, together: the collector has nowhere to upload without the server, and the +# server has nothing to hold without the collector (#2619). See +# docs/user/component-catalog.md#diagnostics-capture. +# +# Before fault-remediation acts on any action other than NONE, it runs a +# privileged Job on the faulty node that captures an nvidia-bug-report and +# uploads it, then remediates once the Job finishes, fails or times out; a Job +# that fails or times out does not block the repair. The collector is skipped +# in dry run, so compose this mixin beside nvsentinel-remediation. +# CheckNVSentinelLogCollectorCoherent rejects it without live remediation, with +# only one of the two settings on, with an upload URL that is not this file +# server, and on a platform where the collector cannot reach the driver: it +# runs nvidia-bug-report in the GPU Operator's driver container, or from GKE's +# managed driver install. +# +# The file server's volume is registered in nvsentinel's storageClassPaths, so +# --storage-class reaches it; without it the claim takes the cluster default. +# +# No shipped recipe composes this mixin (TestNoShippedRecipeAdoptsLogCollector). + +kind: RecipeMixin +apiVersion: aicr.run/v1beta1 +metadata: + name: nvsentinel-log-collector +spec: + componentRefs: + - name: nvsentinel + overrides: + global: + inclusterFileServer: + enabled: true + fault-remediation: + logCollector: + enabled: true + # The file server's Service, named by fullnameOverride below. + uploadURL: "http://nvsentinel-incluster-file-server.nvsentinel.svc.cluster.local/upload" + # Off: must-gather collects from every node in the cluster, not the + # faulty one, so on a large cluster one fault becomes a cluster-wide + # collection that also outlasts the timeout below. + enableGpuOperatorMustGather: false + # Remediation waits for the collector up to this long. The chart + # sizes it for an nvidia-bug-report alone. + timeout: "10m" + incluster-file-server: + # The Service name otherwise follows the Helm release name, which + # differs by deployer (Flux names the release -), and + # uploadURL names this Service. + fullnameOverride: nvsentinel-incluster-file-server + persistence: + # Upstream estimates 5-50 MB per collection, so 50Gi holds at least + # a thousand of the largest within one retention window. + size: 50Gi + logCleanup: + enabled: true + # Long enough to reach a fault from the previous week, short enough + # that the volume does not fill between cleanups. + retentionDays: 7 + # Hourly rather than the chart's daily: each cycle also refreshes + # fileserver_disk_space_free_bytes, which would otherwise be up to + # a day stale, and an expired file goes within the hour. + sleepInterval: 3600 diff --git a/recipes/registry.yaml b/recipes/registry.yaml index 8314330e5f..b0c4b59e2b 100644 --- a/recipes/registry.yaml +++ b/recipes/registry.yaml @@ -518,9 +518,10 @@ components: # Opt-in surface for the nvsentinel-observability (#2617), # nvsentinel-object-monitor (#2612), nvsentinel-preflight (#2610), # nvsentinel-nic-health-monitor, nvsentinel-slurm-drain-monitor (#2611), - # nvsentinel-observe, -quarantine and -remediation, and + # nvsentinel-observe, -quarantine and -remediation, # nvsentinel-slinky-drainer (#2615), nvsentinel-health-events-analyzer - # (#2618) and nvsentinel-event-exporter (#2621) mixins: the exact leaf + # (#2618), nvsentinel-event-exporter (#2621) and nvsentinel-log-collector + # (#2619) mixins: the exact leaf # paths a mixin may set on this # already-base-chained component. global.tracing.endpoint is # deliberately absent -- it must come from outside the mixin (a leaf @@ -641,6 +642,20 @@ components: - event-exporter.exporter.sink.insecureSkipVerify - event-exporter.exporter.oidc.insecureSkipVerify - event-exporter.exporter.filter.expression + - global.inclusterFileServer.enabled + - fault-remediation.logCollector.enabled + - fault-remediation.logCollector.uploadURL + - fault-remediation.logCollector.enableGpuOperatorMustGather + - fault-remediation.logCollector.timeout + - incluster-file-server.fullnameOverride + - incluster-file-server.persistence.size + - incluster-file-server.logCleanup.enabled + - incluster-file-server.logCleanup.retentionDays + - incluster-file-server.logCleanup.sleepInterval + # The in-cluster file server's ReadWriteOnce volume, which the + # nvsentinel-log-collector mixin turns on (#2619). + storageClassPaths: + - incluster-file-server.persistence.storageClassName validations: # Blocking: the NVSentinel labeler decides # nvsentinel.dgxc.nvidia.com/driver.installed by watching for a GPU @@ -734,6 +749,12 @@ components: # chart's placeholders (#2621). The rules are in the function's doc. - function: CheckNVSentinelEventExporterCoherent severity: error + # Blocking: a log collector that cannot capture or upload anything -- + # without the file server, without live remediation, uploading + # elsewhere, or where it cannot reach the driver (#2619). The rules are + # in the function's doc. + - function: CheckNVSentinelLogCollectorCoherent + severity: error # NVSentinel's remediation datastore, added only by the nvsentinel-observe, # -quarantine and -remediation mixins. Bundled apart from NVSentinel's own diff --git a/tests/e2e/nvsentinel-log-collector/run.sh b/tests/e2e/nvsentinel-log-collector/run.sh new file mode 100755 index 0000000000..9403b07563 --- /dev/null +++ b/tests/e2e/nvsentinel-log-collector/run.sh @@ -0,0 +1,483 @@ +#!/usr/bin/env bash +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# ============================================================================= +# nvsentinel-log-collector mixin e2e +# ============================================================================= +# +# Installs the remediation step with the log collector mixin, and the health +# events analyzer that live remediation requires, through the bundle's own +# install.sh on Kind, sends one synthetic GPU fault into +# platform-connectors' socket, and checks that diagnostics are captured before +# the repair: +# +# - the file server's volume binds through the storage class --storage-class +# names, which reaches it through nvsentinel's storageClassPaths; +# - fault-remediation runs the collector Job on the faulty node, and the Job +# completes; +# - the bug report it uploads is listed by the file server at +# /upload///, fetched the way the docs say to, through a +# port-forward; +# - fault-remediation creates the GPU reset only after the Job finished. +# +# TEST-ONLY SETTINGS, passed as --set on the bundle, as in +# tests/e2e/nvsentinel-remediation: +# fault-remediation.logCollector.env.MOCK_MODE=true a Kind node has no GPU +# driver, so the collector uploads its synthetic bug report instead. +# CheckNVSentinelLogCollectorCoherent accepts mock mode in place of a +# reachable driver. +# janitor-provider.csp.provider=kind NVSentinel's simulated provider, so +# nothing can reboot the runner if an action escalates to RESTART_VM. +# node-drainer.systemNamespaces includes cert-manager and local-path-storage, +# so the AllowCompletion drain on Kind's one node finishes. +# fault-quarantine.circuitBreaker.enabled=false on one node the first +# quarantine would trip it. +# nvsentinel-mongodb at one member with smaller requests, so everything fits +# one runner node. +# +# The GPU reset itself cannot run on Kind (no nvidia RuntimeClass); the test +# reads only that it is requested after collection. +# +# PREREQUISITES: go, kind, kubectl, helm, yq, jq, curl +# +# ENVIRONMENT VARIABLES: +# CLUSTER_NAME Kind cluster name (default: aicr-nvsentinel-lc-e2e). +# Always recreated, then deleted at the end unless +# KEEP_CLUSTER=true. +# KEEP_CLUSTER Skip cluster teardown on exit (default: false). +# AICR_BIN Path to a prebuilt aicr binary (default: built fresh). +# ============================================================================= + +set -euo pipefail + +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT="$(cd "${DIR}/../../.." && pwd)" +# shellcheck source=/dev/null +. "${ROOT}/tools/common" + +CLUSTER_NAME="${CLUSTER_NAME:-aicr-nvsentinel-lc-e2e}" +KEEP_CLUSTER="${KEEP_CLUSTER:-false}" +AICR_BIN="${AICR_BIN:-}" +KUBE_CONTEXT="kind-${CLUSTER_NAME}" +NODE="${CLUSTER_NAME}-control-plane" + +GRPCURL_IMAGE="docker.io/fullstorydev/grpcurl:v1.9.3@sha256:085e183ca334eb4e81ca81ee12cbb2b2737505d1d77f5e33dabc5d066593d998" +DRAIN_SYSTEM_NAMESPACES='^(nvsentinel|kube-system|cert-manager|local-path-storage)$' +GPU_UUID="GPU-00000000-0000-0000-0000-00000000e2e9" +# A non-default copy of Kind's local-path class: a claim on it proves +# --storage-class reached it, since the default-class admission plugin would +# otherwise fill in "standard". +STORAGE_CLASS="aicr-lc-e2e" +FILE_SERVER="nvsentinel-incluster-file-server" +GPU_RESETS="gpuresets.janitor.dgxc.nvidia.com" +CERT_MANAGER_WAIT_SECONDS=600 +MONGODB_WAIT_SECONDS=900 +NVSENTINEL_WAIT_SECONDS=900 +# Event delivery through fault-quarantine and the drain to fault-remediation. +PIPELINE_WAIT_SECONDS=300 +# The mixin's collector timeout, which bounds how long remediation waits. +COLLECTOR_WAIT_SECONDS=600 + +WORK="" +CREATED_CLUSTER=false +BUNDLE_DIR="" +CRDS_DIR="" +CERT_MANAGER_DIR="" +PSMDB_OPERATOR_DIR="" +MONGODB_DIR="" +MONGODB_POST_DIR="" +PORT_FORWARD_PID="" +TOTAL_TESTS=0 +PASSED_TESTS=0 +FAILED_TESTS=0 + +cleanup() { + local rc=$? + [[ -n "${PORT_FORWARD_PID}" ]] && kill "${PORT_FORWARD_PID}" 2>/dev/null || true + # Cluster before workdir: $KUBECONFIG lives inside WORK. + if [[ "${CREATED_CLUSTER}" == "true" && "${KEEP_CLUSTER}" != "true" ]]; then + msg "Deleting Kind cluster ${CLUSTER_NAME}..." + kind delete cluster --name "${CLUSTER_NAME}" &>/dev/null || true + fi + [[ -n "${WORK}" && -d "${WORK}" ]] && rm -rf "${WORK}" + exit "${rc}" +} +trap cleanup EXIT + +pass() { TOTAL_TESTS=$((TOTAL_TESTS + 1)); PASSED_TESTS=$((PASSED_TESTS + 1)); msg "PASS: $1"; } +fail() { + TOTAL_TESTS=$((TOTAL_TESTS + 1)) + FAILED_TESTS=$((FAILED_TESTS + 1)) + msg "FAIL: $1 -- $2" +} + +kc() { kubectl --context "${KUBE_CONTEXT}" "$@"; } + +# ── Setup ── + +build_binary() { + if [[ -n "${AICR_BIN}" ]]; then + msg "Using prebuilt AICR_BIN=${AICR_BIN}" + return + fi + AICR_BIN="${WORK}/aicr" + msg "Building aicr binary..." + (cd "${ROOT}" && go build -o "${AICR_BIN}" ./cmd/aicr) || err "failed to build aicr binary" +} + +compose_bundle() { + msg "Composing a bundle with the nvsentinel-remediation, nvsentinel-health-events-analyzer and nvsentinel-log-collector mixins..." + local data_dir="${WORK}/data" + mkdir -p "${data_dir}/overlays" + cat >"${data_dir}/registry.yaml" <<'REGISTRY_EOF' +kind: ComponentRegistry +apiVersion: aicr.run/v1beta1 +metadata: + name: nvsentinel-lc-e2e-registry +components: [] +REGISTRY_EOF + yq '.spec.mixins = ["nvsentinel-remediation", "nvsentinel-health-events-analyzer", "nvsentinel-log-collector"]' \ + "${ROOT}/recipes/overlays/h100-kind-training.yaml" \ + >"${data_dir}/overlays/h100-kind-training.yaml" + + "${AICR_BIN}" recipe --service kind --accelerator h100 --intent training \ + --data "${data_dir}" --output "${WORK}/recipe.yaml" || + err "recipe generation failed" + "${AICR_BIN}" bundle -r "${WORK}/recipe.yaml" -o "${WORK}/bundle" \ + --storage-class "${STORAGE_CLASS}" \ + --set "nvsentinel:fault-remediation.logCollector.env.MOCK_MODE=true" \ + --set "nvsentinel:janitor-provider.csp.provider=kind" \ + --set "nvsentinel:node-drainer.systemNamespaces=${DRAIN_SYSTEM_NAMESPACES}" \ + --set "nvsentinel:fault-quarantine.circuitBreaker.enabled=false" \ + --set "nvsentinelmongodb:unsafeFlags.replsetSize=true" \ + --set "nvsentinelmongodb:replsets.rs0.size=1" \ + --set "nvsentinelmongodb:replsets.rs0.resources.requests.cpu=250m" \ + --set "nvsentinelmongodb:replsets.rs0.resources.requests.memory=512Mi" \ + --system-node-selector kubernetes.io/os=linux || + err "bundle generation failed" + + BUNDLE_DIR=$(find "${WORK}/bundle" -maxdepth 1 -type d -name '*-nvsentinel' -print -quit) + CRDS_DIR=$(find "${WORK}/bundle" -maxdepth 1 -type d -name '*-prometheus-operator-crds' -print -quit) + CERT_MANAGER_DIR=$(find "${WORK}/bundle" -maxdepth 1 -type d -name '*-cert-manager' -print -quit) + PSMDB_OPERATOR_DIR=$(find "${WORK}/bundle" -maxdepth 1 -type d -name '*-psmdb-operator' -print -quit) + MONGODB_DIR=$(find "${WORK}/bundle" -maxdepth 1 -type d -name '*-nvsentinel-mongodb' -print -quit) + MONGODB_POST_DIR=$(find "${WORK}/bundle" -maxdepth 1 -type d -name '*-nvsentinel-mongodb-post' -print -quit) + [[ -n "${BUNDLE_DIR}" ]] || err "bundled nvsentinel component directory not found" + [[ -n "${CRDS_DIR}" ]] || err "bundled prometheus-operator-crds directory not found" + [[ -n "${CERT_MANAGER_DIR}" ]] || err "bundled cert-manager directory not found" + [[ -n "${PSMDB_OPERATOR_DIR}" ]] || err "bundled psmdb-operator directory not found" + [[ -n "${MONGODB_DIR}" ]] || err "bundled nvsentinel-mongodb directory not found" + [[ -n "${MONGODB_POST_DIR}" ]] || err "bundled nvsentinel-mongodb-post directory not found" + + # Fail here, not later as "nothing uploaded": a mixin that stopped composing + # would otherwise read as a collector that never ran. + local values="${BUNDLE_DIR}/values.yaml" + [[ "$(yq '.global.inclusterFileServer.enabled' "${values}")" == "true" ]] || + err "bundle's nvsentinel values do not enable the file server -- composition regressed" + [[ "$(yq '.["fault-remediation"].logCollector.enabled' "${values}")" == "true" ]] || + err "bundle's nvsentinel values do not enable the log collector -- composition regressed" + [[ "$(yq '.global.dryRun' "${values}")" == "false" ]] || + err "bundle's nvsentinel values are not live (global.dryRun != false), which skips the collector" + [[ "$(yq '.["incluster-file-server"].persistence.storageClassName' "${values}")" == "${STORAGE_CLASS}" ]] || + err "--storage-class did not reach the file server's claim -- storageClassPaths regressed" +} + +ensure_cluster() { + if grep -qx "${CLUSTER_NAME}" <<<"$(kind get clusters 2>/dev/null)"; then + msg "Deleting stale Kind cluster ${CLUSTER_NAME}..." + kind delete cluster --name "${CLUSTER_NAME}" >/dev/null 2>&1 || true + fi + msg "Creating Kind cluster ${CLUSTER_NAME}..." + kind create cluster --name "${CLUSTER_NAME}" || err "kind create cluster failed" + CREATED_CLUSTER=true + kind export kubeconfig --name "${CLUSTER_NAME}" --kubeconfig "${KUBECONFIG}" || + err "failed to export kubeconfig for ${CLUSTER_NAME}" + kc wait --for=condition=Ready node --all --timeout=120s || err "cluster nodes never became Ready" + # fault-quarantine counts GPU nodes by this label and refuses to act when + # there are none. + kc label node "${NODE}" nvidia.com/gpu.present=true --overwrite >/dev/null + kc get storageclass standard -o json | + jq --arg name "${STORAGE_CLASS}" '.metadata = {name: $name}' | + kc apply -f - >/dev/null || err "could not create StorageClass ${STORAGE_CLASS}" +} + +# wait_rollout waits until $1/$2 in namespace $3 has rolled out, failing at +# $4, a deadline in bash SECONDS. +wait_rollout() { + local kind="$1" name="$2" ns="$3" deadline="$4" remaining + until kc -n "${ns}" get "${kind}" "${name}" >/dev/null 2>&1; do + ((SECONDS < deadline)) || err "${kind}/${name} was never created in ${ns}" + sleep 2 + done + # kubectl reads --timeout=0s as "wait forever". + remaining=$((deadline - SECONDS)) + ((remaining > 0)) || err "${kind}/${name} never became Ready in ${ns}" + kc -n "${ns}" rollout status "${kind}/${name}" --timeout="${remaining}s" || + err "${kind}/${name} never became Ready in ${ns}" +} + +install_component() { + (cd "$1" && chmod +x install.sh && KUBE_CONTEXT="${KUBE_CONTEXT}" ./install.sh) || + err "$(basename "$1") install failed" +} + +install_bundle() { + msg "Installing cert-manager, prometheus-operator-crds, the Percona datastore and nvsentinel from the bundle..." + local dir d deadline remaining + for dir in "${CERT_MANAGER_DIR}" "${CRDS_DIR}"; do + install_component "${dir}" + done + deadline=$((SECONDS + CERT_MANAGER_WAIT_SECONDS)) + for d in cert-manager cert-manager-webhook cert-manager-cainjector; do + wait_rollout deployment "${d}" cert-manager "${deadline}" + done + + install_component "${PSMDB_OPERATOR_DIR}" + deadline=$((SECONDS + MONGODB_WAIT_SECONDS)) + wait_rollout deployment psmdb-operator nvsentinel "${deadline}" + install_component "${MONGODB_DIR}" + install_component "${MONGODB_POST_DIR}" + remaining=$((deadline - SECONDS)) + ((remaining > 0)) || err "nvsentinel-mongodb never became ready" + kc -n nvsentinel wait --for=jsonpath='{.status.state}'=ready \ + perconaservermongodb/nvsentinel-mongodb --timeout="${remaining}s" || + err "nvsentinel-mongodb never became ready" + remaining=$((deadline - SECONDS)) + ((remaining > 0)) || err "nvsentinel's MongoDB client certificate was never issued" + kc -n nvsentinel wait --for=condition=Ready certificate/nvsentinel-mongodb-app-client \ + --timeout="${remaining}s" || err "nvsentinel's MongoDB client certificate was never issued" + + install_component "${BUNDLE_DIR}" + + deadline=$((SECONDS + NVSENTINEL_WAIT_SECONDS)) + for d in fault-quarantine node-drainer fault-remediation janitor janitor-provider "${FILE_SERVER}"; do + wait_rollout deployment "${d}" nvsentinel "${deadline}" + done + wait_rollout daemonset platform-connectors nvsentinel "${deadline}" +} + +# The pinned version's own proto, less the CRD-generator import and the one +# message that uses it; neither affects the wire format. +publish_proto() { + local version + version=$(yq '.components[] | select(.name == "nvsentinel") | .helm.defaultVersion' "${ROOT}/recipes/registry.yaml") + [[ -n "${version}" && "${version}" != "null" ]] || err "could not read nvsentinel's defaultVersion" + curl -fsSL --connect-timeout 10 --max-time 60 \ + "https://raw.githubusercontent.com/NVIDIA/NVSentinel/${version}/data-models/protobufs/health_event.proto" | + awk '/protoc_gen_crd\/proto\/crd.proto/ {next} /^\/\/ HealthEventResource is the root type/ {exit} {print}' \ + >"${WORK}/health_event.proto" || err "failed to fetch health_event.proto at ${version}" + grep -q "rpc HealthEventOccurredV1" "${WORK}/health_event.proto" || + err "health_event.proto at ${version} has no HealthEventOccurredV1 -- the platform-connector API moved" + kc -n nvsentinel create configmap e2e-health-event-proto \ + --from-file=health_event.proto="${WORK}/health_event.proto" --dry-run=client -o yaml | + kc apply -f - >/dev/null +} + +# fault_event prints one fatal GPU XID HealthEvents request whose action is a +# GPU reset, the action the collector runs before. +fault_event() { + jq -cn --arg node "${NODE}" --arg uuid "${GPU_UUID}" --arg ts "$(date -u +%Y-%m-%dT%H:%M:%SZ)" '{ + version: 1, + events: [{ + version: 1, + agent: "gpu-health-monitor", + componentClass: "GPU", + checkName: "GpuXidError", + isFatal: true, + isHealthy: false, + message: "aicr e2e synthetic XID 79", + recommendedAction: "COMPONENT_RESET", + errorCode: ["79"], + entitiesImpacted: [{entityType: "GPU", entityValue: "0"}, {entityType: "GPU_UUID", entityValue: $uuid}], + generatedTimestamp: $ts, + nodeName: $node, + processingStrategy: "EXECUTE_REMEDIATION" + }] + }' +} + +# send_event delivers $2 through platform-connectors' socket from a pod on the +# node, which is what a node-local monitor does. +send_event() { + local name="$1" payload="$2" + jq -n --arg name "${name}" --arg node "${NODE}" --arg image "${GRPCURL_IMAGE}" --arg data "${payload}" '{ + apiVersion: "v1", kind: "Pod", + metadata: {name: $name, namespace: "nvsentinel"}, + spec: { + nodeName: $node, + restartPolicy: "Never", + tolerations: [{operator: "Exists"}], + containers: [{ + name: "grpcurl", image: $image, + # unix:// rather than -unix, which grpcurl v1.9.3 ignores and dials TCP. + args: ["-plaintext", "-import-path", "/protos", "-proto", "health_event.proto", + "-d", $data, "unix:///sock/nvsentinel.sock", "datamodels.PlatformConnector/HealthEventOccurredV1"], + volumeMounts: [{name: "sock", mountPath: "/sock"}, {name: "protos", mountPath: "/protos"}] + }], + volumes: [ + {name: "sock", hostPath: {path: "/var/run/nvsentinel", type: "Directory"}}, + {name: "protos", configMap: {name: "e2e-health-event-proto"}} + ] + } + }' | kc apply -f - >/dev/null + if ! kc -n nvsentinel wait --for=jsonpath='{.status.phase}'=Succeeded "pod/${name}" --timeout=180s >/dev/null; then + kc -n nvsentinel logs "pod/${name}" 2>&1 | tail -20 >&2 || true + err "event ${name} was not accepted by platform-connectors" + fi +} + +# collector_jobs prints the collector Jobs as a JSON list. A kubectl error +# fails rather than reading as "none". +collector_jobs() { + local json + json=$(kc -n nvsentinel get jobs -l app=log-collector -o json) || return 1 + jq -c '[.items[] | {name: .metadata.name, node: .spec.template.spec.nodeName, + succeeded: (.status.succeeded // 0), failed: (.status.failed // 0), completion: .status.completionTime}]' <<<"${json}" +} + +# wait_for_job_done polls until a collector Job has succeeded or failed, then +# prints that Job. +wait_for_job_done() { + local elapsed=0 jobs + while [[ "${elapsed}" -lt "$((PIPELINE_WAIT_SECONDS + COLLECTOR_WAIT_SECONDS))" ]]; do + if jobs=$(collector_jobs) && jq -e 'any(.[]; .succeeded > 0 or .failed > 0)' <<<"${jobs}" >/dev/null; then + jq -c 'first(.[] | select(.succeeded > 0 or .failed > 0))' <<<"${jobs}" + return 0 + fi + sleep 5 + elapsed=$((elapsed + 5)) + done + return 1 +} + +# wait_for_reset polls for the GPU reset fault-remediation requests and prints +# its creation time. +wait_for_reset() { + local elapsed=0 created + while [[ "${elapsed}" -lt "${PIPELINE_WAIT_SECONDS}" ]]; do + created=$(kc get "${GPU_RESETS}" -A -o json 2>/dev/null | + jq -r '[.items[].metadata.creationTimestamp] | sort | first // empty') + if [[ -n "${created}" ]]; then + echo "${created}" + return 0 + fi + sleep 5 + elapsed=$((elapsed + 5)) + done + return 1 +} + +# file_server_get fetches path $1 from the file server through a port-forward, +# the retrieval the component catalog documents. +file_server_get() { + if [[ -z "${PORT_FORWARD_PID}" ]]; then + kc -n nvsentinel port-forward "svc/${FILE_SERVER}" 18080:80 >"${WORK}/port-forward.log" 2>&1 & + PORT_FORWARD_PID=$! + local i + for ((i = 0; i < 30; i++)); do + curl -fsS --max-time 5 "http://127.0.0.1:18080/healthz" >/dev/null 2>&1 && break + sleep 1 + done + fi + curl -fsS --max-time 30 "http://127.0.0.1:18080$1" +} + +# ── Tests ── + +test_file_server_volume_bound() { + local claim + if ! claim=$(kc -n nvsentinel get pvc "${FILE_SERVER}-data" -o json); then + fail "file server volume" "could not read PVC ${FILE_SERVER}-data" + return + fi + if jq -e --arg sc "${STORAGE_CLASS}" '.spec.storageClassName == $sc and .status.phase == "Bound"' <<<"${claim}" >/dev/null; then + pass "file server volume is Bound on storage class ${STORAGE_CLASS} from --storage-class" + else + fail "file server volume" "want Bound on ${STORAGE_CLASS}; got $(jq -c '{sc: .spec.storageClassName, phase: .status.phase}' <<<"${claim}")" + fi +} + +test_fault_is_collected_before_remediation() { + local jobs + jobs=$(collector_jobs) || err "could not list collector Jobs" + [[ "$(jq 'length' <<<"${jobs}")" == "0" ]] || err "collector Jobs exist before any fault: ${jobs}" + + send_event e2e-lc-fault "$(fault_event)" + + local job + if ! job=$(wait_for_job_done); then + fail "collector Job runs" "no collector Job finished within $((PIPELINE_WAIT_SECONDS + COLLECTOR_WAIT_SECONDS))s; jobs: $(collector_jobs || echo '?')" + return + fi + msg "Collector Job: ${job}" + if jq -e --arg node "${NODE}" '.node == $node' <<<"${job}" >/dev/null; then + pass "collector Job ran on the faulty node" + else + fail "collector Job ran on the faulty node" "got ${job}" + fi + if jq -e '.succeeded > 0' <<<"${job}" >/dev/null; then + pass "collector Job completed" + else + fail "collector Job completed" "it failed: ${job}; logs: $(kc -n nvsentinel logs "job/$(jq -r .name <<<"${job}")" --tail=30 2>&1 || true)" + return + fi + + local listing report page + # The listing also links ../, so match the collector's timestamp directory. + # Each page is captured before grep so grep -q cannot cut curl off mid-write. + if listing=$(file_server_get "/upload/${NODE}/") && + report=$(grep -oE 'href="[0-9]{8}-[0-9]{6}/"' <<<"${listing}" | head -1 | sed -E 's/href="|\/"//g') && + [[ -n "${report}" ]] && page=$(file_server_get "/upload/${NODE}/${report}/") && + grep -q "nvidia-bug-report-${NODE}-" <<<"${page}"; then + pass "bug report uploaded and listed at /upload/${NODE}/${report}/" + else + fail "bug report uploaded" "the file server lists none under /upload/${NODE}/: ${listing:-}" + fi + + local reset_created + if ! reset_created=$(wait_for_reset); then + fail "GPU reset requested after collection" "no ${GPU_RESETS} within ${PIPELINE_WAIT_SECONDS}s" + return + fi + if jq -en --arg finished "$(jq -r .completion <<<"${job}")" --arg reset "${reset_created}" \ + '($finished | fromdateiso8601) <= ($reset | fromdateiso8601)' >/dev/null; then + pass "GPU reset requested at ${reset_created}, after the collector finished" + else + fail "GPU reset requested after collection" "reset at ${reset_created}, collector finished $(jq -r .completion <<<"${job}")" + fi +} + +# ── Main ── + +msg "==========================================" +msg "nvsentinel-log-collector mixin e2e" +msg "==========================================" + +has_tools go kind kubectl helm yq jq curl +WORK=$(mktemp -d) +export KUBECONFIG="${WORK}/kubeconfig" + +build_binary +compose_bundle +ensure_cluster +install_bundle +publish_proto +test_file_server_volume_bound +test_fault_is_collected_before_remediation + +msg "==========================================" +msg "Results: ${PASSED_TESTS}/${TOTAL_TESTS} passed" +[[ "${FAILED_TESTS}" -eq 0 ]] || err "${FAILED_TESTS} test(s) failed" +msg "nvsentinel-log-collector e2e: all tests passed" diff --git a/tools/mirror-e2e b/tools/mirror-e2e index 58a1a38c23..08af4037f3 100755 --- a/tools/mirror-e2e +++ b/tools/mirror-e2e @@ -29,8 +29,8 @@ # nvsentinel-object-monitor, nvsentinel-preflight, # nvsentinel-nic-health-monitor, nvsentinel-slurm-drain-monitor, # nvsentinel-remediation, nvsentinel-slinky-drainer, -# nvsentinel-health-events-analyzer and nvsentinel-event-exporter -# mixins composed, so their mixin-gated images are +# nvsentinel-health-events-analyzer, nvsentinel-event-exporter and +# nvsentinel-log-collector mixins composed, so their mixin-gated images are # asserted in every manifest format and the three small controller images # (object monitor, preflight, slurm drain monitor) are carried through the # transfer @@ -134,6 +134,16 @@ SLINKY_DRAINER_IMAGE="ghcr.io/nvidia/nvsentinel/slinky-drainer:v1.28.0@sha256:bf # The image the nvsentinel-health-events-analyzer mixin pulls in (#2618). HEALTH_EVENTS_ANALYZER_IMAGE="ghcr.io/nvidia/nvsentinel/health-events-analyzer:v1.28.0" +# The images the nvsentinel-log-collector mixin pulls in (#2619): the collector +# Job's, named only inside fault-remediation's ConfigMap, and the in-cluster +# file server's. +LOG_COLLECTOR_IMAGES=( + "ghcr.io/nvidia/nvsentinel/log-collector:v1.28.0" + "ghcr.io/nvidia/nvsentinel/file-server-cleanup:v1.28.0" + "docker.io/bitnamilegacy/nginx:1.27.2-debian-12-r2" + "public.ecr.aws/docker/library/busybox:1.37.0" + "quay.io/martinhelmich/prometheus-nginxlog-exporter:v1.10.0" +) # The image the nvsentinel-event-exporter mixin pulls in (#2621). EVENT_EXPORTER_IMAGE="ghcr.io/nvidia/nvsentinel/event-exporter:v1.28.0" @@ -148,6 +158,7 @@ MIXIN_IMAGES=( "${SLINKY_DRAINER_IMAGE}" "${HEALTH_EVENTS_ANALYZER_IMAGE}" "${EVENT_EXPORTER_IMAGE}" + "${LOG_COLLECTOR_IMAGES[@]}" ) # The subset carried through the Hauler sync, Zarf package and registry @@ -242,7 +253,7 @@ ensure_cluster() { } generate_recipe() { - msg "Generating recipe (with the nvsentinel-object-monitor, nvsentinel-preflight, nvsentinel-nic-health-monitor, nvsentinel-slurm-drain-monitor, nvsentinel-remediation, nvsentinel-slinky-drainer, nvsentinel-health-events-analyzer and nvsentinel-event-exporter mixins composed)..." + msg "Generating recipe (with the nvsentinel-object-monitor, nvsentinel-preflight, nvsentinel-nic-health-monitor, nvsentinel-slurm-drain-monitor, nvsentinel-remediation, nvsentinel-slinky-drainer, nvsentinel-health-events-analyzer, nvsentinel-event-exporter and nvsentinel-log-collector mixins composed)..." # No --os: the Kind overlay tree is OS-agnostic, and a stated OS that no # Kind overlay distinguishes fails criteria coverage (issue #1542). # @@ -250,8 +261,8 @@ generate_recipe() { # the nvsentinel-object-monitor, nvsentinel-preflight, # nvsentinel-nic-health-monitor, nvsentinel-slurm-drain-monitor, # nvsentinel-remediation, nvsentinel-slinky-drainer, - # nvsentinel-health-events-analyzer and nvsentinel-event-exporter - # mixins, so every mixin-gated image is + # nvsentinel-health-events-analyzer, nvsentinel-event-exporter and + # nvsentinel-log-collector mixins, so every mixin-gated image is # discovered by `mirror list` and asserted in the generated Hauler and Zarf # manifests. # Only the three small controller images are carried through the sync, @@ -284,8 +295,10 @@ REGISTRY_EOF # because this lane never bundles: CheckNVSentinelSlurmDrainMonitorRequiresSlinky, # CheckNVSentinelSlurmCustomDrainCoherent and # CheckNVSentinelEventExporterCoherent would reject them at `aicr bundle`, - # but `mirror list` runs no bundle gates. - yq '.spec.mixins = ["nvsentinel-object-monitor", "nvsentinel-preflight", "nvsentinel-nic-health-monitor", "nvsentinel-slurm-drain-monitor", "nvsentinel-remediation", "nvsentinel-slinky-drainer", "nvsentinel-health-events-analyzer", "nvsentinel-event-exporter"]' \ + # but `mirror list` runs no bundle gates. The same holds for + # nvsentinel-log-collector on Kind, whose driver CheckNVSentinelLogCollectorCoherent + # cannot reach. + yq '.spec.mixins = ["nvsentinel-object-monitor", "nvsentinel-preflight", "nvsentinel-nic-health-monitor", "nvsentinel-slurm-drain-monitor", "nvsentinel-remediation", "nvsentinel-slinky-drainer", "nvsentinel-health-events-analyzer", "nvsentinel-event-exporter", "nvsentinel-log-collector"]' \ "${ROOT}/recipes/overlays/h100-kind-training.yaml" \ > "${data_dir}/overlays/h100-kind-training.yaml" @@ -334,6 +347,9 @@ REGISTRY_EOF if ! grep -q "eventExporter" "${WORK}/recipe.yaml"; then err "nvsentinel-event-exporter mixin did not apply -- the recipe carries no eventExporter override" fi + if ! grep -q "inclusterFileServer" "${WORK}/recipe.yaml"; then + err "nvsentinel-log-collector mixin did not apply -- the recipe carries no inclusterFileServer override" + fi msg "Recipe generated at ${WORK}/recipe.yaml" } From 8fb6f22e0ae82075c9dd85c722377cda01c49423 Mon Sep 17 00:00:00 2001 From: Varun Ramesh Date: Wed, 21 Oct 2026 13:06:53 +0530 Subject: [PATCH 2/8] fix(nvsentinel): run log collector e2e on ubuntu-22.04 and check env overrides Signed-off-by: Varun Ramesh --- .../nvsentinel-log-collector-e2e.yaml | 5 +- .../validations/nvsentinel_log_collector.go | 42 +++++++++++++--- .../nvsentinel_log_collector_test.go | 49 +++++++++++++++++++ 3 files changed, 89 insertions(+), 7 deletions(-) diff --git a/.github/workflows/nvsentinel-log-collector-e2e.yaml b/.github/workflows/nvsentinel-log-collector-e2e.yaml index 25e0cd8414..bb133440b5 100644 --- a/.github/workflows/nvsentinel-log-collector-e2e.yaml +++ b/.github/workflows/nvsentinel-log-collector-e2e.yaml @@ -74,7 +74,10 @@ concurrency: jobs: test: name: NVSentinel Log Collector E2E - runs-on: ubuntu-24.04 + # mongod 8.0 before 8.0.35 exits at startup on the ubuntu-24.04 image's + # 7.0 kernel (https://jira.mongodb.org/browse/SERVER-131779). Return to + # ubuntu-24.04 once the Percona MongoDB pin is 8.0.35 or newer. + runs-on: ubuntu-22.04 # The script's bounded waits sum to ~70 minutes worst case (node Ready 2, # cert-manager 10, Percona datastore 15, nvsentinel rollout 15, event # delivery 3, collection 15, reset request 5, proto fetch 1, port-forward diff --git a/pkg/bundler/validations/nvsentinel_log_collector.go b/pkg/bundler/validations/nvsentinel_log_collector.go index 330b504313..be024348bf 100644 --- a/pkg/bundler/validations/nvsentinel_log_collector.go +++ b/pkg/bundler/validations/nvsentinel_log_collector.go @@ -35,6 +35,8 @@ const ( logCollectorUploadURLPath = "fault-remediation.logCollector.uploadURL" logCollectorNamespacesPath = "fault-remediation.logCollector.gpuOperatorNamespaces" logCollectorMockModePath = "fault-remediation.logCollector.env.MOCK_MODE" + logCollectorEnvUploadURLPath = "fault-remediation.logCollector.env.UPLOAD_URL_BASE" + logCollectorEnvNamespacePath = "fault-remediation.logCollector.env.GPU_OPERATOR_NAMESPACE" fileServerPath = "global.inclusterFileServer" fileServerEnabledPath = "global.inclusterFileServer.enabled" fileServerNamePath = "incluster-file-server.fullnameOverride" @@ -69,6 +71,11 @@ const ( // install, so a recipe whose node image carries the driver otherwise (AKS, // OKE, Kind) fails every collection. // +// The Job template renders logCollector.env after the variables it sets from +// uploadURL and gpuOperatorNamespaces, so an env.UPLOAD_URL_BASE or +// env.GPU_OPERATOR_NAMESPACE entry replaces them; the rules read the entry when +// one is set. +// // The collector's MOCK_MODE uploads a synthetic report without a driver, // which the Kind e2e uses, so it skips the last rule. The collector enters it // only on the exact string "true", so that is all this reads as mock. @@ -97,8 +104,8 @@ func CheckNVSentinelLogCollectorCoherent(ctx context.Context, componentName stri collectorRaw, _ := component.GetValueByPath(values, logCollectorEnabledPath) collector := helmTruthy(collectorRaw) fileServer := nvsentinelSubchartRenders(values, "inclusterFileServer") - mockRaw, mockSet := component.GetValueByPath(values, logCollectorMockModePath) - mock := mockSet && fmt.Sprint(mockRaw) == "true" + mockMode, _ := logCollectorEnvValue(values, logCollectorMockModePath) + mock := mockMode == "true" gpuOp := enabledGPUOperator(recipeResult, bundlerConfig) if msgs := logCollectorDynamicViolations(bundlerConfig, componentName, sentinelKeys, collector, fileServer, mock, @@ -160,10 +167,14 @@ func CheckNVSentinelLogCollectorCoherent(ctx context.Context, componentName stri host += ":" + fmt.Sprint(port) } want := "http://" + host + "/upload" - got, _, valid := resolvedStringValue(values, logCollectorUploadURLPath) + urlPath := logCollectorUploadURLPath + got, _, valid := resolvedStringValue(values, urlPath) + if env, ok := logCollectorEnvValue(values, logCollectorEnvUploadURLPath); ok { + urlPath, got, valid = logCollectorEnvUploadURLPath, env, true + } if !valid || strings.Replace(strings.TrimSuffix(got, "/"), ".svc.cluster.local:80/", ".svc.cluster.local/", 1) != want { fail("%s is %q, not the in-cluster file server's Service, so collections upload elsewhere and "+ - "the file server holds nothing -- set it to %q", logCollectorUploadURLPath, got, want) + "the file server holds nothing -- set it to %q", urlPath, got, want) } } } @@ -238,8 +249,13 @@ func logCollectorDriverViolations(ctx context.Context, recipeResult *recipe.Reci "container", gpuOp.name, gpuOperatorDriverEnabledPath)) } - namespace := logCollectorDefaultOperatorNS + nsPath, namespace := logCollectorNamespacesPath, logCollectorDefaultOperatorNS ns, set, valid := resolvedStringValue(values, logCollectorNamespacesPath) + if env, ok := logCollectorEnvValue(values, logCollectorEnvNamespacePath); ok { + // The chart defaults only gpuOperatorNamespaces, so an empty entry + // reaches the collector as an empty namespace. + nsPath, namespace, set, valid = logCollectorEnvNamespacePath, env, false, true + } if !valid { // The chart quotes whatever it is given, so a list renders as the // literal "[gpu-operator]" and names no namespace. @@ -256,11 +272,25 @@ func logCollectorDriverViolations(ctx context.Context, recipeResult *recipe.Reci return []error{aicrerrors.New(aicrerrors.ErrCodeInvalidRequest, fmt.Sprintf("component %q: %s deploys to namespace %q, but %s is %q, the one namespace the collector "+ "searches for the driver container, so every collection fails -- set it to %q", - componentName, gpuOp.name, gpuOp.namespace, logCollectorNamespacesPath, namespace, gpuOp.namespace))} + componentName, gpuOp.name, gpuOp.namespace, nsPath, namespace, gpuOp.namespace))} } return nil } +// logCollectorEnvValue returns the logCollector.env entry at envPath as the +// Job template renders it: any scalar is quoted into a string, and null into +// the empty string. +func logCollectorEnvValue(values map[string]any, envPath string) (string, bool) { + raw, ok := component.GetValueByPath(values, envPath) + if !ok { + return "", false + } + if raw == nil { + return "", true + } + return fmt.Sprint(raw), true +} + // logCollectorDynamicViolations returns a violation for each --dynamic path a // rule of CheckNVSentinelLogCollectorCoherent reads. The two toggles always // matter; the rest only once either is on. diff --git a/pkg/bundler/validations/nvsentinel_log_collector_test.go b/pkg/bundler/validations/nvsentinel_log_collector_test.go index 9577d199cb..b47bb833d5 100644 --- a/pkg/bundler/validations/nvsentinel_log_collector_test.go +++ b/pkg/bundler/validations/nvsentinel_log_collector_test.go @@ -264,6 +264,55 @@ func TestCheckNVSentinelLogCollectorCoherent(t *testing.T) { with(collector(), "gpuOperatorNamespaces", "nvidia-gpu-operator"), recipe.ComponentRef{Name: "gpu-operator", Namespace: "nvidia-gpu-operator"}), }, + { + name: "env GPU_OPERATOR_NAMESPACE naming the GPU Operator's own namespace", + recipeResult: recipeOf(withServer(remediation()), + with(collector(), "env", map[string]any{"GPU_OPERATOR_NAMESPACE": "nvidia-gpu-operator"}), + recipe.ComponentRef{Name: "gpu-operator", Namespace: "nvidia-gpu-operator"}), + }, + { + name: "env GPU_OPERATOR_NAMESPACE replacing a gpuOperatorNamespaces list", + recipeResult: recipeOf(withServer(remediation()), + with(with(collector(), "gpuOperatorNamespaces", []any{"gpu-operator"}), "env", + map[string]any{"GPU_OPERATOR_NAMESPACE": "gpu-operator"}), operator), + }, + { + name: "env GPU_OPERATOR_NAMESPACE overriding a matching gpuOperatorNamespaces", + recipeResult: recipeOf(withServer(remediation()), + with(with(collector(), "gpuOperatorNamespaces", "gpu-operator"), "env", + map[string]any{"GPU_OPERATOR_NAMESPACE": "elsewhere"}), operator), + want: []string{`env.GPU_OPERATOR_NAMESPACE is "elsewhere"`}, + }, + { + name: "env GPU_OPERATOR_NAMESPACE empty, which the chart does not default", + recipeResult: recipeOf(withServer(remediation()), + with(collector(), "env", map[string]any{"GPU_OPERATOR_NAMESPACE": ""}), operator), + want: []string{wantOperatorNS}, + }, + { + name: "env GPU_OPERATOR_NAMESPACE null", + recipeResult: recipeOf(withServer(remediation()), + with(collector(), "env", map[string]any{"GPU_OPERATOR_NAMESPACE": nil}), operator), + want: []string{wantOperatorNS}, + }, + { + name: "env UPLOAD_URL_BASE naming the file server over another uploadURL", + recipeResult: recipeOf(withServer(remediation()), + with(with(collector(), "uploadURL", "http://bucket.example/upload"), "env", + map[string]any{"UPLOAD_URL_BASE": fileServerURL}), operator), + }, + { + name: "env UPLOAD_URL_BASE overriding a matching uploadURL", + recipeResult: recipeOf(withServer(remediation()), + with(collector(), "env", map[string]any{"UPLOAD_URL_BASE": "http://bucket.example/upload"}), operator), + want: []string{`env.UPLOAD_URL_BASE is "http://bucket.example/upload"`}, + }, + { + name: "env UPLOAD_URL_BASE empty", + recipeResult: recipeOf(withServer(remediation()), + with(collector(), "env", map[string]any{"UPLOAD_URL_BASE": ""}), operator), + want: []string{wantUploadURL}, + }, { name: "MOCK_MODE false is not mock mode", recipeResult: recipeOf(withServer(remediation()), with(collector(), "env", map[string]any{"MOCK_MODE": "false"}), hostDriver), From 0ac5059777ab3ec5000fdbdf7de3344f17f2b2b8 Mon Sep 17 00:00:00 2001 From: Varun Ramesh Date: Wed, 21 Oct 2026 14:01:35 +0530 Subject: [PATCH 3/8] fix(nvsentinel): preload file server images and harden log collector checks Signed-off-by: Varun Ramesh --- .../nvsentinel-log-collector-e2e.yaml | 2 + docs/user/component-catalog.md | 2 +- .../validations/nvsentinel_log_collector.go | 25 +++++---- .../nvsentinel_log_collector_test.go | 30 +++++++++-- .../nvsentinel-health-events-analyzer/run.sh | 11 ++-- tests/e2e/nvsentinel-log-collector/run.sh | 51 +++++++++++++++++-- tests/e2e/nvsentinel-remediation/run.sh | 11 ++-- tests/e2e/nvsentinel-slinky-drainer/run.sh | 11 ++-- 8 files changed, 112 insertions(+), 31 deletions(-) diff --git a/.github/workflows/nvsentinel-log-collector-e2e.yaml b/.github/workflows/nvsentinel-log-collector-e2e.yaml index bb133440b5..559a7281f8 100644 --- a/.github/workflows/nvsentinel-log-collector-e2e.yaml +++ b/.github/workflows/nvsentinel-log-collector-e2e.yaml @@ -141,6 +141,8 @@ jobs: kubectl get pods -A kubectl get node -o yaml kubectl -n nvsentinel get pvc,statefulset,deployment,job,perconaservermongodb,certificate + kubectl -n nvsentinel get events --sort-by=.lastTimestamp + kubectl -n nvsentinel describe pods --selector='!job-name' kubectl get gpuresets.janitor.dgxc.nvidia.com -A -o yaml kubectl -n nvsentinel logs -l app=log-collector --tail=200 for d in fault-quarantine node-drainer fault-remediation janitor nvsentinel-incluster-file-server; do diff --git a/docs/user/component-catalog.md b/docs/user/component-catalog.md index 68d475f8c5..afbe94d043 100644 --- a/docs/user/component-catalog.md +++ b/docs/user/component-catalog.md @@ -1102,7 +1102,7 @@ curl -sO http://localhost:8081/upload/gpu-node-1/20261015-101500/nvidia-bug-repo - An `nvidia-bug-report` contains system configuration. The file server speaks plain HTTP inside the cluster and accepts `PUT` and `DELETE` on `/upload/` from any pod that can reach its Service, with no size limit. RBAC does not govern HTTP to a Service, and the chart's own NetworkPolicy excludes the file server, so restricting who can read or delete reports takes a NetworkPolicy you supply. - The file server image is `docker.io/bitnamilegacy/nginx`, from Bitnami's frozen archive, which receives no patches; it is scanned with the other mixin images (see [Container Images](https://github.com/NVIDIA/aicr/blob/main/docs/user/container-images.md)). -**Bundle-time gates.** `CheckNVSentinelLogCollectorCoherent` blocks a bundle with only one of the two settings on; with the collector on but `fault-remediation` off, in dry run, or with a `dryRun` that is not a boolean; without the file server's name pinned by `incluster-file-server.fullnameOverride` (the Service name otherwise follows the Helm release name, which Flux sets to `-`), or with an `uploadURL` that is not that Service; and on a platform where the collector cannot reach the driver, as above. It also rejects `--dynamic` on any path it verifies. The collector's `MOCK_MODE` (exactly `true`), which uploads a synthetic report without a driver, skips the platform rule; AICR's Kind e2e uses it. +**Bundle-time gates.** `CheckNVSentinelLogCollectorCoherent` blocks a bundle with only one of the two settings on; with the collector on but `fault-remediation` off, in dry run, or with a `dryRun` that is not a boolean; unless `incluster-file-server.fullnameOverride` is the mixin's `nvsentinel-incluster-file-server` (unset, the Service name follows the Helm release name, which Flux sets to `-`; renamed, the health check, which asserts the rollout by that name, would pass a failed one), or with an `uploadURL` that is not that Service; and on a platform where the collector cannot reach the driver, as above. An `env.UPLOAD_URL_BASE` or `env.GPU_OPERATOR_NAMESPACE` entry under `logCollector` replaces `uploadURL` or `gpuOperatorNamespaces` in the collector's Job, so the gate checks the entry instead; an empty namespace entry means `gpu-operator`, the collector's own fallback. It also rejects `--dynamic` on any path it verifies. The collector's `MOCK_MODE` (exactly `true`), which uploads a synthetic report without a driver, skips the platform rule; AICR's Kind e2e uses it. **Health check.** The `nvsentinel` health check fails when the `nvsentinel-incluster-file-server` Deployment renders without a complete rollout, and passes where it is absent. It does not prove a collection succeeds; the Kind e2e (`tests/e2e/nvsentinel-log-collector/`) drives one fault through live remediation and checks the uploaded report and that the repair was requested after it. A collection Job that fails leaves its Failed pods in the `nvsentinel` namespace for an hour (`ttlSecondsAfterFinished: 3600`, up to seven pods at the default retry limit), and the health check's failed-pod rule reports them until then. diff --git a/pkg/bundler/validations/nvsentinel_log_collector.go b/pkg/bundler/validations/nvsentinel_log_collector.go index be024348bf..d8030472eb 100644 --- a/pkg/bundler/validations/nvsentinel_log_collector.go +++ b/pkg/bundler/validations/nvsentinel_log_collector.go @@ -42,6 +42,7 @@ const ( fileServerNamePath = "incluster-file-server.fullnameOverride" fileServerPortPath = "incluster-file-server.service.port" logCollectorMixinName = "nvsentinel-log-collector" + logCollectorFileServerName = "nvsentinel-incluster-file-server" gpuOperatorDriverEnabledPath = "driver.enabled" gpuOperatorDriverInstallDirPath = "hostPaths.driverInstallDir" logCollectorDefaultOperatorNS = "gpu-operator" @@ -61,10 +62,11 @@ const ( // skips it in dry run, so it never captures. A logCollector.enabled that // is not a boolean is rejected too: the chart also renders it into // fault-remediation's --enable-log-collector flag, which parses strictly; -// - the file server's name is not pinned by fullnameOverride, so its +// - the file server's name is not the mixin's fullnameOverride: unset, its // Service follows the Helm release name, which differs by deployer (Flux -// names the release -), or the upload URL is not that -// Service; +// names the release -); renamed, the nvsentinel health +// check, which asserts the rollout by that name, passes a failed one. Or +// the upload URL is not that Service; // - the collector cannot reach nvidia-bug-report. It runs the script in the // GPU Operator's driver container, which it looks up in the single // namespace gpuOperatorNamespaces names, or from GKE's managed driver @@ -74,7 +76,7 @@ const ( // The Job template renders logCollector.env after the variables it sets from // uploadURL and gpuOperatorNamespaces, so an env.UPLOAD_URL_BASE or // env.GPU_OPERATOR_NAMESPACE entry replaces them; the rules read the entry when -// one is set. +// one is set, and an empty namespace as the collector's gpu-operator fallback. // // The collector's MOCK_MODE uploads a synthetic report without a driver, // which the Kind e2e uses, so it skips the last rule. The collector enters it @@ -156,11 +158,12 @@ func CheckNVSentinelLogCollectorCoherent(ctx context.Context, componentName stri if fileServer { name, _, _ := resolvedStringValue(values, fileServerNamePath) - if name == "" { + switch name { + case "": fail("%s is unset, so the file server's Service is named after the Helm release, which differs by "+ "deployer (Flux names it -), and %s cannot name it -- compose the %s mixin, which "+ "pins it", fileServerNamePath, logCollectorUploadURLPath, logCollectorMixinName) - } else { + case logCollectorFileServerName: host := fmt.Sprintf("%s.%s.svc.cluster.local", name, componentNamespace(recipeResult, componentName)) // A null or 80 port is the chart's default, which the URL may omit. if port, ok := component.GetValueByPath(values, fileServerPortPath); ok && port != nil && fmt.Sprint(port) != "80" { @@ -176,6 +179,10 @@ func CheckNVSentinelLogCollectorCoherent(ctx context.Context, componentName stri fail("%s is %q, not the in-cluster file server's Service, so collections upload elsewhere and "+ "the file server holds nothing -- set it to %q", urlPath, got, want) } + default: + fail("%s is %q, but the nvsentinel health check asserts the file server's rollout as %q, so a failed "+ + "rollout passes validation -- leave it at %q, as the %s mixin sets it", fileServerNamePath, name, + logCollectorFileServerName, logCollectorFileServerName, logCollectorMixinName) } } @@ -252,9 +259,9 @@ func logCollectorDriverViolations(ctx context.Context, recipeResult *recipe.Reci nsPath, namespace := logCollectorNamespacesPath, logCollectorDefaultOperatorNS ns, set, valid := resolvedStringValue(values, logCollectorNamespacesPath) if env, ok := logCollectorEnvValue(values, logCollectorEnvNamespacePath); ok { - // The chart defaults only gpuOperatorNamespaces, so an empty entry - // reaches the collector as an empty namespace. - nsPath, namespace, set, valid = logCollectorEnvNamespacePath, env, false, true + // The entry replaces gpuOperatorNamespaces outright, so an empty one + // takes the collector's own fallback rather than that value. + nsPath, ns, set, valid = logCollectorEnvNamespacePath, env, true, true } if !valid { // The chart quotes whatever it is given, so a list renders as the diff --git a/pkg/bundler/validations/nvsentinel_log_collector_test.go b/pkg/bundler/validations/nvsentinel_log_collector_test.go index b47bb833d5..e660f8203e 100644 --- a/pkg/bundler/validations/nvsentinel_log_collector_test.go +++ b/pkg/bundler/validations/nvsentinel_log_collector_test.go @@ -97,6 +97,7 @@ func TestCheckNVSentinelLogCollectorCoherent(t *testing.T) { wantEnabledType = "not a boolean, and fault-remediation parses it strictly" wantOperatorNSType = "not a string, so the collector searches a namespace that does not exist" wantUnnamed = "incluster-file-server.fullnameOverride is unset" + wantRenamed = "the nvsentinel health check asserts the file server's rollout" wantDynamic = "--dynamic" wantDynamicCause = "decides whether NVSentinel's log collector runs" ) @@ -198,13 +199,25 @@ func TestCheckNVSentinelLogCollectorCoherent(t *testing.T) { want: []string{wantUnnamed}, }, { - name: "upload URL naming a file server other than the pinned one", + name: "file server renamed without the upload URL", recipeResult: func() *recipe.RecipeResult { r := mixin(operator) r.ComponentRefs[0].Overrides["incluster-file-server"] = map[string]any{"fullnameOverride": "diagnostics"} return r }(), - want: []string{wantUploadURL}, + want: []string{wantRenamed}, + }, + { + // The health check asserts the rollout by the mixin's name, so a + // consistent rename would let a failed rollout pass validation. + name: "file server renamed with a matching upload URL", + recipeResult: func() *recipe.RecipeResult { + r := recipeOf(withServer(remediation()), + with(collector(), "uploadURL", "http://diagnostics.nvsentinel.svc.cluster.local/upload"), operator) + r.ComponentRefs[0].Overrides["incluster-file-server"] = map[string]any{"fullnameOverride": "diagnostics"} + return r + }(), + want: []string{wantRenamed}, }, { name: "file server name dynamic with the mixin", @@ -284,16 +297,23 @@ func TestCheckNVSentinelLogCollectorCoherent(t *testing.T) { want: []string{`env.GPU_OPERATOR_NAMESPACE is "elsewhere"`}, }, { - name: "env GPU_OPERATOR_NAMESPACE empty, which the chart does not default", + // The collector falls back to gpu-operator on an empty namespace. + name: "env GPU_OPERATOR_NAMESPACE empty", recipeResult: recipeOf(withServer(remediation()), with(collector(), "env", map[string]any{"GPU_OPERATOR_NAMESPACE": ""}), operator), - want: []string{wantOperatorNS}, }, { name: "env GPU_OPERATOR_NAMESPACE null", recipeResult: recipeOf(withServer(remediation()), with(collector(), "env", map[string]any{"GPU_OPERATOR_NAMESPACE": nil}), operator), - want: []string{wantOperatorNS}, + }, + { + name: "env GPU_OPERATOR_NAMESPACE empty over gpuOperatorNamespaces naming another namespace", + recipeResult: recipeOf(withServer(remediation()), + with(with(collector(), "gpuOperatorNamespaces", "nvidia-gpu-operator"), "env", + map[string]any{"GPU_OPERATOR_NAMESPACE": ""}), + recipe.ComponentRef{Name: "gpu-operator", Namespace: "nvidia-gpu-operator"}), + want: []string{`env.GPU_OPERATOR_NAMESPACE is "gpu-operator"`}, }, { name: "env UPLOAD_URL_BASE naming the file server over another uploadURL", diff --git a/tests/e2e/nvsentinel-health-events-analyzer/run.sh b/tests/e2e/nvsentinel-health-events-analyzer/run.sh index 4ab4f9ed67..5f24d3740b 100755 --- a/tests/e2e/nvsentinel-health-events-analyzer/run.sh +++ b/tests/e2e/nvsentinel-health-events-analyzer/run.sh @@ -266,10 +266,13 @@ publish_proto() { local version version=$(yq '.components[] | select(.name == "nvsentinel") | .helm.defaultVersion' "${ROOT}/recipes/registry.yaml") [[ -n "${version}" && "${version}" != "null" ]] || err "could not read nvsentinel's defaultVersion" - curl -fsSL --connect-timeout 10 --max-time 60 \ - "https://raw.githubusercontent.com/NVIDIA/NVSentinel/${version}/data-models/protobufs/health_event.proto" | - awk '/protoc_gen_crd\/proto\/crd.proto/ {next} /^\/\/ HealthEventResource is the root type/ {exit} {print}' \ - >"${WORK}/health_event.proto" || err "failed to fetch health_event.proto at ${version}" + # Download before trimming: awk exits at the marker, and a curl still writing + # into the closed pipe fails with "(23) Failed writing body". + curl -fsSL --connect-timeout 10 --max-time 60 -o "${WORK}/health_event.proto.raw" \ + "https://raw.githubusercontent.com/NVIDIA/NVSentinel/${version}/data-models/protobufs/health_event.proto" || + err "failed to fetch health_event.proto at ${version}" + awk '/protoc_gen_crd\/proto\/crd.proto/ {next} /^\/\/ HealthEventResource is the root type/ {exit} {print}' \ + "${WORK}/health_event.proto.raw" >"${WORK}/health_event.proto" grep -q "rpc HealthEventOccurredV1" "${WORK}/health_event.proto" || err "health_event.proto at ${version} has no HealthEventOccurredV1 -- the platform-connector API moved" kc -n nvsentinel create configmap e2e-health-event-proto \ diff --git a/tests/e2e/nvsentinel-log-collector/run.sh b/tests/e2e/nvsentinel-log-collector/run.sh index 9403b07563..798bfeac36 100755 --- a/tests/e2e/nvsentinel-log-collector/run.sh +++ b/tests/e2e/nvsentinel-log-collector/run.sh @@ -233,6 +233,45 @@ install_component() { err "$(basename "$1") install failed" } +# render_nvsentinel_bundle renders the bundle's nvsentinel chart with the +# values its install.sh passes. It runs in a subshell, so upstream.env's +# variables do not leak. +render_nvsentinel_bundle() ( + cd "${BUNDLE_DIR}" || exit 1 + # shellcheck source=/dev/null + . ./upstream.env + # shellcheck disable=SC2153 # CHART, REPO and VERSION come from upstream.env. + helm template nvsentinel "${CHART}" ${REPO:+--repo "${REPO}"} --version "${VERSION}" -n nvsentinel \ + -f values.yaml -f cluster-values.yaml +) + +# preload_file_server_images pulls the file server's images on the host, with +# retries, and imports them into the node: anonymous registry pulls from inside +# Kind on shared CI runners are throttled into ImagePullBackOff. The images come +# from the bundle's own render, so they track the chart pin. Not kind load: +# with Docker's containerd image store it exports an index whose other +# platforms have no layers, which its --all-platforms import rejects. +preload_file_server_images() { + local images image attempt arch + arch=$(docker version -f '{{.Server.Arch}}') || err "could not read the Docker host architecture" + images=$(render_nvsentinel_bundle | + yq -N "select(.kind == \"Deployment\" and .metadata.name == \"${FILE_SERVER}\") | + .spec.template.spec | (.initContainers // []) + .containers | .[].image" | sort -u) || + err "could not render the file server's images from the bundle" + [[ -n "${images}" ]] || err "the bundle renders no ${FILE_SERVER} Deployment" + for image in ${images}; do + for attempt in 1 2 3; do + docker pull -q --platform "linux/${arch}" "${image}" >/dev/null && break + ((attempt < 3)) || err "could not pull ${image}" + sleep $((attempt * 10)) + done + docker save "${image}" | + docker exec -i "${NODE}" ctr -n k8s.io images import --platform "linux/${arch}" --digests \ + --snapshotter=overlayfs - >/dev/null || err "could not import ${image} into ${NODE}" + done + msg "Preloaded the file server's images: $(tr '\n' ' ' <<<"${images}")" +} + install_bundle() { msg "Installing cert-manager, prometheus-operator-crds, the Percona datastore and nvsentinel from the bundle..." local dir d deadline remaining @@ -259,6 +298,7 @@ install_bundle() { kc -n nvsentinel wait --for=condition=Ready certificate/nvsentinel-mongodb-app-client \ --timeout="${remaining}s" || err "nvsentinel's MongoDB client certificate was never issued" + preload_file_server_images install_component "${BUNDLE_DIR}" deadline=$((SECONDS + NVSENTINEL_WAIT_SECONDS)) @@ -274,10 +314,13 @@ publish_proto() { local version version=$(yq '.components[] | select(.name == "nvsentinel") | .helm.defaultVersion' "${ROOT}/recipes/registry.yaml") [[ -n "${version}" && "${version}" != "null" ]] || err "could not read nvsentinel's defaultVersion" - curl -fsSL --connect-timeout 10 --max-time 60 \ - "https://raw.githubusercontent.com/NVIDIA/NVSentinel/${version}/data-models/protobufs/health_event.proto" | - awk '/protoc_gen_crd\/proto\/crd.proto/ {next} /^\/\/ HealthEventResource is the root type/ {exit} {print}' \ - >"${WORK}/health_event.proto" || err "failed to fetch health_event.proto at ${version}" + # Download before trimming: awk exits at the marker, and a curl still writing + # into the closed pipe fails with "(23) Failed writing body". + curl -fsSL --connect-timeout 10 --max-time 60 -o "${WORK}/health_event.proto.raw" \ + "https://raw.githubusercontent.com/NVIDIA/NVSentinel/${version}/data-models/protobufs/health_event.proto" || + err "failed to fetch health_event.proto at ${version}" + awk '/protoc_gen_crd\/proto\/crd.proto/ {next} /^\/\/ HealthEventResource is the root type/ {exit} {print}' \ + "${WORK}/health_event.proto.raw" >"${WORK}/health_event.proto" grep -q "rpc HealthEventOccurredV1" "${WORK}/health_event.proto" || err "health_event.proto at ${version} has no HealthEventOccurredV1 -- the platform-connector API moved" kc -n nvsentinel create configmap e2e-health-event-proto \ diff --git a/tests/e2e/nvsentinel-remediation/run.sh b/tests/e2e/nvsentinel-remediation/run.sh index 5faf4d7bf0..54b3da44c8 100755 --- a/tests/e2e/nvsentinel-remediation/run.sh +++ b/tests/e2e/nvsentinel-remediation/run.sh @@ -257,10 +257,13 @@ publish_proto() { local version version=$(yq '.components[] | select(.name == "nvsentinel") | .helm.defaultVersion' "${ROOT}/recipes/registry.yaml") [[ -n "${version}" && "${version}" != "null" ]] || err "could not read nvsentinel's defaultVersion" - curl -fsSL --connect-timeout 10 --max-time 60 \ - "https://raw.githubusercontent.com/NVIDIA/NVSentinel/${version}/data-models/protobufs/health_event.proto" | - awk '/protoc_gen_crd\/proto\/crd.proto/ {next} /^\/\/ HealthEventResource is the root type/ {exit} {print}' \ - >"${WORK}/health_event.proto" || err "failed to fetch health_event.proto at ${version}" + # Download before trimming: awk exits at the marker, and a curl still writing + # into the closed pipe fails with "(23) Failed writing body". + curl -fsSL --connect-timeout 10 --max-time 60 -o "${WORK}/health_event.proto.raw" \ + "https://raw.githubusercontent.com/NVIDIA/NVSentinel/${version}/data-models/protobufs/health_event.proto" || + err "failed to fetch health_event.proto at ${version}" + awk '/protoc_gen_crd\/proto\/crd.proto/ {next} /^\/\/ HealthEventResource is the root type/ {exit} {print}' \ + "${WORK}/health_event.proto.raw" >"${WORK}/health_event.proto" grep -q "rpc HealthEventOccurredV1" "${WORK}/health_event.proto" || err "health_event.proto at ${version} has no HealthEventOccurredV1 -- the platform-connector API moved" kc -n nvsentinel create configmap e2e-health-event-proto \ diff --git a/tests/e2e/nvsentinel-slinky-drainer/run.sh b/tests/e2e/nvsentinel-slinky-drainer/run.sh index 4b609491f5..282cdd1bff 100755 --- a/tests/e2e/nvsentinel-slinky-drainer/run.sh +++ b/tests/e2e/nvsentinel-slinky-drainer/run.sh @@ -318,10 +318,13 @@ publish_proto() { local version version=$(yq '.components[] | select(.name == "nvsentinel") | .helm.defaultVersion' "${ROOT}/recipes/registry.yaml") [[ -n "${version}" && "${version}" != "null" ]] || err "could not read nvsentinel's defaultVersion" - curl -fsSL --connect-timeout 10 --max-time 60 \ - "https://raw.githubusercontent.com/NVIDIA/NVSentinel/${version}/data-models/protobufs/health_event.proto" | - awk '/protoc_gen_crd\/proto\/crd.proto/ {next} /^\/\/ HealthEventResource is the root type/ {exit} {print}' \ - >"${WORK}/health_event.proto" || err "failed to fetch health_event.proto at ${version}" + # Download before trimming: awk exits at the marker, and a curl still writing + # into the closed pipe fails with "(23) Failed writing body". + curl -fsSL --connect-timeout 10 --max-time 60 -o "${WORK}/health_event.proto.raw" \ + "https://raw.githubusercontent.com/NVIDIA/NVSentinel/${version}/data-models/protobufs/health_event.proto" || + err "failed to fetch health_event.proto at ${version}" + awk '/protoc_gen_crd\/proto\/crd.proto/ {next} /^\/\/ HealthEventResource is the root type/ {exit} {print}' \ + "${WORK}/health_event.proto.raw" >"${WORK}/health_event.proto" grep -q "rpc HealthEventOccurredV1" "${WORK}/health_event.proto" || err "health_event.proto at ${version} has no HealthEventOccurredV1 -- the platform-connector API moved" kc -n nvsentinel create configmap e2e-health-event-proto \ From 108c3b58ef5d2efee648f1cf285e11de94b8e704 Mon Sep 17 00:00:00 2001 From: Varun Ramesh Date: Wed, 21 Oct 2026 14:30:21 +0530 Subject: [PATCH 4/8] fix(nvsentinel): pin the file server init image to docker.io busybox Signed-off-by: Varun Ramesh --- .github/workflows/vuln-scan-images.yaml | 17 +++++++++-------- docs/user/container-images.md | 2 +- .../nvsentinel_log_collector_render_test.go | 2 +- .../nvsentinel_log_collector_mixin_test.go | 9 +++++++-- recipes/mixins/nvsentinel-log-collector.yaml | 7 +++++++ recipes/registry.yaml | 2 ++ tools/mirror-e2e | 4 ++-- 7 files changed, 29 insertions(+), 14 deletions(-) diff --git a/.github/workflows/vuln-scan-images.yaml b/.github/workflows/vuln-scan-images.yaml index 5fc048e798..8a1b46c8b2 100644 --- a/.github/workflows/vuln-scan-images.yaml +++ b/.github/workflows/vuln-scan-images.yaml @@ -358,10 +358,10 @@ jobs: - ghcr.io/nvidia/nvsentinel/log-collector - ghcr.io/nvidia/nvsentinel/file-server-cleanup # The nvsentinel-log-collector mixin's file server, at the nvsentinel - # chart's own pins. bitnamilegacy is Bitnami's frozen archive and is - # never patched, so expect findings here. + # chart's own pins; its init container's busybox is the AICR pin + # below. bitnamilegacy is Bitnami's frozen archive and is never + # patched, so expect findings here. - docker.io/bitnamilegacy/nginx - - public.ecr.aws/docker/library/busybox - quay.io/martinhelmich/prometheus-nginxlog-exporter # Percona, from the remediation mixins' separately bundled datastore # (psmdb-operator, nvsentinel-mongodb). Pinned by AICR, by digest. @@ -374,11 +374,14 @@ jobs: # Third-party, pinned by AICR rather than the chart: # global.initContainerImage, used by chown/permission init # containers in the nic-health-monitor and audit-logging paths, - # and janitor-provider's generic reboot Job image. AICR overrides - # the chart's defaults for both -- the first in + # janitor-provider's generic reboot Job image, and the in-cluster + # file server's init container. AICR overrides the chart's defaults + # for all three -- the first in # recipes/components/nvsentinel/values.yaml, the second # (janitor-provider.csp.generic.rebootImage) in - # recipes/mixins/nvsentinel-remediation.yaml -- so what is scanned + # recipes/mixins/nvsentinel-remediation.yaml, the third + # (incluster-file-server.initImage) in + # recipes/mixins/nvsentinel-log-collector.yaml -- so what is scanned # here is the override; keep them in step. - docker.io/library/busybox include: @@ -435,8 +438,6 @@ jobs: # keeps this list, the mirror lane and the docs on the new tags. - image: docker.io/bitnamilegacy/nginx tag: 1.27.2-debian-12-r2 - - image: public.ecr.aws/docker/library/busybox - tag: 1.37.0 - image: quay.io/martinhelmich/prometheus-nginxlog-exporter tag: v1.10.0 # AICR's own pins in recipes/components/{psmdb-operator,nvsentinel-mongodb}; diff --git a/docs/user/container-images.md b/docs/user/container-images.md index dffd8e16b6..28b994b40b 100644 --- a/docs/user/container-images.md +++ b/docs/user/container-images.md @@ -472,7 +472,7 @@ The trade-off is intentional. Pinning an image gives reproducibility; deferring - The [`nvsentinel-slurm-drain-monitor` mixin](component-catalog.md#slurm-drain-monitor) sets `global.slurmDrainMonitor.enabled`, turning on the chart's `slurm-drain-monitor` subchart and pulling in `ghcr.io/nvidia/nvsentinel/slurm-drain-monitor:v1.28.0`. Every `platform: slurm` recipe composes it, so those recipes deploy this image and the others do not; it is in AICR's weekly image scan for the same reason as the object monitor's. - The [`nvsentinel-health-events-analyzer` mixin](component-catalog.md#health-events-analyzer) sets `global.healthEventsAnalyzer.enabled`, turning on the chart's `health-events-analyzer` subchart and pulling in `ghcr.io/nvidia/nvsentinel/health-events-analyzer:v1.28.0`, with no init container on the MongoDB datastore AICR bundles. No shipped recipe composes it; it is required beside the `nvsentinel-remediation` step. It is in the weekly image scan and the mirror lane, and `TestNVSentinelHealthEventsAnalyzerChartRender` pins it against the rendered chart. - The [`nvsentinel-event-exporter` mixin](component-catalog.md#health-event-export) sets `global.eventExporter.enabled`, turning on the chart's `event-exporter` subchart and pulling in `ghcr.io/nvidia/nvsentinel/event-exporter:v1.28.0`, with no init container on the MongoDB datastore AICR bundles. No shipped recipe composes it; it is in the weekly image scan and the mirror lane, and `TestNVSentinelEventExporterChartRender` pins it against the rendered chart. -- The [`nvsentinel-log-collector` mixin](component-catalog.md#diagnostics-capture) sets `global.inclusterFileServer.enabled` and `fault-remediation.logCollector.enabled`, adding five images: the collection Job's `ghcr.io/nvidia/nvsentinel/log-collector:v1.28.0`, which is named only inside `fault-remediation`'s ConfigMap, and the file server's `ghcr.io/nvidia/nvsentinel/file-server-cleanup:v1.28.0`, `docker.io/bitnamilegacy/nginx:1.27.2-debian-12-r2`, `public.ecr.aws/docker/library/busybox:1.37.0` (its init container) and `quay.io/martinhelmich/prometheus-nginxlog-exporter:v1.10.0`. The three third-party tags are the chart's own pins, not AICR's; `bitnamilegacy` is Bitnami's frozen archive and is never patched. No shipped recipe composes it; all five are in the weekly image scan and the mirror lane, and `TestNVSentinelLogCollectorChartRender` pins them against the rendered chart. +- The [`nvsentinel-log-collector` mixin](component-catalog.md#diagnostics-capture) sets `global.inclusterFileServer.enabled` and `fault-remediation.logCollector.enabled`, adding five images: the collection Job's `ghcr.io/nvidia/nvsentinel/log-collector:v1.28.0`, which is named only inside `fault-remediation`'s ConfigMap, and the file server's `ghcr.io/nvidia/nvsentinel/file-server-cleanup:v1.28.0`, `docker.io/bitnamilegacy/nginx:1.27.2-debian-12-r2`, `docker.io/library/busybox:1.38.0@sha256:fd7dc98638c8e305f4dc34e979f1c0fdfdcaeb0fbf8fcff77ae834b6da3d7e6e` (its init container) and `quay.io/martinhelmich/prometheus-nginxlog-exporter:v1.10.0`. The nginx and exporter tags are the chart's own pins, not AICR's; `bitnamilegacy` is Bitnami's frozen archive and is never patched. The busybox is AICR's: the mixin sets `incluster-file-server.initImage` to the one `nvsentinel-remediation` pins, replacing the chart's `public.ecr.aws/docker/library/busybox:1.37.0`, whose registry caps anonymous pulls by data per IP. No shipped recipe composes it; all five are in the weekly image scan and the mirror lane, and `TestNVSentinelLogCollectorChartRender` pins them against the rendered chart. A recipe composing any of these mixins **with `nvsentinel` still enabled** adds that mixin's images to what it deploys and mirrors; `aicr bundle`/`aicr mirror` on such a recipe surfaces them even though this static BOM cannot. A chain that disables `nvsentinel` (the OCP overlay, for example) can compose a mixin and ship none of them. diff --git a/pkg/bundler/validations/nvsentinel_log_collector_render_test.go b/pkg/bundler/validations/nvsentinel_log_collector_render_test.go index 45dfae3840..c9e479c911 100644 --- a/pkg/bundler/validations/nvsentinel_log_collector_render_test.go +++ b/pkg/bundler/validations/nvsentinel_log_collector_render_test.go @@ -36,7 +36,7 @@ const ( logCollectorImage = "ghcr.io/nvidia/nvsentinel/log-collector:v1.28.0" fileServerCleanupImage = "ghcr.io/nvidia/nvsentinel/file-server-cleanup:v1.28.0" fileServerNginxImage = "docker.io/bitnamilegacy/nginx:1.27.2-debian-12-r2" - fileServerInitImage = "public.ecr.aws/docker/library/busybox:1.37.0" + fileServerInitImage = "docker.io/library/busybox:1.38.0@sha256:fd7dc98638c8e305f4dc34e979f1c0fdfdcaeb0fbf8fcff77ae834b6da3d7e6e" fileServerExporterImage = "quay.io/martinhelmich/prometheus-nginxlog-exporter:v1.10.0" fileServerName = "nvsentinel-incluster-file-server" ) diff --git a/pkg/recipe/nvsentinel_log_collector_mixin_test.go b/pkg/recipe/nvsentinel_log_collector_mixin_test.go index 19d982f255..936c41cca8 100644 --- a/pkg/recipe/nvsentinel_log_collector_mixin_test.go +++ b/pkg/recipe/nvsentinel_log_collector_mixin_test.go @@ -26,7 +26,8 @@ import ( const logCollectorMixin = "nvsentinel-log-collector" // logCollectorChartImages follow nvsentinel's chart version; the file server's -// third-party images are pinned by the chart's own values instead. +// third-party images are pinned by the chart's own values, except the init +// container's busybox, which the mixin pins. var ( logCollectorChartImages = []string{ "ghcr.io/nvidia/nvsentinel/log-collector", @@ -34,7 +35,7 @@ var ( } logCollectorThirdPartyImages = []struct{ image, tag string }{ {"docker.io/bitnamilegacy/nginx", "1.27.2-debian-12-r2"}, - {"public.ecr.aws/docker/library/busybox", "1.37.0"}, + {"docker.io/library/busybox", "1.38.0@sha256:fd7dc98638c8e305f4dc34e979f1c0fdfdcaeb0fbf8fcff77ae834b6da3d7e6e"}, {"quay.io/martinhelmich/prometheus-nginxlog-exporter", "v1.10.0"}, } ) @@ -86,6 +87,10 @@ func TestMixinNVSentinelLogCollector_Composes(t *testing.T) { if !maps.Equal(persistence, map[string]any{"size": "50Gi"}) { t.Errorf("incluster-file-server.persistence = %v, want exactly {size: 50Gi}", persistence) } + initImage, _ := server["initImage"].(map[string]any) + if want := map[string]any{"repository": "docker.io/library/busybox", "tag": "1.38.0@sha256:fd7dc98638c8e305f4dc34e979f1c0fdfdcaeb0fbf8fcff77ae834b6da3d7e6e"}; !maps.Equal(initImage, want) { + t.Errorf("incluster-file-server.initImage = %v, want exactly %v", initImage, want) + } cleanup, _ := server["logCleanup"].(map[string]any) if want := map[string]any{"enabled": true, "retentionDays": 7, "sleepInterval": 3600}; !maps.Equal(cleanup, want) { t.Errorf("incluster-file-server.logCleanup = %v, want exactly %v", cleanup, want) diff --git a/recipes/mixins/nvsentinel-log-collector.yaml b/recipes/mixins/nvsentinel-log-collector.yaml index c57c6e2178..2457be1ee0 100644 --- a/recipes/mixins/nvsentinel-log-collector.yaml +++ b/recipes/mixins/nvsentinel-log-collector.yaml @@ -61,6 +61,13 @@ spec: # differs by deployer (Flux names the release -), and # uploadURL names this Service. fullnameOverride: nvsentinel-incluster-file-server + # The chart's public.ecr.aws busybox caps anonymous pulls by data per + # IP, which shared clusters and CI runners exhaust. This is the + # busybox nvsentinel-remediation pins for the reboot Job, already + # scanned and mirrored. + initImage: + repository: docker.io/library/busybox + tag: "1.38.0@sha256:fd7dc98638c8e305f4dc34e979f1c0fdfdcaeb0fbf8fcff77ae834b6da3d7e6e" persistence: # Upstream estimates 5-50 MB per collection, so 50Gi holds at least # a thousand of the largest within one retention window. diff --git a/recipes/registry.yaml b/recipes/registry.yaml index b0c4b59e2b..6f4cb048fc 100644 --- a/recipes/registry.yaml +++ b/recipes/registry.yaml @@ -648,6 +648,8 @@ components: - fault-remediation.logCollector.enableGpuOperatorMustGather - fault-remediation.logCollector.timeout - incluster-file-server.fullnameOverride + - incluster-file-server.initImage.repository + - incluster-file-server.initImage.tag - incluster-file-server.persistence.size - incluster-file-server.logCleanup.enabled - incluster-file-server.logCleanup.retentionDays diff --git a/tools/mirror-e2e b/tools/mirror-e2e index 08af4037f3..bb9526edac 100755 --- a/tools/mirror-e2e +++ b/tools/mirror-e2e @@ -136,12 +136,12 @@ SLINKY_DRAINER_IMAGE="ghcr.io/nvidia/nvsentinel/slinky-drainer:v1.28.0@sha256:bf HEALTH_EVENTS_ANALYZER_IMAGE="ghcr.io/nvidia/nvsentinel/health-events-analyzer:v1.28.0" # The images the nvsentinel-log-collector mixin pulls in (#2619): the collector # Job's, named only inside fault-remediation's ConfigMap, and the in-cluster -# file server's. +# file server's. Its init container's busybox is left out: the mixin pins the +# one the NIC mixin already brings, so listing it would prove nothing. LOG_COLLECTOR_IMAGES=( "ghcr.io/nvidia/nvsentinel/log-collector:v1.28.0" "ghcr.io/nvidia/nvsentinel/file-server-cleanup:v1.28.0" "docker.io/bitnamilegacy/nginx:1.27.2-debian-12-r2" - "public.ecr.aws/docker/library/busybox:1.37.0" "quay.io/martinhelmich/prometheus-nginxlog-exporter:v1.10.0" ) From 7b9410bcaa77b411b8d173644ee2592ca0f0dd56 Mon Sep 17 00:00:00 2001 From: Varun Ramesh Date: Fri, 23 Oct 2026 14:25:10 +0530 Subject: [PATCH 5/8] fix(nvsentinel): check the file server's StorageClass and harden the log collector e2e - Add CheckNVSentinelFileServerStorageClass (severity: warning): the datastore's snapshot StorageClass rule applied to incluster-file-server.persistence.storageClassName. - Share the StorageClass helper and replay the bundler's override order, so a later --set-json that clears the class is caught for both claims. - e2e: wait on the Job's Complete/Failed condition, install the Prometheus CRDs before cert-manager, and name preloaded images as the kubelet requests them (verified via CRI). - Docs: file server volume lifecycle, EKS os=amazonlinux limits, validator table rows. Signed-off-by: Varun Ramesh --- docs/contributor/validator.md | 2 + docs/integrator/recipe-development.md | 2 +- docs/user/component-catalog.md | 4 +- pkg/bundler/validations/checks.go | 1 + .../nvsentinel_datastore_prerequisites.go | 122 ++++++++++++++---- ...nvsentinel_datastore_prerequisites_test.go | 15 +++ .../validations/nvsentinel_log_collector.go | 61 ++++++++- .../nvsentinel_log_collector_test.go | 100 ++++++++++++++ recipes/registry.yaml | 7 + tests/e2e/nvsentinel-log-collector/run.sh | 69 ++++++++-- 10 files changed, 340 insertions(+), 43 deletions(-) diff --git a/docs/contributor/validator.md b/docs/contributor/validator.md index 22d103308e..a86d5fc9c5 100644 --- a/docs/contributor/validator.md +++ b/docs/contributor/validator.md @@ -1411,6 +1411,8 @@ unrecognized value, which keeps a typo'd severity non-silent. | `CheckNVSentinelSlurmDrainMonitorRequiresSlinky` | NVSentinel's `slurm-drain-monitor` is not enabled without `slinky-slurm` (absent or disabled), or watching a blank namespace or one `slinky-slurm` does not deploy to: the monitor reads the `SlurmNodeStateDrain` condition off Slinky worker pods, which only `slinky-slurm`'s NodeSets create, so it would run and never see a pod. Rejects `--dynamic` on the monitor's toggle and namespace and on `slinky-slurm`'s `enabled`, even with the monitor off. Wired at `severity: error`. | | `CheckNVSentinelSlurmCustomDrainCoherent` | NVSentinel drains Slurm nodes through the slinky-drainer plugin: a recipe with `slinky-slurm` and a live (not dry-run) `fault-remediation` has `node-drainer.customDrain` on, while quarantine alone passes; `customDrain` is only on with a live `node-drainer`, a boolean `global.dryRun` that is false, `slinky-slurm`, the `slinky-drainer-crds` component, and the `slinky-drainer` component in NVSentinel's namespace watching the namespace `slinky-slurm` deploys to; and its values match the plugin's contract (DrainRequest kind, resource and `DrainComplete` condition, the `slinky-drain-template` template, NVSentinel's namespace, a positive whole-seconds `timeout`, and empty `userNamespaces` and `podDrainPolicies` with an empty `nodeSelector`, since a scope could leave slurmd nodes on pod eviction); and the slurmd NodeSet and every Slurm control pod in the Slurm namespace (slinky-slurm's controller, REST API, login and enabled accounting, and `slurm-accounting-mariadb` there) render non-empty node selectors (modelled on the bundler's order: `--set`, then node-selector injection unless the overlay opts out with an empty or null value, then `--set-json`), each control pod's giving some label key a different value from slurmd's, since the plugin waits for every Ready pod in that namespace on a drained node to report Slurm DRAIN. Rejects `--dynamic` on any path a rule that can still fire reads, so a monitoring-only Slurm recipe keeps its `--dynamic` paths. Wired at `severity: error`. | | `CheckNVSentinelEventExporterCoherent` | NVSentinel's Event Exporter can deliver: `sink.endpoint` and `oidc.tokenUrl` are `http` or `https` URLs with a host that is not an `example.com`, `.org` or `.net` placeholder; `oidc.clientId`, `oidc.scope`, `metadata.cluster` (not the chart's `my-cluster`) and `metadata.environment` are set by AICR's values as strings; no metadata value holds a double quote, backslash or control character other than tab, and every metadata key is a TOML bare key, since the chart writes them into its config unescaped; both `insecureSkipVerify` settings are `false`; and `oidcSecretName` is a valid Secret name. A no-op unless `global.eventExporter.enabled`; then rejects `--dynamic` on any `event-exporter` value. Wired at `severity: error`. | +| `CheckNVSentinelLogCollectorCoherent` | NVSentinel's log collector can capture and upload a node's diagnostics: `fault-remediation.logCollector.enabled` and `global.inclusterFileServer.enabled` are on together, `fault-remediation` runs live (not dry run, with a boolean `global.dryRun` and `logCollector.enabled`), the file server keeps the mixin's `fullnameOverride` and the upload URL names its Service, and the collector can reach `nvidia-bug-report` (the GPU Operator installs the driver in the namespace `logCollector.gpuOperatorNamespaces` names, or GKE's managed install; not AKS or OKE at their default `gpuStack`, Kind, or EKS on Amazon Linux). An `env` entry that replaces the upload URL or namespace is read in its place; `MOCK_MODE=true` skips the driver rule. A no-op unless either setting is on; rejects `--dynamic` on the paths it verifies. Wired at `severity: error`. | +| `CheckNVSentinelFileServerStorageClass` | The in-cluster file server's report volume can bind, from snapshot evidence, by the rule `CheckNVSentinelDatastorePrerequisites` applies to the datastore: `metadata.defaultStorageClassState` `absent` and `unknown` block unless `--storage-class` or `incluster-file-server.persistence.storageClassName` names a class; `multiple` and no evidence warn; an unrecognized state blocks. A no-op unless the file server renders with persistence on. Registered apart from `CheckNVSentinelLogCollectorCoherent`, at `severity: warning`, so its warnings stay non-blocking while its errors still block. | | `CheckNPDNotDuplicatingProviderNPD` | `node-problem-detector` is bundled only on an allowlist of verified platforms — never where the provider already runs its own NPD (GKE, AKS), on OpenShift (no privileged SCC binding), or on Talos (restricted namespace) | | `CheckGKETCPXOInterfacesCoherence` | The final resolved kubeflow-trainer `tcpxoInterfaces` value matches the mapping the recipe records in `configuration.gke.tcpxoInterfaces` | | `CheckNVSentinelDriverLabelDetectable` | Where the GPU Operator does not own the driver and no driver pod exists (driver ships in the node image), NVSentinel sets `labeler.assumeDriverInstalled`, so its driver-label-gated DaemonSets do not come up half-rolled-out | diff --git a/docs/integrator/recipe-development.md b/docs/integrator/recipe-development.md index 1de5fc5a6e..b63f72ed4a 100644 --- a/docs/integrator/recipe-development.md +++ b/docs/integrator/recipe-development.md @@ -184,7 +184,7 @@ When authoring a recipe targeting Talos (`criteria.os: talos`), append the `os-t **`nvsentinel-datastore-eks`** declares the storage edge of NVSentinel's datastore on EKS. An EKS leaf that composes a remediation step lists it after the step (`spec.mixins: [nvsentinel-observe, nvsentinel-datastore-eks]`); it adds `aws-ebs-csi-driver` to `nvsentinel-mongodb`'s `dependencyRefs`, so the driver that provisions the datastore's volumes deploys first. It contributes only `dependencyRefs` to an already-chained component, which `mixinComponentRefSafeForMerge` allows; listed before the step it would introduce `nvsentinel-mongodb` itself, and the step's `type` for that component then fails resolution as an identity conflict. On a cluster that uses the EKS-managed EBS addon, `aws-ebs-csi-driver` stays declared with `enabled: false` and the edge counts as satisfied outside the recipe. See [component catalog](../user/component-catalog.md#graded-remediation). -**`nvsentinel-log-collector`** captures a faulty node's diagnostics before NVSentinel repairs it, composed beside the remediation step and the health events analyzer that live remediation requires (`spec.mixins: [nvsentinel-remediation, nvsentinel-health-events-analyzer, nvsentinel-log-collector]`), because only live `fault-remediation` runs the collector. It turns on `fault-remediation.logCollector` and `global.inclusterFileServer` together, pins the file server's name and the upload URL that names it, sets must-gather explicitly off and the volume size and cleanup cycle deliberately, all through `nvsentinel`'s `mixinSafeOverridePaths`; it sets no path a step sets. The file server's claim is in `nvsentinel`'s `storageClassPaths`, so `--storage-class` reaches it. `CheckNVSentinelLogCollectorCoherent` fails the bundle without live remediation, with only one of the two settings on, and on a platform where the collector cannot reach the driver (AKS and OKE at their default `gpuStack`, Kind). See [component catalog](../user/component-catalog.md#diagnostics-capture) for where bundles land, how to retrieve them, and the privileges involved. +**`nvsentinel-log-collector`** captures a faulty node's diagnostics before NVSentinel repairs it, composed beside the remediation step and the health events analyzer that live remediation requires (`spec.mixins: [nvsentinel-remediation, nvsentinel-health-events-analyzer, nvsentinel-log-collector]`), because only live `fault-remediation` runs the collector. It turns on `fault-remediation.logCollector` and `global.inclusterFileServer` together, pins the file server's name and the upload URL that names it, sets must-gather explicitly off and the volume size and cleanup cycle deliberately, all through `nvsentinel`'s `mixinSafeOverridePaths`; it sets no path a step sets. The file server's claim is in `nvsentinel`'s `storageClassPaths`, so `--storage-class` reaches it. `CheckNVSentinelLogCollectorCoherent` fails the bundle without live remediation, with only one of the two settings on, and on a platform where the collector cannot reach the driver (AKS and OKE at their default `gpuStack`, Kind, the EKS `os=amazonlinux` leaves, and the VR200 RKE2 and H200 k0s leaves). See [component catalog](../user/component-catalog.md#diagnostics-capture) for where bundles land, how to retrieve them, and the privileges involved. **Cross-cutting overlays with wildcard criteria** apply across one criteria dimension without being referenced via `spec.base` or listed in `spec.mixins`. The resolver can return multiple independent maximal-leaf overlays for a single query, so a `service: any` overlay is picked up alongside the service-specific maximal leaf and its inheritance chain: diff --git a/docs/user/component-catalog.md b/docs/user/component-catalog.md index afbe94d043..c22c916aaf 100644 --- a/docs/user/component-catalog.md +++ b/docs/user/component-catalog.md @@ -1078,7 +1078,7 @@ The mixin turns on two settings together, because neither is useful alone: `faul **Order and timing.** `fault-remediation` waits for the collection Job before it creates the repair, so each repair is delayed by the collection, up to the mixin's `logCollector.timeout` of 10 minutes. A Job that fails or times out does not block the repair: `fault-remediation` records the failure and remediates, and a timed-out Job is left running. A Job that cannot be created at all does block it, and `fault-remediation` retries the event. The collector is skipped in dry run, so the `nvsentinel-observe` step collects nothing, and `nvsentinel-quarantine` does not run `fault-remediation` at all. It is also skipped once a node reaches the repair cap (`fault-remediation.maxRemediationAttempts: 3`): `fault-remediation` gives up on the event before collecting, so a fault that keeps firing on a capped node does not start a new collection each time. -**Supported platforms.** The collector runs `nvidia-bug-report.sh` inside the GPU Operator's driver container on the node, which it finds by label in the one namespace `logCollector.gpuOperatorNamespaces` names (`gpu-operator`; despite the name, it takes a single namespace), or, on GKE COS, from the managed driver install at `/home/kubernetes/bin/nvidia`. It therefore works where the GPU Operator installs the driver (EKS, BCM, LKE, generic, most RKE2 and k0s leaves, and AKS and OKE with `gpuStack=operator-managed`). The collector can also read GKE COS's managed install, but the remediation step it needs is not supported on GKE COS (see [Graded Remediation](#graded-remediation)), so the mixin cannot be used there today. Where the node image carries the driver and the GPU Operator runs no driver container (AKS and OKE at their default `gpuStack`, Kind, the VR200 RKE2 and H200 k0s leaves), every collection would fail, so the bundle does. Custom cluster domains are not supported: the upload URL names `svc.cluster.local`, as the datastore's does. +**Supported platforms.** The collector runs `nvidia-bug-report.sh` inside the GPU Operator's driver container on the node, which it finds by label in the one namespace `logCollector.gpuOperatorNamespaces` names (`gpu-operator`; despite the name, it takes a single namespace), or, on GKE COS, from the managed driver install at `/home/kubernetes/bin/nvidia`. It therefore works where the GPU Operator installs the driver (EKS except `os=amazonlinux`, BCM, LKE, generic, most RKE2 and k0s leaves, and AKS and OKE with `gpuStack=operator-managed`). The collector can also read GKE COS's managed install, but the remediation step it needs is not supported on GKE COS (see [Graded Remediation](#graded-remediation)), so the mixin cannot be used there today. Where the node image carries the driver and the GPU Operator runs no driver container (AKS and OKE at their default `gpuStack`, Kind, the EKS `os=amazonlinux` leaves, and the VR200 RKE2 and H200 k0s leaves), every collection would fail, so the bundle does. Custom cluster domains are not supported: the upload URL names `svc.cluster.local`, as the datastore's does. **Where bundles land and how to retrieve them.** Each collection is stored on the file server's volume under `/upload///`, as `nvidia-bug-report--.log.gz`. The server lists directories, so browse and download through a port-forward: @@ -1091,7 +1091,7 @@ curl -sO http://localhost:8081/upload/gpu-node-1/20261015-101500/nvidia-bug-repo **Retention and sizing.** A cleanup container runs `find -type f -mtime +7 -delete` (`incluster-file-server.logCleanup.retentionDays: 7`) at start and then every hour (`sleepInterval: 3600`, where the chart's default is daily), so a report is deleted within an hour of being at least eight full days old: `find` counts whole days, so `retentionDays: N` keeps a report for N+1. Only files are deleted; the emptied `//` directories stay. Upstream estimates 5–50 MB per collection, so the 50Gi volume (`persistence.size`) holds at least a thousand of the largest within one retention window. The cleanup container exports `fileserver_disk_space_free_bytes` on the file server's `cleanup-metrics` port (9002), refreshed each cycle; alert on it if collections are frequent. -**Storage.** The file server claims one `ReadWriteOnce` volume. Its `storageClassName` is registered in `nvsentinel`'s `storageClassPaths`, so `--storage-class` sets it; without the flag the claim takes the cluster's default StorageClass, `aicr bundle` warns, and on a cluster with none the claim stays `Pending` and the file server never starts. AICR provisions no storage. +**Storage.** The file server claims one `ReadWriteOnce` volume. Its `storageClassName` is registered in `nvsentinel`'s `storageClassPaths`, so `--storage-class` sets it; without the flag the claim takes the cluster's default StorageClass, and on a cluster with none the claim stays `Pending`, the file server never starts and every upload fails. As for the datastore, `CheckNVSentinelFileServerStorageClass` reads the snapshot's evidence: it fails the bundle when the snapshot found no default StorageClass, or could not tell, and neither `--storage-class` nor `--set nv-sentinel:incluster-file-server.persistence.storageClassName=` names one (naming a class for `nvsentinel-mongodb` alone does not count), and warns when the recipe carries no snapshot evidence or the cluster has more than one default. AICR provisions no storage. The chart does not keep the claim on uninstall, so removing the mixin deletes it, and with a `Delete` reclaim policy the stored reports go with it: copy them off first. **What is not collected.** GPU Operator must-gather is set explicitly off (`enableGpuOperatorMustGather: false`): it collects from every node in the cluster, not the faulty one, so on a large cluster one fault becomes a cluster-wide collection that also outlasts the timeout. The GCP and AWS SOS reports stay at the chart default, off. diff --git a/pkg/bundler/validations/checks.go b/pkg/bundler/validations/checks.go index 6ecdf02896..7939d9b2ad 100644 --- a/pkg/bundler/validations/checks.go +++ b/pkg/bundler/validations/checks.go @@ -69,6 +69,7 @@ func init() { registerCheck("CheckNVSentinelSlurmCustomDrainCoherent", CheckNVSentinelSlurmCustomDrainCoherent) registerCheck("CheckNVSentinelEventExporterCoherent", CheckNVSentinelEventExporterCoherent) registerCheck("CheckNVSentinelLogCollectorCoherent", CheckNVSentinelLogCollectorCoherent) + registerCheck("CheckNVSentinelFileServerStorageClass", CheckNVSentinelFileServerStorageClass) registerCheck("CheckNVSentinelMongoDBCoherent", CheckNVSentinelMongoDBCoherent) registerCheck("CheckNVSentinelDatastorePrerequisites", CheckNVSentinelDatastorePrerequisites) } diff --git a/pkg/bundler/validations/nvsentinel_datastore_prerequisites.go b/pkg/bundler/validations/nvsentinel_datastore_prerequisites.go index 02d58c9e69..4dcf52f36b 100644 --- a/pkg/bundler/validations/nvsentinel_datastore_prerequisites.go +++ b/pkg/bundler/validations/nvsentinel_datastore_prerequisites.go @@ -17,8 +17,10 @@ package validations import ( "context" "fmt" + "slices" "github.com/NVIDIA/aicr/pkg/bundler/config" + "github.com/NVIDIA/aicr/pkg/component" aicrerrors "github.com/NVIDIA/aicr/pkg/errors" "github.com/NVIDIA/aicr/pkg/recipe" ) @@ -67,20 +69,51 @@ func CheckNVSentinelDatastorePrerequisites(ctx context.Context, componentName st return warnings, errs } +// nvsentinelMongoDBClaim is nvsentinel-mongodb's classless volumes. +var nvsentinelMongoDBClaim = storageClassClaim{ + path: nvsentinelMongoDBStorageClassPath, + overrideKey: "nvsentinelmongodb", + volumes: "its volumes", + pending: "its three volumes, which name no StorageClass, would stay Pending and NVSentinel would not start", + purpose: "NVSentinel datastore prerequisites", +} + func nvsentinelStorageClassFinding(ctx context.Context, componentName string, recipeResult *recipe.RecipeResult, bundlerConfig *config.Config, keys []string) (string, error) { + return storageClassFinding(ctx, componentName, recipeResult, bundlerConfig, keys, nvsentinelMongoDBClaim) +} + +// storageClassClaim is a volume claim that names no StorageClass unless the +// bundle does, so whether it binds depends on the cluster's default. +type storageClassClaim struct { + // path is the values path that names the claim's StorageClass, and + // overrideKey the --set key of the component that owns it. + path string + overrideKey string + // volumes names the claim as a sentence subject, and pending what a + // Pending claim stops. + volumes string + pending string + purpose string +} + +// storageClassFinding applies the snapshot's metadata.defaultStorageClassState +// to claim: a warning with no evidence or more than one default, an error when +// the snapshot found none or could not tell, unless the bundle names a class +// for the claim. +func storageClassFinding(ctx context.Context, componentName string, recipeResult *recipe.RecipeResult, bundlerConfig *config.Config, keys []string, claim storageClassClaim) (string, error) { state := recipeResult.Metadata.DefaultStorageClassState switch state { case recipe.DefaultStorageClassStatePresent: return "", nil case "": return fmt.Sprintf("%s: no metadata.defaultStorageClassState snapshot evidence was recorded, so the "+ - "default StorageClass its volumes need was not evaluated. Regenerate the recipe from a current "+ - "snapshot to verify the target cluster before deployment", componentName), nil + "default StorageClass for %s was not evaluated. Regenerate the recipe from a current "+ + "snapshot to verify the target cluster before deployment", componentName, claim.volumes), nil case recipe.DefaultStorageClassStateMultiple: - return fmt.Sprintf("%s: the snapshot found more than one default StorageClass; Kubernetes binds its "+ - "volumes to the newest one. Leave a single StorageClass annotated "+ + return fmt.Sprintf("%s: the snapshot found more than one default StorageClass; Kubernetes binds "+ + "%s to the newest one. Leave a single StorageClass annotated "+ "storageclass.kubernetes.io/is-default-class=true, or set "+ - "--set nvsentinelmongodb:%s=", componentName, nvsentinelMongoDBStorageClassPath), nil + "--set %s:%s=", componentName, claim.volumes, claim.overrideKey, claim.path), nil case recipe.DefaultStorageClassStateAbsent, recipe.DefaultStorageClassStateUnknown: default: return "", aicrerrors.New(aicrerrors.ErrCodeInvalidRequest, fmt.Sprintf( @@ -88,7 +121,7 @@ func nvsentinelStorageClassFinding(ctx context.Context, componentName string, re "AICR version before bundling", componentName, state)) } - named, err := nvsentinelMongoDBNamesStorageClass(ctx, recipeResult, bundlerConfig, componentName, keys) + named, err := bundleNamesStorageClass(ctx, recipeResult, bundlerConfig, componentName, keys, claim) if err != nil { return "", err } @@ -98,39 +131,74 @@ func nvsentinelStorageClassFinding(ctx context.Context, componentName string, re if state == recipe.DefaultStorageClassStateAbsent { return "", aicrerrors.New(aicrerrors.ErrCodeConflict, fmt.Sprintf( - "%s: the snapshot found no default StorageClass, so its three volumes, which name no "+ - "StorageClass, would stay Pending and NVSentinel would not start. Annotate a StorageClass "+ + "%s: the snapshot found no default StorageClass, so %s. Annotate a StorageClass "+ "with storageclass.kubernetes.io/is-default-class=true and regenerate the recipe from a "+ - "fresh snapshot, or name one with --storage-class or --set nvsentinelmongodb:%s=", - componentName, nvsentinelMongoDBStorageClassPath)) + "fresh snapshot, or name one with --storage-class or --set %s:%s=", + componentName, claim.pending, claim.overrideKey, claim.path)) } return "", aicrerrors.New(aicrerrors.ErrCodeConflict, fmt.Sprintf( - "%s: default StorageClass evidence is inconclusive, so its volumes cannot be shown to bind. "+ + "%s: default StorageClass evidence is inconclusive, so %s cannot be shown to bind. "+ "Capture a fresh snapshot with permission to list storageclasses (storage.k8s.io), or name a "+ - "StorageClass with --storage-class or --set nvsentinelmongodb:%s=", - componentName, nvsentinelMongoDBStorageClassPath)) + "StorageClass with --storage-class or --set %s:%s=", + componentName, claim.volumes, claim.overrideKey, claim.path)) } -// nvsentinelMongoDBNamesStorageClass reports whether the bundle names a -// StorageClass for the volumes: through the effective values, or through -// bundle --storage-class, which the bundler injects into the registry's -// storageClassPaths unless a scalar --set names the path explicitly. -func nvsentinelMongoDBNamesStorageClass(ctx context.Context, recipeResult *recipe.RecipeResult, bundlerConfig *config.Config, componentName string, keys []string) (bool, error) { - if bundlerConfig != nil && bundlerConfig.StorageClass() != "" { - explicit := mergeOverridesAcrossKeys(bundlerConfig.ValueOverrides(), keys) - if _, set := explicit[nvsentinelMongoDBStorageClassPath]; !set { - return true, nil +// bundleNamesStorageClass reports whether the bundle names a StorageClass +// for claim. It models the bundler's order: recipe values, then --set, then +// --storage-class injected into the component's registry storageClassPaths +// unless --set names the path, then --set-json/--set-file, which can still +// clear the class (an empty or null value, or a parent object without it). +func bundleNamesStorageClass(ctx context.Context, recipeResult *recipe.RecipeResult, bundlerConfig *config.Config, componentName string, keys []string, claim storageClassClaim) (bool, error) { + values, err := recipeResult.GetValuesForComponentWithContext(ctx, componentName) + if err != nil { + return false, aicrerrors.PropagateOrWrap(err, aicrerrors.ErrCodeInvalidRequest, + fmt.Sprintf("cannot verify %s for component %q: failed to resolve its values", claim.purpose, componentName)) + } + if bundlerConfig == nil { + class, ok, _ := resolvedStringValue(values, claim.path) + return ok && class != "", nil + } + + explicit := mergeOverridesAcrossKeys(bundlerConfig.ValueOverrides(), keys) + delete(explicit, config.ComponentEnabledKey) + if len(explicit) > 0 { + if applyErr := component.ApplyMapOverrides(values, explicit); applyErr != nil { + return false, aicrerrors.Wrap(aicrerrors.ErrCodeInvalidRequest, + fmt.Sprintf("cannot verify %s for component %q: failed to apply --set overrides", claim.purpose, + componentName), applyErr) } } - values, err := effectiveComponentValues(ctx, recipeResult, bundlerConfig, componentName, keys, - "NVSentinel datastore prerequisites") - if err != nil { - return false, err + if class := bundlerConfig.StorageClass(); class != "" { + if _, set := explicit[claim.path]; !set && storageClassPathRegistered(recipeResult, componentName, claim.path) { + // The bundler logs and skips a path it cannot write, so a + // failure here leaves the class unnamed, as it does there. + _ = component.ApplyMapOverrides(values, map[string]string{claim.path: class}) + } + } + typed := mergeOverridesAcrossKeys(bundlerConfig.ValueOverridesTyped(), keys) + delete(typed, config.ComponentEnabledKey) + if len(typed) > 0 { + if applyErr := component.ApplyTypedOverrides(values, typed); applyErr != nil { + return false, aicrerrors.Wrap(aicrerrors.ErrCodeInvalidRequest, + fmt.Sprintf("cannot verify %s for component %q: failed to apply --set-json/--set-file overrides", + claim.purpose, componentName), applyErr) + } } - class, ok, _ := resolvedStringValue(values, nvsentinelMongoDBStorageClassPath) + class, ok, _ := resolvedStringValue(values, claim.path) return ok && class != "", nil } +// storageClassPathRegistered reports whether path is one of the component's +// registry storageClassPaths, the only paths --storage-class reaches. +func storageClassPathRegistered(recipeResult *recipe.RecipeResult, componentName, path string) bool { + registry, err := recipe.GetComponentRegistryFor(recipeResult.DataProvider()) + if err != nil { + return false + } + comp := registry.Get(componentName) + return comp != nil && slices.Contains(comp.GetStorageClassPaths(), path) +} + func nvsentinelPerconaFinding(componentName, state string) (string, error) { switch state { case recipe.PerconaOperatorStateAbsent, recipe.PerconaOperatorStateAICROwned: diff --git a/pkg/bundler/validations/nvsentinel_datastore_prerequisites_test.go b/pkg/bundler/validations/nvsentinel_datastore_prerequisites_test.go index d646139229..1787f89312 100644 --- a/pkg/bundler/validations/nvsentinel_datastore_prerequisites_test.go +++ b/pkg/bundler/validations/nvsentinel_datastore_prerequisites_test.go @@ -46,6 +46,10 @@ func TestCheckNVSentinelDatastorePrerequisites(t *testing.T) { })) } present, absentPercona := recipe.DefaultStorageClassStatePresent, recipe.PerconaOperatorStateAbsent + storageThenTyped := func(path string, value any) *config.Config { + return config.NewConfig(config.WithStorageClass("fast"), config.WithValueOverridesTypedPaths( + []config.TypedComponentPath{{Component: "nvsentinelmongodb", Path: path, Value: value}})) + } tests := []struct { name string @@ -100,6 +104,17 @@ func TestCheckNVSentinelDatastorePrerequisites(t *testing.T) { })), wantErrs: []string{"found no default StorageClass"}, }, + // --set-json runs after --storage-class is injected, so it can still + // clear the class. + {name: "--storage-class then --set-json empty class", rr: result(recipe.DefaultStorageClassStateAbsent, absentPercona, nil), + cfg: storageThenTyped(nvsentinelMongoDBStorageClassPath, ""), wantErrs: []string{"found no default StorageClass"}}, + {name: "--storage-class then --set-json null class", rr: result(recipe.DefaultStorageClassStateAbsent, absentPercona, nil), + cfg: storageThenTyped(nvsentinelMongoDBStorageClassPath, nil), wantErrs: []string{"found no default StorageClass"}}, + {name: "--storage-class then --set-json parent with an empty class", rr: result(recipe.DefaultStorageClassStateAbsent, absentPercona, nil), + cfg: storageThenTyped("replsets.rs0.volumeSpec.pvc", map[string]any{"storageClassName": ""}), + wantErrs: []string{"found no default StorageClass"}}, + {name: "--storage-class then --set-json parent without the key merges", rr: result(recipe.DefaultStorageClassStateAbsent, absentPercona, nil), + cfg: storageThenTyped("replsets.rs0.volumeSpec.pvc", map[string]any{"accessModes": []any{"ReadWriteOnce"}})}, { name: "empty recipe StorageClass does not waive", rr: result(recipe.DefaultStorageClassStateAbsent, absentPercona, emptyClass), diff --git a/pkg/bundler/validations/nvsentinel_log_collector.go b/pkg/bundler/validations/nvsentinel_log_collector.go index d8030472eb..c764ba943e 100644 --- a/pkg/bundler/validations/nvsentinel_log_collector.go +++ b/pkg/bundler/validations/nvsentinel_log_collector.go @@ -71,7 +71,7 @@ const ( // GPU Operator's driver container, which it looks up in the single // namespace gpuOperatorNamespaces names, or from GKE's managed driver // install, so a recipe whose node image carries the driver otherwise (AKS, -// OKE, Kind) fails every collection. +// OKE, Kind, EKS on Amazon Linux) fails every collection. // // The Job template renders logCollector.env after the variables it sets from // uploadURL and gpuOperatorNamespaces, so an env.UPLOAD_URL_BASE or @@ -316,3 +316,62 @@ func logCollectorDynamicViolations(bundlerConfig *config.Config, componentName s } return msgs } + +// fileServerStorageClassPath names the file server's StorageClass; it is in +// nvsentinel's storageClassPaths, so --storage-class reaches it. +const fileServerStorageClassPath = "incluster-file-server.persistence.storageClassName" + +// fileServerClaim is the file server's classless report volume. +var fileServerClaim = storageClassClaim{ + path: fileServerStorageClassPath, + overrideKey: "nv-sentinel", + volumes: "the file server's volume", + pending: "the file server's volume, which names no StorageClass, would stay Pending, the file server " + + "would not start, and every bug report upload would fail", + purpose: "NVSentinel file server storage", +} + +// CheckNVSentinelFileServerStorageClass applies the snapshot's default +// StorageClass evidence to the in-cluster file server's report volume, the +// rule CheckNVSentinelDatastorePrerequisites applies to the datastore's: a +// blocking error when the snapshot found no default StorageClass or could not +// tell, unless --storage-class or incluster-file-server.persistence.storageClassName +// names one; a warning when the recipe carries no evidence or the cluster has +// more than one default. A no-op unless the file server renders with +// persistence on. It is registered apart from CheckNVSentinelLogCollectorCoherent +// because its warnings must stay non-blocking, which that gate's severity +// does not allow. +func CheckNVSentinelFileServerStorageClass(ctx context.Context, componentName string, recipeResult *recipe.RecipeResult, bundlerConfig *config.Config, conditions map[string][]string) ([]string, []error) { + if recipeResult == nil || !checkConditions(recipeResult, conditions) { + return nil, nil + } + ref := recipeResult.GetComponentRef(componentName) + if ref == nil { + return nil, nil + } + keys := componentOverrideKeys(componentName, recipeResult.DataProvider()) + if componentDisabled(ref, bundlerConfig, keys) { + return nil, nil + } + values, err := effectiveComponentValues(ctx, recipeResult, bundlerConfig, componentName, keys, fileServerClaim.purpose) + if err != nil { + return nil, []error{err} + } + if !nvsentinelSubchartRenders(values, "inclusterFileServer") { + return nil, nil + } + // The chart renders the claim only under persistence.enabled, which + // defaults to true. + if enabled, set := component.GetValueByPath(values, "incluster-file-server.persistence.enabled"); set && !helmTruthy(enabled) { + return nil, nil + } + warning, findingErr := storageClassFinding(ctx, componentName, recipeResult, bundlerConfig, keys, fileServerClaim) + var warnings []string + if warning != "" { + warnings = append(warnings, warning) + } + if findingErr != nil { + return warnings, []error{findingErr} + } + return warnings, nil +} diff --git a/pkg/bundler/validations/nvsentinel_log_collector_test.go b/pkg/bundler/validations/nvsentinel_log_collector_test.go index e660f8203e..3d60207550 100644 --- a/pkg/bundler/validations/nvsentinel_log_collector_test.go +++ b/pkg/bundler/validations/nvsentinel_log_collector_test.go @@ -497,3 +497,103 @@ func TestCheckNVSentinelLogCollectorCoherentNilRecipe(t *testing.T) { t.Errorf("no nvsentinel: got %v %v, want nothing", w, e) } } + +// TestCheckNVSentinelFileServerStorageClass covers the file server's +// StorageClass rule, the datastore's applied to the report volume: a remedy +// that names a class for the datastore alone must not pass it. +func TestCheckNVSentinelFileServerStorageClass(t *testing.T) { + t.Parallel() + + fileServer := func(extra map[string]any) map[string]any { + fs := map[string]any{"persistence": map[string]any{"size": "50Gi"}} + for k, v := range extra { + fs[k] = v + } + return map[string]any{ + "global": map[string]any{"inclusterFileServer": map[string]any{"enabled": true}}, + "incluster-file-server": fs, + } + } + result := func(storage string, overrides map[string]any) *recipe.RecipeResult { + r := &recipe.RecipeResult{ComponentRefs: []recipe.ComponentRef{ + {Name: nvsentinelComponent, Overrides: overrides}, + {Name: "nvsentinel-mongodb", Overrides: map[string]any{"replsets": map[string]any{"rs0": map[string]any{ + "volumeSpec": map[string]any{"pvc": map[string]any{"storageClassName": "fast"}}}}}}, + }} + r.Metadata.DefaultStorageClassState = storage + return r + } + set := func(path, value string) *config.Config { + return config.NewConfig(config.WithValueOverrides(map[string]map[string]string{"nv-sentinel": {path: value}})) + } + storageThenTyped := func(path string, value any) *config.Config { + return config.NewConfig(config.WithStorageClass("fast"), config.WithValueOverridesTypedPaths( + []config.TypedComponentPath{{Component: "nv-sentinel", Path: path, Value: value}})) + } + absent := recipe.DefaultStorageClassStateAbsent + + tests := []struct { + name string + rr *recipe.RecipeResult + cfg *config.Config + wantWarnings []string + wantErrs []string + }{ + {name: "nil recipe"}, + {name: "file server off", rr: result(absent, map[string]any{ + "global": map[string]any{"inclusterFileServer": map[string]any{"enabled": false}}})}, + {name: "persistence off renders no claim", rr: result(absent, fileServer(map[string]any{ + "persistence": map[string]any{"enabled": false}}))}, + {name: "nvsentinel disabled", rr: result(absent, fileServer(nil)), cfg: set("enabled", "false")}, + {name: "default StorageClass present", rr: result(recipe.DefaultStorageClassStatePresent, fileServer(nil))}, + {name: "no evidence warns", rr: result("", fileServer(nil)), + wantWarnings: []string{"no metadata.defaultStorageClassState"}}, + {name: "several defaults warn", rr: result(recipe.DefaultStorageClassStateMultiple, fileServer(nil)), + wantWarnings: []string{"--set nv-sentinel:" + fileServerStorageClassPath}}, + {name: "no default, datastore class named only → blocked", rr: result(absent, fileServer(nil)), + wantErrs: []string{"every bug report upload would fail"}}, + {name: "inconclusive evidence → blocked", rr: result(recipe.DefaultStorageClassStateUnknown, fileServer(nil)), + wantErrs: []string{"the file server's volume cannot be shown to bind"}}, + {name: "no default, --storage-class names one", rr: result(absent, fileServer(nil)), + cfg: config.NewConfig(config.WithStorageClass("fast"))}, + {name: "no default, --set names one", rr: result(absent, fileServer(nil)), + cfg: set(fileServerStorageClassPath, "fast")}, + {name: "no default, values name one", rr: result(absent, fileServer(map[string]any{ + "persistence": map[string]any{"storageClassName": "fast"}}))}, + {name: "no default, --storage-class overridden by an empty --set → blocked", rr: result(absent, fileServer(nil)), + cfg: config.NewConfig(config.WithStorageClass("fast"), config.WithValueOverrides(map[string]map[string]string{ + "nv-sentinel": {fileServerStorageClassPath: ""}})), + wantErrs: []string{"found no default StorageClass"}}, + // --set-json runs after --storage-class is injected, so it can still + // clear the class. + {name: "--storage-class then --set-json empty class → blocked", rr: result(absent, fileServer(nil)), + cfg: storageThenTyped(fileServerStorageClassPath, ""), wantErrs: []string{"found no default StorageClass"}}, + {name: "--storage-class then --set-json null class → blocked", rr: result(absent, fileServer(nil)), + cfg: storageThenTyped(fileServerStorageClassPath, nil), wantErrs: []string{"found no default StorageClass"}}, + {name: "--storage-class then --set-json parent with an empty class → blocked", rr: result(absent, fileServer(nil)), + cfg: storageThenTyped("incluster-file-server.persistence", map[string]any{"storageClassName": ""}), + wantErrs: []string{"found no default StorageClass"}}, + {name: "--storage-class then --set-json null parent → blocked", rr: result(absent, fileServer(nil)), + cfg: storageThenTyped("incluster-file-server.persistence", nil), wantErrs: []string{"found no default StorageClass"}}, + {name: "--storage-class then --set-json parent without the key merges → passes", rr: result(absent, fileServer(nil)), + cfg: storageThenTyped("incluster-file-server.persistence", map[string]any{"size": "10Gi"})}, + {name: "unrecognized evidence → blocked", rr: result("bogus", fileServer(nil)), + wantErrs: []string{`defaultStorageClassState="bogus"`}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + cfg := tt.cfg + if cfg == nil { + cfg = config.NewConfig() + } + warnings, errs := CheckNVSentinelFileServerStorageClass(t.Context(), nvsentinelComponent, tt.rr, cfg, nil) + assertDatastoreFindings(t, "warning", warnings, tt.wantWarnings) + errStrings := make([]string, 0, len(errs)) + for _, err := range errs { + errStrings = append(errStrings, err.Error()) + } + assertDatastoreFindings(t, "error", errStrings, tt.wantErrs) + }) + } +} diff --git a/recipes/registry.yaml b/recipes/registry.yaml index 6f4cb048fc..b87597be53 100644 --- a/recipes/registry.yaml +++ b/recipes/registry.yaml @@ -757,6 +757,13 @@ components: # in the function's doc. - function: CheckNVSentinelLogCollectorCoherent severity: error + # Mixed-severity policy, as CheckNVSentinelDatastorePrerequisites: a + # blocking error when the snapshot shows no default StorageClass for + # the file server's classless volume, or evidence it could not read, + # and no class is named; a non-blocking warning when the recipe carries + # no snapshot evidence or the cluster has more than one default. + - function: CheckNVSentinelFileServerStorageClass + severity: warning # NVSentinel's remediation datastore, added only by the nvsentinel-observe, # -quarantine and -remediation mixins. Bundled apart from NVSentinel's own diff --git a/tests/e2e/nvsentinel-log-collector/run.sh b/tests/e2e/nvsentinel-log-collector/run.sh index 798bfeac36..9dc8a54c61 100755 --- a/tests/e2e/nvsentinel-log-collector/run.sh +++ b/tests/e2e/nvsentinel-log-collector/run.sh @@ -250,9 +250,13 @@ render_nvsentinel_bundle() ( # Kind on shared CI runners are throttled into ImagePullBackOff. The images come # from the bundle's own render, so they track the chart pin. Not kind load: # with Docker's containerd image store it exports an index whose other -# platforms have no layers, which its --all-platforms import rejects. +# platforms have no layers, which its --all-platforms import rejects. docker +# save does not record a digest-pinned image's name, so each image is saved +# under a temporary local tag, which classic (manifest.json) and OCI +# (index.json) archives both carry and the import keeps, then named as the +# kubelet requests it and checked through CRI. preload_file_server_images() { - local images image attempt arch + local images image attempt arch staged index=0 repo tag digest arch=$(docker version -f '{{.Server.Arch}}') || err "could not read the Docker host architecture" images=$(render_nvsentinel_bundle | yq -N "select(.kind == \"Deployment\" and .metadata.name == \"${FILE_SERVER}\") | @@ -265,17 +269,54 @@ preload_file_server_images() { ((attempt < 3)) || err "could not pull ${image}" sleep $((attempt * 10)) done - docker save "${image}" | - docker exec -i "${NODE}" ctr -n k8s.io images import --platform "linux/${arch}" --digests \ + index=$((index + 1)) + # Unique per run, so concurrent local runs cannot remove each other's tag. + staged="localhost/aicr-e2e-preload:${CLUSTER_NAME}-$$-${index}" + docker tag "${image}" "${staged}" || err "could not tag ${image} for import" + docker save "${staged}" | + docker exec -i "${NODE}" ctr -n k8s.io images import --platform "linux/${arch}" \ --snapshotter=overlayfs - >/dev/null || err "could not import ${image} into ${NODE}" + docker rmi "${staged}" >/dev/null 2>&1 || true + digest="" + [[ "${image}" == *@* ]] && digest="${image#*@}" + repo=$(normalize_repo "${image%@*}") + tag="${repo##*/}" + if [[ "${tag}" == *:* ]]; then + tag="${tag##*:}" + repo="${repo%:*}" + else + tag=latest + fi + docker exec "${NODE}" ctr -n k8s.io images tag --force "${staged}" "${repo}:${tag}" >/dev/null || + err "could not name ${image} in ${NODE}" + if [[ -n "${digest}" ]]; then + docker exec "${NODE}" ctr -n k8s.io images tag --force "${staged}" "${repo}@${digest}" >/dev/null || + err "could not name ${image} in ${NODE}" + fi + docker exec "${NODE}" ctr -n k8s.io images rm "${staged}" >/dev/null 2>&1 || true + docker exec "${NODE}" crictl inspecti "${image}" >/dev/null 2>&1 || + err "${image} was imported into ${NODE} but the kubelet cannot see it under that name" done msg "Preloaded the file server's images: $(tr '\n' ' ' <<<"${images}")" } +# normalize_repo prefixes a Docker Hub reference with docker.io/ (and +# library/ for an official image), as the kubelet resolves it. +normalize_repo() { + local ref="$1" first="${1%%/*}" + if [[ "${ref}" != */* ]]; then + ref="docker.io/library/${ref}" + elif [[ "${first}" != *.* && "${first}" != *:* && "${first}" != localhost ]]; then + ref="docker.io/${ref}" + fi + echo "${ref}" +} + install_bundle() { - msg "Installing cert-manager, prometheus-operator-crds, the Percona datastore and nvsentinel from the bundle..." + msg "Installing prometheus-operator-crds, cert-manager, the Percona datastore and nvsentinel from the bundle..." local dir d deadline remaining - for dir in "${CERT_MANAGER_DIR}" "${CRDS_DIR}"; do + # CRDs first: cert-manager's chart renders a ServiceMonitor. + for dir in "${CRDS_DIR}" "${CERT_MANAGER_DIR}"; do install_component "${dir}" done deadline=$((SECONDS + CERT_MANAGER_WAIT_SECONDS)) @@ -387,16 +428,20 @@ collector_jobs() { local json json=$(kc -n nvsentinel get jobs -l app=log-collector -o json) || return 1 jq -c '[.items[] | {name: .metadata.name, node: .spec.template.spec.nodeName, - succeeded: (.status.succeeded // 0), failed: (.status.failed // 0), completion: .status.completionTime}]' <<<"${json}" + succeeded: (.status.succeeded // 0), failed: (.status.failed // 0), completion: .status.completionTime, + complete: any(.status.conditions[]?; .type == "Complete" and .status == "True"), + done: any(.status.conditions[]?; (.type == "Complete" or .type == "Failed") and .status == "True")}]' <<<"${json}" } -# wait_for_job_done polls until a collector Job has succeeded or failed, then -# prints that Job. +# wait_for_job_done polls until a collector Job is Complete or Failed, then +# prints that Job. It waits on the Job's condition rather than its pod counts: +# a failed first pod is retried (backoffLimit defaults to 6), and +# completionTime is set only once the Job is Complete. wait_for_job_done() { local elapsed=0 jobs while [[ "${elapsed}" -lt "$((PIPELINE_WAIT_SECONDS + COLLECTOR_WAIT_SECONDS))" ]]; do - if jobs=$(collector_jobs) && jq -e 'any(.[]; .succeeded > 0 or .failed > 0)' <<<"${jobs}" >/dev/null; then - jq -c 'first(.[] | select(.succeeded > 0 or .failed > 0))' <<<"${jobs}" + if jobs=$(collector_jobs) && jq -e 'any(.[]; .done)' <<<"${jobs}" >/dev/null; then + jq -c 'first(.[] | select(.done))' <<<"${jobs}" return 0 fi sleep 5 @@ -470,7 +515,7 @@ test_fault_is_collected_before_remediation() { else fail "collector Job ran on the faulty node" "got ${job}" fi - if jq -e '.succeeded > 0' <<<"${job}" >/dev/null; then + if jq -e '.complete' <<<"${job}" >/dev/null; then pass "collector Job completed" else fail "collector Job completed" "it failed: ${job}; logs: $(kc -n nvsentinel logs "job/$(jq -r .name <<<"${job}")" --tail=30 2>&1 || true)" From d75b7144db60417088b45e87fe97551bd1959800 Mon Sep 17 00:00:00 2001 From: Varun Ramesh Date: Mon, 26 Oct 2026 11:50:25 +0530 Subject: [PATCH 6/8] fix(nvsentinel): skip StorageClass warnings when the bundle names a class Signed-off-by: Varun Ramesh --- docs/contributor/validator.md | 4 +- docs/user/component-catalog.md | 2 +- .../nvsentinel_datastore_prerequisites.go | 37 ++++++++++--------- ...nvsentinel_datastore_prerequisites_test.go | 10 +++++ .../validations/nvsentinel_log_collector.go | 16 ++++---- .../nvsentinel_log_collector_test.go | 6 +++ 6 files changed, 47 insertions(+), 28 deletions(-) diff --git a/docs/contributor/validator.md b/docs/contributor/validator.md index a86d5fc9c5..36b3caf342 100644 --- a/docs/contributor/validator.md +++ b/docs/contributor/validator.md @@ -1405,14 +1405,14 @@ unrecognized value, which keeps a typo'd severity non-silent. | `CheckGB300HostKernelGranule` | Bare-metal GB300 (`service: generic`, `accelerator: gb300`) applies a tuned profile that sizes hugepages for a 64k-granule ARM64 host kernel. A 4k-granule host still boots — Linux rejects the `hugepagesz=512M` clause and drops its paired `hugepages=` count — so the node just runs without that pool. Advisory only (`severity: info`); skipped when the component is disabled or `tuningEnabled` resolves to false on the final effective values (recipe merge plus scalar `--set` and typed `--set-json`/`--set-file`, under the canonical name and its registry aliases). | | `CheckDriverOwnershipCoherence` | GPU driver-ownership coherence on the final effective values (recipe merge + `--set`/`--set-json`/`--set-file` under canonical names and registry aliases): a recipe whose snapshot observed no NVIDIA driver (`metadata.gpuDriverState: absent`) must not bundle with the preinstalled-driver assumption. When GPU Operator manages the driver, `nvidia-dra-driver-gpu.nvidiaDriverRoot` must equal `gpu-operator hostPaths.driverInstallDir`; with a preinstalled driver, the DRA root must avoid the unpopulated operator container root and may intentionally differ from `hostPaths.driverInstallDir` ([#1087](https://github.com/NVIDIA/aicr/issues/1087), [#1757](https://github.com/NVIDIA/aicr/issues/1757)). Wired at `severity: error`. | | `CheckMariaDBOperatorOwnershipCoherence` | MariaDB Operator installation safety for AICR-provided Slurm accounting: `metadata.mariaDBOperatorState` values `crs-detected` and `unknown` block bundling, `api-detected` or omitted evidence warns, and `absent` proceeds silently. Wired at `severity: warning` so warning results remain non-blocking while returned errors still fail the bundle. | -| `CheckNVSentinelDatastorePrerequisites` | Cluster prerequisites of `nvsentinel-mongodb` from snapshot evidence. `metadata.defaultStorageClassState`: `absent` and `unknown` block unless `replsets.rs0.volumeSpec.pvc.storageClassName` or `--storage-class` names a class, `multiple` warns, `present` proceeds. `metadata.perconaOperatorState`: `crs-detected`, `operator-detected`, and `unknown` block, `api-detected` warns that psmdb-operator replaces the existing CRDs, `absent` and `aicr-owned` proceed. Omitted evidence warns for either field. | +| `CheckNVSentinelDatastorePrerequisites` | Cluster prerequisites of `nvsentinel-mongodb` from snapshot evidence. `metadata.defaultStorageClassState`: `absent` and `unknown` block and `multiple` or no evidence warns, unless `replsets.rs0.volumeSpec.pvc.storageClassName` or `--storage-class` names a class; `present` proceeds. `metadata.perconaOperatorState`: `crs-detected`, `operator-detected`, and `unknown` block, `api-detected` warns that psmdb-operator replaces the existing CRDs, `absent` and `aicr-owned` proceed. Omitted evidence warns for either field. | | `CheckNVSentinelRemediationPipelineCoherent` | NVSentinel's fault pipeline can work as configured: no quarantine, drain or remediation stage, `health-events-analyzer` or `event-exporter` with the embedded `mongodbStore` or without a supported `global.datastore.provider`; each stage has the one before it (`node-drainer` needs `fault-quarantine`, `fault-remediation` needs `node-drainer`), `global.dryRun` is a boolean when `fault-remediation` is on, a live `fault-remediation` has `janitor`, `janitor` has `janitor-provider`, and `fault-remediation` has `metadata-collector`; janitor's GPU reset Job reads the driver from the root the GPU Operator populates, with a clean `driverRoot`, a RuntimeClass coherent with the enabled GPU Operator variant (none under CDI with NRI), and a service manager that waits only for operands the GPU Operator runs, in its namespace, with a custom spec's timeouts set; the GPU reset is rejected on GKE COS. With the `nvsentinel-mongodb-uri` datastore it also requires x509 auth with the `nvsentinel-mongodb-app-client` certificate and both Percona components deployed, enabled and in NVSentinel's namespace. Rejects `--dynamic` on any path it verifies. Wired at `severity: error`. | | `CheckNVSentinelMongoDBCoherent` | `nvsentinel-mongodb` is usable by NVSentinel: NVSentinel is deployed, TLS is `requireTLS`, sharding is off, the `$external` x509 app user has `readWrite` on `HealthEventsDatabase` and the `nvsentinelTTLIndex` role, `psmdb-operator` is deployed with `watchAllNamespaces: false` and `disableTelemetry: true`, and `crVersion` and `initImage` are at the operator image's minor. Dependencies are read from the declared component set, so a `bundlers=` subset still sees them. Rejects `--dynamic` on any path it verifies. Wired at `severity: error`. | | `CheckNVSentinelSlurmDrainMonitorRequiresSlinky` | NVSentinel's `slurm-drain-monitor` is not enabled without `slinky-slurm` (absent or disabled), or watching a blank namespace or one `slinky-slurm` does not deploy to: the monitor reads the `SlurmNodeStateDrain` condition off Slinky worker pods, which only `slinky-slurm`'s NodeSets create, so it would run and never see a pod. Rejects `--dynamic` on the monitor's toggle and namespace and on `slinky-slurm`'s `enabled`, even with the monitor off. Wired at `severity: error`. | | `CheckNVSentinelSlurmCustomDrainCoherent` | NVSentinel drains Slurm nodes through the slinky-drainer plugin: a recipe with `slinky-slurm` and a live (not dry-run) `fault-remediation` has `node-drainer.customDrain` on, while quarantine alone passes; `customDrain` is only on with a live `node-drainer`, a boolean `global.dryRun` that is false, `slinky-slurm`, the `slinky-drainer-crds` component, and the `slinky-drainer` component in NVSentinel's namespace watching the namespace `slinky-slurm` deploys to; and its values match the plugin's contract (DrainRequest kind, resource and `DrainComplete` condition, the `slinky-drain-template` template, NVSentinel's namespace, a positive whole-seconds `timeout`, and empty `userNamespaces` and `podDrainPolicies` with an empty `nodeSelector`, since a scope could leave slurmd nodes on pod eviction); and the slurmd NodeSet and every Slurm control pod in the Slurm namespace (slinky-slurm's controller, REST API, login and enabled accounting, and `slurm-accounting-mariadb` there) render non-empty node selectors (modelled on the bundler's order: `--set`, then node-selector injection unless the overlay opts out with an empty or null value, then `--set-json`), each control pod's giving some label key a different value from slurmd's, since the plugin waits for every Ready pod in that namespace on a drained node to report Slurm DRAIN. Rejects `--dynamic` on any path a rule that can still fire reads, so a monitoring-only Slurm recipe keeps its `--dynamic` paths. Wired at `severity: error`. | | `CheckNVSentinelEventExporterCoherent` | NVSentinel's Event Exporter can deliver: `sink.endpoint` and `oidc.tokenUrl` are `http` or `https` URLs with a host that is not an `example.com`, `.org` or `.net` placeholder; `oidc.clientId`, `oidc.scope`, `metadata.cluster` (not the chart's `my-cluster`) and `metadata.environment` are set by AICR's values as strings; no metadata value holds a double quote, backslash or control character other than tab, and every metadata key is a TOML bare key, since the chart writes them into its config unescaped; both `insecureSkipVerify` settings are `false`; and `oidcSecretName` is a valid Secret name. A no-op unless `global.eventExporter.enabled`; then rejects `--dynamic` on any `event-exporter` value. Wired at `severity: error`. | | `CheckNVSentinelLogCollectorCoherent` | NVSentinel's log collector can capture and upload a node's diagnostics: `fault-remediation.logCollector.enabled` and `global.inclusterFileServer.enabled` are on together, `fault-remediation` runs live (not dry run, with a boolean `global.dryRun` and `logCollector.enabled`), the file server keeps the mixin's `fullnameOverride` and the upload URL names its Service, and the collector can reach `nvidia-bug-report` (the GPU Operator installs the driver in the namespace `logCollector.gpuOperatorNamespaces` names, or GKE's managed install; not AKS or OKE at their default `gpuStack`, Kind, or EKS on Amazon Linux). An `env` entry that replaces the upload URL or namespace is read in its place; `MOCK_MODE=true` skips the driver rule. A no-op unless either setting is on; rejects `--dynamic` on the paths it verifies. Wired at `severity: error`. | -| `CheckNVSentinelFileServerStorageClass` | The in-cluster file server's report volume can bind, from snapshot evidence, by the rule `CheckNVSentinelDatastorePrerequisites` applies to the datastore: `metadata.defaultStorageClassState` `absent` and `unknown` block unless `--storage-class` or `incluster-file-server.persistence.storageClassName` names a class; `multiple` and no evidence warn; an unrecognized state blocks. A no-op unless the file server renders with persistence on. Registered apart from `CheckNVSentinelLogCollectorCoherent`, at `severity: warning`, so its warnings stay non-blocking while its errors still block. | +| `CheckNVSentinelFileServerStorageClass` | The in-cluster file server's report volume can bind, from snapshot evidence, by the rule `CheckNVSentinelDatastorePrerequisites` applies to the datastore: `metadata.defaultStorageClassState` `absent` and `unknown` block and `multiple` or no evidence warns, unless `--storage-class` or `incluster-file-server.persistence.storageClassName` names a class; an unrecognized state blocks either way. A no-op unless the file server renders with persistence on. Registered apart from `CheckNVSentinelLogCollectorCoherent`, at `severity: warning`, so its warnings stay non-blocking while its errors still block. | | `CheckNPDNotDuplicatingProviderNPD` | `node-problem-detector` is bundled only on an allowlist of verified platforms — never where the provider already runs its own NPD (GKE, AKS), on OpenShift (no privileged SCC binding), or on Talos (restricted namespace) | | `CheckGKETCPXOInterfacesCoherence` | The final resolved kubeflow-trainer `tcpxoInterfaces` value matches the mapping the recipe records in `configuration.gke.tcpxoInterfaces` | | `CheckNVSentinelDriverLabelDetectable` | Where the GPU Operator does not own the driver and no driver pod exists (driver ships in the node image), NVSentinel sets `labeler.assumeDriverInstalled`, so its driver-label-gated DaemonSets do not come up half-rolled-out | diff --git a/docs/user/component-catalog.md b/docs/user/component-catalog.md index c22c916aaf..593dd1b90e 100644 --- a/docs/user/component-catalog.md +++ b/docs/user/component-catalog.md @@ -1091,7 +1091,7 @@ curl -sO http://localhost:8081/upload/gpu-node-1/20261015-101500/nvidia-bug-repo **Retention and sizing.** A cleanup container runs `find -type f -mtime +7 -delete` (`incluster-file-server.logCleanup.retentionDays: 7`) at start and then every hour (`sleepInterval: 3600`, where the chart's default is daily), so a report is deleted within an hour of being at least eight full days old: `find` counts whole days, so `retentionDays: N` keeps a report for N+1. Only files are deleted; the emptied `//` directories stay. Upstream estimates 5–50 MB per collection, so the 50Gi volume (`persistence.size`) holds at least a thousand of the largest within one retention window. The cleanup container exports `fileserver_disk_space_free_bytes` on the file server's `cleanup-metrics` port (9002), refreshed each cycle; alert on it if collections are frequent. -**Storage.** The file server claims one `ReadWriteOnce` volume. Its `storageClassName` is registered in `nvsentinel`'s `storageClassPaths`, so `--storage-class` sets it; without the flag the claim takes the cluster's default StorageClass, and on a cluster with none the claim stays `Pending`, the file server never starts and every upload fails. As for the datastore, `CheckNVSentinelFileServerStorageClass` reads the snapshot's evidence: it fails the bundle when the snapshot found no default StorageClass, or could not tell, and neither `--storage-class` nor `--set nv-sentinel:incluster-file-server.persistence.storageClassName=` names one (naming a class for `nvsentinel-mongodb` alone does not count), and warns when the recipe carries no snapshot evidence or the cluster has more than one default. AICR provisions no storage. The chart does not keep the claim on uninstall, so removing the mixin deletes it, and with a `Delete` reclaim policy the stored reports go with it: copy them off first. +**Storage.** The file server claims one `ReadWriteOnce` volume. Its `storageClassName` is registered in `nvsentinel`'s `storageClassPaths`, so `--storage-class` sets it; without the flag the claim takes the cluster's default StorageClass, and on a cluster with none the claim stays `Pending`, the file server never starts and every upload fails. As for the datastore, `CheckNVSentinelFileServerStorageClass` reads the snapshot's evidence: it fails the bundle when the snapshot found no default StorageClass, or could not tell, and neither `--storage-class` nor `--set nv-sentinel:incluster-file-server.persistence.storageClassName=` names one (naming a class for `nvsentinel-mongodb` alone does not count). Without a named class it also warns when the recipe carries no snapshot evidence or the cluster has more than one default. AICR provisions no storage. The chart does not keep the claim on uninstall, so removing the mixin deletes it, and with a `Delete` reclaim policy the stored reports go with it: copy them off first. **What is not collected.** GPU Operator must-gather is set explicitly off (`enableGpuOperatorMustGather: false`): it collects from every node in the cluster, not the faulty one, so on a large cluster one fault becomes a cluster-wide collection that also outlasts the timeout. The GCP and AWS SOS reports stay at the chart default, off. diff --git a/pkg/bundler/validations/nvsentinel_datastore_prerequisites.go b/pkg/bundler/validations/nvsentinel_datastore_prerequisites.go index 4dcf52f36b..bc9a6b6506 100644 --- a/pkg/bundler/validations/nvsentinel_datastore_prerequisites.go +++ b/pkg/bundler/validations/nvsentinel_datastore_prerequisites.go @@ -32,9 +32,9 @@ const nvsentinelMongoDBStorageClassPath = "replsets.rs0.volumeSpec.pvc.storageCl // classless volumes, and no other Percona Operator for MongoDB whose CRDs // psmdb-operator would replace. A non-empty // replsets.rs0.volumeSpec.pvc.storageClassName or bundle --storage-class -// waives the StorageClass requirement. Empty metadata means the recipe was -// not resolved from a snapshot carrying this evidence and produces -// non-blocking warnings. +// waives the StorageClass requirement and its warnings. Empty metadata means +// the recipe was not resolved from a snapshot carrying this evidence and +// produces non-blocking warnings. func CheckNVSentinelDatastorePrerequisites(ctx context.Context, componentName string, recipeResult *recipe.RecipeResult, bundlerConfig *config.Config, conditions map[string][]string) ([]string, []error) { if recipeResult == nil || !checkConditions(recipeResult, conditions) { return nil, nil @@ -97,24 +97,17 @@ type storageClassClaim struct { } // storageClassFinding applies the snapshot's metadata.defaultStorageClassState -// to claim: a warning with no evidence or more than one default, an error when -// the snapshot found none or could not tell, unless the bundle names a class -// for the claim. +// to claim. A bundle that names a class for the claim has no finding; +// otherwise no evidence or more than one default warns, and none or +// inconclusive evidence is an error. An unrecognized state is an error either +// way. func storageClassFinding(ctx context.Context, componentName string, recipeResult *recipe.RecipeResult, bundlerConfig *config.Config, keys []string, claim storageClassClaim) (string, error) { state := recipeResult.Metadata.DefaultStorageClassState switch state { case recipe.DefaultStorageClassStatePresent: return "", nil - case "": - return fmt.Sprintf("%s: no metadata.defaultStorageClassState snapshot evidence was recorded, so the "+ - "default StorageClass for %s was not evaluated. Regenerate the recipe from a current "+ - "snapshot to verify the target cluster before deployment", componentName, claim.volumes), nil - case recipe.DefaultStorageClassStateMultiple: - return fmt.Sprintf("%s: the snapshot found more than one default StorageClass; Kubernetes binds "+ - "%s to the newest one. Leave a single StorageClass annotated "+ - "storageclass.kubernetes.io/is-default-class=true, or set "+ - "--set %s:%s=", componentName, claim.volumes, claim.overrideKey, claim.path), nil - case recipe.DefaultStorageClassStateAbsent, recipe.DefaultStorageClassStateUnknown: + case "", recipe.DefaultStorageClassStateMultiple, recipe.DefaultStorageClassStateAbsent, + recipe.DefaultStorageClassStateUnknown: default: return "", aicrerrors.New(aicrerrors.ErrCodeInvalidRequest, fmt.Sprintf( "%s: metadata.defaultStorageClassState=%q is not recognized. Regenerate the recipe with this "+ @@ -129,7 +122,17 @@ func storageClassFinding(ctx context.Context, componentName string, recipeResult return "", nil } - if state == recipe.DefaultStorageClassStateAbsent { + switch state { + case "": + return fmt.Sprintf("%s: no metadata.defaultStorageClassState snapshot evidence was recorded, so the "+ + "default StorageClass for %s was not evaluated. Regenerate the recipe from a current "+ + "snapshot to verify the target cluster before deployment", componentName, claim.volumes), nil + case recipe.DefaultStorageClassStateMultiple: + return fmt.Sprintf("%s: the snapshot found more than one default StorageClass; Kubernetes binds "+ + "%s to the newest one. Leave a single StorageClass annotated "+ + "storageclass.kubernetes.io/is-default-class=true, or set "+ + "--set %s:%s=", componentName, claim.volumes, claim.overrideKey, claim.path), nil + case recipe.DefaultStorageClassStateAbsent: return "", aicrerrors.New(aicrerrors.ErrCodeConflict, fmt.Sprintf( "%s: the snapshot found no default StorageClass, so %s. Annotate a StorageClass "+ "with storageclass.kubernetes.io/is-default-class=true and regenerate the recipe from a "+ diff --git a/pkg/bundler/validations/nvsentinel_datastore_prerequisites_test.go b/pkg/bundler/validations/nvsentinel_datastore_prerequisites_test.go index 1787f89312..2d172b8d48 100644 --- a/pkg/bundler/validations/nvsentinel_datastore_prerequisites_test.go +++ b/pkg/bundler/validations/nvsentinel_datastore_prerequisites_test.go @@ -90,6 +90,16 @@ func TestCheckNVSentinelDatastorePrerequisites(t *testing.T) { wantCode: aicrerrors.ErrCodeConflict, }, {name: "recipe-named StorageClass waives absent", rr: result(recipe.DefaultStorageClassStateAbsent, absentPercona, namedClass)}, + {name: "--storage-class silences missing evidence", rr: result("", absentPercona, nil), + cfg: config.NewConfig(config.WithStorageClass("fast"))}, + {name: "recipe-named StorageClass silences multiple defaults", + rr: result(recipe.DefaultStorageClassStateMultiple, absentPercona, namedClass)}, + { + name: "a named StorageClass does not waive an unrecognized state", + rr: result("bogus", absentPercona, namedClass), + wantErrs: []string{`defaultStorageClassState="bogus"`}, + wantCode: aicrerrors.ErrCodeInvalidRequest, + }, {name: "--set StorageClass waives unknown", rr: result(recipe.DefaultStorageClassStateUnknown, absentPercona, nil), cfg: setClass("fast")}, { name: "--storage-class waives absent", diff --git a/pkg/bundler/validations/nvsentinel_log_collector.go b/pkg/bundler/validations/nvsentinel_log_collector.go index c764ba943e..e1166a3e98 100644 --- a/pkg/bundler/validations/nvsentinel_log_collector.go +++ b/pkg/bundler/validations/nvsentinel_log_collector.go @@ -333,14 +333,14 @@ var fileServerClaim = storageClassClaim{ // CheckNVSentinelFileServerStorageClass applies the snapshot's default // StorageClass evidence to the in-cluster file server's report volume, the -// rule CheckNVSentinelDatastorePrerequisites applies to the datastore's: a -// blocking error when the snapshot found no default StorageClass or could not -// tell, unless --storage-class or incluster-file-server.persistence.storageClassName -// names one; a warning when the recipe carries no evidence or the cluster has -// more than one default. A no-op unless the file server renders with -// persistence on. It is registered apart from CheckNVSentinelLogCollectorCoherent -// because its warnings must stay non-blocking, which that gate's severity -// does not allow. +// rule CheckNVSentinelDatastorePrerequisites applies to the datastore's. +// Unless --storage-class or incluster-file-server.persistence.storageClassName +// names a class, it is a blocking error when the snapshot found no default +// StorageClass or could not tell, and a warning when the recipe carries no +// evidence or the cluster has more than one default. A no-op unless the file +// server renders with persistence on. It is registered apart from +// CheckNVSentinelLogCollectorCoherent because its warnings must stay +// non-blocking, which that gate's severity does not allow. func CheckNVSentinelFileServerStorageClass(ctx context.Context, componentName string, recipeResult *recipe.RecipeResult, bundlerConfig *config.Config, conditions map[string][]string) ([]string, []error) { if recipeResult == nil || !checkConditions(recipeResult, conditions) { return nil, nil diff --git a/pkg/bundler/validations/nvsentinel_log_collector_test.go b/pkg/bundler/validations/nvsentinel_log_collector_test.go index 3d60207550..42392dc929 100644 --- a/pkg/bundler/validations/nvsentinel_log_collector_test.go +++ b/pkg/bundler/validations/nvsentinel_log_collector_test.go @@ -579,6 +579,12 @@ func TestCheckNVSentinelFileServerStorageClass(t *testing.T) { cfg: storageThenTyped("incluster-file-server.persistence", map[string]any{"size": "10Gi"})}, {name: "unrecognized evidence → blocked", rr: result("bogus", fileServer(nil)), wantErrs: []string{`defaultStorageClassState="bogus"`}}, + {name: "no evidence, --storage-class names one → silent", rr: result("", fileServer(nil)), + cfg: config.NewConfig(config.WithStorageClass("fast"))}, + {name: "several defaults, --set names one → silent", rr: result(recipe.DefaultStorageClassStateMultiple, fileServer(nil)), + cfg: set(fileServerStorageClassPath, "fast")}, + {name: "unrecognized evidence with a class named → blocked", rr: result("bogus", fileServer(nil)), + cfg: config.NewConfig(config.WithStorageClass("fast")), wantErrs: []string{`defaultStorageClassState="bogus"`}}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { From d8f1be3f931ce02b28cffee16981a9538a1c07d8 Mon Sep 17 00:00:00 2001 From: Varun Ramesh Date: Tue, 27 Oct 2026 00:10:51 +0530 Subject: [PATCH 7/8] fix(nvsentinel): address log collector review comments Signed-off-by: Varun Ramesh --- docs/contributor/validator.md | 4 +-- docs/user/container-images.md | 6 ++-- .../validations/nvsentinel_log_collector.go | 7 +++-- tests/e2e/nvsentinel-log-collector/run.sh | 28 +++++++++++-------- 4 files changed, 26 insertions(+), 19 deletions(-) diff --git a/docs/contributor/validator.md b/docs/contributor/validator.md index 36b3caf342..f21cf44ff4 100644 --- a/docs/contributor/validator.md +++ b/docs/contributor/validator.md @@ -1405,13 +1405,13 @@ unrecognized value, which keeps a typo'd severity non-silent. | `CheckGB300HostKernelGranule` | Bare-metal GB300 (`service: generic`, `accelerator: gb300`) applies a tuned profile that sizes hugepages for a 64k-granule ARM64 host kernel. A 4k-granule host still boots — Linux rejects the `hugepagesz=512M` clause and drops its paired `hugepages=` count — so the node just runs without that pool. Advisory only (`severity: info`); skipped when the component is disabled or `tuningEnabled` resolves to false on the final effective values (recipe merge plus scalar `--set` and typed `--set-json`/`--set-file`, under the canonical name and its registry aliases). | | `CheckDriverOwnershipCoherence` | GPU driver-ownership coherence on the final effective values (recipe merge + `--set`/`--set-json`/`--set-file` under canonical names and registry aliases): a recipe whose snapshot observed no NVIDIA driver (`metadata.gpuDriverState: absent`) must not bundle with the preinstalled-driver assumption. When GPU Operator manages the driver, `nvidia-dra-driver-gpu.nvidiaDriverRoot` must equal `gpu-operator hostPaths.driverInstallDir`; with a preinstalled driver, the DRA root must avoid the unpopulated operator container root and may intentionally differ from `hostPaths.driverInstallDir` ([#1087](https://github.com/NVIDIA/aicr/issues/1087), [#1757](https://github.com/NVIDIA/aicr/issues/1757)). Wired at `severity: error`. | | `CheckMariaDBOperatorOwnershipCoherence` | MariaDB Operator installation safety for AICR-provided Slurm accounting: `metadata.mariaDBOperatorState` values `crs-detected` and `unknown` block bundling, `api-detected` or omitted evidence warns, and `absent` proceeds silently. Wired at `severity: warning` so warning results remain non-blocking while returned errors still fail the bundle. | -| `CheckNVSentinelDatastorePrerequisites` | Cluster prerequisites of `nvsentinel-mongodb` from snapshot evidence. `metadata.defaultStorageClassState`: `absent` and `unknown` block and `multiple` or no evidence warns, unless `replsets.rs0.volumeSpec.pvc.storageClassName` or `--storage-class` names a class; `present` proceeds. `metadata.perconaOperatorState`: `crs-detected`, `operator-detected`, and `unknown` block, `api-detected` warns that psmdb-operator replaces the existing CRDs, `absent` and `aicr-owned` proceed. Omitted evidence warns for either field. | +| `CheckNVSentinelDatastorePrerequisites` | Cluster prerequisites of `nvsentinel-mongodb` from snapshot evidence. `metadata.defaultStorageClassState`: `absent` and `unknown` block and `multiple` or no evidence warns, unless `replsets.rs0.volumeSpec.pvc.storageClassName` or `--storage-class` names a class; `present` proceeds. `metadata.perconaOperatorState`: `crs-detected`, `operator-detected`, and `unknown` block, `api-detected` warns that psmdb-operator replaces the existing CRDs, `absent` and `aicr-owned` proceed; omitted `perconaOperatorState` evidence warns. | | `CheckNVSentinelRemediationPipelineCoherent` | NVSentinel's fault pipeline can work as configured: no quarantine, drain or remediation stage, `health-events-analyzer` or `event-exporter` with the embedded `mongodbStore` or without a supported `global.datastore.provider`; each stage has the one before it (`node-drainer` needs `fault-quarantine`, `fault-remediation` needs `node-drainer`), `global.dryRun` is a boolean when `fault-remediation` is on, a live `fault-remediation` has `janitor`, `janitor` has `janitor-provider`, and `fault-remediation` has `metadata-collector`; janitor's GPU reset Job reads the driver from the root the GPU Operator populates, with a clean `driverRoot`, a RuntimeClass coherent with the enabled GPU Operator variant (none under CDI with NRI), and a service manager that waits only for operands the GPU Operator runs, in its namespace, with a custom spec's timeouts set; the GPU reset is rejected on GKE COS. With the `nvsentinel-mongodb-uri` datastore it also requires x509 auth with the `nvsentinel-mongodb-app-client` certificate and both Percona components deployed, enabled and in NVSentinel's namespace. Rejects `--dynamic` on any path it verifies. Wired at `severity: error`. | | `CheckNVSentinelMongoDBCoherent` | `nvsentinel-mongodb` is usable by NVSentinel: NVSentinel is deployed, TLS is `requireTLS`, sharding is off, the `$external` x509 app user has `readWrite` on `HealthEventsDatabase` and the `nvsentinelTTLIndex` role, `psmdb-operator` is deployed with `watchAllNamespaces: false` and `disableTelemetry: true`, and `crVersion` and `initImage` are at the operator image's minor. Dependencies are read from the declared component set, so a `bundlers=` subset still sees them. Rejects `--dynamic` on any path it verifies. Wired at `severity: error`. | | `CheckNVSentinelSlurmDrainMonitorRequiresSlinky` | NVSentinel's `slurm-drain-monitor` is not enabled without `slinky-slurm` (absent or disabled), or watching a blank namespace or one `slinky-slurm` does not deploy to: the monitor reads the `SlurmNodeStateDrain` condition off Slinky worker pods, which only `slinky-slurm`'s NodeSets create, so it would run and never see a pod. Rejects `--dynamic` on the monitor's toggle and namespace and on `slinky-slurm`'s `enabled`, even with the monitor off. Wired at `severity: error`. | | `CheckNVSentinelSlurmCustomDrainCoherent` | NVSentinel drains Slurm nodes through the slinky-drainer plugin: a recipe with `slinky-slurm` and a live (not dry-run) `fault-remediation` has `node-drainer.customDrain` on, while quarantine alone passes; `customDrain` is only on with a live `node-drainer`, a boolean `global.dryRun` that is false, `slinky-slurm`, the `slinky-drainer-crds` component, and the `slinky-drainer` component in NVSentinel's namespace watching the namespace `slinky-slurm` deploys to; and its values match the plugin's contract (DrainRequest kind, resource and `DrainComplete` condition, the `slinky-drain-template` template, NVSentinel's namespace, a positive whole-seconds `timeout`, and empty `userNamespaces` and `podDrainPolicies` with an empty `nodeSelector`, since a scope could leave slurmd nodes on pod eviction); and the slurmd NodeSet and every Slurm control pod in the Slurm namespace (slinky-slurm's controller, REST API, login and enabled accounting, and `slurm-accounting-mariadb` there) render non-empty node selectors (modelled on the bundler's order: `--set`, then node-selector injection unless the overlay opts out with an empty or null value, then `--set-json`), each control pod's giving some label key a different value from slurmd's, since the plugin waits for every Ready pod in that namespace on a drained node to report Slurm DRAIN. Rejects `--dynamic` on any path a rule that can still fire reads, so a monitoring-only Slurm recipe keeps its `--dynamic` paths. Wired at `severity: error`. | | `CheckNVSentinelEventExporterCoherent` | NVSentinel's Event Exporter can deliver: `sink.endpoint` and `oidc.tokenUrl` are `http` or `https` URLs with a host that is not an `example.com`, `.org` or `.net` placeholder; `oidc.clientId`, `oidc.scope`, `metadata.cluster` (not the chart's `my-cluster`) and `metadata.environment` are set by AICR's values as strings; no metadata value holds a double quote, backslash or control character other than tab, and every metadata key is a TOML bare key, since the chart writes them into its config unescaped; both `insecureSkipVerify` settings are `false`; and `oidcSecretName` is a valid Secret name. A no-op unless `global.eventExporter.enabled`; then rejects `--dynamic` on any `event-exporter` value. Wired at `severity: error`. | -| `CheckNVSentinelLogCollectorCoherent` | NVSentinel's log collector can capture and upload a node's diagnostics: `fault-remediation.logCollector.enabled` and `global.inclusterFileServer.enabled` are on together, `fault-remediation` runs live (not dry run, with a boolean `global.dryRun` and `logCollector.enabled`), the file server keeps the mixin's `fullnameOverride` and the upload URL names its Service, and the collector can reach `nvidia-bug-report` (the GPU Operator installs the driver in the namespace `logCollector.gpuOperatorNamespaces` names, or GKE's managed install; not AKS or OKE at their default `gpuStack`, Kind, or EKS on Amazon Linux). An `env` entry that replaces the upload URL or namespace is read in its place; `MOCK_MODE=true` skips the driver rule. A no-op unless either setting is on; rejects `--dynamic` on the paths it verifies. Wired at `severity: error`. | +| `CheckNVSentinelLogCollectorCoherent` | NVSentinel's log collector can capture and upload a node's diagnostics: `fault-remediation.logCollector.enabled` and `global.inclusterFileServer.enabled` are on together, `fault-remediation` runs live (not dry run, with a boolean `global.dryRun` and `logCollector.enabled`), the file server keeps the mixin's `fullnameOverride` and the upload URL names its Service, and the collector can reach `nvidia-bug-report` (the GPU Operator installs the driver in the namespace `logCollector.gpuOperatorNamespaces` names, or GKE's managed install; not AKS or OKE at their default `gpuStack`, Kind, EKS on Amazon Linux, or the VR200 RKE2 and H200 k0s leaves). An `env` entry that replaces the upload URL or namespace is read in its place; `MOCK_MODE=true` skips the driver rule. A no-op unless either setting is on; rejects `--dynamic` on the paths it verifies. Wired at `severity: error`. | | `CheckNVSentinelFileServerStorageClass` | The in-cluster file server's report volume can bind, from snapshot evidence, by the rule `CheckNVSentinelDatastorePrerequisites` applies to the datastore: `metadata.defaultStorageClassState` `absent` and `unknown` block and `multiple` or no evidence warns, unless `--storage-class` or `incluster-file-server.persistence.storageClassName` names a class; an unrecognized state blocks either way. A no-op unless the file server renders with persistence on. Registered apart from `CheckNVSentinelLogCollectorCoherent`, at `severity: warning`, so its warnings stay non-blocking while its errors still block. | | `CheckNPDNotDuplicatingProviderNPD` | `node-problem-detector` is bundled only on an allowlist of verified platforms — never where the provider already runs its own NPD (GKE, AKS), on OpenShift (no privileged SCC binding), or on Talos (restricted namespace) | | `CheckGKETCPXOInterfacesCoherence` | The final resolved kubeflow-trainer `tcpxoInterfaces` value matches the mapping the recipe records in `configuration.gke.tcpxoInterfaces` | diff --git a/docs/user/container-images.md b/docs/user/container-images.md index 28b994b40b..7c089ce96a 100644 --- a/docs/user/container-images.md +++ b/docs/user/container-images.md @@ -464,7 +464,7 @@ Air-gapped OpenShift deployments must separately mirror the relevant Red Hat cer The trade-off is intentional. Pinning an image gives reproducibility; deferring to the upstream chart lets security patches flow without an AICR release. The split is policy, not oversight — see the [supply chain epic](https://github.com/NVIDIA/aicr/issues/739) for how each component's policy is being made explicit. -**Opt-in values enabled by a leaf override or mixin are a fourth gap.** A handful of images only appear once a component's *values*, not just its enablement, are overridden outside the shared `recipes/components//values.yaml` this BOM renders (`tools/bom/main.go`'s `renderHelmComponent` resolves each component against only its base values file, so it cannot see leaf or mixin overrides). Eight known cases, none counted in the `nvsentinel` row's image count above. Six set a `global.*` toggle: +**Opt-in values enabled by a leaf override or mixin are a fourth gap.** A handful of images only appear once a component's *values*, not just its enablement, are overridden outside the shared `recipes/components//values.yaml` this BOM renders (`tools/bom/main.go`'s `renderHelmComponent` resolves each component against only its base values file, so it cannot see leaf or mixin overrides). Nine known cases, none counted in the `nvsentinel` row's image count above. Seven set a `global.*` toggle: - The [`nvsentinel-observability` mixin](component-catalog.md#audit-logging-and-tracing) sets `global.auditLogging.enabled: true`, which conditionally adds a `fix-audit-log-permissions` init container (`docker.io/library/busybox:1.38.0@sha256:fd7dc98638c8e305f4dc34e979f1c0fdfdcaeb0fbf8fcff77ae834b6da3d7e6e`) to the `platform-connectors` DaemonSet and `labeler` Deployment. The chart's own default for this init container is `docker.io/bitnamilegacy/os-shell:12-debian-12-r30`, which sits in Bitnami's frozen archive and will never be patched; AICR overrides `global.initContainerImage` in `recipes/components/nvsentinel/values.yaml` to the same digest-pinned busybox it already ships in the `network-operator` and `gpu-operator` manifests. - The [`nvsentinel-object-monitor` mixin](component-catalog.md#kubernetes-object-monitor) sets `global.kubernetesObjectMonitor.enabled`, turning on the chart's `kubernetes-object-monitor` subchart and pulling in `ghcr.io/nvidia/nvsentinel/kubernetes-object-monitor:v1.28.0`. That image is in AICR's weekly image scan despite not being built here, since nothing else would surface a CVE in it. @@ -476,7 +476,7 @@ The trade-off is intentional. Pinning an image gives reproducibility; deferring A recipe composing any of these mixins **with `nvsentinel` still enabled** adds that mixin's images to what it deploys and mirrors; `aicr bundle`/`aicr mirror` on such a recipe surfaces them even though this static BOM cannot. A chain that disables `nvsentinel` (the OCP overlay, for example) can compose a mixin and ship none of them. -The `nvsentinel-preflight` mixin (see [Preflight Checks](component-catalog.md#preflight-checks)) is the seventh case. Setting `global.preflight.enabled: true` on `nvsentinel` adds four images, all from `ghcr.io/nvidia/nvsentinel/` at the chart's own version and therefore already covered by the NVIDIA mirroring path — but none of them appear in the `nvsentinel` row above: +The `nvsentinel-preflight` mixin (see [Preflight Checks](component-catalog.md#preflight-checks)) is the eighth case. Setting `global.preflight.enabled: true` on `nvsentinel` adds four images, all from `ghcr.io/nvidia/nvsentinel/` at the chart's own version and therefore already covered by the NVIDIA mirroring path — but none of them appear in the `nvsentinel` row above: | Image | Role | |---|---| @@ -487,7 +487,7 @@ The `nvsentinel-preflight` mixin (see [Preflight Checks](component-catalog.md#pr `TestNVSentinelPreflightChartRender` pins all four against the rendered chart, so a bump that changes a repository fails there rather than silently diverging from this table. -The three remediation mixins (see [Graded Remediation](component-catalog.md#graded-remediation)) are the eighth case. They turn on NVSentinel's remediation subcharts, so they add images rather than values. `nvsentinel-remediation`, the widest step, pulls all seven below; `nvsentinel-observe` pulls all but `janitor`, `janitor-provider`, `gpu-reset` and the reboot Job's busybox, and `nvsentinel-quarantine` additionally drops `fault-remediation`. +The three remediation mixins (see [Graded Remediation](component-catalog.md#graded-remediation)) are the ninth case. They turn on NVSentinel's remediation subcharts, so they add images rather than values. `nvsentinel-remediation`, the widest step, pulls all seven below; `nvsentinel-observe` pulls all but `janitor`, `janitor-provider`, `gpu-reset` and the reboot Job's busybox, and `nvsentinel-quarantine` additionally drops `fault-remediation`. | Image | Role | |---|---| diff --git a/pkg/bundler/validations/nvsentinel_log_collector.go b/pkg/bundler/validations/nvsentinel_log_collector.go index e1166a3e98..cec3f9192e 100644 --- a/pkg/bundler/validations/nvsentinel_log_collector.go +++ b/pkg/bundler/validations/nvsentinel_log_collector.go @@ -71,7 +71,8 @@ const ( // GPU Operator's driver container, which it looks up in the single // namespace gpuOperatorNamespaces names, or from GKE's managed driver // install, so a recipe whose node image carries the driver otherwise (AKS, -// OKE, Kind, EKS on Amazon Linux) fails every collection. +// OKE, Kind, EKS on Amazon Linux, the VR200 RKE2 and H200 k0s leaves) +// fails every collection. // // The Job template renders logCollector.env after the variables it sets from // uploadURL and gpuOperatorNamespaces, so an env.UPLOAD_URL_BASE or @@ -227,8 +228,8 @@ func logCollectorDriverViolations(ctx context.Context, recipeResult *recipe.Reci return []error{aicrerrors.New(aicrerrors.ErrCodeInvalidRequest, fmt.Sprintf("component %q: %s is on but %s, so every collection fails and no bug report is captured. "+ "The collector runs nvidia-bug-report in the GPU Operator's driver container, or from GKE's "+ - "managed driver install -- let the GPU Operator install the driver (on AKS and OKE, "+ - "--profile gpuStack=operator-managed), or do not compose %s", componentName, + "managed driver install -- let the GPU Operator install the driver (on AKS and OKE, regenerate the "+ + "recipe with aicr recipe --profile gpuStack=operator-managed), or do not compose %s", componentName, logCollectorEnabledPath, why, logCollectorMixinName))} } if gpuOp == nil { diff --git a/tests/e2e/nvsentinel-log-collector/run.sh b/tests/e2e/nvsentinel-log-collector/run.sh index 9dc8a54c61..714079eb38 100755 --- a/tests/e2e/nvsentinel-log-collector/run.sh +++ b/tests/e2e/nvsentinel-log-collector/run.sh @@ -467,18 +467,23 @@ wait_for_reset() { return 1 } -# file_server_get fetches path $1 from the file server through a port-forward, -# the retrieval the component catalog documents. +# start_file_server_forward opens the port-forward the component catalog +# documents for retrieving reports. Call it in the parent shell: started inside +# $(...), PORT_FORWARD_PID would never reach cleanup. +start_file_server_forward() { + [[ -n "${PORT_FORWARD_PID}" ]] && return 0 + kc -n nvsentinel port-forward "svc/${FILE_SERVER}" 18080:80 >"${WORK}/port-forward.log" 2>&1 & + PORT_FORWARD_PID=$! + local i + for ((i = 0; i < 30; i++)); do + curl -fsS --max-time 5 "http://127.0.0.1:18080/healthz" >/dev/null 2>&1 && return 0 + sleep 1 + done +} + +# file_server_get fetches path $1 through the forward start_file_server_forward +# opened. file_server_get() { - if [[ -z "${PORT_FORWARD_PID}" ]]; then - kc -n nvsentinel port-forward "svc/${FILE_SERVER}" 18080:80 >"${WORK}/port-forward.log" 2>&1 & - PORT_FORWARD_PID=$! - local i - for ((i = 0; i < 30; i++)); do - curl -fsS --max-time 5 "http://127.0.0.1:18080/healthz" >/dev/null 2>&1 && break - sleep 1 - done - fi curl -fsS --max-time 30 "http://127.0.0.1:18080$1" } @@ -523,6 +528,7 @@ test_fault_is_collected_before_remediation() { fi local listing report page + start_file_server_forward # The listing also links ../, so match the collector's timestamp directory. # Each page is captured before grep so grep -q cannot cut curl off mid-write. if listing=$(file_server_get "/upload/${NODE}/") && From 7f7d44d745d861ff26030126928bddfe3cfa0c6b Mon Sep 17 00:00:00 2001 From: Varun Ramesh Date: Tue, 27 Oct 2026 11:03:45 +0530 Subject: [PATCH 8/8] test(bundler): regenerate render golden after rebasing on main Signed-off-by: Varun Ramesh --- pkg/bundler/testdata/stock_render_golden.yaml | 568 +++++++++--------- 1 file changed, 284 insertions(+), 284 deletions(-) diff --git a/pkg/bundler/testdata/stock_render_golden.yaml b/pkg/bundler/testdata/stock_render_golden.yaml index 9e0c29fe99..11cce26fd0 100644 --- a/pkg/bundler/testdata/stock_render_golden.yaml +++ b/pkg/bundler/testdata/stock_render_golden.yaml @@ -78,17 +78,17 @@ a100-aks-ubuntu-training-kubeflow: 016-nvsentinel/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 016-nvsentinel/install.sh: 290c0907c263ac3c1a976be76da13528e36fe7f1db79945783b6ac7da4588857 016-nvsentinel/upstream.env: 7fc07c3fb32a6e569a151e4c12f2834386b4c7026cda6cd1c86e29592ad8936e - 016-nvsentinel/values.yaml: 843a8e5cb1d7e52b0b2a1a6f4366dd15d1e275d6cea64bbf73d33f52ccfafd0c + 016-nvsentinel/values.yaml: aa27bae925e838d0ba1a1b8a9d9ce6460ccfc20cfb29d9d52d85f9d1f927f008 017-prometheus-adapter/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 017-prometheus-adapter/install.sh: 688ee4274fe06c001af6835d7965b5da33ce3fbeb724263622737d0bb88249e3 017-prometheus-adapter/upstream.env: 9ab918b32522baca13cd37bad7f420a206a19bb763b81041e174fffffe3da724 017-prometheus-adapter/values.yaml: 95231d7a46d650f96f844ca3734614271152a41f1230022c9d79960e44647c00 README.md: 49b66b06038dcfd6bc764014217259c93475ad2609f2545bd30d768febb06dc3 UPGRADING.md: 9444e3b2469985d89ae8fd6bb0ca9116e56c2904f9aff6929487211120b75484 - bundle-info.yaml: a9d01dbae9753f866d549d3ff5056cad9588aae1d9439d024bb69f999e860345 - checksums.txt: 9e446d70501937b9dca395c79cf2371889e940dd378ff01dda10cbf252fe88b7 + bundle-info.yaml: 7a9c83cbaaf61bff1cfe9ba453234fbf8747608bfb7cf73b8b5282c2d9430342 + checksums.txt: d0a92d03d92e72b3f08cf58fc1e021936828edd0b4a81f3c6cf1c9ce16fd92c0 deploy.sh: cfb12fbb4315ebaadd1378f55f81e866e9f5faaa909b57e569bab1c0ce558c0b - recipe.yaml: 79f969af7cbc03343b7e2c7f7caf169e3c7e753eac9d1793ca90e9be21fc71bd + recipe.yaml: 423142e6efa0d311d96ae0c797af7648082b615bfc14d1a679fa85658a83d637 a100-any: 001-nodewright-operator/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 001-nodewright-operator/install.sh: 5bb299d13650bdb1baa19ea8047b33baa94eb03a48a8b5b0102f1ba533b392c2 @@ -130,17 +130,17 @@ a100-any: 010-nvsentinel/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 010-nvsentinel/install.sh: 290c0907c263ac3c1a976be76da13528e36fe7f1db79945783b6ac7da4588857 010-nvsentinel/upstream.env: 7fc07c3fb32a6e569a151e4c12f2834386b4c7026cda6cd1c86e29592ad8936e - 010-nvsentinel/values.yaml: 5dbdd974de19bda092dff19de6a6fbb9489b7e7629abdeab2d1a8fe0bff69247 + 010-nvsentinel/values.yaml: 14400ae55936f06cc2025495b7d2efbb933e388a985abf2061b3b4fcbc36eeea 011-prometheus-adapter/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 011-prometheus-adapter/install.sh: 688ee4274fe06c001af6835d7965b5da33ce3fbeb724263622737d0bb88249e3 011-prometheus-adapter/upstream.env: 9ab918b32522baca13cd37bad7f420a206a19bb763b81041e174fffffe3da724 011-prometheus-adapter/values.yaml: 95231d7a46d650f96f844ca3734614271152a41f1230022c9d79960e44647c00 README.md: e6f877e48adaaba9cb1c22d48671c36d737fe2ba5882aebf2de1308c0d250793 UPGRADING.md: 4b285b6e5866a3582feef1afa6cc5906b79b275e94b0033d33f752c442d1256b - bundle-info.yaml: 4ee795ba22318f11a150f0f7fb1789e45aa71059b64c80c2cf537d6bc5cb3356 - checksums.txt: 0a7eafd2ea8acf8736cb0e06ed45434aeddbbd5c9939cee4c089c2821e430e2f + bundle-info.yaml: 35a13723ec9a4081121dc99b9e1b3fa92b45f05118ea338aeadf1585e0527573 + checksums.txt: 525f5cb28a2199ea883de64df03e4041c4d9a1848e70e5a00941d71f12a2138d deploy.sh: 187bf11702c83fdf08c22fbdd72a6b16c937314d87f85da747c4753cbe5d5de4 - recipe.yaml: 30ad2f0d3f2970086c259cd375de354a2febff330ddf7bd7bab8883594134d1d + recipe.yaml: 7112b4d3c752fb3c3e8bb0cf8145db56dc8bb69971ca5234d2be313502d36bbb a100-eks-ubuntu-training-kubeflow: 001-aws-efa/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 001-aws-efa/install.sh: 00574b8307fa54f90f18cd999a2b175a75a547942fd18cb242bd83b7af836d5a @@ -204,17 +204,17 @@ a100-eks-ubuntu-training-kubeflow: 015-nvsentinel/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 015-nvsentinel/install.sh: 290c0907c263ac3c1a976be76da13528e36fe7f1db79945783b6ac7da4588857 015-nvsentinel/upstream.env: 7fc07c3fb32a6e569a151e4c12f2834386b4c7026cda6cd1c86e29592ad8936e - 015-nvsentinel/values.yaml: 5dbdd974de19bda092dff19de6a6fbb9489b7e7629abdeab2d1a8fe0bff69247 + 015-nvsentinel/values.yaml: 14400ae55936f06cc2025495b7d2efbb933e388a985abf2061b3b4fcbc36eeea 016-prometheus-adapter/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 016-prometheus-adapter/install.sh: 688ee4274fe06c001af6835d7965b5da33ce3fbeb724263622737d0bb88249e3 016-prometheus-adapter/upstream.env: 9ab918b32522baca13cd37bad7f420a206a19bb763b81041e174fffffe3da724 016-prometheus-adapter/values.yaml: 95231d7a46d650f96f844ca3734614271152a41f1230022c9d79960e44647c00 README.md: c17ec81d9cb89e06d9498de83723fb0d36c818907a018cebb2ba9ee77e402bb8 UPGRADING.md: 4b285b6e5866a3582feef1afa6cc5906b79b275e94b0033d33f752c442d1256b - bundle-info.yaml: d3dfe057a7a0f955ca0d060eb553a38c99eb38acbe06ad0fdfd7ccb6cf77536f - checksums.txt: 3069dfa9de1b5070d9bac79fca60e41f9fb8fc1210c7fe1acb5bebd548660a64 + bundle-info.yaml: 0be6349087872344ab5d51b96f048ef2c29906733a5df91ca68fab2f7355f200 + checksums.txt: e31142843ded09d0ace374185672de018a723e5ceb163ca509659c0d33c07cc5 deploy.sh: 949466ed3363eb59ea6ff5a61a29ca150215a2041c47323cada206dcf3cdc0a8 - recipe.yaml: 00718f9ca55c48853f9e329d51a623580b30c8d55ecc22b93c9a722e59b06e75 + recipe.yaml: fd8dd760a25e1a66b3780c0617c42b8d2f0ce38273e15ac909896ed498caa350 a100-gke-cos-training-kubeflow: 001-gcp-driver-installer/Chart.yaml: b0476226ebbc9a7738c4e5c2828ce5a73dbd371f3fa7cf1139c5fe9b2e825c02 001-gcp-driver-installer/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 @@ -279,17 +279,17 @@ a100-gke-cos-training-kubeflow: 015-nvsentinel/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 015-nvsentinel/install.sh: 290c0907c263ac3c1a976be76da13528e36fe7f1db79945783b6ac7da4588857 015-nvsentinel/upstream.env: 7fc07c3fb32a6e569a151e4c12f2834386b4c7026cda6cd1c86e29592ad8936e - 015-nvsentinel/values.yaml: 3374b3a52c2466d902a85e6d5e4b2fc02d4f824e690094772cb6dbe9ee6962b2 + 015-nvsentinel/values.yaml: 7f5f9edb985771f1b7c01e5ee67edba4a2a6182b990a940a5e89601e5d589ae0 016-prometheus-adapter/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 016-prometheus-adapter/install.sh: 688ee4274fe06c001af6835d7965b5da33ce3fbeb724263622737d0bb88249e3 016-prometheus-adapter/upstream.env: 9ab918b32522baca13cd37bad7f420a206a19bb763b81041e174fffffe3da724 016-prometheus-adapter/values.yaml: 95231d7a46d650f96f844ca3734614271152a41f1230022c9d79960e44647c00 README.md: 66006bb70dabcb856b8c248813eae56961b704b5b33659f0386fe945d255976e UPGRADING.md: 4b285b6e5866a3582feef1afa6cc5906b79b275e94b0033d33f752c442d1256b - bundle-info.yaml: 1b804af57621bfe0d7b79fb9ec79b794275a79a3bd5fb13fcd924476d67a1d42 - checksums.txt: 557d4c5b1189a6258497d7ae0535afe133174d181f96a0a5581bc5e6d3549ad6 + bundle-info.yaml: 90cb7ffd21fc19a69f1f158779eb7be503b5dc90a811115804d81de8b261746e + checksums.txt: 2d1192d5267840eec4eadcf876ccd256762fa70e862d4ddad8da306ac02ad677 deploy.sh: 50736743f1a6091c2dcb82baedfed99d4ae2f5bfe3325f7e7ab89995353f81bd - recipe.yaml: 9d682b0ac6acd68d583f818b70437eb7b6e6bc9fa3a505cb87b90d0129c80340 + recipe.yaml: d3100db2dbc30205704026e31a0d09fcb839061459c5d14142852f72daea7fc1 a100-oke-ubuntu-training-kubeflow: 001-nodewright-operator/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 001-nodewright-operator/install.sh: 5bb299d13650bdb1baa19ea8047b33baa94eb03a48a8b5b0102f1ba533b392c2 @@ -340,17 +340,17 @@ a100-oke-ubuntu-training-kubeflow: 012-nvsentinel/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 012-nvsentinel/install.sh: 290c0907c263ac3c1a976be76da13528e36fe7f1db79945783b6ac7da4588857 012-nvsentinel/upstream.env: 7fc07c3fb32a6e569a151e4c12f2834386b4c7026cda6cd1c86e29592ad8936e - 012-nvsentinel/values.yaml: 36e1a505cabb70a1cd795d582ac2a0b550d4fdd7875114cc49f9129bb1e3e867 + 012-nvsentinel/values.yaml: 6f417215aceac1cf2154ab9ac04a92eea2cfd631b116e2642fc8446b7cbca974 013-prometheus-adapter/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 013-prometheus-adapter/install.sh: 688ee4274fe06c001af6835d7965b5da33ce3fbeb724263622737d0bb88249e3 013-prometheus-adapter/upstream.env: 9ab918b32522baca13cd37bad7f420a206a19bb763b81041e174fffffe3da724 013-prometheus-adapter/values.yaml: 95231d7a46d650f96f844ca3734614271152a41f1230022c9d79960e44647c00 README.md: 74e5f514d77f133967c12e594461c7626b2b4bf14a24410e42068d53a1babcf7 UPGRADING.md: 4b285b6e5866a3582feef1afa6cc5906b79b275e94b0033d33f752c442d1256b - bundle-info.yaml: 66f80917e2d3bce361ded1990efba6b0ce0cb56392fe52060ef4786be9af4d26 - checksums.txt: 1efa6542d190c11f657541623bf8fb5ec78000a4679976e9e78b565627ad9072 + bundle-info.yaml: 734d0f11746fcf4456892cce87ae38d59185077167fdb181e1b6b32ac76dbfb4 + checksums.txt: 4d26789717329cc1e0aab142092ee2391b77a1b0453425a6151db12f6ff034c5 deploy.sh: e3fbb41b352a410cced1659ded2f3c4e42afc681fac4a241a8b61c7e22c14f30 - recipe.yaml: 7e65b014025f4d3c518c7710539aac8cbc86fa1ac8c93995f89c724b76fac5c0 + recipe.yaml: 2f5d9e0168d547b10496a730952a58ff2f297d1f4d9b786d9459704994135002 aks-ubuntu: 001-nodewright-operator/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 001-nodewright-operator/install.sh: 5bb299d13650bdb1baa19ea8047b33baa94eb03a48a8b5b0102f1ba533b392c2 @@ -408,17 +408,17 @@ aks-ubuntu: 013-nvsentinel/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 013-nvsentinel/install.sh: 290c0907c263ac3c1a976be76da13528e36fe7f1db79945783b6ac7da4588857 013-nvsentinel/upstream.env: 7fc07c3fb32a6e569a151e4c12f2834386b4c7026cda6cd1c86e29592ad8936e - 013-nvsentinel/values.yaml: 843a8e5cb1d7e52b0b2a1a6f4366dd15d1e275d6cea64bbf73d33f52ccfafd0c + 013-nvsentinel/values.yaml: aa27bae925e838d0ba1a1b8a9d9ce6460ccfc20cfb29d9d52d85f9d1f927f008 014-prometheus-adapter/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 014-prometheus-adapter/install.sh: 688ee4274fe06c001af6835d7965b5da33ce3fbeb724263622737d0bb88249e3 014-prometheus-adapter/upstream.env: 9ab918b32522baca13cd37bad7f420a206a19bb763b81041e174fffffe3da724 014-prometheus-adapter/values.yaml: 95231d7a46d650f96f844ca3734614271152a41f1230022c9d79960e44647c00 README.md: ffb9bcaa3b7a07d6d1ded468640b55a4d1b08ce198ec78e0d2f9374a742a41a6 UPGRADING.md: 9444e3b2469985d89ae8fd6bb0ca9116e56c2904f9aff6929487211120b75484 - bundle-info.yaml: 942001ff865ebfcb5c6a2aec686065a253406ab190a82bd00547a914beea1d50 - checksums.txt: 07b18a834270d94aeff68cc8b35a19afaf0a3cbfeca5dec4866fa6b7d79108c1 + bundle-info.yaml: 4cb34cea015297f8789e2c8b5487ea28f0bbcf891b7ce9e99fe1e4aacba2334d + checksums.txt: 7e7405145fd2d907ffa440a4d432bf2a4a4b15dd233880536b01c410488576d2 deploy.sh: 8f468711ed209c09e6c8e5d220abbdc6f273ddba9c9634f103bd57d6514136cf - recipe.yaml: bcbc25ccc530a9d0d93b6c504a88a4d1a4244213f0bf96dc263e3d5622ae5102 + recipe.yaml: 0fca3998062a98196e5e56ff0cf9b7819b35a970ce6cdf455326febd01580c14 b200-any: 001-nodewright-operator/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 001-nodewright-operator/install.sh: 5bb299d13650bdb1baa19ea8047b33baa94eb03a48a8b5b0102f1ba533b392c2 @@ -460,17 +460,17 @@ b200-any: 010-nvsentinel/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 010-nvsentinel/install.sh: 290c0907c263ac3c1a976be76da13528e36fe7f1db79945783b6ac7da4588857 010-nvsentinel/upstream.env: 7fc07c3fb32a6e569a151e4c12f2834386b4c7026cda6cd1c86e29592ad8936e - 010-nvsentinel/values.yaml: 5dbdd974de19bda092dff19de6a6fbb9489b7e7629abdeab2d1a8fe0bff69247 + 010-nvsentinel/values.yaml: 14400ae55936f06cc2025495b7d2efbb933e388a985abf2061b3b4fcbc36eeea 011-prometheus-adapter/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 011-prometheus-adapter/install.sh: 688ee4274fe06c001af6835d7965b5da33ce3fbeb724263622737d0bb88249e3 011-prometheus-adapter/upstream.env: 9ab918b32522baca13cd37bad7f420a206a19bb763b81041e174fffffe3da724 011-prometheus-adapter/values.yaml: 95231d7a46d650f96f844ca3734614271152a41f1230022c9d79960e44647c00 README.md: bebe62e795349aaec643f384823f28b5855413ee7fcbb499c6289cd52ca4e324 UPGRADING.md: 4b285b6e5866a3582feef1afa6cc5906b79b275e94b0033d33f752c442d1256b - bundle-info.yaml: 344418c942a16ad9db5489ff125a30028c529addd23e32e265f0b3b8e5922bc8 - checksums.txt: df8457a65f5888e8cbbf7c56b117fbb707e7099952ca82d13c935e455909451c + bundle-info.yaml: bdf86146f90c95cde2a6e93d9505187886efe84bff74fbc513b5322633f295d8 + checksums.txt: 2a7a6465c50764c20c9c7fed89a3615762d9a70fa7c0ec88aea005f10dbee35b deploy.sh: 187bf11702c83fdf08c22fbdd72a6b16c937314d87f85da747c4753cbe5d5de4 - recipe.yaml: aac6f191d497c7083145c8f522b42ce1e78fb636043d7bc7971c894ec1aea27f + recipe.yaml: c260d9c918fb69810399b5a83a5c4a0f14e5859fea1eb2e12f02f06e44d0530c b200-gke-cos-inference-dynamo: 001-agentgateway-crds/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 001-agentgateway-crds/install.sh: f0f864c4eb1f1d647af123953d5df5aa1fa03226e1f8c5feb724ea42d304eb3f @@ -553,17 +553,17 @@ b200-gke-cos-inference-dynamo: 019-nvsentinel/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 019-nvsentinel/install.sh: 290c0907c263ac3c1a976be76da13528e36fe7f1db79945783b6ac7da4588857 019-nvsentinel/upstream.env: 7fc07c3fb32a6e569a151e4c12f2834386b4c7026cda6cd1c86e29592ad8936e - 019-nvsentinel/values.yaml: 3374b3a52c2466d902a85e6d5e4b2fc02d4f824e690094772cb6dbe9ee6962b2 + 019-nvsentinel/values.yaml: 7f5f9edb985771f1b7c01e5ee67edba4a2a6182b990a940a5e89601e5d589ae0 020-prometheus-adapter/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 020-prometheus-adapter/install.sh: 688ee4274fe06c001af6835d7965b5da33ce3fbeb724263622737d0bb88249e3 020-prometheus-adapter/upstream.env: 9ab918b32522baca13cd37bad7f420a206a19bb763b81041e174fffffe3da724 020-prometheus-adapter/values.yaml: 95231d7a46d650f96f844ca3734614271152a41f1230022c9d79960e44647c00 README.md: 3c3d5ef47a3dc3b8184065f6cdb0e7babccf02dd655390313ffc32a57f76f510 UPGRADING.md: a2aa0d1cd4b9e3a564ecf713c5fb05bb9e002f8e28cf51251d74d9f2962b2401 - bundle-info.yaml: c8212c1aac5d408eebf71d2392b3c80fbdcac2226926eed748ea258b84adea1e - checksums.txt: 1f8fa925f8115cc4643ecc6bd75613bd3f30d587e089d15ea42272c904a6f7c5 + bundle-info.yaml: 3dfe37d2242315dc8e1c5515c52c46b1139bc1d02fd0bb12a11330deeab7d728 + checksums.txt: dbe0bcb8d63169456fa87c8c2a1cb25d6b8b71373081007c5efbb9ba6e75a306 deploy.sh: 841a92636bb0a65da6fa0162b67084f68f362fa0e41837b1c57e6560c10a7a63 - recipe.yaml: 12ca3abab7dc3392f0d7e9bb6e5e4f6a1eb864befcfe72c71949670206c71e2c + recipe.yaml: 3ae309d3918dd339002e09093e038c15be33a3c2ce67c1d66d606a21344b0b7d b200-gke-cos-training-kubeflow: 001-gcp-driver-installer/Chart.yaml: b0476226ebbc9a7738c4e5c2828ce5a73dbd371f3fa7cf1139c5fe9b2e825c02 001-gcp-driver-installer/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 @@ -628,17 +628,17 @@ b200-gke-cos-training-kubeflow: 015-nvsentinel/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 015-nvsentinel/install.sh: 290c0907c263ac3c1a976be76da13528e36fe7f1db79945783b6ac7da4588857 015-nvsentinel/upstream.env: 7fc07c3fb32a6e569a151e4c12f2834386b4c7026cda6cd1c86e29592ad8936e - 015-nvsentinel/values.yaml: 3374b3a52c2466d902a85e6d5e4b2fc02d4f824e690094772cb6dbe9ee6962b2 + 015-nvsentinel/values.yaml: 7f5f9edb985771f1b7c01e5ee67edba4a2a6182b990a940a5e89601e5d589ae0 016-prometheus-adapter/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 016-prometheus-adapter/install.sh: 688ee4274fe06c001af6835d7965b5da33ce3fbeb724263622737d0bb88249e3 016-prometheus-adapter/upstream.env: 9ab918b32522baca13cd37bad7f420a206a19bb763b81041e174fffffe3da724 016-prometheus-adapter/values.yaml: 95231d7a46d650f96f844ca3734614271152a41f1230022c9d79960e44647c00 README.md: 63792fc186196d746069a56bf39e72e71814ad011d7697378c052810aff54105 UPGRADING.md: 4b285b6e5866a3582feef1afa6cc5906b79b275e94b0033d33f752c442d1256b - bundle-info.yaml: c9ec92b4f6e8bb06f6118d3da6b40aad6f86e0937a60f4b0224753e656ebfbbe - checksums.txt: 6ead233b9547009c984ea4ecb5d714fbd5181ced907c3f3bc88d110d298250e5 + bundle-info.yaml: 4cf53dc55d5b5cf0bfbe900451c863e4a918c9c0d0f4f3b8b6e55374e2fd3f90 + checksums.txt: 719b5144b2750c6e43d2e11b9e14bdc5cc7325718edde91c82b2ee649ac4a185 deploy.sh: 50736743f1a6091c2dcb82baedfed99d4ae2f5bfe3325f7e7ab89995353f81bd - recipe.yaml: 6826289281d40d0c28e044c367831fe0f72550e54a16fa839dd16861b08524c6 + recipe.yaml: 6e9b988a8c0f15c40cb419a45b0b25bcd5a78738eb44a8151d0d3d60dab16a41 b300-mk8s-ubuntu-inference-dynamo: 001-cert-manager/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 001-cert-manager/install.sh: d8043b8845f8225dc46f54e05a3a86362e30487775b59ac74f47be104e8555bd @@ -744,17 +744,17 @@ bcm-inference: 015-nvsentinel/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 015-nvsentinel/install.sh: 290c0907c263ac3c1a976be76da13528e36fe7f1db79945783b6ac7da4588857 015-nvsentinel/upstream.env: 7fc07c3fb32a6e569a151e4c12f2834386b4c7026cda6cd1c86e29592ad8936e - 015-nvsentinel/values.yaml: 5dbdd974de19bda092dff19de6a6fbb9489b7e7629abdeab2d1a8fe0bff69247 + 015-nvsentinel/values.yaml: 14400ae55936f06cc2025495b7d2efbb933e388a985abf2061b3b4fcbc36eeea 016-prometheus-adapter/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 016-prometheus-adapter/install.sh: 688ee4274fe06c001af6835d7965b5da33ce3fbeb724263622737d0bb88249e3 016-prometheus-adapter/upstream.env: 9ab918b32522baca13cd37bad7f420a206a19bb763b81041e174fffffe3da724 016-prometheus-adapter/values.yaml: 95231d7a46d650f96f844ca3734614271152a41f1230022c9d79960e44647c00 README.md: d04732cab3d450d8744260f6e4393941c7876424e90beeecddb7e9dab4d3db13 UPGRADING.md: 4b285b6e5866a3582feef1afa6cc5906b79b275e94b0033d33f752c442d1256b - bundle-info.yaml: fcff6bf5870529a6ec4d39d302d73e811e69552aec14db56b44d5efb0966128e - checksums.txt: bb7e1c0b96d5d8b0c3d156f8ea0164cb170c80d264150c8ab62bcad6bcd2e5ab + bundle-info.yaml: 54b08d5a15218de3da59c5d789852fd63e9fd244399a561d5ccb34ce23de1272 + checksums.txt: 83cad8e1afdc70e3418c83d60cf033c39ce684c31b5a97970764295272bdaa63 deploy.sh: 1ccc85228bcbafaf6345c9da5753ab24dac14c7edec7f22e99c53c2535b4653b - recipe.yaml: af5097bbc10b6ff5ad105ed1972a02eabf1bd7ae8c3ee86d7a0a1ee31f8d6715 + recipe.yaml: 3ba0fc4933552fa5d6659ef53fc51b117409235ae7952546e6220a4f8393b526 eks-ubuntu: 001-aws-efa/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 001-aws-efa/install.sh: 00574b8307fa54f90f18cd999a2b175a75a547942fd18cb242bd83b7af836d5a @@ -804,17 +804,17 @@ eks-ubuntu: 012-nvsentinel/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 012-nvsentinel/install.sh: 290c0907c263ac3c1a976be76da13528e36fe7f1db79945783b6ac7da4588857 012-nvsentinel/upstream.env: 7fc07c3fb32a6e569a151e4c12f2834386b4c7026cda6cd1c86e29592ad8936e - 012-nvsentinel/values.yaml: 5dbdd974de19bda092dff19de6a6fbb9489b7e7629abdeab2d1a8fe0bff69247 + 012-nvsentinel/values.yaml: 14400ae55936f06cc2025495b7d2efbb933e388a985abf2061b3b4fcbc36eeea 013-prometheus-adapter/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 013-prometheus-adapter/install.sh: 688ee4274fe06c001af6835d7965b5da33ce3fbeb724263622737d0bb88249e3 013-prometheus-adapter/upstream.env: 9ab918b32522baca13cd37bad7f420a206a19bb763b81041e174fffffe3da724 013-prometheus-adapter/values.yaml: 95231d7a46d650f96f844ca3734614271152a41f1230022c9d79960e44647c00 README.md: 78facb3ff07888af226723b512f6735975c898ced360f18e811e06ba42c42fc2 UPGRADING.md: 4b285b6e5866a3582feef1afa6cc5906b79b275e94b0033d33f752c442d1256b - bundle-info.yaml: 0f1e16203688e87fd8d794c3a12a5a6497ab4292eae5ef949dcdaa55c7b84a87 - checksums.txt: 621e3989df0f41922d5d97536eeccb429433de0883c35c438c2042a6e47b9960 + bundle-info.yaml: 2bd1fe426b610293b05f6bbfdce92c74559971064ec7a14495d9119a53f74f0f + checksums.txt: 27c05c68ae8b1fad7d6eb6d8541af31d370a9e4bd6de8cc1a0d922a6e0abb504 deploy.sh: 4e2e552d2fdb5b8485a1b3abba65a1cac19d50c59a3fa000796224fe59eb72e5 - recipe.yaml: c18c87a4e6b5d4eb5dc88e080086b320102dc98a84208762b16edb82d88e63f3 + recipe.yaml: d4de9fbe6f4bfd043c224b91254c4814293164e488594aa73f4bac463fd1dc92 gb200-any: 001-nodewright-operator/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 001-nodewright-operator/install.sh: 5bb299d13650bdb1baa19ea8047b33baa94eb03a48a8b5b0102f1ba533b392c2 @@ -856,17 +856,17 @@ gb200-any: 010-nvsentinel/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 010-nvsentinel/install.sh: 290c0907c263ac3c1a976be76da13528e36fe7f1db79945783b6ac7da4588857 010-nvsentinel/upstream.env: 7fc07c3fb32a6e569a151e4c12f2834386b4c7026cda6cd1c86e29592ad8936e - 010-nvsentinel/values.yaml: 5dbdd974de19bda092dff19de6a6fbb9489b7e7629abdeab2d1a8fe0bff69247 + 010-nvsentinel/values.yaml: 14400ae55936f06cc2025495b7d2efbb933e388a985abf2061b3b4fcbc36eeea 011-prometheus-adapter/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 011-prometheus-adapter/install.sh: 688ee4274fe06c001af6835d7965b5da33ce3fbeb724263622737d0bb88249e3 011-prometheus-adapter/upstream.env: 9ab918b32522baca13cd37bad7f420a206a19bb763b81041e174fffffe3da724 011-prometheus-adapter/values.yaml: 95231d7a46d650f96f844ca3734614271152a41f1230022c9d79960e44647c00 README.md: 8eb66c4e76000a9717f025a90a75670e8c0dac446234530909960daaa8940867 UPGRADING.md: 4b285b6e5866a3582feef1afa6cc5906b79b275e94b0033d33f752c442d1256b - bundle-info.yaml: aa14b33076f37579ba22bfca26f7494bb95c7f4ca683534e7a844305ce1bff10 - checksums.txt: 14f16b68bcc5501ff9fc393d312bf63df03f10e337d961fe9b1e90e7ce203408 + bundle-info.yaml: 122e3e2ac409142b7d2e50fdd79ee1954462e5cc5b0f9f5a09fcf62873254f91 + checksums.txt: 553751807fd89908a4ce7036ca8286aab5ab109694f47aca7819de55a858f9bc deploy.sh: 187bf11702c83fdf08c22fbdd72a6b16c937314d87f85da747c4753cbe5d5de4 - recipe.yaml: 3fa1b284bad88c74d1820dd914cd61e6c211aef536016e28807797a0a3adf411 + recipe.yaml: d47dd789f2517261bf197698cd0bf26bd0ffc09f244e9c3a7ddb9f7e17b40fe8 gb200-eks-amazonlinux-inference: 001-agentgateway-crds/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 001-agentgateway-crds/install.sh: f0f864c4eb1f1d647af123953d5df5aa1fa03226e1f8c5feb724ea42d304eb3f @@ -940,17 +940,17 @@ gb200-eks-amazonlinux-inference: 017-nvsentinel/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 017-nvsentinel/install.sh: 290c0907c263ac3c1a976be76da13528e36fe7f1db79945783b6ac7da4588857 017-nvsentinel/upstream.env: 7fc07c3fb32a6e569a151e4c12f2834386b4c7026cda6cd1c86e29592ad8936e - 017-nvsentinel/values.yaml: ede1bf610929dcf019396356eec2e1a0bf9c1417c181595e86f03d6035fe27bb + 017-nvsentinel/values.yaml: 376d322d05dfab359cef13a99ded134b567f8100905a39caaf4af614efefa223 018-prometheus-adapter/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 018-prometheus-adapter/install.sh: 688ee4274fe06c001af6835d7965b5da33ce3fbeb724263622737d0bb88249e3 018-prometheus-adapter/upstream.env: 9ab918b32522baca13cd37bad7f420a206a19bb763b81041e174fffffe3da724 018-prometheus-adapter/values.yaml: 95231d7a46d650f96f844ca3734614271152a41f1230022c9d79960e44647c00 README.md: 0357a5c5fc693eb0d70c8de2c92f5305c1b1a5183cbf75850b7a7a59c8306137 UPGRADING.md: 4b285b6e5866a3582feef1afa6cc5906b79b275e94b0033d33f752c442d1256b - bundle-info.yaml: 0cff7c68275bbd43f80fc8b4aa45c5e4e6772fe72cb865a35ad7bee1e6c9b23e - checksums.txt: d248d9da83f8402d715ed35865181ee29dbeebbcad737af38c3fe624e2d7ebfd + bundle-info.yaml: f05453de9aa09bdd1e3583feed534a3aaabda9f124fee98c0b5387f3537ef33d + checksums.txt: 992a90536f05a77f300ea9cfec7a214675f59b3e07ccacde75a9435bec9ab05d deploy.sh: 0248fabe5a8e4dbdbe60fc3f0cbf1aaab47a637402bec6eca1191fa601e8bfd6 - recipe.yaml: 34f24e413814d74fcb32cf9aa3cd4a1f9992f00768fb973b7a7c2f6c92a75427 + recipe.yaml: aec1ae9b75ec8f025cc574eeddbefa69eb187008361a0fc35a0521114bca569a gb200-eks-ubuntu-inference-dynamo: 001-agentgateway-crds/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 001-agentgateway-crds/install.sh: f0f864c4eb1f1d647af123953d5df5aa1fa03226e1f8c5feb724ea42d304eb3f @@ -1037,17 +1037,17 @@ gb200-eks-ubuntu-inference-dynamo: 020-nvsentinel/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 020-nvsentinel/install.sh: 290c0907c263ac3c1a976be76da13528e36fe7f1db79945783b6ac7da4588857 020-nvsentinel/upstream.env: 7fc07c3fb32a6e569a151e4c12f2834386b4c7026cda6cd1c86e29592ad8936e - 020-nvsentinel/values.yaml: 5dbdd974de19bda092dff19de6a6fbb9489b7e7629abdeab2d1a8fe0bff69247 + 020-nvsentinel/values.yaml: 14400ae55936f06cc2025495b7d2efbb933e388a985abf2061b3b4fcbc36eeea 021-prometheus-adapter/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 021-prometheus-adapter/install.sh: 688ee4274fe06c001af6835d7965b5da33ce3fbeb724263622737d0bb88249e3 021-prometheus-adapter/upstream.env: 9ab918b32522baca13cd37bad7f420a206a19bb763b81041e174fffffe3da724 021-prometheus-adapter/values.yaml: 95231d7a46d650f96f844ca3734614271152a41f1230022c9d79960e44647c00 README.md: 764d05f990ffb5b7340c2dc9228e63f597977730da9a2b38529881896fedc1ff UPGRADING.md: a2aa0d1cd4b9e3a564ecf713c5fb05bb9e002f8e28cf51251d74d9f2962b2401 - bundle-info.yaml: ff8a3531bae119f4c64d4b67654db9e5f5253e4bf0f71c56672a68fc91f5dcac - checksums.txt: 735debeac467deb5a40c0efc11e5954164552ce29b7bb9aaf3809d8dcc81ec2b + bundle-info.yaml: b41888126775f85c0cbfdafe8f0bd0e07d74f17046ab11fbf1f832b62b447fed + checksums.txt: 9f7237ceb3c82bb53a946793dc3bdc6f0fc4a79c823741fbaee748aa6c7957aa deploy.sh: 5463ca8ed80e72d789bfb987637d85e2d3da5e4212a6a20e52b3640465c24da1 - recipe.yaml: 69d7f306d8084310bd3e765fbf4312819ab1c686a215be250ad8ebe2637ef230 + recipe.yaml: 9675084995650308c337080d55fcc0f8e1e43a762867cc6f37d9d9de414db863 gb200-eks-ubuntu-training-kubeflow: 001-aws-efa/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 001-aws-efa/install.sh: 00574b8307fa54f90f18cd999a2b175a75a547942fd18cb242bd83b7af836d5a @@ -1116,17 +1116,17 @@ gb200-eks-ubuntu-training-kubeflow: 016-nvsentinel/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 016-nvsentinel/install.sh: 290c0907c263ac3c1a976be76da13528e36fe7f1db79945783b6ac7da4588857 016-nvsentinel/upstream.env: 7fc07c3fb32a6e569a151e4c12f2834386b4c7026cda6cd1c86e29592ad8936e - 016-nvsentinel/values.yaml: 5dbdd974de19bda092dff19de6a6fbb9489b7e7629abdeab2d1a8fe0bff69247 + 016-nvsentinel/values.yaml: 14400ae55936f06cc2025495b7d2efbb933e388a985abf2061b3b4fcbc36eeea 017-prometheus-adapter/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 017-prometheus-adapter/install.sh: 688ee4274fe06c001af6835d7965b5da33ce3fbeb724263622737d0bb88249e3 017-prometheus-adapter/upstream.env: 9ab918b32522baca13cd37bad7f420a206a19bb763b81041e174fffffe3da724 017-prometheus-adapter/values.yaml: 95231d7a46d650f96f844ca3734614271152a41f1230022c9d79960e44647c00 README.md: 9cf889ca44d9fb84bd1d94658e2db2bb516394ea8d8481f6ee99ce3c8add49d6 UPGRADING.md: 4b285b6e5866a3582feef1afa6cc5906b79b275e94b0033d33f752c442d1256b - bundle-info.yaml: 5759720780b5bcba5e0ee2b2c842caaaed692f273b23839b821241ac646b8b25 - checksums.txt: ab8449fa6282bed9dc27c4fcbfdb41d34492f9a82b7784b424334e6ab44c22e8 + bundle-info.yaml: 3837707e4e8a2b8c0920b3de7c2cd1023dc811048ccc6c9192edd896520c0dba + checksums.txt: 6b9eccbadeb32f4ebb7f5b1ebb671f11bc69355ff46870552597bf205fefabcc deploy.sh: 949466ed3363eb59ea6ff5a61a29ca150215a2041c47323cada206dcf3cdc0a8 - recipe.yaml: dbbca7e69db298870fbbe79a2980711a26178dde7e5d4459972297093f2eff98 + recipe.yaml: 04b1de334ab6aba0682985d9f0349fe51f88960252b29764c761329afa171ff1 gb200-eks-ubuntu-training-slurm: 001-aws-efa/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 001-aws-efa/install.sh: 00574b8307fa54f90f18cd999a2b175a75a547942fd18cb242bd83b7af836d5a @@ -1186,7 +1186,7 @@ gb200-eks-ubuntu-training-slurm: 014-nvsentinel/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 014-nvsentinel/install.sh: 290c0907c263ac3c1a976be76da13528e36fe7f1db79945783b6ac7da4588857 014-nvsentinel/upstream.env: 7fc07c3fb32a6e569a151e4c12f2834386b4c7026cda6cd1c86e29592ad8936e - 014-nvsentinel/values.yaml: 14a4d2de25f5e20bf1b9cc3f9964d1be5e9b388e3c40bd01b5bbed6b7dc09c52 + 014-nvsentinel/values.yaml: 8d8fc8ff4f18cad5c88227aaf425e76cfe57781de040cf96cd10bb285dcbf403 015-prometheus-adapter/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 015-prometheus-adapter/install.sh: 688ee4274fe06c001af6835d7965b5da33ce3fbeb724263622737d0bb88249e3 015-prometheus-adapter/upstream.env: 9ab918b32522baca13cd37bad7f420a206a19bb763b81041e174fffffe3da724 @@ -1211,10 +1211,10 @@ gb200-eks-ubuntu-training-slurm: 019-slinky-slurm/values.yaml: 3a2dd1888df27899c573b67f06a75fc8eff8cb4ae9f0efd681c0fa09b57c0117 README.md: f9071833adaf913bd917cd0df3b94c4b40c9e4f4dff5a2d9528f8b297bdec80b UPGRADING.md: 4b285b6e5866a3582feef1afa6cc5906b79b275e94b0033d33f752c442d1256b - bundle-info.yaml: f73ead7e959a3536080ba72f21390fd0821a3fadc9d6b1a3b286574370874ed8 - checksums.txt: 64785ac6eb8357360b9e9a573b753abcdea4fec5d72a06d6e8b34fd0482fa77b + bundle-info.yaml: bb14e8fd1301acdacec1040d9415af3d6f0196f5f9492353a763898d3cd2a8b1 + checksums.txt: 990f4b439dc832ed874037d29c34884148012939293f06b77ea711a5c7bfb45f deploy.sh: 49009b25d90dd3b0c344008506134dc39d12e4b80582b1ce3356216f7cfb871d - recipe.yaml: ba06a6a2187e8f9294b99e6027f81deb0263494d0864e282a71d444d0d0bd96d + recipe.yaml: 9446cfa93732d7453c1ed33e52e4db042e2a6ea40be7c0b0555d0fc0b8e20bd4 gb200-gke-cos-inference-dynamo: 001-agentgateway-crds/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 001-agentgateway-crds/install.sh: f0f864c4eb1f1d647af123953d5df5aa1fa03226e1f8c5feb724ea42d304eb3f @@ -1308,17 +1308,17 @@ gb200-gke-cos-inference-dynamo: 021-nvsentinel/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 021-nvsentinel/install.sh: 290c0907c263ac3c1a976be76da13528e36fe7f1db79945783b6ac7da4588857 021-nvsentinel/upstream.env: 7fc07c3fb32a6e569a151e4c12f2834386b4c7026cda6cd1c86e29592ad8936e - 021-nvsentinel/values.yaml: 3374b3a52c2466d902a85e6d5e4b2fc02d4f824e690094772cb6dbe9ee6962b2 + 021-nvsentinel/values.yaml: 7f5f9edb985771f1b7c01e5ee67edba4a2a6182b990a940a5e89601e5d589ae0 022-prometheus-adapter/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 022-prometheus-adapter/install.sh: 688ee4274fe06c001af6835d7965b5da33ce3fbeb724263622737d0bb88249e3 022-prometheus-adapter/upstream.env: 9ab918b32522baca13cd37bad7f420a206a19bb763b81041e174fffffe3da724 022-prometheus-adapter/values.yaml: 95231d7a46d650f96f844ca3734614271152a41f1230022c9d79960e44647c00 README.md: 1d0b18970d702bb131d8a93a6964b200616d2b4db5d85f12937bc9d08703b5e0 UPGRADING.md: a2aa0d1cd4b9e3a564ecf713c5fb05bb9e002f8e28cf51251d74d9f2962b2401 - bundle-info.yaml: 0c6a367ee29be1c699ae9c9aacbb8b468abd25436b7988d60d3c5f85a35ffd8e - checksums.txt: 3cb0c922046c6a17a42c1b26e8225feee79aa16337ad090e0a542048c9777969 + bundle-info.yaml: f27af3bfafb7c21bdc77483cd81298908a1ef14e2552739925cfb39fc3fd0fbe + checksums.txt: 2655801b65445b205d7134f7849f8200edb0653fc9bdcef58920e33a2879be25 deploy.sh: 169ebed142cd9adf4392755b843629729d2d26e0cfd613dfce97b816b1069827 - recipe.yaml: e7ac65c79a84bdbd345309ee63ab5ddbddb5d0f451e71b89ff4e877479eb0e45 + recipe.yaml: 70604b0223ad6c222deb0a9fc115e8cb32f0f9874c9daacdfb232ff26c148669 gb200-gke-cos-training-kubeflow: 001-gcp-driver-installer/Chart.yaml: b0476226ebbc9a7738c4e5c2828ce5a73dbd371f3fa7cf1139c5fe9b2e825c02 001-gcp-driver-installer/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 @@ -1390,17 +1390,17 @@ gb200-gke-cos-training-kubeflow: 016-nvsentinel/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 016-nvsentinel/install.sh: 290c0907c263ac3c1a976be76da13528e36fe7f1db79945783b6ac7da4588857 016-nvsentinel/upstream.env: 7fc07c3fb32a6e569a151e4c12f2834386b4c7026cda6cd1c86e29592ad8936e - 016-nvsentinel/values.yaml: 3374b3a52c2466d902a85e6d5e4b2fc02d4f824e690094772cb6dbe9ee6962b2 + 016-nvsentinel/values.yaml: 7f5f9edb985771f1b7c01e5ee67edba4a2a6182b990a940a5e89601e5d589ae0 017-prometheus-adapter/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 017-prometheus-adapter/install.sh: 688ee4274fe06c001af6835d7965b5da33ce3fbeb724263622737d0bb88249e3 017-prometheus-adapter/upstream.env: 9ab918b32522baca13cd37bad7f420a206a19bb763b81041e174fffffe3da724 017-prometheus-adapter/values.yaml: 95231d7a46d650f96f844ca3734614271152a41f1230022c9d79960e44647c00 README.md: bd2ce60a060322fb791198a3aa84ad14f9239a0b10316015db05d773758be541 UPGRADING.md: 4b285b6e5866a3582feef1afa6cc5906b79b275e94b0033d33f752c442d1256b - bundle-info.yaml: 91f755a178ac23036d9a75f1fbcea4f26ae0752db663a53c7345bcd569bf7bb6 - checksums.txt: 9878b41d6841aa48747ed8688afc1d436b3e21883c93eb96b56de640e801e2a0 + bundle-info.yaml: 0ae7fc0261e2e9c4b9a06bb4bd566d20cf6c6f3ed404d4396fce90489525227f + checksums.txt: 4b22d9c23376003325cd386f6330327b5fb6c1417403129c4a487a0c5103aa91 deploy.sh: 4fa420735f0c5403b7e2bb44e37f180f928c157aab09355d29be499cdffb27d2 - recipe.yaml: e5db9b7669bcbe7845d55cfd63aae0ea92a9378cacad184e7ef3979bf99626f0 + recipe.yaml: 92a370fc2028d044b7449316546d2b9187ea8078e0ff70c789ba4285f054c619 gb200-gke-cos-training-slurm: 001-gcp-driver-installer/Chart.yaml: b0476226ebbc9a7738c4e5c2828ce5a73dbd371f3fa7cf1139c5fe9b2e825c02 001-gcp-driver-installer/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 @@ -1462,7 +1462,7 @@ gb200-gke-cos-training-slurm: 014-nvsentinel/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 014-nvsentinel/install.sh: 290c0907c263ac3c1a976be76da13528e36fe7f1db79945783b6ac7da4588857 014-nvsentinel/upstream.env: 7fc07c3fb32a6e569a151e4c12f2834386b4c7026cda6cd1c86e29592ad8936e - 014-nvsentinel/values.yaml: ef308e8e0286992ad25a311fd88960e537431f28140d953dfdf0c869fb2b86fe + 014-nvsentinel/values.yaml: 631c76fda879256d07d8e329e6f4582d7680e10b316444be4f7215356ca5c655 015-prometheus-adapter/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 015-prometheus-adapter/install.sh: 688ee4274fe06c001af6835d7965b5da33ce3fbeb724263622737d0bb88249e3 015-prometheus-adapter/upstream.env: 9ab918b32522baca13cd37bad7f420a206a19bb763b81041e174fffffe3da724 @@ -1487,10 +1487,10 @@ gb200-gke-cos-training-slurm: 019-slinky-slurm/values.yaml: 3a2dd1888df27899c573b67f06a75fc8eff8cb4ae9f0efd681c0fa09b57c0117 README.md: d787fc1e20f3abdd3160cdd6a84c5aa7cdfac94134849832c9014bcdbb4106e3 UPGRADING.md: 4b285b6e5866a3582feef1afa6cc5906b79b275e94b0033d33f752c442d1256b - bundle-info.yaml: 592981e962232e587f1e3350aca0da130f70de187cb36b27813173d03969926b - checksums.txt: 23ac3951ec03b93a855904ec4b625c117b3ffc48e2fa4520e6360c12bee02860 + bundle-info.yaml: 117008920a59df2cfcdb80e0e75cfb51007bba977373eca4b64465d965118d27 + checksums.txt: 2bf0591381f1419f904bf2793372dff73a1383afaee268a6fe954055c9b098d1 deploy.sh: d54909f4d258435a4f5a944a2624c6ab0adf954532cebb9c8c27321f08d0d0bc - recipe.yaml: df7e19d1346c0c76d831e521f1d9e36a13e28815a6b51fa3815bec73456870df + recipe.yaml: 6b935a338686ae093d88bc144f086e7200b71c806b47ae571bab1580a60a4458 gb200-oke-ubuntu-inference-dynamo: 001-agentgateway-crds/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 001-agentgateway-crds/install.sh: f0f864c4eb1f1d647af123953d5df5aa1fa03226e1f8c5feb724ea42d304eb3f @@ -1559,17 +1559,17 @@ gb200-oke-ubuntu-inference-dynamo: 016-nvsentinel/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 016-nvsentinel/install.sh: 290c0907c263ac3c1a976be76da13528e36fe7f1db79945783b6ac7da4588857 016-nvsentinel/upstream.env: 7fc07c3fb32a6e569a151e4c12f2834386b4c7026cda6cd1c86e29592ad8936e - 016-nvsentinel/values.yaml: 36e1a505cabb70a1cd795d582ac2a0b550d4fdd7875114cc49f9129bb1e3e867 + 016-nvsentinel/values.yaml: 6f417215aceac1cf2154ab9ac04a92eea2cfd631b116e2642fc8446b7cbca974 017-prometheus-adapter/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 017-prometheus-adapter/install.sh: 688ee4274fe06c001af6835d7965b5da33ce3fbeb724263622737d0bb88249e3 017-prometheus-adapter/upstream.env: 9ab918b32522baca13cd37bad7f420a206a19bb763b81041e174fffffe3da724 017-prometheus-adapter/values.yaml: 95231d7a46d650f96f844ca3734614271152a41f1230022c9d79960e44647c00 README.md: 4a684558488c1bc0bff9bf18c3ec9d5695e5c85cdddb67d1632ca48a54d7de26 UPGRADING.md: a2aa0d1cd4b9e3a564ecf713c5fb05bb9e002f8e28cf51251d74d9f2962b2401 - bundle-info.yaml: 9fb38bcfa602a60a4fd3e9fdee237e52844ac01f8e2d03887b78b7e55dfeb856 - checksums.txt: b474f069f418137adbdcdcb40437f3e2b49767e75cba427af0135460be12bc33 + bundle-info.yaml: 854de61bfa16cda01c5859f9ae0edb86b2ed356f4147a8db13922f5ed4dba38c + checksums.txt: 8cf9f06e4300c9037415cf1441fcc7d459820803c7a989a51c07e32e29b905c6 deploy.sh: 1e42e0e0bc9c9ddc7f45c00104c12763d0ebf6ace12098813023d30512df1fdc - recipe.yaml: 3391b973bf69be198c92db5ef254fbac2c031f6616c880db2fa50413e05b14ec + recipe.yaml: 81a36141f5039ea6ff220bbec1c5feb3c9ab7ffb6171bff8244e819444b2951f gb200-oke-ubuntu-training-kubeflow: 001-nodewright-operator/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 001-nodewright-operator/install.sh: 5bb299d13650bdb1baa19ea8047b33baa94eb03a48a8b5b0102f1ba533b392c2 @@ -1634,17 +1634,17 @@ gb200-oke-ubuntu-training-kubeflow: 015-nvsentinel/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 015-nvsentinel/install.sh: 290c0907c263ac3c1a976be76da13528e36fe7f1db79945783b6ac7da4588857 015-nvsentinel/upstream.env: 7fc07c3fb32a6e569a151e4c12f2834386b4c7026cda6cd1c86e29592ad8936e - 015-nvsentinel/values.yaml: 36e1a505cabb70a1cd795d582ac2a0b550d4fdd7875114cc49f9129bb1e3e867 + 015-nvsentinel/values.yaml: 6f417215aceac1cf2154ab9ac04a92eea2cfd631b116e2642fc8446b7cbca974 016-prometheus-adapter/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 016-prometheus-adapter/install.sh: 688ee4274fe06c001af6835d7965b5da33ce3fbeb724263622737d0bb88249e3 016-prometheus-adapter/upstream.env: 9ab918b32522baca13cd37bad7f420a206a19bb763b81041e174fffffe3da724 016-prometheus-adapter/values.yaml: 95231d7a46d650f96f844ca3734614271152a41f1230022c9d79960e44647c00 README.md: e509fba4f7aa4f1919d1ad1c39b535d36b7f972fad8b011f154736ed679cb103 UPGRADING.md: 9444e3b2469985d89ae8fd6bb0ca9116e56c2904f9aff6929487211120b75484 - bundle-info.yaml: cdfca269210dbcd82099923837dcfc0c8f057811ccb4f7366f0cd92c674ca49e - checksums.txt: dda51d576de1f709745f76788fe96ffa792aae62aae4479f66e58e3e034fac84 + bundle-info.yaml: 4d4d78b428451a8dfcb57e0dec463b10f53f6087c6303effb758827029e2c941 + checksums.txt: b6fc48e24996399a08515f2ebd19991ff76a722d1bb104ebfe6060fd3ad859c1 deploy.sh: 585028988d2cdbd7fad24e6a6aa04e416399fb8c675ac560c1ea2cf01e76def6 - recipe.yaml: 8684024a4038998b9a4ac94cc81b3ea2ebb8e60f5ea4e4cafc42e4412bef1e92 + recipe.yaml: a6777a3da516b9072a9eb7fd21495bf2f0f7320ee62110ca341f566b5c6ca2a1 gb300-any: 001-nodewright-operator/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 001-nodewright-operator/install.sh: 5bb299d13650bdb1baa19ea8047b33baa94eb03a48a8b5b0102f1ba533b392c2 @@ -1686,17 +1686,17 @@ gb300-any: 010-nvsentinel/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 010-nvsentinel/install.sh: 290c0907c263ac3c1a976be76da13528e36fe7f1db79945783b6ac7da4588857 010-nvsentinel/upstream.env: 7fc07c3fb32a6e569a151e4c12f2834386b4c7026cda6cd1c86e29592ad8936e - 010-nvsentinel/values.yaml: 5dbdd974de19bda092dff19de6a6fbb9489b7e7629abdeab2d1a8fe0bff69247 + 010-nvsentinel/values.yaml: 14400ae55936f06cc2025495b7d2efbb933e388a985abf2061b3b4fcbc36eeea 011-prometheus-adapter/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 011-prometheus-adapter/install.sh: 688ee4274fe06c001af6835d7965b5da33ce3fbeb724263622737d0bb88249e3 011-prometheus-adapter/upstream.env: 9ab918b32522baca13cd37bad7f420a206a19bb763b81041e174fffffe3da724 011-prometheus-adapter/values.yaml: 95231d7a46d650f96f844ca3734614271152a41f1230022c9d79960e44647c00 README.md: 6b0d066f7018d9207467978c8f7920283908962e24d6136e9c72501a1d1f0612 UPGRADING.md: 4b285b6e5866a3582feef1afa6cc5906b79b275e94b0033d33f752c442d1256b - bundle-info.yaml: cbb02fd7f135dbb46e9758fe584e960ec15881c39ad8c7ac9275b5800f343327 - checksums.txt: 5ec4069e579c6ec291418114c86cc179dce71300305e163d9c3254e012d6afe6 + bundle-info.yaml: 3152d9768a8045d6ddcebc4311cbde08ab70e2b672ecf89b994e87028941a00a + checksums.txt: c21640cba990146f9196487737744c1e95a25c38bc4cb914457e4ba31f0f0308 deploy.sh: 187bf11702c83fdf08c22fbdd72a6b16c937314d87f85da747c4753cbe5d5de4 - recipe.yaml: db3af520b6ee01ff50d439ea91a2837341371ce04f94dd857b726f6e14ab5495 + recipe.yaml: f44ff8b87ed4fcce60ff961cbf00c5495b09cbfadcfcad63b9435f7b257990c1 gb300-eks-amazonlinux-inference: 001-agentgateway-crds/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 001-agentgateway-crds/install.sh: f0f864c4eb1f1d647af123953d5df5aa1fa03226e1f8c5feb724ea42d304eb3f @@ -1770,17 +1770,17 @@ gb300-eks-amazonlinux-inference: 017-nvsentinel/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 017-nvsentinel/install.sh: 290c0907c263ac3c1a976be76da13528e36fe7f1db79945783b6ac7da4588857 017-nvsentinel/upstream.env: 7fc07c3fb32a6e569a151e4c12f2834386b4c7026cda6cd1c86e29592ad8936e - 017-nvsentinel/values.yaml: ede1bf610929dcf019396356eec2e1a0bf9c1417c181595e86f03d6035fe27bb + 017-nvsentinel/values.yaml: 376d322d05dfab359cef13a99ded134b567f8100905a39caaf4af614efefa223 018-prometheus-adapter/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 018-prometheus-adapter/install.sh: 688ee4274fe06c001af6835d7965b5da33ce3fbeb724263622737d0bb88249e3 018-prometheus-adapter/upstream.env: 9ab918b32522baca13cd37bad7f420a206a19bb763b81041e174fffffe3da724 018-prometheus-adapter/values.yaml: 95231d7a46d650f96f844ca3734614271152a41f1230022c9d79960e44647c00 README.md: bdcaec80c92acb9c3d2b9f92b80a03247849c0658ae0a300338913c1a341f477 UPGRADING.md: 4b285b6e5866a3582feef1afa6cc5906b79b275e94b0033d33f752c442d1256b - bundle-info.yaml: 13c06657a8ce363cac710ce15a473f7d3c946ee6c055db36b5c956fe0a59b3a0 - checksums.txt: 5d5916ec473a892d0e8fb67dc571434fe19ba770443dd220626345191025e55a + bundle-info.yaml: c30b4c22d3d2d479e4c7731fca6a98b20ce74fd0133317b1d6925d9aafa086b0 + checksums.txt: 324d8771fbd014d9bfe977cb873d796a3e68ca59cbb8e33001cc1376d6cfe244 deploy.sh: 0248fabe5a8e4dbdbe60fc3f0cbf1aaab47a637402bec6eca1191fa601e8bfd6 - recipe.yaml: c8b115c786883712301524e4526504bfa30b6eb95719f8cefa04ffc2f973b779 + recipe.yaml: 2486f84827044c4251b2dae5a4918d373dcb4334fa11ae52659c5f33761b1fbb gb300-eks-ubuntu-inference-dynamo: 001-agentgateway-crds/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 001-agentgateway-crds/install.sh: f0f864c4eb1f1d647af123953d5df5aa1fa03226e1f8c5feb724ea42d304eb3f @@ -1867,17 +1867,17 @@ gb300-eks-ubuntu-inference-dynamo: 020-nvsentinel/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 020-nvsentinel/install.sh: 290c0907c263ac3c1a976be76da13528e36fe7f1db79945783b6ac7da4588857 020-nvsentinel/upstream.env: 7fc07c3fb32a6e569a151e4c12f2834386b4c7026cda6cd1c86e29592ad8936e - 020-nvsentinel/values.yaml: 5dbdd974de19bda092dff19de6a6fbb9489b7e7629abdeab2d1a8fe0bff69247 + 020-nvsentinel/values.yaml: 14400ae55936f06cc2025495b7d2efbb933e388a985abf2061b3b4fcbc36eeea 021-prometheus-adapter/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 021-prometheus-adapter/install.sh: 688ee4274fe06c001af6835d7965b5da33ce3fbeb724263622737d0bb88249e3 021-prometheus-adapter/upstream.env: 9ab918b32522baca13cd37bad7f420a206a19bb763b81041e174fffffe3da724 021-prometheus-adapter/values.yaml: 95231d7a46d650f96f844ca3734614271152a41f1230022c9d79960e44647c00 README.md: c27715674b5a7a66952d956e231327228ab3bb96b5604c70b02d8e7f9bedfe7a UPGRADING.md: a2aa0d1cd4b9e3a564ecf713c5fb05bb9e002f8e28cf51251d74d9f2962b2401 - bundle-info.yaml: c8ad592fb1cc6a7ec77a0c56a81054190ea52553e6faa3e7d892b0eb294d6b61 - checksums.txt: 789fce2a269489052766971d1db2dcee8ad4b4499b51671ebe860d26d66aff1e + bundle-info.yaml: 51e1349fe463d7b20b9beecd1071d67e74bacac3d3b1e79e8bf738412793dd1c + checksums.txt: 03c1e7041038198715b4be71bb0b50512065f8d39e6afcd640c485b3cf513824 deploy.sh: 5463ca8ed80e72d789bfb987637d85e2d3da5e4212a6a20e52b3640465c24da1 - recipe.yaml: d8353d592144a896ad7caafa71a07c5370506e2f19f6f5b3c5005659751c21cd + recipe.yaml: b39ce5984291c502d81f96c3a8777a843521c27b78d9f210961536f6f3c1b26c gb300-eks-ubuntu-training-kubeflow: 001-aws-efa/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 001-aws-efa/install.sh: 00574b8307fa54f90f18cd999a2b175a75a547942fd18cb242bd83b7af836d5a @@ -1946,17 +1946,17 @@ gb300-eks-ubuntu-training-kubeflow: 016-nvsentinel/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 016-nvsentinel/install.sh: 290c0907c263ac3c1a976be76da13528e36fe7f1db79945783b6ac7da4588857 016-nvsentinel/upstream.env: 7fc07c3fb32a6e569a151e4c12f2834386b4c7026cda6cd1c86e29592ad8936e - 016-nvsentinel/values.yaml: 5dbdd974de19bda092dff19de6a6fbb9489b7e7629abdeab2d1a8fe0bff69247 + 016-nvsentinel/values.yaml: 14400ae55936f06cc2025495b7d2efbb933e388a985abf2061b3b4fcbc36eeea 017-prometheus-adapter/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 017-prometheus-adapter/install.sh: 688ee4274fe06c001af6835d7965b5da33ce3fbeb724263622737d0bb88249e3 017-prometheus-adapter/upstream.env: 9ab918b32522baca13cd37bad7f420a206a19bb763b81041e174fffffe3da724 017-prometheus-adapter/values.yaml: 95231d7a46d650f96f844ca3734614271152a41f1230022c9d79960e44647c00 README.md: 9a35a538c27fc4be8a07a399f2b8c36ba2f0d341948d7ef1772c44b4b1038b9a UPGRADING.md: 4b285b6e5866a3582feef1afa6cc5906b79b275e94b0033d33f752c442d1256b - bundle-info.yaml: ad831e745e70d22ab3337162a301bff485eca337c6ee5c235744fe00ab3615c7 - checksums.txt: 143bb8303824e0908324dd093b435da694ba7108f3bfce6ce46b5e161c8426df + bundle-info.yaml: f8e02b675a404f0f557181e5e6af1cbedc5e39318d5757732c8a85700bd6b972 + checksums.txt: 9050aab1cb0472b2f6083dfcdb2fc9f4429e9cb9716be64a4459fc2d326c2ae9 deploy.sh: 949466ed3363eb59ea6ff5a61a29ca150215a2041c47323cada206dcf3cdc0a8 - recipe.yaml: 5daa5c2ba1fc7663ab09a2c6e4a6849755fb455496ba767085c833c67c233f78 + recipe.yaml: de5d866ef7a3a54cca00718d667412773eccaf299a61ee9be76881c97b6caaa3 gb300-eks-ubuntu-training-slurm: 001-aws-efa/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 001-aws-efa/install.sh: 00574b8307fa54f90f18cd999a2b175a75a547942fd18cb242bd83b7af836d5a @@ -2016,7 +2016,7 @@ gb300-eks-ubuntu-training-slurm: 014-nvsentinel/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 014-nvsentinel/install.sh: 290c0907c263ac3c1a976be76da13528e36fe7f1db79945783b6ac7da4588857 014-nvsentinel/upstream.env: 7fc07c3fb32a6e569a151e4c12f2834386b4c7026cda6cd1c86e29592ad8936e - 014-nvsentinel/values.yaml: 14a4d2de25f5e20bf1b9cc3f9964d1be5e9b388e3c40bd01b5bbed6b7dc09c52 + 014-nvsentinel/values.yaml: 8d8fc8ff4f18cad5c88227aaf425e76cfe57781de040cf96cd10bb285dcbf403 015-prometheus-adapter/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 015-prometheus-adapter/install.sh: 688ee4274fe06c001af6835d7965b5da33ce3fbeb724263622737d0bb88249e3 015-prometheus-adapter/upstream.env: 9ab918b32522baca13cd37bad7f420a206a19bb763b81041e174fffffe3da724 @@ -2041,10 +2041,10 @@ gb300-eks-ubuntu-training-slurm: 019-slinky-slurm/values.yaml: 7156ebcc98d8c91ea1f3792ccbf7c21dfc612ad2652909d3d626f5499e4eb16f README.md: a78e5647424521feb4e5c881b0758a403bc0a763bb271c1eb28631a6e64a8a5b UPGRADING.md: 4b285b6e5866a3582feef1afa6cc5906b79b275e94b0033d33f752c442d1256b - bundle-info.yaml: b031b55baffb1b5cd714eed1897c69061fb6a453b64a08dc2b3d3ab0a273b973 - checksums.txt: d96724cba400570a82583114d3dbf2e37f8c4be26c292eaf23b3f9a9e7671109 + bundle-info.yaml: c0a0d946ee5f5e4327407da81ab750b6213e3819035bf40428c03b9843fd7d13 + checksums.txt: 2b41ce4a1fa56064c46de99cd83fd5eef3070fd6ca630834acb64647305cb744 deploy.sh: 49009b25d90dd3b0c344008506134dc39d12e4b80582b1ce3356216f7cfb871d - recipe.yaml: 1a171f78fe8ef4b29db7c7bb4de11733951265cc426ad7d205ff644d0da710e1 + recipe.yaml: ce8fb128d3dc81633afa9cab16ae95645f824de084a30f3014d2f0ba1baca5ae gb300-generic-ubuntu-training: 001-nodewright-operator/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 001-nodewright-operator/install.sh: 5bb299d13650bdb1baa19ea8047b33baa94eb03a48a8b5b0102f1ba533b392c2 @@ -2100,17 +2100,17 @@ gb300-generic-ubuntu-training: 013-nvsentinel/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 013-nvsentinel/install.sh: 290c0907c263ac3c1a976be76da13528e36fe7f1db79945783b6ac7da4588857 013-nvsentinel/upstream.env: 7fc07c3fb32a6e569a151e4c12f2834386b4c7026cda6cd1c86e29592ad8936e - 013-nvsentinel/values.yaml: 5dbdd974de19bda092dff19de6a6fbb9489b7e7629abdeab2d1a8fe0bff69247 + 013-nvsentinel/values.yaml: 14400ae55936f06cc2025495b7d2efbb933e388a985abf2061b3b4fcbc36eeea 014-prometheus-adapter/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 014-prometheus-adapter/install.sh: 688ee4274fe06c001af6835d7965b5da33ce3fbeb724263622737d0bb88249e3 014-prometheus-adapter/upstream.env: 9ab918b32522baca13cd37bad7f420a206a19bb763b81041e174fffffe3da724 014-prometheus-adapter/values.yaml: 95231d7a46d650f96f844ca3734614271152a41f1230022c9d79960e44647c00 README.md: 5a122084cd1985ba9076ffaecd4d20491a49aa0d4b30074d382ab19da40c8832 UPGRADING.md: 9444e3b2469985d89ae8fd6bb0ca9116e56c2904f9aff6929487211120b75484 - bundle-info.yaml: 1fa226ec4030c86656527e84da7523451c214078eeebee3638d8b07cddd3ccbf - checksums.txt: 55b5af45c0a47684edb7e9fd150870a2e4e42e60c35b6536b829a3b2a7b8427b + bundle-info.yaml: ab6cff5d17d0934bc32d600840956fc9bdb84268a995ee5ff5c35af054156709 + checksums.txt: 0be6102289b23ceccbf04c8e21b5523b761fc7078840232fc619d35a2e561fe5 deploy.sh: 4196d34c287d2bdc2086c77410c18acfc254f60a86d60cc8d6ba400e4837f3b0 - recipe.yaml: a01216e3f78fecd914a5a178beb9489eaa4382b6f7c139304b0932b132c80112 + recipe.yaml: a102dced244b06c39482990b39ce6046f20471aaf132edd7d3409b00afd61d32 gb300-gke-cos-inference-dynamo: 001-agentgateway-crds/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 001-agentgateway-crds/install.sh: f0f864c4eb1f1d647af123953d5df5aa1fa03226e1f8c5feb724ea42d304eb3f @@ -2193,17 +2193,17 @@ gb300-gke-cos-inference-dynamo: 019-nvsentinel/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 019-nvsentinel/install.sh: 290c0907c263ac3c1a976be76da13528e36fe7f1db79945783b6ac7da4588857 019-nvsentinel/upstream.env: 7fc07c3fb32a6e569a151e4c12f2834386b4c7026cda6cd1c86e29592ad8936e - 019-nvsentinel/values.yaml: 3374b3a52c2466d902a85e6d5e4b2fc02d4f824e690094772cb6dbe9ee6962b2 + 019-nvsentinel/values.yaml: 7f5f9edb985771f1b7c01e5ee67edba4a2a6182b990a940a5e89601e5d589ae0 020-prometheus-adapter/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 020-prometheus-adapter/install.sh: 688ee4274fe06c001af6835d7965b5da33ce3fbeb724263622737d0bb88249e3 020-prometheus-adapter/upstream.env: 9ab918b32522baca13cd37bad7f420a206a19bb763b81041e174fffffe3da724 020-prometheus-adapter/values.yaml: 95231d7a46d650f96f844ca3734614271152a41f1230022c9d79960e44647c00 README.md: b2bf3798f845c3db319aff562e4a492ac31f3a4fced9f5dc87c3a24dcf94f331 UPGRADING.md: a2aa0d1cd4b9e3a564ecf713c5fb05bb9e002f8e28cf51251d74d9f2962b2401 - bundle-info.yaml: e3a40670dc8d77dcfb5230df8ced3322ae1435730cddb97366b18aabb07c79aa - checksums.txt: c014b99fb4123faa5730f29a2aabe4cedcc2ba5f7b52c9e83b7331d9af5dbc57 + bundle-info.yaml: c0204a2ab18a2decd073171b7a4a0045e34a83d802b30f1ac49494fd244f7a34 + checksums.txt: 16080affe077424249d5d73fd3cb4e507cf8104c328fc093bee22c14b5542a35 deploy.sh: 841a92636bb0a65da6fa0162b67084f68f362fa0e41837b1c57e6560c10a7a63 - recipe.yaml: 2c48e3f3f2c9312c0c18a40ed22800de7b6e76eac9e78d043dd5946d740596e9 + recipe.yaml: 220474ca9a43604d394a510f47ace0596f664601983a86155923138a3bdeabba gb300-gke-cos-training-kubeflow: 001-gcp-driver-installer/Chart.yaml: b0476226ebbc9a7738c4e5c2828ce5a73dbd371f3fa7cf1139c5fe9b2e825c02 001-gcp-driver-installer/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 @@ -2268,17 +2268,17 @@ gb300-gke-cos-training-kubeflow: 015-nvsentinel/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 015-nvsentinel/install.sh: 290c0907c263ac3c1a976be76da13528e36fe7f1db79945783b6ac7da4588857 015-nvsentinel/upstream.env: 7fc07c3fb32a6e569a151e4c12f2834386b4c7026cda6cd1c86e29592ad8936e - 015-nvsentinel/values.yaml: 3374b3a52c2466d902a85e6d5e4b2fc02d4f824e690094772cb6dbe9ee6962b2 + 015-nvsentinel/values.yaml: 7f5f9edb985771f1b7c01e5ee67edba4a2a6182b990a940a5e89601e5d589ae0 016-prometheus-adapter/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 016-prometheus-adapter/install.sh: 688ee4274fe06c001af6835d7965b5da33ce3fbeb724263622737d0bb88249e3 016-prometheus-adapter/upstream.env: 9ab918b32522baca13cd37bad7f420a206a19bb763b81041e174fffffe3da724 016-prometheus-adapter/values.yaml: 95231d7a46d650f96f844ca3734614271152a41f1230022c9d79960e44647c00 README.md: 957abece673a92572c34cb0f622ed08d74c5829db71ab8aadb78153c20e23532 UPGRADING.md: 4b285b6e5866a3582feef1afa6cc5906b79b275e94b0033d33f752c442d1256b - bundle-info.yaml: 0cff08cd990bbbaea2414eb3958908b677a5a369df9e8ce57d037a3917462c3a - checksums.txt: 343a96a2fb03b99df30a716507014d3e1567d4bfcfbe35abd2209f14a587aeff + bundle-info.yaml: 1448d90d402f567acb9a68b8091dba2067df1cccd62fb5b8a2beb3adac527195 + checksums.txt: d8fa7290e9dbf12000bcfa12fad2682d6ed6feb77c1bbe528738cf0d9d0625c0 deploy.sh: 50736743f1a6091c2dcb82baedfed99d4ae2f5bfe3325f7e7ab89995353f81bd - recipe.yaml: 865c6a12a3092e3f531d352af5635d49b27e388561f690010d4afced09d9d9e3 + recipe.yaml: 83798d84cf9161084bfb56b587e4d53d6b034bc0d9144e290cc73e117d9928e4 h100-aks-ubuntu-inference-dynamo: 001-agentgateway-crds/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 001-agentgateway-crds/install.sh: f0f864c4eb1f1d647af123953d5df5aa1fa03226e1f8c5feb724ea42d304eb3f @@ -2368,17 +2368,17 @@ h100-aks-ubuntu-inference-dynamo: 020-nvsentinel/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 020-nvsentinel/install.sh: 290c0907c263ac3c1a976be76da13528e36fe7f1db79945783b6ac7da4588857 020-nvsentinel/upstream.env: 7fc07c3fb32a6e569a151e4c12f2834386b4c7026cda6cd1c86e29592ad8936e - 020-nvsentinel/values.yaml: 843a8e5cb1d7e52b0b2a1a6f4366dd15d1e275d6cea64bbf73d33f52ccfafd0c + 020-nvsentinel/values.yaml: aa27bae925e838d0ba1a1b8a9d9ce6460ccfc20cfb29d9d52d85f9d1f927f008 021-prometheus-adapter/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 021-prometheus-adapter/install.sh: 688ee4274fe06c001af6835d7965b5da33ce3fbeb724263622737d0bb88249e3 021-prometheus-adapter/upstream.env: 9ab918b32522baca13cd37bad7f420a206a19bb763b81041e174fffffe3da724 021-prometheus-adapter/values.yaml: 95231d7a46d650f96f844ca3734614271152a41f1230022c9d79960e44647c00 README.md: f7bb49bcbf2c7abca66359c0bdfbb9518df9faa8d5a34e3636f46ff0a323ea3a UPGRADING.md: 668b1aba11148b80ac92ea8e793d263cabba6a615aad52f1b3df7f83007e446f - bundle-info.yaml: 07b85fdef229b373914e33d6cef2cafa8284c5e113d0c9b3a03d469b9fc810e1 - checksums.txt: 5d0066900d64d5b1616342d1d6e5d138f056835e55f3a23e54436ec4044257ad + bundle-info.yaml: 69bff5fac9dbc7442537d37fe7dfb0c1faaef2435a7d4640ad0a22da8baa1b24 + checksums.txt: 3cd1cd90b515bfcd811812648bcb2271b236aef7dd540040dac257f742d2daee deploy.sh: 1a78677ffdf4ba1f861de19dbb30bffaab64b77aa62b48a87bd929a564bce55a - recipe.yaml: af507402be6dabaac792b252d28c42cc7b593369f283c7d068953ec487731abf + recipe.yaml: e7b5bbfab53fa15c9899b85bb8b6ae7262bf3ea927fb24b46c1a2d2bfc6e066f h100-aks-ubuntu-training-kubeflow: 001-nodewright-operator/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 001-nodewright-operator/install.sh: 5bb299d13650bdb1baa19ea8047b33baa94eb03a48a8b5b0102f1ba533b392c2 @@ -2450,17 +2450,17 @@ h100-aks-ubuntu-training-kubeflow: 016-nvsentinel/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 016-nvsentinel/install.sh: 290c0907c263ac3c1a976be76da13528e36fe7f1db79945783b6ac7da4588857 016-nvsentinel/upstream.env: 7fc07c3fb32a6e569a151e4c12f2834386b4c7026cda6cd1c86e29592ad8936e - 016-nvsentinel/values.yaml: 843a8e5cb1d7e52b0b2a1a6f4366dd15d1e275d6cea64bbf73d33f52ccfafd0c + 016-nvsentinel/values.yaml: aa27bae925e838d0ba1a1b8a9d9ce6460ccfc20cfb29d9d52d85f9d1f927f008 017-prometheus-adapter/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 017-prometheus-adapter/install.sh: 688ee4274fe06c001af6835d7965b5da33ce3fbeb724263622737d0bb88249e3 017-prometheus-adapter/upstream.env: 9ab918b32522baca13cd37bad7f420a206a19bb763b81041e174fffffe3da724 017-prometheus-adapter/values.yaml: 95231d7a46d650f96f844ca3734614271152a41f1230022c9d79960e44647c00 README.md: 1ecbbae1c80ecc8527f887435d46c29f3075c3916d352561c28a815a3a643cf5 UPGRADING.md: 9444e3b2469985d89ae8fd6bb0ca9116e56c2904f9aff6929487211120b75484 - bundle-info.yaml: 2bff12c3616a3eb3ee074166bda0175b32f995bca74731c7d841c713a81cd9a6 - checksums.txt: 3dacc729f1104fccdaebd06da2945425bdd85472e610b5a1ba67a0a16fbe9d2f + bundle-info.yaml: 3fcbb85474c37762279f2c30c11ed8924c3b14066af8b140ba4a96e3bd440de5 + checksums.txt: 5cca4ca502e714c4919f3f69a660365d8b1b35474329bc9305d55ba2a79669ad deploy.sh: cfb12fbb4315ebaadd1378f55f81e866e9f5faaa909b57e569bab1c0ce558c0b - recipe.yaml: be59d067d1cae2fce5b380ce3e0afad85de7dc3103967f76cac7328627904ee0 + recipe.yaml: 81fae95dbb7ca7662c1806f40c71e9820a7342671f07cdf49b9684b0fc20d14b h100-aks-ubuntu-training-slurm: 001-nodewright-operator/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 001-nodewright-operator/install.sh: 5bb299d13650bdb1baa19ea8047b33baa94eb03a48a8b5b0102f1ba533b392c2 @@ -2523,7 +2523,7 @@ h100-aks-ubuntu-training-slurm: 014-nvsentinel/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 014-nvsentinel/install.sh: 290c0907c263ac3c1a976be76da13528e36fe7f1db79945783b6ac7da4588857 014-nvsentinel/upstream.env: 7fc07c3fb32a6e569a151e4c12f2834386b4c7026cda6cd1c86e29592ad8936e - 014-nvsentinel/values.yaml: dab88137f357029b34dcdb0a99964cbc0b23d7b36458c708a022fa162afac822 + 014-nvsentinel/values.yaml: beec1d6f08d5ceb658ea48d2a70e9fae46a7f0cb58a0e4d352501949b074e292 015-prometheus-adapter/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 015-prometheus-adapter/install.sh: 688ee4274fe06c001af6835d7965b5da33ce3fbeb724263622737d0bb88249e3 015-prometheus-adapter/upstream.env: 9ab918b32522baca13cd37bad7f420a206a19bb763b81041e174fffffe3da724 @@ -2547,10 +2547,10 @@ h100-aks-ubuntu-training-slurm: 019-slinky-slurm/values.yaml: efd9525d04fb69ffea430a76aae43654b7ac9ab8970fcff9f32dbf9b01c06de5 README.md: 642e206c8a643ebdbf0a0a642ce70a89898de7c783fe951b7109d958c0e6a2da UPGRADING.md: 9444e3b2469985d89ae8fd6bb0ca9116e56c2904f9aff6929487211120b75484 - bundle-info.yaml: 58a936bdf31475521af41ef1883de3e055d84ac0ccc5a5711553068ea6422010 - checksums.txt: cc34ff3e4c8a7ab88c7dda9a35ea1dd714493c7e9af11c4bc7c64a4705a9e450 + bundle-info.yaml: 2073af454adbafb651dc2fb055d56ddecb67e193f89441b881a605298d54df78 + checksums.txt: 90fe647053aa17125e04dc200ef18429ba83a965f987c1d28282b287e07ef068 deploy.sh: 8d85df5a2b5665074cf1efa1980f5b1de87dc8be91f8c6c7ae5132aef410cabc - recipe.yaml: 0a924c222efcd4de50af4e7478aa3bed83988854a78e78970d89aacf76f34ef4 + recipe.yaml: abdfaaa63fc5d96988cbcbd40e416e8d507fed814efaeb2ab358558c4e6bbcd6 h100-any: 001-nodewright-operator/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 001-nodewright-operator/install.sh: 5bb299d13650bdb1baa19ea8047b33baa94eb03a48a8b5b0102f1ba533b392c2 @@ -2592,17 +2592,17 @@ h100-any: 010-nvsentinel/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 010-nvsentinel/install.sh: 290c0907c263ac3c1a976be76da13528e36fe7f1db79945783b6ac7da4588857 010-nvsentinel/upstream.env: 7fc07c3fb32a6e569a151e4c12f2834386b4c7026cda6cd1c86e29592ad8936e - 010-nvsentinel/values.yaml: 5dbdd974de19bda092dff19de6a6fbb9489b7e7629abdeab2d1a8fe0bff69247 + 010-nvsentinel/values.yaml: 14400ae55936f06cc2025495b7d2efbb933e388a985abf2061b3b4fcbc36eeea 011-prometheus-adapter/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 011-prometheus-adapter/install.sh: 688ee4274fe06c001af6835d7965b5da33ce3fbeb724263622737d0bb88249e3 011-prometheus-adapter/upstream.env: 9ab918b32522baca13cd37bad7f420a206a19bb763b81041e174fffffe3da724 011-prometheus-adapter/values.yaml: 95231d7a46d650f96f844ca3734614271152a41f1230022c9d79960e44647c00 README.md: ddb369e4f78c771fb739627e2e03c7695fad0680461e2933494bbfbbaec13981 UPGRADING.md: 4b285b6e5866a3582feef1afa6cc5906b79b275e94b0033d33f752c442d1256b - bundle-info.yaml: c7fbf4b3546651fa6e675a3b44dab04ccf6372a45ee943321aa01d02378881c4 - checksums.txt: 7309feeca5c40a5998c232a8ea48f5bd88093b735de8e5b7bf2bb2815d06a33e + bundle-info.yaml: 8adda23e096eafa75bdf9f09d3e2b481a740f47b4a7d5f0d1f268089d16c2f36 + checksums.txt: 98c1db92427e7106208f1ac4ac84a0dd5bd4273d8b4264bd695e78cb6ced76b3 deploy.sh: 187bf11702c83fdf08c22fbdd72a6b16c937314d87f85da747c4753cbe5d5de4 - recipe.yaml: 8db3e194288d8190572d625d5654e9f3dbd73aa313fa96b7fcb48b76cab0cd7e + recipe.yaml: 2d5d7655fcdbd43dbe6e51f7a4210a437eb43a138f5c578efd623ea438c34f22 h100-bcm-ubuntu-training-kubeflow: 001-nodewright-operator/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 001-nodewright-operator/install.sh: 5bb299d13650bdb1baa19ea8047b33baa94eb03a48a8b5b0102f1ba533b392c2 @@ -2658,17 +2658,17 @@ h100-bcm-ubuntu-training-kubeflow: 013-nvsentinel/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 013-nvsentinel/install.sh: 290c0907c263ac3c1a976be76da13528e36fe7f1db79945783b6ac7da4588857 013-nvsentinel/upstream.env: 7fc07c3fb32a6e569a151e4c12f2834386b4c7026cda6cd1c86e29592ad8936e - 013-nvsentinel/values.yaml: 5dbdd974de19bda092dff19de6a6fbb9489b7e7629abdeab2d1a8fe0bff69247 + 013-nvsentinel/values.yaml: 14400ae55936f06cc2025495b7d2efbb933e388a985abf2061b3b4fcbc36eeea 014-prometheus-adapter/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 014-prometheus-adapter/install.sh: 688ee4274fe06c001af6835d7965b5da33ce3fbeb724263622737d0bb88249e3 014-prometheus-adapter/upstream.env: 9ab918b32522baca13cd37bad7f420a206a19bb763b81041e174fffffe3da724 014-prometheus-adapter/values.yaml: 95231d7a46d650f96f844ca3734614271152a41f1230022c9d79960e44647c00 README.md: 54233f40746309d270641c721a59414878f32da024b65c359b6a4b2204919ee9 UPGRADING.md: 4b285b6e5866a3582feef1afa6cc5906b79b275e94b0033d33f752c442d1256b - bundle-info.yaml: 525d3528b049c08a3db42aace19732b83cfd254bb1e1816014542bb560202541 - checksums.txt: 40a164543ea85e3dc3db51b4287644440bdd1659e71bd9d84c5d947186a0f023 + bundle-info.yaml: e6865cd3888e2c64997a727569f808534c8cb1003bf4e6c1f8550f05d71391dd + checksums.txt: 51b56c6a7d1d0f6acb65e02ea23ec3c258de03c7722acff76c6407b93bee9227 deploy.sh: cf8476042245b8769a0db57a19a57b9d874d957734fb0a778adc5e238a7988e8 - recipe.yaml: b07b640cef9cbdcf7271169bb06cf32abd12f50db2f87d6f24f2aabbe569d9e1 + recipe.yaml: eb64a5223cdf139180aab81dc08786f7f7976f38ad6d3729972547b503d1722f h100-eks-amazonlinux-inference: 001-agentgateway-crds/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 001-agentgateway-crds/install.sh: f0f864c4eb1f1d647af123953d5df5aa1fa03226e1f8c5feb724ea42d304eb3f @@ -2737,17 +2737,17 @@ h100-eks-amazonlinux-inference: 016-nvsentinel/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 016-nvsentinel/install.sh: 290c0907c263ac3c1a976be76da13528e36fe7f1db79945783b6ac7da4588857 016-nvsentinel/upstream.env: 7fc07c3fb32a6e569a151e4c12f2834386b4c7026cda6cd1c86e29592ad8936e - 016-nvsentinel/values.yaml: ede1bf610929dcf019396356eec2e1a0bf9c1417c181595e86f03d6035fe27bb + 016-nvsentinel/values.yaml: 376d322d05dfab359cef13a99ded134b567f8100905a39caaf4af614efefa223 017-prometheus-adapter/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 017-prometheus-adapter/install.sh: 688ee4274fe06c001af6835d7965b5da33ce3fbeb724263622737d0bb88249e3 017-prometheus-adapter/upstream.env: 9ab918b32522baca13cd37bad7f420a206a19bb763b81041e174fffffe3da724 017-prometheus-adapter/values.yaml: 95231d7a46d650f96f844ca3734614271152a41f1230022c9d79960e44647c00 README.md: cf8e4f1db4f8d0cad61344fa1d4ddf9577409037c3516f31634fc35cad10895c UPGRADING.md: 4b285b6e5866a3582feef1afa6cc5906b79b275e94b0033d33f752c442d1256b - bundle-info.yaml: 9649e46cdbd78509e4f21abc0a9e4806c18a2c155e129945a00d0218c960513d - checksums.txt: 5adef1cd2f7a4368c3e80bad805a51dc25cb2e1529d90df0c1af6c56824721d7 + bundle-info.yaml: 92ffc947f7483512d44cd7f110352f9d716436073b35204b87859c748a64815a + checksums.txt: dac38fc5cd3e1521d0bcd1635cfed3b1bf59bbfa3ccda31197d1309aa1cdfe71 deploy.sh: 0248fabe5a8e4dbdbe60fc3f0cbf1aaab47a637402bec6eca1191fa601e8bfd6 - recipe.yaml: 3cd8281718c80c1e1df4ce2fab6080b1886266d3bbe617ccfb10f72e3c2ca67b + recipe.yaml: 325be06c0a1aade74b2ee19152dee1b5977d94e9130e3a4ca4dedc6bd8297d52 h100-eks-ubuntu-inference-dynamo: 001-agentgateway-crds/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 001-agentgateway-crds/install.sh: f0f864c4eb1f1d647af123953d5df5aa1fa03226e1f8c5feb724ea42d304eb3f @@ -2829,17 +2829,17 @@ h100-eks-ubuntu-inference-dynamo: 019-nvsentinel/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 019-nvsentinel/install.sh: 290c0907c263ac3c1a976be76da13528e36fe7f1db79945783b6ac7da4588857 019-nvsentinel/upstream.env: 7fc07c3fb32a6e569a151e4c12f2834386b4c7026cda6cd1c86e29592ad8936e - 019-nvsentinel/values.yaml: 5dbdd974de19bda092dff19de6a6fbb9489b7e7629abdeab2d1a8fe0bff69247 + 019-nvsentinel/values.yaml: 14400ae55936f06cc2025495b7d2efbb933e388a985abf2061b3b4fcbc36eeea 020-prometheus-adapter/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 020-prometheus-adapter/install.sh: 688ee4274fe06c001af6835d7965b5da33ce3fbeb724263622737d0bb88249e3 020-prometheus-adapter/upstream.env: 9ab918b32522baca13cd37bad7f420a206a19bb763b81041e174fffffe3da724 020-prometheus-adapter/values.yaml: 95231d7a46d650f96f844ca3734614271152a41f1230022c9d79960e44647c00 README.md: 31f0b37d2e38db44076487159a018f0aa933fe4284e112533832a18453aa70a8 UPGRADING.md: a2aa0d1cd4b9e3a564ecf713c5fb05bb9e002f8e28cf51251d74d9f2962b2401 - bundle-info.yaml: 5ce53c505859c0b54afd2cdfa3b368c10ce8a5438b488067e77605b1cb8e9f0b - checksums.txt: 07c3043ccd24f5c5ee88491185691bfcb03e9b9b1fffed96d1f7ff6bc4a49be2 + bundle-info.yaml: 0a606ad5d1020bfc61521c87cfadee8fc432cc5d27b28437a03221a92fccf1ed + checksums.txt: c3460016b0c3fbf9f4afc6670780ef3596fe5da8084eb0855433ba4cea546608 deploy.sh: 5463ca8ed80e72d789bfb987637d85e2d3da5e4212a6a20e52b3640465c24da1 - recipe.yaml: 9c398e8040cc19423c980da1fba00f6f5cb6b32636a7a07d4d967751aaa06bde + recipe.yaml: 43a82e16e49419daad8f9bfa4d6fc37f5c814912d52f3da7f12811b63afd72b3 h100-eks-ubuntu-inference-nim: 001-agentgateway-crds/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 001-agentgateway-crds/install.sh: f0f864c4eb1f1d647af123953d5df5aa1fa03226e1f8c5feb724ea42d304eb3f @@ -2917,17 +2917,17 @@ h100-eks-ubuntu-inference-nim: 018-nvsentinel/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 018-nvsentinel/install.sh: 290c0907c263ac3c1a976be76da13528e36fe7f1db79945783b6ac7da4588857 018-nvsentinel/upstream.env: 7fc07c3fb32a6e569a151e4c12f2834386b4c7026cda6cd1c86e29592ad8936e - 018-nvsentinel/values.yaml: 5dbdd974de19bda092dff19de6a6fbb9489b7e7629abdeab2d1a8fe0bff69247 + 018-nvsentinel/values.yaml: 14400ae55936f06cc2025495b7d2efbb933e388a985abf2061b3b4fcbc36eeea 019-prometheus-adapter/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 019-prometheus-adapter/install.sh: 688ee4274fe06c001af6835d7965b5da33ce3fbeb724263622737d0bb88249e3 019-prometheus-adapter/upstream.env: 9ab918b32522baca13cd37bad7f420a206a19bb763b81041e174fffffe3da724 019-prometheus-adapter/values.yaml: 95231d7a46d650f96f844ca3734614271152a41f1230022c9d79960e44647c00 README.md: da9981d7ca835c63b8857aeda1c1c1c379676e7572e83f5872d20d8a7433c19d UPGRADING.md: 4b285b6e5866a3582feef1afa6cc5906b79b275e94b0033d33f752c442d1256b - bundle-info.yaml: e8aa86e6f86cb1e434b4ac587d6e9cc85000c22accedb9756d152bf1c09089e1 - checksums.txt: d9dda38b07f49f714e977b808603ee2d1c1545f3c537b1feabb5b91f9419d34f + bundle-info.yaml: a0bbf5a166286a059012f2c343004487289ef84d58a352196e0bb23755b5539e + checksums.txt: d872ea66e6c6788983f9d25d46f3d4b3b8e77120e6faf6ec6af581d0458faeaf deploy.sh: 93abef48d4cfd33ed295b33a924b01bb02e810595d7dd63575ae58c29c0122be - recipe.yaml: b9069ed7303d9f678383b770c2d0b80467b2fa7d10c4fb7bf0518b7f94fcb408 + recipe.yaml: a49b609ad6a0bc6f4f6c4aacdc948e131f68af714371be0e9ac0be5f2c6998ea h100-eks-ubuntu-training-kubeflow: 001-aws-efa/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 001-aws-efa/install.sh: 00574b8307fa54f90f18cd999a2b175a75a547942fd18cb242bd83b7af836d5a @@ -2991,17 +2991,17 @@ h100-eks-ubuntu-training-kubeflow: 015-nvsentinel/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 015-nvsentinel/install.sh: 290c0907c263ac3c1a976be76da13528e36fe7f1db79945783b6ac7da4588857 015-nvsentinel/upstream.env: 7fc07c3fb32a6e569a151e4c12f2834386b4c7026cda6cd1c86e29592ad8936e - 015-nvsentinel/values.yaml: 5dbdd974de19bda092dff19de6a6fbb9489b7e7629abdeab2d1a8fe0bff69247 + 015-nvsentinel/values.yaml: 14400ae55936f06cc2025495b7d2efbb933e388a985abf2061b3b4fcbc36eeea 016-prometheus-adapter/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 016-prometheus-adapter/install.sh: 688ee4274fe06c001af6835d7965b5da33ce3fbeb724263622737d0bb88249e3 016-prometheus-adapter/upstream.env: 9ab918b32522baca13cd37bad7f420a206a19bb763b81041e174fffffe3da724 016-prometheus-adapter/values.yaml: 95231d7a46d650f96f844ca3734614271152a41f1230022c9d79960e44647c00 README.md: 8ccb9814a4d03fa3cdcbcc4fc0a23aa0bf1e90dd155d88bdbc1749e49e9059ba UPGRADING.md: 4b285b6e5866a3582feef1afa6cc5906b79b275e94b0033d33f752c442d1256b - bundle-info.yaml: 4f01ea76e81e7df4c8db25a1a63ec77b332f2e22e65895ce3fdf0de17ff0b6ea - checksums.txt: fbb88bfdf1f5c627a1894e1a0017e6a11ac35b98ce27db1df1e50a3112844c0c + bundle-info.yaml: 773655e394c4e0dbe3a77112318a01338716151b5880ffece7b814fc0067f54d + checksums.txt: 7a3105bb3e33fd8d365a79ae3172cee95cac222c9b9e93779702a8fc942be1a9 deploy.sh: 949466ed3363eb59ea6ff5a61a29ca150215a2041c47323cada206dcf3cdc0a8 - recipe.yaml: 34a157e3d2277e1698efcce88ae7a127138828788dbe3a1d2119ecb23dc5cda1 + recipe.yaml: 217be13db08b39bf7e728da5ba9349072e45db9722274ee0ddd7a7468cfe4413 h100-eks-ubuntu-training-kubeflow+dra-eviction-node-label: 001-aws-efa/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 001-aws-efa/install.sh: 00574b8307fa54f90f18cd999a2b175a75a547942fd18cb242bd83b7af836d5a @@ -3070,17 +3070,17 @@ h100-eks-ubuntu-training-kubeflow+dra-eviction-node-label: 016-nvsentinel/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 016-nvsentinel/install.sh: 290c0907c263ac3c1a976be76da13528e36fe7f1db79945783b6ac7da4588857 016-nvsentinel/upstream.env: 7fc07c3fb32a6e569a151e4c12f2834386b4c7026cda6cd1c86e29592ad8936e - 016-nvsentinel/values.yaml: 5dbdd974de19bda092dff19de6a6fbb9489b7e7629abdeab2d1a8fe0bff69247 + 016-nvsentinel/values.yaml: 14400ae55936f06cc2025495b7d2efbb933e388a985abf2061b3b4fcbc36eeea 017-prometheus-adapter/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 017-prometheus-adapter/install.sh: 688ee4274fe06c001af6835d7965b5da33ce3fbeb724263622737d0bb88249e3 017-prometheus-adapter/upstream.env: 9ab918b32522baca13cd37bad7f420a206a19bb763b81041e174fffffe3da724 017-prometheus-adapter/values.yaml: 95231d7a46d650f96f844ca3734614271152a41f1230022c9d79960e44647c00 README.md: 1a2a114db56cd0060ef6ff81450a4268c3b89433ca1f0e335e4b1912602df003 UPGRADING.md: 4b285b6e5866a3582feef1afa6cc5906b79b275e94b0033d33f752c442d1256b - bundle-info.yaml: 0354b7415dd2112fb0ae7491f5bb3923ac5d8140cf1873e27d9a876e61ee39f1 - checksums.txt: 313e3ada9361d275ca51147d7b549ed67b368023ebbec7096ba4c53b7fde48c2 + bundle-info.yaml: 8eaa8647f4851ae7a56fe6dd95157b6e7b34469e2ce0ffd6644bf37a74b9e9bb + checksums.txt: dd3183d823ba38630abdc4f17f734696ff37be703c545bee057203b1e50895bf deploy.sh: c8a75ff8a8aa453cd38287c76c42294ce706a2f4e04b3e157dd676789117a1ef - recipe.yaml: 3fd8664926e80a3830fedc2e234fb808558db170d08d3d9517e1065fdb5f2c2f + recipe.yaml: 95f3695c5aa17cb9284d6072890ec2ce09186923ef0dfad9d860a369621f34f5 h100-eks-ubuntu-training-kubeflow+user-overlay-components: 001-aws-efa/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 001-aws-efa/install.sh: 00574b8307fa54f90f18cd999a2b175a75a547942fd18cb242bd83b7af836d5a @@ -3158,19 +3158,19 @@ h100-eks-ubuntu-training-kubeflow+user-overlay-components: 018-nvsentinel/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 018-nvsentinel/install.sh: 290c0907c263ac3c1a976be76da13528e36fe7f1db79945783b6ac7da4588857 018-nvsentinel/upstream.env: 7fc07c3fb32a6e569a151e4c12f2834386b4c7026cda6cd1c86e29592ad8936e - 018-nvsentinel/values.yaml: 5dbdd974de19bda092dff19de6a6fbb9489b7e7629abdeab2d1a8fe0bff69247 + 018-nvsentinel/values.yaml: 14400ae55936f06cc2025495b7d2efbb933e388a985abf2061b3b4fcbc36eeea 019-prometheus-adapter/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 019-prometheus-adapter/install.sh: 688ee4274fe06c001af6835d7965b5da33ce3fbeb724263622737d0bb88249e3 019-prometheus-adapter/upstream.env: 9ab918b32522baca13cd37bad7f420a206a19bb763b81041e174fffffe3da724 019-prometheus-adapter/values.yaml: 95231d7a46d650f96f844ca3734614271152a41f1230022c9d79960e44647c00 README.md: 3aac8b6c06671af3da3772f340b61e73fcf4509a8d699ca7768d064fdfdfd332 UPGRADING.md: d9a77939a154566f087d9fe1296aeccc42399186c7436e444bcf600e755f1648 - bundle-info.yaml: 483455793e14ec103933a14e29c499359c2391c1b99decba134aa737164721a8 - checksums.txt: 6c591d305df845491f133d2947e016e87bf8ff0a77e7995e90d2d7e3f6654196 + bundle-info.yaml: 2d5ab4b5c58bdf9fbd033ef304f76456d5e7cdd8c467a810cdbf5cf6fd8b2b52 + checksums.txt: e7fd476a65915ddbf5719993f948adb7a1df3edcb862d17316cd360e0f89f71e data/overlays/stock-render-opt-in.yaml: 100a230e36a3fe11b9678bb4ad0dcd59fe8ae0a51534814dace25056bb9a2ba3 data/registry.yaml: f9dbb0c0e12b4f7932d93c44aded6be227bdb838c5be22e9ef0df93b3386ae1c deploy.sh: 71e3992efdc69f520f5e7f4c824964c1611f59ac4ea0ec4b0223025464445a44 - recipe.yaml: 9b4bd9f31384e16563248610a407fcc6502dea07c60207a5d07f44ce72888e08 + recipe.yaml: 02bc016ddf908cf8a7641b2039fd9d968dfcb041b084e006e40eb90b34917dc7 h100-eks-ubuntu-training-kueue: 001-aws-efa/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 001-aws-efa/install.sh: 00574b8307fa54f90f18cd999a2b175a75a547942fd18cb242bd83b7af836d5a @@ -3225,7 +3225,7 @@ h100-eks-ubuntu-training-kueue: 013-nvsentinel/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 013-nvsentinel/install.sh: 290c0907c263ac3c1a976be76da13528e36fe7f1db79945783b6ac7da4588857 013-nvsentinel/upstream.env: 7fc07c3fb32a6e569a151e4c12f2834386b4c7026cda6cd1c86e29592ad8936e - 013-nvsentinel/values.yaml: 5dbdd974de19bda092dff19de6a6fbb9489b7e7629abdeab2d1a8fe0bff69247 + 013-nvsentinel/values.yaml: 14400ae55936f06cc2025495b7d2efbb933e388a985abf2061b3b4fcbc36eeea 014-prometheus-adapter/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 014-prometheus-adapter/install.sh: 688ee4274fe06c001af6835d7965b5da33ce3fbeb724263622737d0bb88249e3 014-prometheus-adapter/upstream.env: 9ab918b32522baca13cd37bad7f420a206a19bb763b81041e174fffffe3da724 @@ -3248,10 +3248,10 @@ h100-eks-ubuntu-training-kueue: 017-kueue-post/values.yaml: 9ae0af1abd28154ef1efc514bd20ee1daf35cfccbfaa0c6d97e08092b211d6fc README.md: 24f71150a1008fe913da3d8793f8e164cc6ea8f6c1827475d2f08e22601dbfad UPGRADING.md: 27af534b3ca82d0a5e1135c70e185aafa9566df84b328f494f2f96168af9d18c - bundle-info.yaml: b551a8dbd8ccef0d3668e8a94d668509bf9ac3e49aecd3b1bb8f3f4c82f23a1a - checksums.txt: d1750ddf6c3cf2cb32c178866efe2bd2d35559057c133c748e96a5c2662eedfc + bundle-info.yaml: 54a0d9bdc45cd6d28f84c20b642caad2932859a61a50d5c946a0b0a1b3bc9aa4 + checksums.txt: cfc7e318b30095960e00523840bf5fbd5d35827403d556a404b08a5d9bbf36f0 deploy.sh: be46432db13f4a3fc337d6271516f97efaad65cc61f7e21a3af308e3d56d3735 - recipe.yaml: 5984293b09970ca611344d259180ee81327651d1a0fceb1e03df62a4dacd2ef9 + recipe.yaml: c50ae433ab8c41e392001ac1a2515aff6392a930cced8f191cd2c392b32c2bf8 h100-eks-ubuntu-training-slurm: 001-aws-efa/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 001-aws-efa/install.sh: 00574b8307fa54f90f18cd999a2b175a75a547942fd18cb242bd83b7af836d5a @@ -3306,7 +3306,7 @@ h100-eks-ubuntu-training-slurm: 013-nvsentinel/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 013-nvsentinel/install.sh: 290c0907c263ac3c1a976be76da13528e36fe7f1db79945783b6ac7da4588857 013-nvsentinel/upstream.env: 7fc07c3fb32a6e569a151e4c12f2834386b4c7026cda6cd1c86e29592ad8936e - 013-nvsentinel/values.yaml: 14a4d2de25f5e20bf1b9cc3f9964d1be5e9b388e3c40bd01b5bbed6b7dc09c52 + 013-nvsentinel/values.yaml: 8d8fc8ff4f18cad5c88227aaf425e76cfe57781de040cf96cd10bb285dcbf403 014-prometheus-adapter/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 014-prometheus-adapter/install.sh: 688ee4274fe06c001af6835d7965b5da33ce3fbeb724263622737d0bb88249e3 014-prometheus-adapter/upstream.env: 9ab918b32522baca13cd37bad7f420a206a19bb763b81041e174fffffe3da724 @@ -3330,10 +3330,10 @@ h100-eks-ubuntu-training-slurm: 018-slinky-slurm/values.yaml: efd9525d04fb69ffea430a76aae43654b7ac9ab8970fcff9f32dbf9b01c06de5 README.md: 0f96f35d7cfb21013bee8a9c9ce3d56bf5298e35ec71cef00e6bf2f4a3c40908 UPGRADING.md: 4b285b6e5866a3582feef1afa6cc5906b79b275e94b0033d33f752c442d1256b - bundle-info.yaml: 3be96dc9920449241e5ffc01f023427e498974422d95ba7e40132111d590fcc7 - checksums.txt: 01ef6567d86abd32387c49bf9e54591b5c89f55463ee75eeb8a29251bc699035 + bundle-info.yaml: 67b036b98f64a509e7f786c5a63a4be048b4a49ae7c7099f6ed13d0803ceb3cb + checksums.txt: 7c399bc9daa1c46d982f4e6b6910de3dde6614d0a6094ae7302bc942cff3de80 deploy.sh: 49009b25d90dd3b0c344008506134dc39d12e4b80582b1ce3356216f7cfb871d - recipe.yaml: 9beb6c2d9f5c26170af8a8bc0fd6dd636a2f3a9752323e7bd20da7b43692b4d1 + recipe.yaml: 174897079ec5dec246173560520eeb0edea43de724271ea583a4088627e583a5 h100-eks-ubuntu-training-slurm+nvsentinel-slinky-drainer: 001-aws-efa/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 001-aws-efa/install.sh: 00574b8307fa54f90f18cd999a2b175a75a547942fd18cb242bd83b7af836d5a @@ -3418,7 +3418,7 @@ h100-eks-ubuntu-training-slurm+nvsentinel-slinky-drainer: 019-nvsentinel/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 019-nvsentinel/install.sh: 290c0907c263ac3c1a976be76da13528e36fe7f1db79945783b6ac7da4588857 019-nvsentinel/upstream.env: 7fc07c3fb32a6e569a151e4c12f2834386b4c7026cda6cd1c86e29592ad8936e - 019-nvsentinel/values.yaml: c5f3a104f3f3b6af4e3ac02c96718ba99f7c6ccbb448efc9e23e52f5ba0efd6c + 019-nvsentinel/values.yaml: 26a877937ad15a70d9131c0ee1db7fc2adcc46a63f8ac43fdef0dd64f0d07c52 020-slinky-slurm-operator-crds/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 020-slinky-slurm-operator-crds/install.sh: b763afb19a730721e43f3a5be956d4ff183eb0623c8e3c168544ad26e8fbb46f 020-slinky-slurm-operator-crds/upstream.env: 4d0a2be7546b7bf8bc9469de1609fdaca2af94c51f77882e050878f5820e2b88 @@ -3438,12 +3438,12 @@ h100-eks-ubuntu-training-slurm+nvsentinel-slinky-drainer: 023-slinky-slurm/values.yaml: 24b4418eae1e7dec15ae641661f5a21b925dd268b28304b18ad7639269e288a2 README.md: 27c801264139aa87d2ec0e821dc7c37cf91992b68ee23acabb7a80f00a35a666 UPGRADING.md: dbc6cf7a20850e98b34dd49b3ebdc4530b4da52d12b96aa1632700babf92354d - bundle-info.yaml: 12a0a239ffa457cbde42c5c0f7686bdce47c053435090fa4339eb4df0b1bc040 - checksums.txt: c9a3013d3ef25a6315ec0ace2a2d26aed0ba2ae398cffc38e8cbd9c3a2b0c6de + bundle-info.yaml: 4fbb64b0581bda93494900bcb72f5001ac4cdf74bbc7580b76f319e3c5fe516d + checksums.txt: 86a4ff419df15a6a3afe0fa14671607096e13156202ffe31c9a78b75def74135 data/overlays/stock-render-opt-in.yaml: 328e7217055d4c00c950d690a1579830a95456e9953214a1ec0803416edaaaf3 data/registry.yaml: f9dbb0c0e12b4f7932d93c44aded6be227bdb838c5be22e9ef0df93b3386ae1c deploy.sh: 03dc63942cd1283225f2d20f0651f318bbf346b29f376876204c849a1af150ea - recipe.yaml: 532417696732d49181129336e616d1fa1bf6ca2fc2e0bfea5f72823bada7761f + recipe.yaml: b688bf7ba7437d9c5db2172a5f424d7a71ad38a06ce2d53d1a1badc3b08361cf h100-eks-ubuntu-training-slurm+slurm-accounting-aicr-provided: 001-aws-efa/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 001-aws-efa/install.sh: 00574b8307fa54f90f18cd999a2b175a75a547942fd18cb242bd83b7af836d5a @@ -3506,7 +3506,7 @@ h100-eks-ubuntu-training-slurm+slurm-accounting-aicr-provided: 015-nvsentinel/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 015-nvsentinel/install.sh: 290c0907c263ac3c1a976be76da13528e36fe7f1db79945783b6ac7da4588857 015-nvsentinel/upstream.env: 7fc07c3fb32a6e569a151e4c12f2834386b4c7026cda6cd1c86e29592ad8936e - 015-nvsentinel/values.yaml: 14a4d2de25f5e20bf1b9cc3f9964d1be5e9b388e3c40bd01b5bbed6b7dc09c52 + 015-nvsentinel/values.yaml: 8d8fc8ff4f18cad5c88227aaf425e76cfe57781de040cf96cd10bb285dcbf403 016-prometheus-adapter/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 016-prometheus-adapter/install.sh: 688ee4274fe06c001af6835d7965b5da33ce3fbeb724263622737d0bb88249e3 016-prometheus-adapter/upstream.env: 9ab918b32522baca13cd37bad7f420a206a19bb763b81041e174fffffe3da724 @@ -3534,10 +3534,10 @@ h100-eks-ubuntu-training-slurm+slurm-accounting-aicr-provided: 021-slinky-slurm/values.yaml: b97df3268a17dd1b4d1d963c6e003ab30b6b5ead510207f2a1476140143f6ea8 README.md: 045cd61481e66c6007b91223d63d73bcbdb3979c424931ce16d867be3227bc77 UPGRADING.md: 8f0af9c6fb2dc0bed077dae89732259a3f5478af5fb572e25465ee91ac4aecb1 - bundle-info.yaml: 5ea63209b30c23bc6518b01da31cad6adac939c3916bca80c7a0b4b9b5183cc2 - checksums.txt: 2e1a0c620b10f5edc7e2a4c4c49b853a2b92a0e22f9b0293c4c4847a2cc6d828 + bundle-info.yaml: d8541702e92175217663ac8b4897201e4a0eb117464bfe5671c2fe903095d0b5 + checksums.txt: 2984bc72a50d7fdfb52b27512e165bea86b4a201fee11dda9f49a7bb22022383 deploy.sh: 5eb3138723061d7639b956d70d9a9997a85e4d35a5924382d17db8f4e66b7b15 - recipe.yaml: ea67445b60991466d4f7a779691991a549c2f24e16aaedf7a738fb9718c3c8ed + recipe.yaml: e26dbf198f5652221de539e24c405615d86adb6a06f0c2392d809a4e2b75b846 h100-gke-cos-inference-dynamo: 001-agentgateway-crds/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 001-agentgateway-crds/install.sh: f0f864c4eb1f1d647af123953d5df5aa1fa03226e1f8c5feb724ea42d304eb3f @@ -3625,17 +3625,17 @@ h100-gke-cos-inference-dynamo: 020-nvsentinel/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 020-nvsentinel/install.sh: 290c0907c263ac3c1a976be76da13528e36fe7f1db79945783b6ac7da4588857 020-nvsentinel/upstream.env: 7fc07c3fb32a6e569a151e4c12f2834386b4c7026cda6cd1c86e29592ad8936e - 020-nvsentinel/values.yaml: 3374b3a52c2466d902a85e6d5e4b2fc02d4f824e690094772cb6dbe9ee6962b2 + 020-nvsentinel/values.yaml: 7f5f9edb985771f1b7c01e5ee67edba4a2a6182b990a940a5e89601e5d589ae0 021-prometheus-adapter/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 021-prometheus-adapter/install.sh: 688ee4274fe06c001af6835d7965b5da33ce3fbeb724263622737d0bb88249e3 021-prometheus-adapter/upstream.env: 9ab918b32522baca13cd37bad7f420a206a19bb763b81041e174fffffe3da724 021-prometheus-adapter/values.yaml: 95231d7a46d650f96f844ca3734614271152a41f1230022c9d79960e44647c00 README.md: 4117a076afe9904f4e66b120a6f4da0d0f36aeeba32b8606b31cfd586bc09f1f UPGRADING.md: c3ad3d3f95b3754dc8d85cde3c35a4a2717bd643f8d3cb7add47f6d327b7de95 - bundle-info.yaml: 2fd26b75a203f2ab169bf6d99d21d453301fea7a34d8e95598028b84f3bee115 - checksums.txt: 5341b1dea0053d0623084fd89d3509e78b6248af32acecf2b94a867fceb91c1f + bundle-info.yaml: 7dbb054e828c1266fdb77e508ac14dbd999a8a4682b1e34ba6221b9c79ae6215 + checksums.txt: 597031cb94fde718b827e0ae16a8d4368b58dc9b03d7c48e40cf8240fcdd1ff1 deploy.sh: 34d9d6c33e26fe690cb388ba01ea383b3444fa18c63e525c83f143f5c41f613e - recipe.yaml: 797b479444489702cd5bfd681d2f3fb0badd5e67faf638cd977a8c81c99cd904 + recipe.yaml: c5c3e06d8d0caea61c1464e61cc6eea81015739c1bdd2f6caa3dd53d29cb54c9 h100-gke-cos-training-kubeflow: 001-gcp-driver-installer/Chart.yaml: b0476226ebbc9a7738c4e5c2828ce5a73dbd371f3fa7cf1139c5fe9b2e825c02 001-gcp-driver-installer/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 @@ -3707,17 +3707,17 @@ h100-gke-cos-training-kubeflow: 016-nvsentinel/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 016-nvsentinel/install.sh: 290c0907c263ac3c1a976be76da13528e36fe7f1db79945783b6ac7da4588857 016-nvsentinel/upstream.env: 7fc07c3fb32a6e569a151e4c12f2834386b4c7026cda6cd1c86e29592ad8936e - 016-nvsentinel/values.yaml: 3374b3a52c2466d902a85e6d5e4b2fc02d4f824e690094772cb6dbe9ee6962b2 + 016-nvsentinel/values.yaml: 7f5f9edb985771f1b7c01e5ee67edba4a2a6182b990a940a5e89601e5d589ae0 017-prometheus-adapter/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 017-prometheus-adapter/install.sh: 688ee4274fe06c001af6835d7965b5da33ce3fbeb724263622737d0bb88249e3 017-prometheus-adapter/upstream.env: 9ab918b32522baca13cd37bad7f420a206a19bb763b81041e174fffffe3da724 017-prometheus-adapter/values.yaml: 95231d7a46d650f96f844ca3734614271152a41f1230022c9d79960e44647c00 README.md: 483fd9860f7e2636942a2f6cca7d183331ccdfbeedd38c487db45c09ab8be1ad UPGRADING.md: 4b285b6e5866a3582feef1afa6cc5906b79b275e94b0033d33f752c442d1256b - bundle-info.yaml: 35d12d11588fedbe1d46173311a07777046cafeacf04b076630da37c1e6fb42a - checksums.txt: 2f4b3fda4bf9c7205e8c619ab1d73035344ef26132eb6b6ef074673e296203b9 + bundle-info.yaml: 0da0316afecf9f9a2586d073835d5bbec23c3fb66168f28c8673b9505dcc84ec + checksums.txt: 027fde74e9faad12906f996b2b1583ee7c687410dcb4c45892fa4b3608091e7e deploy.sh: 0f6a2c61ae21d657eacbd33dcbdb5224b6148dbd42c8182f6dd56d7d06c9be8a - recipe.yaml: 0a77e386dfb388054385ad7bd88c2d93e81e62acb7ad7684b2dabe1733cbb60a + recipe.yaml: 520f750cc167df8cb79cdb7c18cc7af1ca631f0f4643acd6d07df347e9d5c8d5 h100-gke-cos-training-kueue: 001-gcp-driver-installer/Chart.yaml: b0476226ebbc9a7738c4e5c2828ce5a73dbd371f3fa7cf1139c5fe9b2e825c02 001-gcp-driver-installer/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 @@ -3779,7 +3779,7 @@ h100-gke-cos-training-kueue: 014-nvsentinel/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 014-nvsentinel/install.sh: 290c0907c263ac3c1a976be76da13528e36fe7f1db79945783b6ac7da4588857 014-nvsentinel/upstream.env: 7fc07c3fb32a6e569a151e4c12f2834386b4c7026cda6cd1c86e29592ad8936e - 014-nvsentinel/values.yaml: 3374b3a52c2466d902a85e6d5e4b2fc02d4f824e690094772cb6dbe9ee6962b2 + 014-nvsentinel/values.yaml: 7f5f9edb985771f1b7c01e5ee67edba4a2a6182b990a940a5e89601e5d589ae0 015-prometheus-adapter/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 015-prometheus-adapter/install.sh: 688ee4274fe06c001af6835d7965b5da33ce3fbeb724263622737d0bb88249e3 015-prometheus-adapter/upstream.env: 9ab918b32522baca13cd37bad7f420a206a19bb763b81041e174fffffe3da724 @@ -3802,10 +3802,10 @@ h100-gke-cos-training-kueue: 018-kueue-post/values.yaml: 9ae0af1abd28154ef1efc514bd20ee1daf35cfccbfaa0c6d97e08092b211d6fc README.md: cb310e160580cccbd6ed92fd814a1bafe988669159210ac569dffe9b1a4e9d9e UPGRADING.md: 27af534b3ca82d0a5e1135c70e185aafa9566df84b328f494f2f96168af9d18c - bundle-info.yaml: f0eb8b504d585ce1a177132b7cd5936784dc5f5472cc893380a2097ffc409297 - checksums.txt: 12532b18caa7f3904a39c765a59114ee5dc0738e465531d465daf42cff1e735f + bundle-info.yaml: 0e36ec7596990617819ff53f6111e610159eacc3c3f76dbd5679a02c77d9caa4 + checksums.txt: 36e0b2a96acedcaa2d08fb6d540362272503e0b1ba7281ae5ccf6fd2781a548f deploy.sh: 70e0b277ed17dfc69744f487905b679d1f9f3820bf647248e87f2e0e95493f8a - recipe.yaml: e42d6fff59d6af8d514239068ef0821d451580b693f893e69a78e97d05ed0d58 + recipe.yaml: 86480428434f7a92bb2b01a3c15e3dc866a1d0f9ce2ab7ba4f9e26201b77b9a2 h100-gke-cos-training-slurm: 001-gcp-driver-installer/Chart.yaml: b0476226ebbc9a7738c4e5c2828ce5a73dbd371f3fa7cf1139c5fe9b2e825c02 001-gcp-driver-installer/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 @@ -3867,7 +3867,7 @@ h100-gke-cos-training-slurm: 014-nvsentinel/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 014-nvsentinel/install.sh: 290c0907c263ac3c1a976be76da13528e36fe7f1db79945783b6ac7da4588857 014-nvsentinel/upstream.env: 7fc07c3fb32a6e569a151e4c12f2834386b4c7026cda6cd1c86e29592ad8936e - 014-nvsentinel/values.yaml: ef308e8e0286992ad25a311fd88960e537431f28140d953dfdf0c869fb2b86fe + 014-nvsentinel/values.yaml: 631c76fda879256d07d8e329e6f4582d7680e10b316444be4f7215356ca5c655 015-prometheus-adapter/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 015-prometheus-adapter/install.sh: 688ee4274fe06c001af6835d7965b5da33ce3fbeb724263622737d0bb88249e3 015-prometheus-adapter/upstream.env: 9ab918b32522baca13cd37bad7f420a206a19bb763b81041e174fffffe3da724 @@ -3895,10 +3895,10 @@ h100-gke-cos-training-slurm: 020-slinky-topograph/values.yaml: f11e2c423e180afd7141528eabed1ea9f0c1a81839f9b8218e194b7a61726bdb README.md: 95f564b494923cddf6a4f88b15cb71ffe798d1213653df7b473426ca1ebbce28 UPGRADING.md: 4b285b6e5866a3582feef1afa6cc5906b79b275e94b0033d33f752c442d1256b - bundle-info.yaml: 62968df1fde51ff2cf9d97b99f1a207873760cd236a7bbf45e46b0f318797bbc - checksums.txt: 24ec35842d20bcab9951bbc4eff9f4baa190eedb3ff677015f7e734333589814 + bundle-info.yaml: 1f434b4a55066d011311073efc252fd9db17651c1f5ca210741619df35e672d0 + checksums.txt: f7b9f8520676c165a617ec7e92bf36fa52e49dd1d616a3ab5706496589898a42 deploy.sh: 3c91727718e2f0d106a7dd220171cdb28fda16113de7394a668c62fa9388a197 - recipe.yaml: 799a3dcd74ec41b4394d47dd385c34af793a84071ad886cc67628f2667a49cd1 + recipe.yaml: 72dfdb5c32e64194c1a4b0893bb72a4fe8b2aae9da6286501362acb506712d07 h100-kind-inference-dynamo: 001-agentgateway-crds/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 001-agentgateway-crds/install.sh: f0f864c4eb1f1d647af123953d5df5aa1fa03226e1f8c5feb724ea42d304eb3f @@ -3971,17 +3971,17 @@ h100-kind-inference-dynamo: 017-nvsentinel/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 017-nvsentinel/install.sh: 290c0907c263ac3c1a976be76da13528e36fe7f1db79945783b6ac7da4588857 017-nvsentinel/upstream.env: 7fc07c3fb32a6e569a151e4c12f2834386b4c7026cda6cd1c86e29592ad8936e - 017-nvsentinel/values.yaml: 64f3d3b8ccd78cf7cd50db43dcac8fa2d510fd0195fa8680cbae8236f7ff52fc + 017-nvsentinel/values.yaml: 7dea0ee93e9c35d925e7db9120898cc8a572558823b11bc434fe95000a30c6a5 018-prometheus-adapter/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 018-prometheus-adapter/install.sh: 688ee4274fe06c001af6835d7965b5da33ce3fbeb724263622737d0bb88249e3 018-prometheus-adapter/upstream.env: 9ab918b32522baca13cd37bad7f420a206a19bb763b81041e174fffffe3da724 018-prometheus-adapter/values.yaml: 95231d7a46d650f96f844ca3734614271152a41f1230022c9d79960e44647c00 README.md: c98cad2dfd4cd47eee21a67efbaac2129b769643f30c7ab1c6eb03ecef92bede UPGRADING.md: 668b1aba11148b80ac92ea8e793d263cabba6a615aad52f1b3df7f83007e446f - bundle-info.yaml: dcf3f09dfef0720f6791038dd199d59bdbc4b4188004f74d2859c4fc48b28fc9 - checksums.txt: 54e56d628c6920cd1dd4e9a6cb4d6e44f75fea537b3dc52dfa52b4a8ae8ddda9 + bundle-info.yaml: 1b6a6a35d6c87cce8c0fa8c8196320953e72ebaab69366cf8b250ac6d25edecc + checksums.txt: a49517b010e0dd4248f8d4b1fc11e7375d2389947db022935d6641cdce1865ac deploy.sh: fc332287efdf91cc0ea399c6b5227ed2bb2fac9577a14311a44f24646cd09ad2 - recipe.yaml: 849b99eb3c7cdfbb9752272c16f4ea47201bb8f0736e5ed0f52c14d870dd5b42 + recipe.yaml: 5f258a7950e8b10ddc249233e078690636c2ac6d32dbde642cee3f7ac7a7f430 h100-kind-training-kubeflow: 001-nodewright-operator/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 001-nodewright-operator/install.sh: 5bb299d13650bdb1baa19ea8047b33baa94eb03a48a8b5b0102f1ba533b392c2 @@ -4036,17 +4036,17 @@ h100-kind-training-kubeflow: 013-nvsentinel/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 013-nvsentinel/install.sh: 290c0907c263ac3c1a976be76da13528e36fe7f1db79945783b6ac7da4588857 013-nvsentinel/upstream.env: 7fc07c3fb32a6e569a151e4c12f2834386b4c7026cda6cd1c86e29592ad8936e - 013-nvsentinel/values.yaml: 64f3d3b8ccd78cf7cd50db43dcac8fa2d510fd0195fa8680cbae8236f7ff52fc + 013-nvsentinel/values.yaml: 7dea0ee93e9c35d925e7db9120898cc8a572558823b11bc434fe95000a30c6a5 014-prometheus-adapter/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 014-prometheus-adapter/install.sh: 688ee4274fe06c001af6835d7965b5da33ce3fbeb724263622737d0bb88249e3 014-prometheus-adapter/upstream.env: 9ab918b32522baca13cd37bad7f420a206a19bb763b81041e174fffffe3da724 014-prometheus-adapter/values.yaml: 95231d7a46d650f96f844ca3734614271152a41f1230022c9d79960e44647c00 README.md: 8b8e7dcf4aa5595f8332b85821c37985bfe69e86b76a37ada909e0fc05c7beeb UPGRADING.md: 9444e3b2469985d89ae8fd6bb0ca9116e56c2904f9aff6929487211120b75484 - bundle-info.yaml: dedfa6868852690d6b0289cdc41296253529ee2cf9f59b9469a4d4ee56354956 - checksums.txt: 6a8d43eedb60381b72a672229fd3dee4fb45bc9b915b146d2c2e575a902f3f7b + bundle-info.yaml: 0760d5a7e142400f0ae3957ac565afed07bba6ce1eb26290edec433f69f5229f + checksums.txt: 5854e373b9d246dcb4410ed8a3ba48d479fed18375488ebeb8b3e15fcbda1de0 deploy.sh: 585028988d2cdbd7fad24e6a6aa04e416399fb8c675ac560c1ea2cf01e76def6 - recipe.yaml: ca66e3a524a5d340ec400b684d0ff7f4c9ec15655a409f34b4fc8b8ec7568964 + recipe.yaml: 7e540579e38948cd8aeb8a59767802f82c1db4134f598be8f7b3bf6af66acb4f h100-kind-training-kueue: 001-nodewright-operator/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 001-nodewright-operator/install.sh: 5bb299d13650bdb1baa19ea8047b33baa94eb03a48a8b5b0102f1ba533b392c2 @@ -4092,7 +4092,7 @@ h100-kind-training-kueue: 011-nvsentinel/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 011-nvsentinel/install.sh: 290c0907c263ac3c1a976be76da13528e36fe7f1db79945783b6ac7da4588857 011-nvsentinel/upstream.env: 7fc07c3fb32a6e569a151e4c12f2834386b4c7026cda6cd1c86e29592ad8936e - 011-nvsentinel/values.yaml: 64f3d3b8ccd78cf7cd50db43dcac8fa2d510fd0195fa8680cbae8236f7ff52fc + 011-nvsentinel/values.yaml: 7dea0ee93e9c35d925e7db9120898cc8a572558823b11bc434fe95000a30c6a5 012-prometheus-adapter/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 012-prometheus-adapter/install.sh: 688ee4274fe06c001af6835d7965b5da33ce3fbeb724263622737d0bb88249e3 012-prometheus-adapter/upstream.env: 9ab918b32522baca13cd37bad7f420a206a19bb763b81041e174fffffe3da724 @@ -4115,10 +4115,10 @@ h100-kind-training-kueue: 015-kueue-post/values.yaml: 9ae0af1abd28154ef1efc514bd20ee1daf35cfccbfaa0c6d97e08092b211d6fc README.md: f1e44cb3ac64c9a238129f81df52ef4f91296f1a7a621209977b27cc65c98986 UPGRADING.md: 3da0e0d2a3d4866a1518c6703cd8f862cb5186961d9448b0e75b9a9461bb6707 - bundle-info.yaml: 794e6d19b59b4e330c1a2c9f2b21e9f335c465b73afe42e7c7d491ab4d47b9ec - checksums.txt: 64f8b9c00ac35d005ab26b9e18e305ef277b5eb0fb60e40138453f4b1fcc4d46 + bundle-info.yaml: d579cb1e73633945e9aac6f4f9908049c61403930b1ae704b274f8741c95fb4a + checksums.txt: d77df7d18d1bef585da076e408686f80fd33ddd67de98b4fd27d70f8fdb99ed1 deploy.sh: 706a8d58c6b713b9376215029a49a213fc943bdecab7cd7d4ea7d7ef2d86e692 - recipe.yaml: 9a3a9f467ab7c639f6c65de043652dce9e4d6ed2ac987d6f7adf949f2b2f145e + recipe.yaml: 829b8651dde68183d33772d9e72bf27f36926d30b4fa6578b4194bf857002317 h100-kind-training-slurm: 001-nodewright-operator/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 001-nodewright-operator/install.sh: 5bb299d13650bdb1baa19ea8047b33baa94eb03a48a8b5b0102f1ba533b392c2 @@ -4164,7 +4164,7 @@ h100-kind-training-slurm: 011-nvsentinel/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 011-nvsentinel/install.sh: 290c0907c263ac3c1a976be76da13528e36fe7f1db79945783b6ac7da4588857 011-nvsentinel/upstream.env: 7fc07c3fb32a6e569a151e4c12f2834386b4c7026cda6cd1c86e29592ad8936e - 011-nvsentinel/values.yaml: 937ccfad99c7ed9383b6bae92d81204aee00f6e77f4c661a497e0cd556832ff7 + 011-nvsentinel/values.yaml: 9c8d231e7d34f6e9f0ed152083e9cd404cb7a6316b85f7198cc89106031e0f0b 012-prometheus-adapter/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 012-prometheus-adapter/install.sh: 688ee4274fe06c001af6835d7965b5da33ce3fbeb724263622737d0bb88249e3 012-prometheus-adapter/upstream.env: 9ab918b32522baca13cd37bad7f420a206a19bb763b81041e174fffffe3da724 @@ -4192,10 +4192,10 @@ h100-kind-training-slurm: 017-slinky-topograph/values.yaml: c9015074818102d8727cfd38e96941c5c285175852fa71f81182c4323f751e05 README.md: 5ea67d9b43752a8251ced4e0585f15e6f2eb0b05ea14b846d9b9a8c66e978c8f UPGRADING.md: 9444e3b2469985d89ae8fd6bb0ca9116e56c2904f9aff6929487211120b75484 - bundle-info.yaml: 667fb78adc7207a9fed799507235476faf304d56430358600fb0e5188f1794d2 - checksums.txt: 1c2ff6a70c5129823170eb0e72574c2ff732bdb9b9c311d176eb16f144056c38 + bundle-info.yaml: 0a2feeb32206bfb20c4dfdcc6a98e028d31f65597dab94aa1b7c1871bf2c9d77 + checksums.txt: cfe20ff79cefb44498a6778efeedb077c955259b68f9f18eb0a5a837826b6956 deploy.sh: cd7ccfe96441d0d0a17d9b7ef13f18e2938613e70854fc367a7bd8e64039019c - recipe.yaml: 506b960ca245744ea245581a8098ab8d9d6fee1d49f8a49c85845c3373b8bd73 + recipe.yaml: 0609c3d92b2e1f1bea57d51b99873f0f10ce3b884f4fb3a5dd9ab1587850142f h200-any: 001-nodewright-operator/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 001-nodewright-operator/install.sh: 5bb299d13650bdb1baa19ea8047b33baa94eb03a48a8b5b0102f1ba533b392c2 @@ -4237,17 +4237,17 @@ h200-any: 010-nvsentinel/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 010-nvsentinel/install.sh: 290c0907c263ac3c1a976be76da13528e36fe7f1db79945783b6ac7da4588857 010-nvsentinel/upstream.env: 7fc07c3fb32a6e569a151e4c12f2834386b4c7026cda6cd1c86e29592ad8936e - 010-nvsentinel/values.yaml: 5dbdd974de19bda092dff19de6a6fbb9489b7e7629abdeab2d1a8fe0bff69247 + 010-nvsentinel/values.yaml: 14400ae55936f06cc2025495b7d2efbb933e388a985abf2061b3b4fcbc36eeea 011-prometheus-adapter/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 011-prometheus-adapter/install.sh: 688ee4274fe06c001af6835d7965b5da33ce3fbeb724263622737d0bb88249e3 011-prometheus-adapter/upstream.env: 9ab918b32522baca13cd37bad7f420a206a19bb763b81041e174fffffe3da724 011-prometheus-adapter/values.yaml: 95231d7a46d650f96f844ca3734614271152a41f1230022c9d79960e44647c00 README.md: f5acf4a4aa2415bbeb70ea0b7742bef39e6bf3195b5425e518cec261a6519ae6 UPGRADING.md: 4b285b6e5866a3582feef1afa6cc5906b79b275e94b0033d33f752c442d1256b - bundle-info.yaml: eb186788bcc3db83180f2da0cb9dd204aeec81bf67c060f726d4690f4b6ec24c - checksums.txt: 8c2a27609f93f42e5851f45e9039242376d715a3552cb0cada9d39488d634411 + bundle-info.yaml: 1ccd4378e85fd31c0e416700c56e4b47de573f323f2e4f97a392a70f95a43ef1 + checksums.txt: 87c3fb6defde56f4ca800da7105481c234546c8b63b208049664cfda963d3340 deploy.sh: 187bf11702c83fdf08c22fbdd72a6b16c937314d87f85da747c4753cbe5d5de4 - recipe.yaml: 9f83b8fdbf465ec7020a366d12e6ad8b845afb0181c348369fcf58cc6c86629a + recipe.yaml: 2482c94f728bca303aa714337942789bbdd7f02c291a5d81981c53ccc6979d32 h200-eks-inference: 001-agentgateway-crds/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 001-agentgateway-crds/install.sh: f0f864c4eb1f1d647af123953d5df5aa1fa03226e1f8c5feb724ea42d304eb3f @@ -4321,17 +4321,17 @@ h200-eks-inference: 017-nvsentinel/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 017-nvsentinel/install.sh: 290c0907c263ac3c1a976be76da13528e36fe7f1db79945783b6ac7da4588857 017-nvsentinel/upstream.env: 7fc07c3fb32a6e569a151e4c12f2834386b4c7026cda6cd1c86e29592ad8936e - 017-nvsentinel/values.yaml: 5dbdd974de19bda092dff19de6a6fbb9489b7e7629abdeab2d1a8fe0bff69247 + 017-nvsentinel/values.yaml: 14400ae55936f06cc2025495b7d2efbb933e388a985abf2061b3b4fcbc36eeea 018-prometheus-adapter/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 018-prometheus-adapter/install.sh: 688ee4274fe06c001af6835d7965b5da33ce3fbeb724263622737d0bb88249e3 018-prometheus-adapter/upstream.env: 9ab918b32522baca13cd37bad7f420a206a19bb763b81041e174fffffe3da724 018-prometheus-adapter/values.yaml: 95231d7a46d650f96f844ca3734614271152a41f1230022c9d79960e44647c00 README.md: 8d3f8c72d2409dc42b46c1aa12bceb4840522dfe8ff71c842c7aa98c6fcead0f UPGRADING.md: 4b285b6e5866a3582feef1afa6cc5906b79b275e94b0033d33f752c442d1256b - bundle-info.yaml: 3a203be6fd5d31402815b01597c1979c7cdc43e760ac90e14229f94637deb39e - checksums.txt: 94816c94750c905d2eee285438b9b9754d5b627c35e414c8f54d07e291948c6e + bundle-info.yaml: 7069dfdaac3ddeec9ba3e7db36d686b2cd2ca844fb388073c5d94b83015b1bf8 + checksums.txt: 307690905eb170503c8a9f0c8faf400be1a2c598b82604b8122a4552839c56be deploy.sh: 4dc09cb6d1d5fe7cc484477ddc854ac7156a258c39f5885eac02c2fe8b1afe54 - recipe.yaml: 281a9e5b9ff22fd3a0e87989bba9d72d04b718bca6ad9cadea8fddcc28d2bf3d + recipe.yaml: 77aa43e03acfc295003139c01f68286a7ecb5bb22ef8fe8337640855fefc6093 h200-eks-training-kubeflow: 001-aws-efa/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 001-aws-efa/install.sh: 00574b8307fa54f90f18cd999a2b175a75a547942fd18cb242bd83b7af836d5a @@ -4395,17 +4395,17 @@ h200-eks-training-kubeflow: 015-nvsentinel/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 015-nvsentinel/install.sh: 290c0907c263ac3c1a976be76da13528e36fe7f1db79945783b6ac7da4588857 015-nvsentinel/upstream.env: 7fc07c3fb32a6e569a151e4c12f2834386b4c7026cda6cd1c86e29592ad8936e - 015-nvsentinel/values.yaml: 5dbdd974de19bda092dff19de6a6fbb9489b7e7629abdeab2d1a8fe0bff69247 + 015-nvsentinel/values.yaml: 14400ae55936f06cc2025495b7d2efbb933e388a985abf2061b3b4fcbc36eeea 016-prometheus-adapter/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 016-prometheus-adapter/install.sh: 688ee4274fe06c001af6835d7965b5da33ce3fbeb724263622737d0bb88249e3 016-prometheus-adapter/upstream.env: 9ab918b32522baca13cd37bad7f420a206a19bb763b81041e174fffffe3da724 016-prometheus-adapter/values.yaml: 95231d7a46d650f96f844ca3734614271152a41f1230022c9d79960e44647c00 README.md: b65b7934272f5455cbd9927c7dd866e4b1fa64c661a7ed310eb1da7327d9fa6b UPGRADING.md: 4b285b6e5866a3582feef1afa6cc5906b79b275e94b0033d33f752c442d1256b - bundle-info.yaml: cbc24b2aeee9ac3bb92a0bf1b4b223fe4dfc9c56202a9cb24a17e0bae4056889 - checksums.txt: 54a467555e705cee44eb3e8da6ecfe74530a4deaf5f8d81dc5900cc8ef173c2a + bundle-info.yaml: 445bdc4165fb7073ac279017fe48e1452ed2ea1009d84001e89657bf36162e0f + checksums.txt: 26b90070a1028ddb411952c4876fb611acda2f6a8c3f285bcaa4ba32f9c1dd8b deploy.sh: 949466ed3363eb59ea6ff5a61a29ca150215a2041c47323cada206dcf3cdc0a8 - recipe.yaml: 54ccc40078a9ad61a2ec6fddb29da286aa96cf77604bac25bca297b4d50c61a7 + recipe.yaml: 99b18eff9ce22adb64d631f5f50af2105609df928c2b6bd6bf5fa2b84e8d441c h200-k0s-ubuntu-training: 001-nodewright-operator/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 001-nodewright-operator/install.sh: 5bb299d13650bdb1baa19ea8047b33baa94eb03a48a8b5b0102f1ba533b392c2 @@ -4447,17 +4447,17 @@ h200-k0s-ubuntu-training: 010-nvsentinel/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 010-nvsentinel/install.sh: 290c0907c263ac3c1a976be76da13528e36fe7f1db79945783b6ac7da4588857 010-nvsentinel/upstream.env: 7fc07c3fb32a6e569a151e4c12f2834386b4c7026cda6cd1c86e29592ad8936e - 010-nvsentinel/values.yaml: ede1bf610929dcf019396356eec2e1a0bf9c1417c181595e86f03d6035fe27bb + 010-nvsentinel/values.yaml: 376d322d05dfab359cef13a99ded134b567f8100905a39caaf4af614efefa223 011-prometheus-adapter/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 011-prometheus-adapter/install.sh: 688ee4274fe06c001af6835d7965b5da33ce3fbeb724263622737d0bb88249e3 011-prometheus-adapter/upstream.env: 9ab918b32522baca13cd37bad7f420a206a19bb763b81041e174fffffe3da724 011-prometheus-adapter/values.yaml: 95231d7a46d650f96f844ca3734614271152a41f1230022c9d79960e44647c00 README.md: 951c553815ea718031602b9bd78c0691c425e978cf97744b5d3aae016174e636 UPGRADING.md: 4b285b6e5866a3582feef1afa6cc5906b79b275e94b0033d33f752c442d1256b - bundle-info.yaml: 7c36cdc60f58bc33373f92ee894e9b2fc5703c6558d3bc1d44936dbee5f80396 - checksums.txt: bd0da79e0b5f919d9a7bce0cd45af22277ece7cd2fbe7a66ae7dbf9c2f6175bd + bundle-info.yaml: 8f64742c995a6a3704ca0e14898836d44f650ba3e5d62307dc6f44dd57072b3f + checksums.txt: 8f29840a0e6031459a33ecc1790f7cc5f590e70a74d9e9871ac8ae32e6feb8f3 deploy.sh: cc316b6457856fa5913d4da2b95e9345278177c284e554a8c45072f3aabad4bd - recipe.yaml: 8fb936b585678c913efe3266cf88b284f3fecdfaa7b218d4af0142e7a5236643 + recipe.yaml: 46ff0cfe45379489434001dbf4541dd0fb362401759969a9e6139643adf5f247 l40-any: 001-nodewright-operator/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 001-nodewright-operator/install.sh: 5bb299d13650bdb1baa19ea8047b33baa94eb03a48a8b5b0102f1ba533b392c2 @@ -4499,17 +4499,17 @@ l40-any: 010-nvsentinel/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 010-nvsentinel/install.sh: 290c0907c263ac3c1a976be76da13528e36fe7f1db79945783b6ac7da4588857 010-nvsentinel/upstream.env: 7fc07c3fb32a6e569a151e4c12f2834386b4c7026cda6cd1c86e29592ad8936e - 010-nvsentinel/values.yaml: 5dbdd974de19bda092dff19de6a6fbb9489b7e7629abdeab2d1a8fe0bff69247 + 010-nvsentinel/values.yaml: 14400ae55936f06cc2025495b7d2efbb933e388a985abf2061b3b4fcbc36eeea 011-prometheus-adapter/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 011-prometheus-adapter/install.sh: 688ee4274fe06c001af6835d7965b5da33ce3fbeb724263622737d0bb88249e3 011-prometheus-adapter/upstream.env: 9ab918b32522baca13cd37bad7f420a206a19bb763b81041e174fffffe3da724 011-prometheus-adapter/values.yaml: 95231d7a46d650f96f844ca3734614271152a41f1230022c9d79960e44647c00 README.md: f93af52ccc72c626ef12232339dff4cc3f3b28ce09e9276ce61a3c1ea049df00 UPGRADING.md: 4b285b6e5866a3582feef1afa6cc5906b79b275e94b0033d33f752c442d1256b - bundle-info.yaml: dfb38ea4975e1e85e8b3f1a500946c55baee1bcfe96bcdac58774038c21d9295 - checksums.txt: ee1ce5289eda05346882ced56f94ff038e0b4562da5f6b7e828077137021aab5 + bundle-info.yaml: a0933d3e7df47b103bd259e052d83a749bf1840caacedd2622da42db849162c8 + checksums.txt: c53aae450edb2a3b2d5343dab875cec7f227d9b888dc99156ecb6dc8cacff4aa deploy.sh: 187bf11702c83fdf08c22fbdd72a6b16c937314d87f85da747c4753cbe5d5de4 - recipe.yaml: 0e28cff83d20cd3334649e83a42ec88f85832b499d0211b81d1cf16c0ead60c0 + recipe.yaml: 1a87308135bca89127d3a726127c45fc554569b178a6322c71dbe2b9bda7170e l40s-any: 001-nodewright-operator/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 001-nodewright-operator/install.sh: 5bb299d13650bdb1baa19ea8047b33baa94eb03a48a8b5b0102f1ba533b392c2 @@ -4551,17 +4551,17 @@ l40s-any: 010-nvsentinel/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 010-nvsentinel/install.sh: 290c0907c263ac3c1a976be76da13528e36fe7f1db79945783b6ac7da4588857 010-nvsentinel/upstream.env: 7fc07c3fb32a6e569a151e4c12f2834386b4c7026cda6cd1c86e29592ad8936e - 010-nvsentinel/values.yaml: 5dbdd974de19bda092dff19de6a6fbb9489b7e7629abdeab2d1a8fe0bff69247 + 010-nvsentinel/values.yaml: 14400ae55936f06cc2025495b7d2efbb933e388a985abf2061b3b4fcbc36eeea 011-prometheus-adapter/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 011-prometheus-adapter/install.sh: 688ee4274fe06c001af6835d7965b5da33ce3fbeb724263622737d0bb88249e3 011-prometheus-adapter/upstream.env: 9ab918b32522baca13cd37bad7f420a206a19bb763b81041e174fffffe3da724 011-prometheus-adapter/values.yaml: 95231d7a46d650f96f844ca3734614271152a41f1230022c9d79960e44647c00 README.md: ed4445e492f674cc09c02e8aebe7d55d9d92d2a3bcb5acae18abf87912dca42e UPGRADING.md: 4b285b6e5866a3582feef1afa6cc5906b79b275e94b0033d33f752c442d1256b - bundle-info.yaml: 21f33b0f9a33a8fc50cccacac074c66df42be33f2987f9fc7831bfc7598e246c - checksums.txt: 361ebd9830445b1a4f68eab4208f9f1e843988ddb81d69ac905d71c67b640bcf + bundle-info.yaml: e24db840c7c70f0e1d2d803e1cdb8acc8eab03a214cf917d77468449530c1534 + checksums.txt: 9e4f92ee26ec403186e801975be3b8f85d3766d93c2cb9ac09ad4843277dd118 deploy.sh: 187bf11702c83fdf08c22fbdd72a6b16c937314d87f85da747c4753cbe5d5de4 - recipe.yaml: 5154afcaaed7bfeb9a41883ee944e1a291935237e9cbd6b6d66ba0affe1f9448 + recipe.yaml: 365c93ad7e1d9ebcbaa7b4618fb70b553435bd1d6eb2231896848ade48986130 l40s-oke-inference: 001-agentgateway-crds/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 001-agentgateway-crds/install.sh: f0f864c4eb1f1d647af123953d5df5aa1fa03226e1f8c5feb724ea42d304eb3f @@ -4622,17 +4622,17 @@ l40s-oke-inference: 014-nvsentinel/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 014-nvsentinel/install.sh: 290c0907c263ac3c1a976be76da13528e36fe7f1db79945783b6ac7da4588857 014-nvsentinel/upstream.env: 7fc07c3fb32a6e569a151e4c12f2834386b4c7026cda6cd1c86e29592ad8936e - 014-nvsentinel/values.yaml: 36e1a505cabb70a1cd795d582ac2a0b550d4fdd7875114cc49f9129bb1e3e867 + 014-nvsentinel/values.yaml: 6f417215aceac1cf2154ab9ac04a92eea2cfd631b116e2642fc8446b7cbca974 015-prometheus-adapter/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 015-prometheus-adapter/install.sh: 688ee4274fe06c001af6835d7965b5da33ce3fbeb724263622737d0bb88249e3 015-prometheus-adapter/upstream.env: 9ab918b32522baca13cd37bad7f420a206a19bb763b81041e174fffffe3da724 015-prometheus-adapter/values.yaml: 95231d7a46d650f96f844ca3734614271152a41f1230022c9d79960e44647c00 README.md: c25c7d2559ee16d5d1c2abfb8f2de3b66de8e1c3bf16e4c2da8beb1406a02475 UPGRADING.md: 4b285b6e5866a3582feef1afa6cc5906b79b275e94b0033d33f752c442d1256b - bundle-info.yaml: 028df8dd2fa3c5fa0391da47dfa38206a261b84ce376520cced68022f27d0a70 - checksums.txt: 7e243acd50876cb7c5dee0cf61f5be762cd44521cef8d1cfe652201427379fe0 + bundle-info.yaml: 5bd3d8f516014283cada32b1a599867f718e6d32d9f61c34808294aea949d513 + checksums.txt: f0e0d556657b3bb25cfb9d0b142481fe5642786696c774912bdbabde3ef4375d deploy.sh: e576ddc4b52f43843cee5b8b516961df9e34b6566cb6df72f34d01a5f09173fe - recipe.yaml: e8497c6f1da0fa7568835b8f53f178d68842ce884fae327922556b4892271270 + recipe.yaml: 701841c8e3cd50391085fd42eed150fe63ad9112369b57662765cabad84a7b5c l40s-oke-training-kubeflow: 001-nodewright-operator/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 001-nodewright-operator/install.sh: 5bb299d13650bdb1baa19ea8047b33baa94eb03a48a8b5b0102f1ba533b392c2 @@ -4692,17 +4692,17 @@ l40s-oke-training-kubeflow: 014-nvsentinel/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 014-nvsentinel/install.sh: 290c0907c263ac3c1a976be76da13528e36fe7f1db79945783b6ac7da4588857 014-nvsentinel/upstream.env: 7fc07c3fb32a6e569a151e4c12f2834386b4c7026cda6cd1c86e29592ad8936e - 014-nvsentinel/values.yaml: 36e1a505cabb70a1cd795d582ac2a0b550d4fdd7875114cc49f9129bb1e3e867 + 014-nvsentinel/values.yaml: 6f417215aceac1cf2154ab9ac04a92eea2cfd631b116e2642fc8446b7cbca974 015-prometheus-adapter/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 015-prometheus-adapter/install.sh: 688ee4274fe06c001af6835d7965b5da33ce3fbeb724263622737d0bb88249e3 015-prometheus-adapter/upstream.env: 9ab918b32522baca13cd37bad7f420a206a19bb763b81041e174fffffe3da724 015-prometheus-adapter/values.yaml: 95231d7a46d650f96f844ca3734614271152a41f1230022c9d79960e44647c00 README.md: 49ff81283ed3f6df7ad9e6588e43c06c7afc37c1b8b01ad23507dc8c38a53a94 UPGRADING.md: 9444e3b2469985d89ae8fd6bb0ca9116e56c2904f9aff6929487211120b75484 - bundle-info.yaml: 45629db05daaa889bc48e9967aff5d2cf2898b2fff3fe7307ccbd12637222285 - checksums.txt: 6ccbc8b091e6ba59b5ddaaf4ccbb9b96f3160dfad145fe3724297509f92495aa + bundle-info.yaml: 3d667b93058bddfc1b3d743d29927a64b6f3f319548072c0998cd5ec28d3e4fe + checksums.txt: 0db1ac4c657047c04eb5af0e684737ae350cc36114a16f1126353a5ca8d2ecd9 deploy.sh: 585028988d2cdbd7fad24e6a6aa04e416399fb8c675ac560c1ea2cf01e76def6 - recipe.yaml: 4bd965222d08c2064e5762f110e0a5a492b78ae85e45f452cbabdbc004430488 + recipe.yaml: d7e8becbc53e49ad057fb47c480ae819efe05a8b51e72856ecd18f47b1083391 monitoring-hpa: 001-nodewright-operator/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 001-nodewright-operator/install.sh: 5bb299d13650bdb1baa19ea8047b33baa94eb03a48a8b5b0102f1ba533b392c2 @@ -4744,17 +4744,17 @@ monitoring-hpa: 010-nvsentinel/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 010-nvsentinel/install.sh: 290c0907c263ac3c1a976be76da13528e36fe7f1db79945783b6ac7da4588857 010-nvsentinel/upstream.env: 7fc07c3fb32a6e569a151e4c12f2834386b4c7026cda6cd1c86e29592ad8936e - 010-nvsentinel/values.yaml: 5dbdd974de19bda092dff19de6a6fbb9489b7e7629abdeab2d1a8fe0bff69247 + 010-nvsentinel/values.yaml: 14400ae55936f06cc2025495b7d2efbb933e388a985abf2061b3b4fcbc36eeea 011-prometheus-adapter/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 011-prometheus-adapter/install.sh: 688ee4274fe06c001af6835d7965b5da33ce3fbeb724263622737d0bb88249e3 011-prometheus-adapter/upstream.env: 9ab918b32522baca13cd37bad7f420a206a19bb763b81041e174fffffe3da724 011-prometheus-adapter/values.yaml: 95231d7a46d650f96f844ca3734614271152a41f1230022c9d79960e44647c00 README.md: 2285a558e34f749873fd3749b70d3ae7fb0b64031450909846c467fbeee1c202 UPGRADING.md: 4b285b6e5866a3582feef1afa6cc5906b79b275e94b0033d33f752c442d1256b - bundle-info.yaml: 3b370f7c4a210ec2e33ac35b220db95b57841d39cba3385a99d6f0877ce497fe - checksums.txt: d7c17cdf89a6c6965824adc4b2f340fffe011c9a504a4b0b3ca2461b36d19567 + bundle-info.yaml: f10a0b4b2628595549500ac36140e954d1a63047a50286eae437a8ff67f34513 + checksums.txt: 7be0afe247837900cbb5840d119ffecc4f61552da5f5675e2711f95a926bf210 deploy.sh: 187bf11702c83fdf08c22fbdd72a6b16c937314d87f85da747c4753cbe5d5de4 - recipe.yaml: 3adb69b98a13b729d37ba66c07bddddf3b1ea20dddc8915050145b179ad43f25 + recipe.yaml: fa9cace5c3a9e51b3690f35994ac582b8d4dcd023e9524534e050dc5fd529c0b ocp-inference-nim: 001-nfd-ocp-olm/Chart.yaml: af13850294d68e9730e11e9b15c329a1935088f4e07c0ffb65a6d8a5915b78e0 001-nfd-ocp-olm/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 @@ -4960,17 +4960,17 @@ rke2-sles: 011-nvsentinel/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 011-nvsentinel/install.sh: 290c0907c263ac3c1a976be76da13528e36fe7f1db79945783b6ac7da4588857 011-nvsentinel/upstream.env: 7fc07c3fb32a6e569a151e4c12f2834386b4c7026cda6cd1c86e29592ad8936e - 011-nvsentinel/values.yaml: 1195972dea995a69dea5bbe9a0fe288f20b27d02fb643e7adea12d54ae3b4ec1 + 011-nvsentinel/values.yaml: 750c6f6ca9acbac9ef054da0480064e4e909e813aede75fc9392028d956e53a4 012-prometheus-adapter/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 012-prometheus-adapter/install.sh: 688ee4274fe06c001af6835d7965b5da33ce3fbeb724263622737d0bb88249e3 012-prometheus-adapter/upstream.env: 9ab918b32522baca13cd37bad7f420a206a19bb763b81041e174fffffe3da724 012-prometheus-adapter/values.yaml: 95231d7a46d650f96f844ca3734614271152a41f1230022c9d79960e44647c00 README.md: fddd04a34a4eeed73818083c1be8b115c3371f5999235e2590a981409e402e1a UPGRADING.md: 4b285b6e5866a3582feef1afa6cc5906b79b275e94b0033d33f752c442d1256b - bundle-info.yaml: f96ea3c851805f3ef0eaa2f2272c2583b2600622bf0ddaff92b8efb0d28a3a6f - checksums.txt: 7c509ecb8bfc076c6c56313bb8e1adcc07e0c216ad4f8c7a9d4b8d82d39d2bac + bundle-info.yaml: 224f29f0b85b67656a7d4aa9a85abf9cfd973a7e78023932c9f659f9e4e4e6c6 + checksums.txt: cd668768ced22e326394327ad8972fa7d73a66654176e4eaa09f79c3d03e795e deploy.sh: c9b22d53d6f5d5b6964dd6bca38379e3cf7585685dc3bf07970aaa73328b885a - recipe.yaml: 947b55522ce632e2f227c4615671e1588cbf60b3d1c1363a0034b0714342ef0c + recipe.yaml: 9181eba1f4687355baaeede416f792fd45a442f1b3fe3eaf7b579e81709a350d rtx-pro-6000-any: 001-nodewright-operator/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 001-nodewright-operator/install.sh: 5bb299d13650bdb1baa19ea8047b33baa94eb03a48a8b5b0102f1ba533b392c2 @@ -5012,17 +5012,17 @@ rtx-pro-6000-any: 010-nvsentinel/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 010-nvsentinel/install.sh: 290c0907c263ac3c1a976be76da13528e36fe7f1db79945783b6ac7da4588857 010-nvsentinel/upstream.env: 7fc07c3fb32a6e569a151e4c12f2834386b4c7026cda6cd1c86e29592ad8936e - 010-nvsentinel/values.yaml: 5dbdd974de19bda092dff19de6a6fbb9489b7e7629abdeab2d1a8fe0bff69247 + 010-nvsentinel/values.yaml: 14400ae55936f06cc2025495b7d2efbb933e388a985abf2061b3b4fcbc36eeea 011-prometheus-adapter/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 011-prometheus-adapter/install.sh: 688ee4274fe06c001af6835d7965b5da33ce3fbeb724263622737d0bb88249e3 011-prometheus-adapter/upstream.env: 9ab918b32522baca13cd37bad7f420a206a19bb763b81041e174fffffe3da724 011-prometheus-adapter/values.yaml: 95231d7a46d650f96f844ca3734614271152a41f1230022c9d79960e44647c00 README.md: cec67c7e5b7baf1553e7b9cd7d6c09c9f3e95881d883ceb3007de63553b98739 UPGRADING.md: 4b285b6e5866a3582feef1afa6cc5906b79b275e94b0033d33f752c442d1256b - bundle-info.yaml: a9f12e13112ffc100bf8d1c7f1213045fb827fbc00ee643429bde2af4ff01c88 - checksums.txt: 785f7d287adbf2151475fe4862f5c70665ff57e4c2039f04372bb434fad2a2a6 + bundle-info.yaml: 1f42de41593201bc80029d35a98293a33b4677c890bd4c9fd6a5db262d049032 + checksums.txt: 3aadab8746b2cbd851db0c4a9572892c34bd81d285a87e9afe3e1d978342d48c deploy.sh: 187bf11702c83fdf08c22fbdd72a6b16c937314d87f85da747c4753cbe5d5de4 - recipe.yaml: a8600a45a5b8d36c018c171acc58d9a941f1e73d2b38636b98ce1db99ed973a0 + recipe.yaml: 8136aa4c61cccd87da0e47f492938f6c398009343b5d02022e290711eb0f9a2d rtx-pro-6000-eks-amazonlinux-inference: 001-agentgateway-crds/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 001-agentgateway-crds/install.sh: f0f864c4eb1f1d647af123953d5df5aa1fa03226e1f8c5feb724ea42d304eb3f @@ -5091,17 +5091,17 @@ rtx-pro-6000-eks-amazonlinux-inference: 016-nvsentinel/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 016-nvsentinel/install.sh: 290c0907c263ac3c1a976be76da13528e36fe7f1db79945783b6ac7da4588857 016-nvsentinel/upstream.env: 7fc07c3fb32a6e569a151e4c12f2834386b4c7026cda6cd1c86e29592ad8936e - 016-nvsentinel/values.yaml: ede1bf610929dcf019396356eec2e1a0bf9c1417c181595e86f03d6035fe27bb + 016-nvsentinel/values.yaml: 376d322d05dfab359cef13a99ded134b567f8100905a39caaf4af614efefa223 017-prometheus-adapter/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 017-prometheus-adapter/install.sh: 688ee4274fe06c001af6835d7965b5da33ce3fbeb724263622737d0bb88249e3 017-prometheus-adapter/upstream.env: 9ab918b32522baca13cd37bad7f420a206a19bb763b81041e174fffffe3da724 017-prometheus-adapter/values.yaml: 95231d7a46d650f96f844ca3734614271152a41f1230022c9d79960e44647c00 README.md: 238c986ba092e68c493ee63b5fea56635e5d491798dc70e932c30c588f112f26 UPGRADING.md: 4b285b6e5866a3582feef1afa6cc5906b79b275e94b0033d33f752c442d1256b - bundle-info.yaml: d1c6d720eae3399cdc561a2ae12ddeb87800b618c2a761e338158bef738de25b - checksums.txt: 2a8941fa87a4c6eb3103ddb7f4ca4fd3c3290fc7ca07dc0eb5603a0266d5b6de + bundle-info.yaml: 1748e7eb969c7040c99c752d1aa09d55cd9b94098d1397bf704ca1aaafd80fb4 + checksums.txt: 078e109510b3ea7a6e42a83efef8c26c4128191606ec61a44417f20153279c42 deploy.sh: 0248fabe5a8e4dbdbe60fc3f0cbf1aaab47a637402bec6eca1191fa601e8bfd6 - recipe.yaml: 21a6a532beafef18fd79ffc5b8d74e748b4871bca1f76626d836ec92031522b0 + recipe.yaml: a9f457663915ad74ab0eb597358ceaf83486b1afab8481a4d35d19ce26faab11 rtx-pro-6000-eks-ubuntu-inference-dynamo: 001-agentgateway-crds/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 001-agentgateway-crds/install.sh: f0f864c4eb1f1d647af123953d5df5aa1fa03226e1f8c5feb724ea42d304eb3f @@ -5183,17 +5183,17 @@ rtx-pro-6000-eks-ubuntu-inference-dynamo: 019-nvsentinel/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 019-nvsentinel/install.sh: 290c0907c263ac3c1a976be76da13528e36fe7f1db79945783b6ac7da4588857 019-nvsentinel/upstream.env: 7fc07c3fb32a6e569a151e4c12f2834386b4c7026cda6cd1c86e29592ad8936e - 019-nvsentinel/values.yaml: 5dbdd974de19bda092dff19de6a6fbb9489b7e7629abdeab2d1a8fe0bff69247 + 019-nvsentinel/values.yaml: 14400ae55936f06cc2025495b7d2efbb933e388a985abf2061b3b4fcbc36eeea 020-prometheus-adapter/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 020-prometheus-adapter/install.sh: 688ee4274fe06c001af6835d7965b5da33ce3fbeb724263622737d0bb88249e3 020-prometheus-adapter/upstream.env: 9ab918b32522baca13cd37bad7f420a206a19bb763b81041e174fffffe3da724 020-prometheus-adapter/values.yaml: 95231d7a46d650f96f844ca3734614271152a41f1230022c9d79960e44647c00 README.md: fb257a672854bc8bba49c383baf4d643d8ffb0f0c6f947949a1afe4244b14efb UPGRADING.md: a2aa0d1cd4b9e3a564ecf713c5fb05bb9e002f8e28cf51251d74d9f2962b2401 - bundle-info.yaml: 81393783d6a4651cd4337ffd994befba9d4f5228dc9cdf89f26c318cbb90c63b - checksums.txt: 67d0fc32f320d91600422527959b5c0cd2731a7db4c3fcb0292a12190e3aff91 + bundle-info.yaml: 134d3b6f25f023652420aaf40a47d6514f2456096198a2cdbd3955f3d108f2d3 + checksums.txt: 786a07fe9ab4fecccddca5ed372c10ac6205346bb36b74dcc223bf27775b6e8b deploy.sh: 5463ca8ed80e72d789bfb987637d85e2d3da5e4212a6a20e52b3640465c24da1 - recipe.yaml: bef93a23a40e660e2ba200cdf422020f8c8a313e72a7cb2856e1b2d10a44872c + recipe.yaml: 0930a55f000e06a24fccb410264cf2f67f44421bc62578679fb92ab4099cba16 rtx-pro-6000-eks-ubuntu-inference-nim: 001-agentgateway-crds/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 001-agentgateway-crds/install.sh: f0f864c4eb1f1d647af123953d5df5aa1fa03226e1f8c5feb724ea42d304eb3f @@ -5271,17 +5271,17 @@ rtx-pro-6000-eks-ubuntu-inference-nim: 018-nvsentinel/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 018-nvsentinel/install.sh: 290c0907c263ac3c1a976be76da13528e36fe7f1db79945783b6ac7da4588857 018-nvsentinel/upstream.env: 7fc07c3fb32a6e569a151e4c12f2834386b4c7026cda6cd1c86e29592ad8936e - 018-nvsentinel/values.yaml: 5dbdd974de19bda092dff19de6a6fbb9489b7e7629abdeab2d1a8fe0bff69247 + 018-nvsentinel/values.yaml: 14400ae55936f06cc2025495b7d2efbb933e388a985abf2061b3b4fcbc36eeea 019-prometheus-adapter/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 019-prometheus-adapter/install.sh: 688ee4274fe06c001af6835d7965b5da33ce3fbeb724263622737d0bb88249e3 019-prometheus-adapter/upstream.env: 9ab918b32522baca13cd37bad7f420a206a19bb763b81041e174fffffe3da724 019-prometheus-adapter/values.yaml: 95231d7a46d650f96f844ca3734614271152a41f1230022c9d79960e44647c00 README.md: 2cb21ec40386cb74dcd65d074c1a48894d22955e96d9a678f651bf998b751de1 UPGRADING.md: 4b285b6e5866a3582feef1afa6cc5906b79b275e94b0033d33f752c442d1256b - bundle-info.yaml: c0a162fa7bd6b16ddbbaa0e2822ae5d1e69ba74d05f7a18637fa3bb0a34f0353 - checksums.txt: ebae1a1d330c52bce578c94ccc7242d0bbca106f955385bdef19df841ebc731f + bundle-info.yaml: 047eb0069b62515ec657ec73785badebbcd8a8b68f53225dd5c087881b90d4ba + checksums.txt: 690ff29e9b4abe48f93b7666b009e47875f85b642ca8aebf28fcf670ae565cde deploy.sh: 93abef48d4cfd33ed295b33a924b01bb02e810595d7dd63575ae58c29c0122be - recipe.yaml: 751290881caacb33b8adba92d66ab79bfcc00c2266c410367feae7c8a0d6eb7b + recipe.yaml: f90e4dda8c0222115edd3529fb4280a67f89f214fa9b98054ace634fd5d123b2 rtx-pro-6000-eks-ubuntu-training-kubeflow: 001-aws-efa/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 001-aws-efa/install.sh: 00574b8307fa54f90f18cd999a2b175a75a547942fd18cb242bd83b7af836d5a @@ -5345,17 +5345,17 @@ rtx-pro-6000-eks-ubuntu-training-kubeflow: 015-nvsentinel/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 015-nvsentinel/install.sh: 290c0907c263ac3c1a976be76da13528e36fe7f1db79945783b6ac7da4588857 015-nvsentinel/upstream.env: 7fc07c3fb32a6e569a151e4c12f2834386b4c7026cda6cd1c86e29592ad8936e - 015-nvsentinel/values.yaml: 5dbdd974de19bda092dff19de6a6fbb9489b7e7629abdeab2d1a8fe0bff69247 + 015-nvsentinel/values.yaml: 14400ae55936f06cc2025495b7d2efbb933e388a985abf2061b3b4fcbc36eeea 016-prometheus-adapter/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 016-prometheus-adapter/install.sh: 688ee4274fe06c001af6835d7965b5da33ce3fbeb724263622737d0bb88249e3 016-prometheus-adapter/upstream.env: 9ab918b32522baca13cd37bad7f420a206a19bb763b81041e174fffffe3da724 016-prometheus-adapter/values.yaml: 95231d7a46d650f96f844ca3734614271152a41f1230022c9d79960e44647c00 README.md: 83365353dd1c2f39ece97eddb86fa9e1914d0218c921c799e997cd40f2789d28 UPGRADING.md: 4b285b6e5866a3582feef1afa6cc5906b79b275e94b0033d33f752c442d1256b - bundle-info.yaml: 4164edd6a3fe5601c3daa136ee66cdeb5eac004c0db2c7eb5514dd8f0d32af37 - checksums.txt: 3e4bbc5824c42dc90b1638ed325c83a2665f658ceae3df3c8838ebae7a0e0679 + bundle-info.yaml: 18a6becc603257de6adaec74ae12aa5ff4272922743d124436e6897b4ccab82b + checksums.txt: 3265110c7b999e3cf3c14194d09b10f244f09eb964b98baa94c28bab50fe41d8 deploy.sh: 949466ed3363eb59ea6ff5a61a29ca150215a2041c47323cada206dcf3cdc0a8 - recipe.yaml: a3d0100939e2ff181f27d7bddac7fd4029e4a7754b0b782110490f74a20016fb + recipe.yaml: d5227225338a26f8f24e80e0780f47f79a27e7a86b3834ac5a4f11d97ae45735 rtx-pro-6000-lke-ubuntu-inference: 001-agentgateway-crds/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 001-agentgateway-crds/install.sh: f0f864c4eb1f1d647af123953d5df5aa1fa03226e1f8c5feb724ea42d304eb3f @@ -5416,17 +5416,17 @@ rtx-pro-6000-lke-ubuntu-inference: 014-nvsentinel/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 014-nvsentinel/install.sh: 290c0907c263ac3c1a976be76da13528e36fe7f1db79945783b6ac7da4588857 014-nvsentinel/upstream.env: 7fc07c3fb32a6e569a151e4c12f2834386b4c7026cda6cd1c86e29592ad8936e - 014-nvsentinel/values.yaml: 5dbdd974de19bda092dff19de6a6fbb9489b7e7629abdeab2d1a8fe0bff69247 + 014-nvsentinel/values.yaml: 14400ae55936f06cc2025495b7d2efbb933e388a985abf2061b3b4fcbc36eeea 015-prometheus-adapter/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 015-prometheus-adapter/install.sh: 688ee4274fe06c001af6835d7965b5da33ce3fbeb724263622737d0bb88249e3 015-prometheus-adapter/upstream.env: 9ab918b32522baca13cd37bad7f420a206a19bb763b81041e174fffffe3da724 015-prometheus-adapter/values.yaml: 95231d7a46d650f96f844ca3734614271152a41f1230022c9d79960e44647c00 README.md: a0c30eea0b80ce21495ee4784037d21ab62c0ba59d44f0e54128c46f185f7937 UPGRADING.md: 4b285b6e5866a3582feef1afa6cc5906b79b275e94b0033d33f752c442d1256b - bundle-info.yaml: 4aa74c0e95ed7124bf4e3a51dcd1ca16735f09a6fa9c422f7147485ab10f351a - checksums.txt: 8af02dfe444ae592e92983ebfedf68c2a6af8dda46f275b188f93c9b8e81bd44 + bundle-info.yaml: ee934186c4bc883202739b70b295a06db89cd22cfe142fe4a5e6c1963669ee66 + checksums.txt: f5aee9fa80291f71071f855cb58855defd7034161994b2a3974ffcbcac93b530 deploy.sh: 68dbbe44b809aaa9dd14281f7b43cf6c55a7006e3931b03dff27b5bcb6744ca3 - recipe.yaml: c08bc55e5fbfc97ce3e4a7cc816d78fb1ff7d60b308ab7c2faf0c68451bd45da + recipe.yaml: 27775fd9623bf99e908e9230263e567443a89d363771b36add5f61b16c8a256b rtx-pro-6000-lke-ubuntu-training-kubeflow: 001-nodewright-operator/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 001-nodewright-operator/install.sh: 5bb299d13650bdb1baa19ea8047b33baa94eb03a48a8b5b0102f1ba533b392c2 @@ -5477,17 +5477,17 @@ rtx-pro-6000-lke-ubuntu-training-kubeflow: 012-nvsentinel/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 012-nvsentinel/install.sh: 290c0907c263ac3c1a976be76da13528e36fe7f1db79945783b6ac7da4588857 012-nvsentinel/upstream.env: 7fc07c3fb32a6e569a151e4c12f2834386b4c7026cda6cd1c86e29592ad8936e - 012-nvsentinel/values.yaml: 5dbdd974de19bda092dff19de6a6fbb9489b7e7629abdeab2d1a8fe0bff69247 + 012-nvsentinel/values.yaml: 14400ae55936f06cc2025495b7d2efbb933e388a985abf2061b3b4fcbc36eeea 013-prometheus-adapter/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 013-prometheus-adapter/install.sh: 688ee4274fe06c001af6835d7965b5da33ce3fbeb724263622737d0bb88249e3 013-prometheus-adapter/upstream.env: 9ab918b32522baca13cd37bad7f420a206a19bb763b81041e174fffffe3da724 013-prometheus-adapter/values.yaml: 95231d7a46d650f96f844ca3734614271152a41f1230022c9d79960e44647c00 README.md: f1c9dafba19d3731ebe94a53bc555ba70568fb3306e477d30f1eded6c800d727 UPGRADING.md: 4b285b6e5866a3582feef1afa6cc5906b79b275e94b0033d33f752c442d1256b - bundle-info.yaml: ada576e9bcf527f0a3053365177e562a3905e276cdff21a77f157f06cb4a8308 - checksums.txt: 5667de414a444d0cb709910f426fca10c41e8a17c73abf256fb8870ec401f789 + bundle-info.yaml: 41b747d3a3cef8f2f740be8839b7b0849e17aeb189b20b03270c62cd332d342a + checksums.txt: 7f819198ad28774ada637f45ba4fbd1156993a0679656e128ffe2d7a83039d11 deploy.sh: 79df4a5d6ddf25879ec925d7ba35a33815b9c929451c4cb88ed85fbe42e09b6d - recipe.yaml: 97b87b9c73cab168f88c53b7bd564491692af5345aa1673967556fb1a682964e + recipe.yaml: e85527518c7d7f89dcde0e95b0b93698d5a84e51a10b6160909512241cb2f2ea vr200-rke2-ubuntu-inference-dynamo: 001-agentgateway-crds/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 001-agentgateway-crds/install.sh: f0f864c4eb1f1d647af123953d5df5aa1fa03226e1f8c5feb724ea42d304eb3f @@ -5567,17 +5567,17 @@ vr200-rke2-ubuntu-inference-dynamo: 018-nvsentinel/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 018-nvsentinel/install.sh: 290c0907c263ac3c1a976be76da13528e36fe7f1db79945783b6ac7da4588857 018-nvsentinel/upstream.env: 7fc07c3fb32a6e569a151e4c12f2834386b4c7026cda6cd1c86e29592ad8936e - 018-nvsentinel/values.yaml: 8747e0ae149e20a60b74b16b991bb1ec11c3c3f55fb3684c960c1641192e085c + 018-nvsentinel/values.yaml: 58e4b4c7078de0520cfeea044d1110e84d532aba7d4a5dd7dafba554ec65921d 019-prometheus-adapter/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 019-prometheus-adapter/install.sh: 688ee4274fe06c001af6835d7965b5da33ce3fbeb724263622737d0bb88249e3 019-prometheus-adapter/upstream.env: 9ab918b32522baca13cd37bad7f420a206a19bb763b81041e174fffffe3da724 019-prometheus-adapter/values.yaml: 95231d7a46d650f96f844ca3734614271152a41f1230022c9d79960e44647c00 README.md: 97f6f01f1c67d7e41d4b0268f0ab69865142484ed276da57638e7fae3f1d8b6e UPGRADING.md: a2aa0d1cd4b9e3a564ecf713c5fb05bb9e002f8e28cf51251d74d9f2962b2401 - bundle-info.yaml: 9b5ea8299945807bd52793ed3d6baf33f6686d73f6f00eee8d4386a7972a8461 - checksums.txt: 0e24abfd8db553cc63d9cd30727187275260d70388d80a41940fcb24c8fdcc88 + bundle-info.yaml: 75355d10c32ce9c3768b7a05489fa1ae5594a8f7210c4fc350e2aa7bd56dc162 + checksums.txt: 933dc07db8bea6e2eab893de8f72924d73c67b40d040d75276505d3304c32cee deploy.sh: 841a92636bb0a65da6fa0162b67084f68f362fa0e41837b1c57e6560c10a7a63 - recipe.yaml: b3d717cefa88f7e6524446b26a6e4b9e12db3ee01738433cace2a46777ca1484 + recipe.yaml: cbc120691dcfea17e87e41a48a27e42ce3c087c70e079e5a5ac181eee5d78023 vr200-rke2-ubuntu-training-kubeflow: 001-dranet/Chart.yaml: afeb8fc7d2d11ad349641c63cb13d55d7528027571c2061f0754bd96fcef0e16 001-dranet/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 @@ -5638,14 +5638,14 @@ vr200-rke2-ubuntu-training-kubeflow: 014-nvsentinel/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 014-nvsentinel/install.sh: 290c0907c263ac3c1a976be76da13528e36fe7f1db79945783b6ac7da4588857 014-nvsentinel/upstream.env: 7fc07c3fb32a6e569a151e4c12f2834386b4c7026cda6cd1c86e29592ad8936e - 014-nvsentinel/values.yaml: 8747e0ae149e20a60b74b16b991bb1ec11c3c3f55fb3684c960c1641192e085c + 014-nvsentinel/values.yaml: 58e4b4c7078de0520cfeea044d1110e84d532aba7d4a5dd7dafba554ec65921d 015-prometheus-adapter/cluster-values.yaml: 49f560d440b0b58c4ec35c7f72fc187f23b94a69f5f7135482fa7070310bae26 015-prometheus-adapter/install.sh: 688ee4274fe06c001af6835d7965b5da33ce3fbeb724263622737d0bb88249e3 015-prometheus-adapter/upstream.env: 9ab918b32522baca13cd37bad7f420a206a19bb763b81041e174fffffe3da724 015-prometheus-adapter/values.yaml: 95231d7a46d650f96f844ca3734614271152a41f1230022c9d79960e44647c00 README.md: ce17de03ece4ecf23c7f88acc916ba8e64fd11b037fff274ead3017807c6d365 UPGRADING.md: 4b285b6e5866a3582feef1afa6cc5906b79b275e94b0033d33f752c442d1256b - bundle-info.yaml: 46486f451eb823516e2b45c48cee0f13584bc653ab045c31064585b47b1ad846 - checksums.txt: d4bb6b503f34f23f0a144612de00787ba19b89131b578b3d2f002680f0fd304e + bundle-info.yaml: 2b2adc8662080d8793f7fef5263e63f370945c7552a945534ef537b82276f3fa + checksums.txt: d7cdad08f6265e585615dae78e1d91e7375210c6e4e137835e4ca3be6b3350df deploy.sh: 50736743f1a6091c2dcb82baedfed99d4ae2f5bfe3325f7e7ab89995353f81bd - recipe.yaml: 2a9c328b4458e36cc417dde07a16bfa93d13e23e421223bc279286238f6a5511 + recipe.yaml: 3d2d9fb22e2623b9d3d3001099a9da578d45632691a80a5118461de0d63cf1b5