From f71813b056781fa53a850aac31da427792b5d417 Mon Sep 17 00:00:00 2001
From: Divya <divyasingh@tracebloc.io>
Date: Thu, 23 Jul 2026 17:47:59 +0530
Subject: [PATCH 1/9] feat(chart): one-time metadata backfill as a post-upgrade
 Helm hook
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Adds a post-upgrade/post-install hook Job that runs the ingestor image's
`tracebloc-backfill` entrypoint (data-ingestors #393, backend #1166/#1198) once
per chart upgrade. The auto-upgrade CronJob runs `helm upgrade` when a new chart
publishes, which fires the hook — so a client that auto-updates to this release
runs the backfill automatically, no operator step.

The sweep brings every already-registered pre-cutover dataset up to the
federated-alignment metadata shape (enriched schema + feature_stats, recomputed
from the persisted MySQL rows) and re-folds any competition built from them, so
old competitions/experiments keep working under the new backend alignment code.

- Reuses the same credentials Secret (CLIENT_ID/CLIENT_PASSWORD) and in-cluster
  MySQL (mysql-client) as jobs-manager, plus tracebloc.proxyEnv for corporate
  egress; resolves the same ingestor image (repo/tag/digest) the ingestion path
  spawns.
- Best-effort: retries a few times in-pod then exits 0 regardless, so a backfill
  hiccup never fails the auto-upgrade (which would strand the cluster on the old
  chart). Operators read the outcome via `kubectl logs`.
- One-time/idempotent: before-hook-creation replaces the prior run; the sweep
  skips datasets already carrying new-shape metadata and the upsert is a safe
  overwrite, so re-runs are cheap. Opt out with metadataBackfill.enabled=false.
- Hardened pod: non-root, no SA-token, read-only rootfs, caps dropped.

Tests: 7-case helm-unittest (renders by default, hook annotations, image
tag/digest modes, cred+MySQL env, hardened pod, disabled-by-flag). Full chart
suite green (274).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
---
 client/templates/metadata-backfill-hook.yaml  | 125 ++++++++++++++++++
 client/tests/metadata_backfill_hook_test.yaml |  91 +++++++++++++
 client/values.yaml                            |  26 ++++
 3 files changed, 242 insertions(+)
 create mode 100644 client/templates/metadata-backfill-hook.yaml
 create mode 100644 client/tests/metadata_backfill_hook_test.yaml

diff --git a/client/templates/metadata-backfill-hook.yaml b/client/templates/metadata-backfill-hook.yaml
new file mode 100644
index 00000000..98d875c4
--- /dev/null
+++ b/client/templates/metadata-backfill-hook.yaml
@@ -0,0 +1,125 @@
+{{- if .Values.metadataBackfill.enabled }}
+{{- $ing := default dict .Values.images.ingestor -}}
+{{- $repo := $ing.repository | default "ghcr.io/tracebloc/ingestor" -}}
+{{- $digest := $ing.digest | default "" -}}
+{{- $tag := $ing.tag | default "0.5" -}}
+# One-time pre-cutover metadata backfill (data-ingestors #393, backend
+# #1166/#1198). Runs the ingestor image's `tracebloc-backfill` entrypoint, which
+# sweeps THIS client's already-registered datasets: GET each from the backend,
+# recompute the enriched {schema, meta_data} from the persisted MySQL rows, and
+# POST it back (which also re-folds any competition built from them). Datasets
+# ingested before the federated-alignment cutover otherwise carry no enriched
+# schema / feature_stats, so the backend's combine-time alignment checks and the
+# edge metadata API can't reconcile them — old competitions/experiments that
+# rely on that metadata would degrade under the new code.
+#
+# Trigger: a Helm post-upgrade/post-install hook. The auto-upgrade CronJob runs
+# `helm upgrade` when a new chart is published, which fires this hook — so a
+# client that auto-updates to this release runs the backfill once, automatically,
+# with no operator step.
+#
+# One-time / idempotent: `before-hook-creation` replaces the prior hook Job on
+# each upgrade; the sweep skips datasets already carrying new-shape metadata and
+# the backend upsert is a safe overwrite, so re-runs are cheap no-ops. It is
+# deliberately BEST-EFFORT — the entrypoint's exit code is swallowed so a
+# backfill hiccup can never fail the `helm upgrade` (which would strand the
+# cluster on the old chart). Operators read the outcome via `kubectl logs`.
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: {{ .Release.Name }}-metadata-backfill
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "tracebloc.labels" . | nindent 4 }}
+    app.kubernetes.io/component: metadata-backfill
+  annotations:
+    helm.sh/hook: post-upgrade,post-install
+    # Run late, after the release's core resources (MySQL, jobs-manager) are in
+    # place — the sweep talks to in-cluster MySQL and the backend API.
+    helm.sh/hook-weight: "5"
+    helm.sh/hook-delete-policy: before-hook-creation
+spec:
+  # Best-effort: the container always exits 0 (see args), so backoff/deadline
+  # only bound a genuinely stuck pod rather than gating correctness.
+  backoffLimit: 1
+  activeDeadlineSeconds: {{ .Values.metadataBackfill.deadlineSeconds | default 1800 }}
+  ttlSecondsAfterFinished: 86400
+  template:
+    metadata:
+      labels:
+        {{- include "tracebloc.selectorLabels" . | nindent 8 }}
+        app.kubernetes.io/component: metadata-backfill
+    spec:
+      restartPolicy: Never
+      # The sweep never calls the Kubernetes API — it only reaches MySQL and the
+      # backend — so no ServiceAccount token is mounted.
+      automountServiceAccountToken: false
+      {{- if include "tracebloc.useImagePullSecrets" . }}
+      imagePullSecrets:
+        - name: {{ include "tracebloc.registrySecretName" . }}
+      {{- end }}
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 65534
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: metadata-backfill
+          image: {{ if $digest }}{{ printf "%s@%s" $repo $digest | quote }}{{ else }}{{ printf "%s:%s" $repo $tag | quote }}{{ end }}
+          imagePullPolicy: {{ if $digest }}IfNotPresent{{ else }}Always{{ end }}
+          # Run the backfill runner, but never fail the release on its exit code:
+          # a few in-pod retries cover a transient MySQL/network blip, then we
+          # exit 0 regardless so a backfill problem can't block the auto-upgrade.
+          command: ["/bin/sh", "-c"]
+          args:
+            - |
+              set -u
+              for attempt in 1 2 3; do
+                echo "[metadata-backfill] attempt $attempt"
+                if tracebloc-backfill; then
+                  echo "[metadata-backfill] completed"
+                  exit 0
+                fi
+                echo "[metadata-backfill] attempt $attempt failed; retrying in 15s"
+                sleep 15
+              done
+              echo "[metadata-backfill] still failing after retries — leaving pre-cutover metadata for the next upgrade (non-fatal)"
+              exit 0
+          env:
+            {{- include "tracebloc.proxyEnv" . | nindent 12 }}
+            # Backend auth: the ingestor's APIClient mints a token from these
+            # (same credentials Secret jobs-manager uses). BACKEND_TOKEN would be
+            # preferred, but the chart holds no static backend token — jobs-manager
+            # mints its own at runtime — so the backfill uses the client creds.
+            - name: CLIENT_ID
+              valueFrom:
+                secretKeyRef:
+                  name: {{ include "tracebloc.secretName" . }}
+                  key: CLIENT_ID
+            - name: CLIENT_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: {{ include "tracebloc.secretName" . }}
+                  key: CLIENT_PASSWORD
+            # In-cluster MySQL + backend endpoint, mirroring the jobs-manager /
+            # ingestion-Job env so the ingestor's Config resolves the same store.
+            - name: MYSQL_HOST
+              value: "mysql-client"
+            - name: CLIENT_ENV
+              value: {{ .Values.env.CLIENT_ENV | default "prod" | quote }}
+            - name: HOME
+              value: "/tmp"
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop: ["ALL"]
+          resources:
+            {{- toYaml .Values.metadataBackfill.resources | nindent 12 }}
+          volumeMounts:
+            - name: tmp
+              mountPath: /tmp
+      volumes:
+        - name: tmp
+          emptyDir: {}
+{{- end }}
diff --git a/client/tests/metadata_backfill_hook_test.yaml b/client/tests/metadata_backfill_hook_test.yaml
new file mode 100644
index 00000000..a205cff9
--- /dev/null
+++ b/client/tests/metadata_backfill_hook_test.yaml
@@ -0,0 +1,91 @@
+suite: metadata-backfill post-upgrade hook Job
+templates:
+  - templates/metadata-backfill-hook.yaml
+release:
+  name: stg
+  namespace: tracebloc
+set:
+  clientId: "test-id"
+  clientPassword: "test"
+tests:
+  - it: renders a hook Job by default
+    asserts:
+      - hasDocuments:
+          count: 1
+      - isKind:
+          of: Job
+      - equal:
+          path: metadata.name
+          value: stg-metadata-backfill
+
+  - it: is a post-upgrade/post-install hook that replaces its prior run
+    asserts:
+      - equal:
+          path: metadata.annotations["helm.sh/hook"]
+          value: post-upgrade,post-install
+      - equal:
+          path: metadata.annotations["helm.sh/hook-delete-policy"]
+          value: before-hook-creation
+
+  - it: runs the backfill entrypoint on the ingestor image, best-effort (exit 0)
+    set:
+      images.ingestor.tag: "0.9"
+    asserts:
+      - equal:
+          path: spec.template.spec.containers[0].image
+          value: ghcr.io/tracebloc/ingestor:0.9
+      - equal:
+          path: spec.template.spec.containers[0].imagePullPolicy
+          value: Always
+      - matchRegex:
+          path: spec.template.spec.containers[0].args[0]
+          pattern: tracebloc-backfill
+
+  - it: pins the ingestor image by digest when one is set
+    set:
+      images.ingestor.digest: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+    asserts:
+      - equal:
+          path: spec.template.spec.containers[0].image
+          value: ghcr.io/tracebloc/ingestor@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
+      - equal:
+          path: spec.template.spec.containers[0].imagePullPolicy
+          value: IfNotPresent
+
+  - it: sources backend creds from the release Secret and points at in-cluster MySQL
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: CLIENT_ID
+            valueFrom:
+              secretKeyRef:
+                name: stg-secrets
+                key: CLIENT_ID
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: MYSQL_HOST
+            value: "mysql-client"
+
+  - it: hardened pod — non-root, no SA token, read-only rootfs, caps dropped
+    asserts:
+      - equal:
+          path: spec.template.spec.securityContext.runAsNonRoot
+          value: true
+      - equal:
+          path: spec.template.spec.automountServiceAccountToken
+          value: false
+      - equal:
+          path: spec.template.spec.containers[0].securityContext.readOnlyRootFilesystem
+          value: true
+      - equal:
+          path: spec.template.spec.containers[0].securityContext.allowPrivilegeEscalation
+          value: false
+
+  - it: renders nothing when metadataBackfill.enabled=false
+    set:
+      metadataBackfill.enabled: false
+    asserts:
+      - hasDocuments:
+          count: 0
diff --git a/client/values.yaml b/client/values.yaml
index 22f3f378..c52d91d1 100644
--- a/client/values.yaml
+++ b/client/values.yaml
@@ -544,6 +544,32 @@ autoUpgrade:
       cpu: "500m"
       memory: "256Mi"
 
+# -- One-time pre-cutover metadata backfill (data-ingestors #393, backend
+# #1166/#1198). A post-upgrade/post-install Helm hook Job that runs the ingestor
+# image's `tracebloc-backfill` entrypoint once per chart upgrade. It sweeps this
+# client's already-registered datasets and brings their global-metadata up to
+# the federated-alignment shape (enriched schema + feature_stats), recomputed
+# from the persisted MySQL rows — so old competitions/experiments that predate
+# the cutover keep working under the new backend alignment code.
+#
+# Safe to leave enabled: the sweep skips datasets already carrying new-shape
+# metadata and the backend upsert is idempotent, so after the fleet has
+# backfilled once it is a cheap no-op on every subsequent upgrade. Set
+# `enabled: false` to opt a cluster out (e.g. one with no pre-cutover data).
+metadataBackfill:
+  enabled: true
+  # Upper bound on a single run before the pod is killed (best-effort — a
+  # timeout does NOT fail the upgrade; it just leaves the backlog for next time).
+  deadlineSeconds: 1800
+  # Small: the sweep is I/O-bound on MySQL aggregates + backend calls.
+  resources:
+    requests:
+      cpu: "250m"
+      memory: "512Mi"
+    limits:
+      cpu: "1"
+      memory: "2Gi"
+
 # -- Image-refresh CronJob (closes tracebloc/client#154).
 # Auto-restarts the jobs-manager Deployment when a new image digest is
 # published to Docker Hub under the floating CLIENT_ENV tag. Without

From 8a5239d75c05130b38a1fce4af7e124e5e87f136 Mon Sep 17 00:00:00 2001
From: Divya <divyasingh@tracebloc.io>
Date: Thu, 23 Jul 2026 17:53:53 +0530
Subject: [PATCH 2/9] fix(chart): nil-guard metadataBackfill for --reuse-values
 upgrades
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The auto-upgrade E2E upgrades from the last published release via --reuse-values,
whose stored values predate the metadataBackfill key — so .Values.metadataBackfill
is nil and `.enabled` failed template rendering ("nil pointer evaluating
interface {}.enabled"), the same hazard the images.ingestor keys already guard.

Guard the whole block with `$mb := default (dict) .Values.metadataBackfill` and
read deadlineSeconds/resources through $mb. Absent key => feature off on the
--reuse-values path; the production auto-upgrade uses --reset-then-reuse-values,
which layers in the new default (enabled: true), so the backfill still runs on
the real upgrade path.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
---
 client/templates/metadata-backfill-hook.yaml | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/client/templates/metadata-backfill-hook.yaml b/client/templates/metadata-backfill-hook.yaml
index 98d875c4..6c98167e 100644
--- a/client/templates/metadata-backfill-hook.yaml
+++ b/client/templates/metadata-backfill-hook.yaml
@@ -1,4 +1,12 @@
-{{- if .Values.metadataBackfill.enabled }}
+{{- /* Nil-guard the whole block: an `--reuse-values` upgrade from a release
+       that predates the `metadataBackfill` key stores no such value, so
+       `.Values.metadataBackfill` is nil and a bare `.enabled` access would fail
+       template rendering (same hazard the images.ingestor keys guard against).
+       Absent key => feature off on that path; the production auto-upgrade uses
+       `--reset-then-reuse-values`, which layers in the new default (enabled:
+       true), so the backfill still runs on the real upgrade path. */ -}}
+{{- $mb := default (dict) .Values.metadataBackfill -}}
+{{- if $mb.enabled }}
 {{- $ing := default dict .Values.images.ingestor -}}
 {{- $repo := $ing.repository | default "ghcr.io/tracebloc/ingestor" -}}
 {{- $digest := $ing.digest | default "" -}}
@@ -42,7 +50,7 @@ spec:
   # Best-effort: the container always exits 0 (see args), so backoff/deadline
   # only bound a genuinely stuck pod rather than gating correctness.
   backoffLimit: 1
-  activeDeadlineSeconds: {{ .Values.metadataBackfill.deadlineSeconds | default 1800 }}
+  activeDeadlineSeconds: {{ $mb.deadlineSeconds | default 1800 }}
   ttlSecondsAfterFinished: 86400
   template:
     metadata:
@@ -115,7 +123,7 @@ spec:
             capabilities:
               drop: ["ALL"]
           resources:
-            {{- toYaml .Values.metadataBackfill.resources | nindent 12 }}
+            {{- toYaml ($mb.resources | default dict) | nindent 12 }}
           volumeMounts:
             - name: tmp
               mountPath: /tmp

From d17fc7a526f9b082a19656f9363be9f697365268 Mon Sep 17 00:00:00 2001
From: Divya <divyasingh@tracebloc.io>
Date: Thu, 23 Jul 2026 18:20:21 +0530
Subject: [PATCH 3/9] fix(chart): stop the backfill hook from failing the
 auto-upgrade (Bugbot #380)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Two Bugbot findings on the metadata-backfill hook:

- Medium: the ingestor-tag fallback was "0.5", but values.yaml and jobs-manager's
  INGESTOR_IMAGE_TAG default to "0.7"; on the fallback path the Job would pull an
  ingestor predating the tracebloc-backfill entrypoint and no-op. Aligned to "0.7".

- High: swallowing the entrypoint exit code didn't make the hook best-effort —
  activeDeadlineSeconds (1800, and > autoUpgrade.timeout 10m), image pull/OOM, or
  Helm's wait timeout still fail the Job and thus `helm upgrade`. Now the
  container bounds ITS OWN runtime with `timeout $backfillTimeoutSeconds`
  (default 300s, under the 10m Helm timeout) and ALWAYS exits 0, so the pod
  completes successfully regardless of how the sweep goes. Removed
  activeDeadlineSeconds (a deadline kill = failed Job = failed upgrade) and set
  backoffLimit 0. A run that hits the bound stops cleanly and resumes on the next
  upgrade (the sweep is idempotent / skips already-backfilled datasets).

Residual: a container that never starts (image pull) or is OOM-killed can still
fail the hook; those are rare, retried on the next hourly auto-upgrade, and
mitigated by the generous memory limit + floating-tag Always pull.

Test: asserts no activeDeadlineSeconds, backoffLimit 0, and the timeout-wrapped
always-exit-0 command.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
---
 client/templates/metadata-backfill-hook.yaml  | 44 +++++++++++--------
 client/tests/metadata_backfill_hook_test.yaml | 16 +++++++
 client/values.yaml                            |  9 ++--
 3 files changed, 48 insertions(+), 21 deletions(-)

diff --git a/client/templates/metadata-backfill-hook.yaml b/client/templates/metadata-backfill-hook.yaml
index 6c98167e..7ab07f40 100644
--- a/client/templates/metadata-backfill-hook.yaml
+++ b/client/templates/metadata-backfill-hook.yaml
@@ -10,7 +10,12 @@
 {{- $ing := default dict .Values.images.ingestor -}}
 {{- $repo := $ing.repository | default "ghcr.io/tracebloc/ingestor" -}}
 {{- $digest := $ing.digest | default "" -}}
-{{- $tag := $ing.tag | default "0.5" -}}
+{{- /* Fallback tag matches values.yaml images.ingestor.tag and jobs-manager's
+       INGESTOR_IMAGE_TAG default — an older tag here would pull an ingestor that
+       predates the `tracebloc-backfill` entrypoint and the sweep would no-op.
+       (bugbot) The default only applies on a --reuse-values upgrade whose stored
+       values predate the images.ingestor key; keep it in lockstep with those. */ -}}
+{{- $tag := $ing.tag | default "0.7" -}}
 # One-time pre-cutover metadata backfill (data-ingestors #393, backend
 # #1166/#1198). Runs the ingestor image's `tracebloc-backfill` entrypoint, which
 # sweeps THIS client's already-registered datasets: GET each from the backend,
@@ -47,10 +52,13 @@ metadata:
     helm.sh/hook-weight: "5"
     helm.sh/hook-delete-policy: before-hook-creation
 spec:
-  # Best-effort: the container always exits 0 (see args), so backoff/deadline
-  # only bound a genuinely stuck pod rather than gating correctness.
-  backoffLimit: 1
-  activeDeadlineSeconds: {{ $mb.deadlineSeconds | default 1800 }}
+  # No activeDeadlineSeconds: a deadline kill marks the Job FAILED, which fails
+  # the `helm upgrade` (and thus the auto-upgrade). Instead the container bounds
+  # its OWN runtime with `timeout` and always exits 0 (see args), so the pod
+  # completes successfully well within Helm's --timeout regardless of how the
+  # sweep goes. backoffLimit 0 so a genuine infra failure (image pull / OOM)
+  # fails fast rather than retry-looping toward Helm's timeout. (bugbot)
+  backoffLimit: 0
   ttlSecondsAfterFinished: 86400
   template:
     metadata:
@@ -75,23 +83,23 @@ spec:
         - name: metadata-backfill
           image: {{ if $digest }}{{ printf "%s@%s" $repo $digest | quote }}{{ else }}{{ printf "%s:%s" $repo $tag | quote }}{{ end }}
           imagePullPolicy: {{ if $digest }}IfNotPresent{{ else }}Always{{ end }}
-          # Run the backfill runner, but never fail the release on its exit code:
-          # a few in-pod retries cover a transient MySQL/network blip, then we
-          # exit 0 regardless so a backfill problem can't block the auto-upgrade.
+          # Bound the sweep with `timeout` and ALWAYS exit 0, so neither a slow
+          # run nor a backfill error can fail the hook (and thus the upgrade).
+          # backfillTimeoutSeconds must stay under autoUpgrade.timeout (10m) so
+          # the pod finishes before Helm's --timeout; a run that hits the bound
+          # is killed cleanly and resumes on the next upgrade (the sweep is
+          # idempotent and skips datasets already backfilled). (bugbot)
           command: ["/bin/sh", "-c"]
           args:
             - |
               set -u
-              for attempt in 1 2 3; do
-                echo "[metadata-backfill] attempt $attempt"
-                if tracebloc-backfill; then
-                  echo "[metadata-backfill] completed"
-                  exit 0
-                fi
-                echo "[metadata-backfill] attempt $attempt failed; retrying in 15s"
-                sleep 15
-              done
-              echo "[metadata-backfill] still failing after retries — leaving pre-cutover metadata for the next upgrade (non-fatal)"
+              TIMEOUT={{ $mb.backfillTimeoutSeconds | default 300 }}
+              echo "[metadata-backfill] running (timeout ${TIMEOUT}s)"
+              if timeout "${TIMEOUT}" tracebloc-backfill; then
+                echo "[metadata-backfill] completed"
+              else
+                echo "[metadata-backfill] did not finish (exit $?); remaining tables resume on the next upgrade (non-fatal)"
+              fi
               exit 0
           env:
             {{- include "tracebloc.proxyEnv" . | nindent 12 }}
diff --git a/client/tests/metadata_backfill_hook_test.yaml b/client/tests/metadata_backfill_hook_test.yaml
index a205cff9..c6ad1119 100644
--- a/client/tests/metadata_backfill_hook_test.yaml
+++ b/client/tests/metadata_backfill_hook_test.yaml
@@ -41,6 +41,22 @@ tests:
           path: spec.template.spec.containers[0].args[0]
           pattern: tracebloc-backfill
 
+  - it: is best-effort — self-bounded by timeout, always exits 0, no Job deadline
+    asserts:
+      # No activeDeadlineSeconds: a deadline kill would FAIL the Job and thus the
+      # upgrade; the container bounds itself and exits 0 instead.
+      - notExists:
+          path: spec.activeDeadlineSeconds
+      - equal:
+          path: spec.backoffLimit
+          value: 0
+      - matchRegex:
+          path: spec.template.spec.containers[0].args[0]
+          pattern: 'timeout "\$\{TIMEOUT\}" tracebloc-backfill'
+      - matchRegex:
+          path: spec.template.spec.containers[0].args[0]
+          pattern: exit 0
+
   - it: pins the ingestor image by digest when one is set
     set:
       images.ingestor.digest: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
diff --git a/client/values.yaml b/client/values.yaml
index c52d91d1..2fdf33a2 100644
--- a/client/values.yaml
+++ b/client/values.yaml
@@ -558,9 +558,12 @@ autoUpgrade:
 # `enabled: false` to opt a cluster out (e.g. one with no pre-cutover data).
 metadataBackfill:
   enabled: true
-  # Upper bound on a single run before the pod is killed (best-effort — a
-  # timeout does NOT fail the upgrade; it just leaves the backlog for next time).
-  deadlineSeconds: 1800
+  # Wall-clock bound the sweep gives ITSELF (`timeout` in the hook), after which
+  # it stops and exits 0 cleanly — the pod never fails, so the upgrade never
+  # fails. Remaining tables resume on the next upgrade (the sweep is idempotent
+  # and skips already-backfilled datasets). MUST stay under `autoUpgrade.timeout`
+  # (10m) so the pod finishes before Helm's --timeout gives up on the hook.
+  backfillTimeoutSeconds: 300
   # Small: the sweep is I/O-bound on MySQL aggregates + backend calls.
   resources:
     requests:

From 22ef7dd8cb4d901f4def82206b9b28776fa1d28f Mon Sep 17 00:00:00 2001
From: Divya <divyasingh@tracebloc.io>
Date: Thu, 23 Jul 2026 18:32:08 +0530
Subject: [PATCH 4/9] fix(chart): backfill timeout headroom + values.schema
 entry (Bugbot #380)

- Medium: backfillTimeoutSeconds was 300s = Helm's default --timeout (5m) that a
  manual `helm install/upgrade` uses. Since Helm counts image-pull + scheduling
  against its hook wait (outside the in-container `timeout`), a 300s sweep could
  push a manual upgrade past 5m and mark it failed. Lowered to 180s, leaving ~2m
  headroom under the 5m default (and far more under the auto-upgrade's 10m).
- Low: schematized the new metadataBackfill block in values.schema.json
  (enabled: bool, backfillTimeoutSeconds: int>=1, resources), matching the
  autoUpgrade/imageRefresh blocks so helm type-checks it.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
---
 client/templates/metadata-backfill-hook.yaml |  2 +-
 client/values.schema.json                    | 47 ++++++++++++++++++++
 client/values.yaml                           | 11 +++--
 3 files changed, 56 insertions(+), 4 deletions(-)

diff --git a/client/templates/metadata-backfill-hook.yaml b/client/templates/metadata-backfill-hook.yaml
index 7ab07f40..fc55877c 100644
--- a/client/templates/metadata-backfill-hook.yaml
+++ b/client/templates/metadata-backfill-hook.yaml
@@ -93,7 +93,7 @@ spec:
           args:
             - |
               set -u
-              TIMEOUT={{ $mb.backfillTimeoutSeconds | default 300 }}
+              TIMEOUT={{ $mb.backfillTimeoutSeconds | default 180 }}
               echo "[metadata-backfill] running (timeout ${TIMEOUT}s)"
               if timeout "${TIMEOUT}" tracebloc-backfill; then
                 echo "[metadata-backfill] completed"
diff --git a/client/values.schema.json b/client/values.schema.json
index 5ce28bfe..8e6b34b0 100644
--- a/client/values.schema.json
+++ b/client/values.schema.json
@@ -628,6 +628,53 @@
         }
       }
     },
+    "metadataBackfill": {
+      "type": "object",
+      "description": "One-time pre-cutover metadata backfill (data-ingestors #393, backend #1166/#1198). A post-upgrade/post-install hook Job runs the ingestor image's `tracebloc-backfill` entrypoint to bring already-registered datasets up to the federated-alignment metadata shape. Idempotent (skips already-backfilled datasets) and best-effort (never fails the upgrade). Disable to opt a cluster out.",
+      "properties": {
+        "enabled": {
+          "type": "boolean",
+          "default": true,
+          "description": "When false, the backfill hook Job is not rendered."
+        },
+        "backfillTimeoutSeconds": {
+          "type": "integer",
+          "minimum": 1,
+          "description": "Wall-clock bound the sweep gives itself before stopping and exiting 0. Must stay under Helm's --timeout (5m for a manual `helm install/upgrade`) with headroom for image pull + pod scheduling."
+        },
+        "resources": {
+          "type": "object",
+          "properties": {
+            "requests": {
+              "type": "object",
+              "properties": {
+                "cpu": {
+                  "type": "string",
+                  "pattern": "^[0-9]+m?$"
+                },
+                "memory": {
+                  "type": "string",
+                  "pattern": "^[0-9]+(Ki|Mi|Gi|Ti)$"
+                }
+              }
+            },
+            "limits": {
+              "type": "object",
+              "properties": {
+                "cpu": {
+                  "type": "string",
+                  "pattern": "^[0-9]+m?$"
+                },
+                "memory": {
+                  "type": "string",
+                  "pattern": "^[0-9]+(Ki|Mi|Gi|Ti)$"
+                }
+              }
+            }
+          }
+        }
+      }
+    },
     "imageRefresh": {
       "type": "object",
       "description": "Image-refresh CronJob (issue tracebloc/client#154). Polls Docker Hub for jobs-manager and pods-monitor digests under the floating CLIENT_ENV tag and rolls the deployment when a new image is published. Disable to keep the running image in place until manual restart.",
diff --git a/client/values.yaml b/client/values.yaml
index 2fdf33a2..5101a6b2 100644
--- a/client/values.yaml
+++ b/client/values.yaml
@@ -561,9 +561,14 @@ metadataBackfill:
   # Wall-clock bound the sweep gives ITSELF (`timeout` in the hook), after which
   # it stops and exits 0 cleanly — the pod never fails, so the upgrade never
   # fails. Remaining tables resume on the next upgrade (the sweep is idempotent
-  # and skips already-backfilled datasets). MUST stay under `autoUpgrade.timeout`
-  # (10m) so the pod finishes before Helm's --timeout gives up on the hook.
-  backfillTimeoutSeconds: 300
+  # and skips already-backfilled datasets).
+  #
+  # MUST stay comfortably under Helm's --timeout with headroom for image pull +
+  # pod scheduling (which Helm counts against its hook wait, but the in-container
+  # `timeout` does not). The binding case is a MANUAL `helm install/upgrade`,
+  # whose default --timeout is 5m — not the auto-upgrade CronJob's 10m. 180s
+  # leaves ~2m of that 5m for pull/scheduling. (bugbot)
+  backfillTimeoutSeconds: 180
   # Small: the sweep is I/O-bound on MySQL aggregates + backend calls.
   resources:
     requests:

From 35038c9be0d04663a55b161d95d41624019cca94 Mon Sep 17 00:00:00 2001
From: Divya <divyasingh@tracebloc.io>
Date: Thu, 23 Jul 2026 18:39:59 +0530
Subject: [PATCH 5/9] fix(chart): hard-stop the backfill timeout with -k
 (Bugbot #380)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

`timeout "$TIMEOUT" tracebloc-backfill` only sends SIGTERM; if the entrypoint
ignores it or is wedged, timeout waits indefinitely, the wrapper never reaches
exit 0, and the hook hangs until Helm's --timeout — failing the upgrade, the
very mode removing activeDeadlineSeconds was meant to avoid. Add `-k 30` so
timeout escalates to SIGKILL 30s after the SIGTERM. Worst-case wrapper runtime
is TIMEOUT+30s (210s at the default), still well under the 5m manual --timeout.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
---
 client/templates/metadata-backfill-hook.yaml  | 6 +++++-
 client/tests/metadata_backfill_hook_test.yaml | 2 +-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/client/templates/metadata-backfill-hook.yaml b/client/templates/metadata-backfill-hook.yaml
index fc55877c..2f29f854 100644
--- a/client/templates/metadata-backfill-hook.yaml
+++ b/client/templates/metadata-backfill-hook.yaml
@@ -95,7 +95,11 @@ spec:
               set -u
               TIMEOUT={{ $mb.backfillTimeoutSeconds | default 180 }}
               echo "[metadata-backfill] running (timeout ${TIMEOUT}s)"
-              if timeout "${TIMEOUT}" tracebloc-backfill; then
+              # -k 30: if the backfill ignores the SIGTERM at TIMEOUT (or is
+              # wedged), escalate to SIGKILL 30s later so `timeout` can't wait
+              # forever and hang the hook past Helm's --timeout. Worst-case
+              # wrapper runtime is TIMEOUT+30s, still well under the 5m default. (bugbot)
+              if timeout -k 30 "${TIMEOUT}" tracebloc-backfill; then
                 echo "[metadata-backfill] completed"
               else
                 echo "[metadata-backfill] did not finish (exit $?); remaining tables resume on the next upgrade (non-fatal)"
diff --git a/client/tests/metadata_backfill_hook_test.yaml b/client/tests/metadata_backfill_hook_test.yaml
index c6ad1119..ecf99df3 100644
--- a/client/tests/metadata_backfill_hook_test.yaml
+++ b/client/tests/metadata_backfill_hook_test.yaml
@@ -52,7 +52,7 @@ tests:
           value: 0
       - matchRegex:
           path: spec.template.spec.containers[0].args[0]
-          pattern: 'timeout "\$\{TIMEOUT\}" tracebloc-backfill'
+          pattern: 'timeout -k 30 "\$\{TIMEOUT\}" tracebloc-backfill'
       - matchRegex:
           path: spec.template.spec.containers[0].args[0]
           pattern: exit 0

From d1399aa72ad584be7d8979600fc7dbf9ae007fa7 Mon Sep 17 00:00:00 2001
From: Divya <divyasingh@tracebloc.io>
Date: Thu, 23 Jul 2026 18:45:16 +0530
Subject: [PATCH 6/9] fix(chart): widen backfill Helm headroom to 120s (Bugbot
 #380)

180s sweep + 30s SIGKILL grace = 210s worst-case in-container, leaving only ~90s
of Helm's default 5m --timeout for a cold ghcr.io pull + scheduling on a manual
install/upgrade. Lower backfillTimeoutSeconds to 120s so worst-case in-container
is 150s, leaving ~150s for pull/scheduling under the 5m default (and ~450s under
the auto-upgrade's 10m). The sweep is idempotent/incremental, so a smaller
per-run budget just spreads a large backlog across upgrades.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
---
 client/templates/metadata-backfill-hook.yaml | 2 +-
 client/values.yaml                           | 8 +++++---
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/client/templates/metadata-backfill-hook.yaml b/client/templates/metadata-backfill-hook.yaml
index 2f29f854..023edaa9 100644
--- a/client/templates/metadata-backfill-hook.yaml
+++ b/client/templates/metadata-backfill-hook.yaml
@@ -93,7 +93,7 @@ spec:
           args:
             - |
               set -u
-              TIMEOUT={{ $mb.backfillTimeoutSeconds | default 180 }}
+              TIMEOUT={{ $mb.backfillTimeoutSeconds | default 120 }}
               echo "[metadata-backfill] running (timeout ${TIMEOUT}s)"
               # -k 30: if the backfill ignores the SIGTERM at TIMEOUT (or is
               # wedged), escalate to SIGKILL 30s later so `timeout` can't wait
diff --git a/client/values.yaml b/client/values.yaml
index 5101a6b2..8a40668f 100644
--- a/client/values.yaml
+++ b/client/values.yaml
@@ -566,9 +566,11 @@ metadataBackfill:
   # MUST stay comfortably under Helm's --timeout with headroom for image pull +
   # pod scheduling (which Helm counts against its hook wait, but the in-container
   # `timeout` does not). The binding case is a MANUAL `helm install/upgrade`,
-  # whose default --timeout is 5m — not the auto-upgrade CronJob's 10m. 180s
-  # leaves ~2m of that 5m for pull/scheduling. (bugbot)
-  backfillTimeoutSeconds: 180
+  # whose default --timeout is 5m — not the auto-upgrade CronJob's 10m. Budget:
+  # worst-case in-container = this + the 30s SIGKILL grace, so 120 -> 150s, which
+  # leaves ~150s of the 5m default for a cold ghcr.io pull + scheduling (and
+  # ~450s under the auto-upgrade's 10m). (bugbot)
+  backfillTimeoutSeconds: 120
   # Small: the sweep is I/O-bound on MySQL aggregates + backend calls.
   resources:
     requests:

From 046a516d7f9934a763fc69c6aaa9b5cd692959aa Mon Sep 17 00:00:00 2001
From: Divya <divyasingh@tracebloc.io>
Date: Thu, 23 Jul 2026 19:09:01 +0530
Subject: [PATCH 7/9] fix(chart): backstop ImagePullBackOff with
 activeDeadlineSeconds (Bugbot #380)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

backoffLimit 0 doesn't bound an ImagePullBackOff pod — it stays Pending (never a
failed attempt), so with no activeDeadlineSeconds the Job neither completes nor
fails and Helm waits its full --timeout before failing the release. The
in-container timeout/exit-0 only apply once the container starts, so they don't
cover a pod that never runs.

Restore activeDeadlineSeconds as a Job-level backstop = backfillTimeoutSeconds +
120s (240s at the default), well under Helm's 5m default, so a stuck/Pending pod
(pull failure) fails FAST and the hourly auto-upgrade retries instead of hanging
the whole wait. It does not gate a healthy run: the in-container `timeout` exits
~150s in, before the deadline. A genuinely un-pullable image still fails the
upgrade (irreducible for a synchronous hook running the ingestor image) but now
quickly and self-healing once the registry recovers.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
---
 client/templates/metadata-backfill-hook.yaml  | 20 +++++++++++++------
 client/tests/metadata_backfill_hook_test.yaml | 10 ++++++----
 2 files changed, 20 insertions(+), 10 deletions(-)

diff --git a/client/templates/metadata-backfill-hook.yaml b/client/templates/metadata-backfill-hook.yaml
index 023edaa9..24354f59 100644
--- a/client/templates/metadata-backfill-hook.yaml
+++ b/client/templates/metadata-backfill-hook.yaml
@@ -52,12 +52,20 @@ metadata:
     helm.sh/hook-weight: "5"
     helm.sh/hook-delete-policy: before-hook-creation
 spec:
-  # No activeDeadlineSeconds: a deadline kill marks the Job FAILED, which fails
-  # the `helm upgrade` (and thus the auto-upgrade). Instead the container bounds
-  # its OWN runtime with `timeout` and always exits 0 (see args), so the pod
-  # completes successfully well within Helm's --timeout regardless of how the
-  # sweep goes. backoffLimit 0 so a genuine infra failure (image pull / OOM)
-  # fails fast rather than retry-looping toward Helm's timeout. (bugbot)
+  # activeDeadlineSeconds is a Job-level BACKSTOP for a pod that never runs its
+  # container — chiefly ImagePullBackOff, which sits Pending forever (backoffLimit
+  # can't catch it, since Pending isn't a failed attempt). Without it, such a pod
+  # would hang the hook until Helm's own --timeout before failing. Set it well
+  # under Helm's 5m default (backfillTimeoutSeconds + 120s = 240s at the default:
+  # +30s kill grace, +90s pull/scheduling), so a stuck pull fails FAST and the
+  # hourly auto-upgrade retries. It does NOT gate a healthy run: the in-container
+  # `timeout` (see args) exits ~150s in, well before this deadline, so a running
+  # sweep is never killed by it. A genuinely un-pullable image still fails the
+  # upgrade — irreducible for a synchronous hook running the ingestor image — but
+  # now quickly and self-healing once the registry recovers. (bugbot)
+  activeDeadlineSeconds: {{ add (int ($mb.backfillTimeoutSeconds | default 120)) 120 }}
+  # backoffLimit 0: never retry a failed pod (a retry loop would only burn more
+  # of Helm's wait). The deadline above is the real bound.
   backoffLimit: 0
   ttlSecondsAfterFinished: 86400
   template:
diff --git a/client/tests/metadata_backfill_hook_test.yaml b/client/tests/metadata_backfill_hook_test.yaml
index ecf99df3..e62bbdc9 100644
--- a/client/tests/metadata_backfill_hook_test.yaml
+++ b/client/tests/metadata_backfill_hook_test.yaml
@@ -41,12 +41,14 @@ tests:
           path: spec.template.spec.containers[0].args[0]
           pattern: tracebloc-backfill
 
-  - it: is best-effort — self-bounded by timeout, always exits 0, no Job deadline
+  - it: is best-effort — self-bounded by timeout, always exits 0, deadline backstops a stuck pull
     asserts:
-      # No activeDeadlineSeconds: a deadline kill would FAIL the Job and thus the
-      # upgrade; the container bounds itself and exits 0 instead.
-      - notExists:
+      # Job-level backstop bounds an ImagePullBackOff/Pending pod below Helm's
+      # timeout (backfillTimeoutSeconds 120 + 120 = 240); the in-container timeout
+      # exits a healthy run well before it.
+      - equal:
           path: spec.activeDeadlineSeconds
+          value: 240
       - equal:
           path: spec.backoffLimit
           value: 0

From 4be435766ee67a55c257eb32c888feb860fcfae9 Mon Sep 17 00:00:00 2001
From: Divya <divyasingh@tracebloc.io>
Date: Thu, 23 Jul 2026 19:15:32 +0530
Subject: [PATCH 8/9] fix(chart): make backfill deadline consistent with the
 pull budget (Bugbot #380)

The Job deadline (backfillTimeoutSeconds + 120 = 240) contradicted the values
comment's ~150s cold-pull budget: a pull of 90-150s plus a full in-container run
(150s) exceeded 240s and DeadlineExceeded a HEALTHY sweep. Since
activeDeadlineSeconds is Job wall-clock and includes the pull, it must exceed
(cold pull + full run).

Re-derive the deadline as backfillTimeoutSeconds + 30s grace + 120s pull = 270s
and reconcile the docs: in-container <=150s, cold-pull budget 120s, deadline
270s. A healthy run (warm pull ~instant, or a cold pull up to ~120s + <=150s
container) fits; a Pending/stuck pod is still bounded (270s). Sized for the
auto-upgrade's 10m --timeout (the target path); manual installs on a cold
cluster should pass --timeout 10m. A pull slower than the budget fails that
upgrade and self-heals on the next (image cached by then).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
---
 client/templates/metadata-backfill-hook.yaml  | 26 ++++++++++++-------
 client/tests/metadata_backfill_hook_test.yaml |  7 +++--
 client/values.yaml                            | 15 ++++++-----
 3 files changed, 27 insertions(+), 21 deletions(-)

diff --git a/client/templates/metadata-backfill-hook.yaml b/client/templates/metadata-backfill-hook.yaml
index 24354f59..e97ecbe5 100644
--- a/client/templates/metadata-backfill-hook.yaml
+++ b/client/templates/metadata-backfill-hook.yaml
@@ -54,16 +54,22 @@ metadata:
 spec:
   # activeDeadlineSeconds is a Job-level BACKSTOP for a pod that never runs its
   # container — chiefly ImagePullBackOff, which sits Pending forever (backoffLimit
-  # can't catch it, since Pending isn't a failed attempt). Without it, such a pod
-  # would hang the hook until Helm's own --timeout before failing. Set it well
-  # under Helm's 5m default (backfillTimeoutSeconds + 120s = 240s at the default:
-  # +30s kill grace, +90s pull/scheduling), so a stuck pull fails FAST and the
-  # hourly auto-upgrade retries. It does NOT gate a healthy run: the in-container
-  # `timeout` (see args) exits ~150s in, well before this deadline, so a running
-  # sweep is never killed by it. A genuinely un-pullable image still fails the
-  # upgrade — irreducible for a synchronous hook running the ingestor image — but
-  # now quickly and self-healing once the registry recovers. (bugbot)
-  activeDeadlineSeconds: {{ add (int ($mb.backfillTimeoutSeconds | default 120)) 120 }}
+  # can't catch it, since Pending isn't a failed attempt). Without it such a pod
+  # hangs the hook until Helm's own --timeout before failing.
+  #
+  # It is Job wall-clock and INCLUDES the image pull, so it must exceed
+  # (cold pull + full in-container run) or it would kill a HEALTHY sweep. Budget:
+  #   backfillTimeoutSeconds (in-container sweep) + 30s kill grace + 120s cold
+  #   pull/scheduling = 270s at the default.
+  # A healthy run therefore fits (warm pull ≈ instant; a cold pull up to ~120s
+  # plus the ≤150s container still lands on the deadline), while a Pending/stuck
+  # pod is bounded at 270s. That is well under the auto-upgrade CronJob's 10m
+  # --timeout (the path this feature targets); on a MANUAL `helm install/upgrade`
+  # use `--timeout 10m` on a cold cluster, since the 5m default leaves little room
+  # once resource rollout + a first-ever pull are counted. A pull slower than the
+  # budget hits the deadline and fails that upgrade, then self-heals on the next
+  # one (the image is cached by then). (bugbot)
+  activeDeadlineSeconds: {{ add (int ($mb.backfillTimeoutSeconds | default 120)) 150 }}
   # backoffLimit 0: never retry a failed pod (a retry loop would only burn more
   # of Helm's wait). The deadline above is the real bound.
   backoffLimit: 0
diff --git a/client/tests/metadata_backfill_hook_test.yaml b/client/tests/metadata_backfill_hook_test.yaml
index e62bbdc9..800e3c97 100644
--- a/client/tests/metadata_backfill_hook_test.yaml
+++ b/client/tests/metadata_backfill_hook_test.yaml
@@ -43,12 +43,11 @@ tests:
 
   - it: is best-effort — self-bounded by timeout, always exits 0, deadline backstops a stuck pull
     asserts:
-      # Job-level backstop bounds an ImagePullBackOff/Pending pod below Helm's
-      # timeout (backfillTimeoutSeconds 120 + 120 = 240); the in-container timeout
-      # exits a healthy run well before it.
+      # Job-level backstop bounds an ImagePullBackOff/Pending pod (backfillTimeout
+      # 120 + 30 grace + 120 pull = 270), sized to exceed a healthy cold-pull run.
       - equal:
           path: spec.activeDeadlineSeconds
-          value: 240
+          value: 270
       - equal:
           path: spec.backoffLimit
           value: 0
diff --git a/client/values.yaml b/client/values.yaml
index 8a40668f..71e315f1 100644
--- a/client/values.yaml
+++ b/client/values.yaml
@@ -563,13 +563,14 @@ metadataBackfill:
   # fails. Remaining tables resume on the next upgrade (the sweep is idempotent
   # and skips already-backfilled datasets).
   #
-  # MUST stay comfortably under Helm's --timeout with headroom for image pull +
-  # pod scheduling (which Helm counts against its hook wait, but the in-container
-  # `timeout` does not). The binding case is a MANUAL `helm install/upgrade`,
-  # whose default --timeout is 5m — not the auto-upgrade CronJob's 10m. Budget:
-  # worst-case in-container = this + the 30s SIGKILL grace, so 120 -> 150s, which
-  # leaves ~150s of the 5m default for a cold ghcr.io pull + scheduling (and
-  # ~450s under the auto-upgrade's 10m). (bugbot)
+  # In-container sweep budget. Worst-case container runtime is this + the 30s
+  # SIGKILL grace (150s at the default). The Job's activeDeadlineSeconds is
+  # derived as this + 30s grace + 120s cold-pull/scheduling allowance (270s at
+  # the default) and is what bounds a stuck/Pending pod — see the hook template.
+  # Keep this small enough that (this + 30 + 120) stays under your Helm --timeout:
+  # comfortable under the auto-upgrade's 10m (the target path); on a MANUAL
+  # `helm install/upgrade` on a cold cluster pass `--timeout 10m` rather than
+  # relying on the 5m default. (bugbot)
   backfillTimeoutSeconds: 120
   # Small: the sweep is I/O-bound on MySQL aggregates + backend calls.
   resources:

From 17ec7cba19f41ee967a95d05888217b0d2cd69a7 Mon Sep 17 00:00:00 2001
From: Divya <divyasingh@tracebloc.io>
Date: Thu, 23 Jul 2026 19:21:55 +0530
Subject: [PATCH 9/9] fix(chart): backfill hook is post-upgrade only, not
 post-install (Bugbot #380)

The hook was post-upgrade,post-install. On a fresh install there are no
pre-cutover datasets to backfill, yet Helm waits on hooks even without --wait, so
a cold ingestor-image pull / ImagePullBackOff could fail the whole client install
(the bash/Windows installers run `helm upgrade --install` under the default 5m
timeout and assume a ~10-15s apply). Drop post-install so the hook never runs at
install time; a cluster that actually carries pre-cutover data reaches it on its
first auto-upgrade (post-upgrade).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
---
 client/templates/metadata-backfill-hook.yaml  | 15 ++++++++++-----
 client/tests/metadata_backfill_hook_test.yaml |  4 ++--
 2 files changed, 12 insertions(+), 7 deletions(-)

diff --git a/client/templates/metadata-backfill-hook.yaml b/client/templates/metadata-backfill-hook.yaml
index e97ecbe5..b98cb08e 100644
--- a/client/templates/metadata-backfill-hook.yaml
+++ b/client/templates/metadata-backfill-hook.yaml
@@ -26,10 +26,10 @@
 # edge metadata API can't reconcile them — old competitions/experiments that
 # rely on that metadata would degrade under the new code.
 #
-# Trigger: a Helm post-upgrade/post-install hook. The auto-upgrade CronJob runs
-# `helm upgrade` when a new chart is published, which fires this hook — so a
-# client that auto-updates to this release runs the backfill once, automatically,
-# with no operator step.
+# Trigger: a Helm post-upgrade hook. The auto-upgrade CronJob runs `helm upgrade`
+# when a new chart is published, which fires this hook — so a client that
+# auto-updates to this release runs the backfill once, automatically, with no
+# operator step. (Not post-install: a fresh install has nothing to backfill.)
 #
 # One-time / idempotent: `before-hook-creation` replaces the prior hook Job on
 # each upgrade; the sweep skips datasets already carrying new-shape metadata and
@@ -46,7 +46,12 @@ metadata:
     {{- include "tracebloc.labels" . | nindent 4 }}
     app.kubernetes.io/component: metadata-backfill
   annotations:
-    helm.sh/hook: post-upgrade,post-install
+    # post-upgrade ONLY — deliberately NOT post-install. A fresh install has no
+    # pre-cutover datasets to backfill, so running there is pure risk (Helm waits
+    # on hooks even without --wait, so a cold ingestor-image pull / ImagePullBackOff
+    # could fail the client install for no benefit). A cluster that does carry old
+    # data reaches this on its FIRST auto-upgrade. (bugbot)
+    helm.sh/hook: post-upgrade
     # Run late, after the release's core resources (MySQL, jobs-manager) are in
     # place — the sweep talks to in-cluster MySQL and the backend API.
     helm.sh/hook-weight: "5"
diff --git a/client/tests/metadata_backfill_hook_test.yaml b/client/tests/metadata_backfill_hook_test.yaml
index 800e3c97..cacfbe69 100644
--- a/client/tests/metadata_backfill_hook_test.yaml
+++ b/client/tests/metadata_backfill_hook_test.yaml
@@ -18,11 +18,11 @@ tests:
           path: metadata.name
           value: stg-metadata-backfill
 
-  - it: is a post-upgrade/post-install hook that replaces its prior run
+  - it: is a post-upgrade (only) hook that replaces its prior run
     asserts:
       - equal:
           path: metadata.annotations["helm.sh/hook"]
-          value: post-upgrade,post-install
+          value: post-upgrade
       - equal:
           path: metadata.annotations["helm.sh/hook-delete-policy"]
           value: before-hook-creation