diff --git a/client/templates/metadata-backfill-hook.yaml b/client/templates/metadata-backfill-hook.yaml new file mode 100644 index 0000000..b98cb08 --- /dev/null +++ b/client/templates/metadata-backfill-hook.yaml @@ -0,0 +1,164 @@ +{{- /* Nil-guard the whole block: an `--reuse-values` upgrade from a release + that predates the `metadataBackfill` key stores no such value, so + `.Values.metadataBackfill` is nil and a bare `.enabled` access would fail + template rendering (same hazard the images.ingestor keys guard against). + Absent key => feature off on that path; the production auto-upgrade uses + `--reset-then-reuse-values`, which layers in the new default (enabled: + true), so the backfill still runs on the real upgrade path. */ -}} +{{- $mb := default (dict) .Values.metadataBackfill -}} +{{- if $mb.enabled }} +{{- $ing := default dict .Values.images.ingestor -}} +{{- $repo := $ing.repository | default "ghcr.io/tracebloc/ingestor" -}} +{{- $digest := $ing.digest | default "" -}} +{{- /* Fallback tag matches values.yaml images.ingestor.tag and jobs-manager's + INGESTOR_IMAGE_TAG default — an older tag here would pull an ingestor that + predates the `tracebloc-backfill` entrypoint and the sweep would no-op. + (bugbot) The default only applies on a --reuse-values upgrade whose stored + values predate the images.ingestor key; keep it in lockstep with those. */ -}} +{{- $tag := $ing.tag | default "0.7" -}} +# One-time pre-cutover metadata backfill (data-ingestors #393, backend +# #1166/#1198). Runs the ingestor image's `tracebloc-backfill` entrypoint, which +# sweeps THIS client's already-registered datasets: GET each from the backend, +# recompute the enriched {schema, meta_data} from the persisted MySQL rows, and +# POST it back (which also re-folds any competition built from them). Datasets +# ingested before the federated-alignment cutover otherwise carry no enriched +# schema / feature_stats, so the backend's combine-time alignment checks and the +# edge metadata API can't reconcile them — old competitions/experiments that +# rely on that metadata would degrade under the new code. +# +# Trigger: a Helm post-upgrade hook. The auto-upgrade CronJob runs `helm upgrade` +# when a new chart is published, which fires this hook — so a client that +# auto-updates to this release runs the backfill once, automatically, with no +# operator step. (Not post-install: a fresh install has nothing to backfill.) +# +# One-time / idempotent: `before-hook-creation` replaces the prior hook Job on +# each upgrade; the sweep skips datasets already carrying new-shape metadata and +# the backend upsert is a safe overwrite, so re-runs are cheap no-ops. It is +# deliberately BEST-EFFORT — the entrypoint's exit code is swallowed so a +# backfill hiccup can never fail the `helm upgrade` (which would strand the +# cluster on the old chart). Operators read the outcome via `kubectl logs`. +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ .Release.Name }}-metadata-backfill + namespace: {{ .Release.Namespace }} + labels: + {{- include "tracebloc.labels" . | nindent 4 }} + app.kubernetes.io/component: metadata-backfill + annotations: + # post-upgrade ONLY — deliberately NOT post-install. A fresh install has no + # pre-cutover datasets to backfill, so running there is pure risk (Helm waits + # on hooks even without --wait, so a cold ingestor-image pull / ImagePullBackOff + # could fail the client install for no benefit). A cluster that does carry old + # data reaches this on its FIRST auto-upgrade. (bugbot) + helm.sh/hook: post-upgrade + # Run late, after the release's core resources (MySQL, jobs-manager) are in + # place — the sweep talks to in-cluster MySQL and the backend API. + helm.sh/hook-weight: "5" + helm.sh/hook-delete-policy: before-hook-creation +spec: + # activeDeadlineSeconds is a Job-level BACKSTOP for a pod that never runs its + # container — chiefly ImagePullBackOff, which sits Pending forever (backoffLimit + # can't catch it, since Pending isn't a failed attempt). Without it such a pod + # hangs the hook until Helm's own --timeout before failing. + # + # It is Job wall-clock and INCLUDES the image pull, so it must exceed + # (cold pull + full in-container run) or it would kill a HEALTHY sweep. Budget: + # backfillTimeoutSeconds (in-container sweep) + 30s kill grace + 120s cold + # pull/scheduling = 270s at the default. + # A healthy run therefore fits (warm pull ≈ instant; a cold pull up to ~120s + # plus the ≤150s container still lands on the deadline), while a Pending/stuck + # pod is bounded at 270s. That is well under the auto-upgrade CronJob's 10m + # --timeout (the path this feature targets); on a MANUAL `helm install/upgrade` + # use `--timeout 10m` on a cold cluster, since the 5m default leaves little room + # once resource rollout + a first-ever pull are counted. A pull slower than the + # budget hits the deadline and fails that upgrade, then self-heals on the next + # one (the image is cached by then). (bugbot) + activeDeadlineSeconds: {{ add (int ($mb.backfillTimeoutSeconds | default 120)) 150 }} + # backoffLimit 0: never retry a failed pod (a retry loop would only burn more + # of Helm's wait). The deadline above is the real bound. + backoffLimit: 0 + ttlSecondsAfterFinished: 86400 + template: + metadata: + labels: + {{- include "tracebloc.selectorLabels" . | nindent 8 }} + app.kubernetes.io/component: metadata-backfill + spec: + restartPolicy: Never + # The sweep never calls the Kubernetes API — it only reaches MySQL and the + # backend — so no ServiceAccount token is mounted. + automountServiceAccountToken: false + {{- if include "tracebloc.useImagePullSecrets" . }} + imagePullSecrets: + - name: {{ include "tracebloc.registrySecretName" . }} + {{- end }} + securityContext: + runAsNonRoot: true + runAsUser: 65534 + seccompProfile: + type: RuntimeDefault + containers: + - name: metadata-backfill + image: {{ if $digest }}{{ printf "%s@%s" $repo $digest | quote }}{{ else }}{{ printf "%s:%s" $repo $tag | quote }}{{ end }} + imagePullPolicy: {{ if $digest }}IfNotPresent{{ else }}Always{{ end }} + # Bound the sweep with `timeout` and ALWAYS exit 0, so neither a slow + # run nor a backfill error can fail the hook (and thus the upgrade). + # backfillTimeoutSeconds must stay under autoUpgrade.timeout (10m) so + # the pod finishes before Helm's --timeout; a run that hits the bound + # is killed cleanly and resumes on the next upgrade (the sweep is + # idempotent and skips datasets already backfilled). (bugbot) + command: ["/bin/sh", "-c"] + args: + - | + set -u + TIMEOUT={{ $mb.backfillTimeoutSeconds | default 120 }} + echo "[metadata-backfill] running (timeout ${TIMEOUT}s)" + # -k 30: if the backfill ignores the SIGTERM at TIMEOUT (or is + # wedged), escalate to SIGKILL 30s later so `timeout` can't wait + # forever and hang the hook past Helm's --timeout. Worst-case + # wrapper runtime is TIMEOUT+30s, still well under the 5m default. (bugbot) + if timeout -k 30 "${TIMEOUT}" tracebloc-backfill; then + echo "[metadata-backfill] completed" + else + echo "[metadata-backfill] did not finish (exit $?); remaining tables resume on the next upgrade (non-fatal)" + fi + exit 0 + env: + {{- include "tracebloc.proxyEnv" . | nindent 12 }} + # Backend auth: the ingestor's APIClient mints a token from these + # (same credentials Secret jobs-manager uses). BACKEND_TOKEN would be + # preferred, but the chart holds no static backend token — jobs-manager + # mints its own at runtime — so the backfill uses the client creds. + - name: CLIENT_ID + valueFrom: + secretKeyRef: + name: {{ include "tracebloc.secretName" . }} + key: CLIENT_ID + - name: CLIENT_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "tracebloc.secretName" . }} + key: CLIENT_PASSWORD + # In-cluster MySQL + backend endpoint, mirroring the jobs-manager / + # ingestion-Job env so the ingestor's Config resolves the same store. + - name: MYSQL_HOST + value: "mysql-client" + - name: CLIENT_ENV + value: {{ .Values.env.CLIENT_ENV | default "prod" | quote }} + - name: HOME + value: "/tmp" + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + resources: + {{- toYaml ($mb.resources | default dict) | nindent 12 }} + volumeMounts: + - name: tmp + mountPath: /tmp + volumes: + - name: tmp + emptyDir: {} +{{- end }} diff --git a/client/tests/metadata_backfill_hook_test.yaml b/client/tests/metadata_backfill_hook_test.yaml new file mode 100644 index 0000000..cacfbe6 --- /dev/null +++ b/client/tests/metadata_backfill_hook_test.yaml @@ -0,0 +1,108 @@ +suite: metadata-backfill post-upgrade hook Job +templates: + - templates/metadata-backfill-hook.yaml +release: + name: stg + namespace: tracebloc +set: + clientId: "test-id" + clientPassword: "test" +tests: + - it: renders a hook Job by default + asserts: + - hasDocuments: + count: 1 + - isKind: + of: Job + - equal: + path: metadata.name + value: stg-metadata-backfill + + - it: is a post-upgrade (only) hook that replaces its prior run + asserts: + - equal: + path: metadata.annotations["helm.sh/hook"] + value: post-upgrade + - equal: + path: metadata.annotations["helm.sh/hook-delete-policy"] + value: before-hook-creation + + - it: runs the backfill entrypoint on the ingestor image, best-effort (exit 0) + set: + images.ingestor.tag: "0.9" + asserts: + - equal: + path: spec.template.spec.containers[0].image + value: ghcr.io/tracebloc/ingestor:0.9 + - equal: + path: spec.template.spec.containers[0].imagePullPolicy + value: Always + - matchRegex: + path: spec.template.spec.containers[0].args[0] + pattern: tracebloc-backfill + + - it: is best-effort — self-bounded by timeout, always exits 0, deadline backstops a stuck pull + asserts: + # Job-level backstop bounds an ImagePullBackOff/Pending pod (backfillTimeout + # 120 + 30 grace + 120 pull = 270), sized to exceed a healthy cold-pull run. + - equal: + path: spec.activeDeadlineSeconds + value: 270 + - equal: + path: spec.backoffLimit + value: 0 + - matchRegex: + path: spec.template.spec.containers[0].args[0] + pattern: 'timeout -k 30 "\$\{TIMEOUT\}" tracebloc-backfill' + - matchRegex: + path: spec.template.spec.containers[0].args[0] + pattern: exit 0 + + - it: pins the ingestor image by digest when one is set + set: + images.ingestor.digest: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + asserts: + - equal: + path: spec.template.spec.containers[0].image + value: ghcr.io/tracebloc/ingestor@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + - equal: + path: spec.template.spec.containers[0].imagePullPolicy + value: IfNotPresent + + - it: sources backend creds from the release Secret and points at in-cluster MySQL + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: CLIENT_ID + valueFrom: + secretKeyRef: + name: stg-secrets + key: CLIENT_ID + - contains: + path: spec.template.spec.containers[0].env + content: + name: MYSQL_HOST + value: "mysql-client" + + - it: hardened pod — non-root, no SA token, read-only rootfs, caps dropped + asserts: + - equal: + path: spec.template.spec.securityContext.runAsNonRoot + value: true + - equal: + path: spec.template.spec.automountServiceAccountToken + value: false + - equal: + path: spec.template.spec.containers[0].securityContext.readOnlyRootFilesystem + value: true + - equal: + path: spec.template.spec.containers[0].securityContext.allowPrivilegeEscalation + value: false + + - it: renders nothing when metadataBackfill.enabled=false + set: + metadataBackfill.enabled: false + asserts: + - hasDocuments: + count: 0 diff --git a/client/values.schema.json b/client/values.schema.json index 5ce28bf..8e6b34b 100644 --- a/client/values.schema.json +++ b/client/values.schema.json @@ -628,6 +628,53 @@ } } }, + "metadataBackfill": { + "type": "object", + "description": "One-time pre-cutover metadata backfill (data-ingestors #393, backend #1166/#1198). A post-upgrade/post-install hook Job runs the ingestor image's `tracebloc-backfill` entrypoint to bring already-registered datasets up to the federated-alignment metadata shape. Idempotent (skips already-backfilled datasets) and best-effort (never fails the upgrade). Disable to opt a cluster out.", + "properties": { + "enabled": { + "type": "boolean", + "default": true, + "description": "When false, the backfill hook Job is not rendered." + }, + "backfillTimeoutSeconds": { + "type": "integer", + "minimum": 1, + "description": "Wall-clock bound the sweep gives itself before stopping and exiting 0. Must stay under Helm's --timeout (5m for a manual `helm install/upgrade`) with headroom for image pull + pod scheduling." + }, + "resources": { + "type": "object", + "properties": { + "requests": { + "type": "object", + "properties": { + "cpu": { + "type": "string", + "pattern": "^[0-9]+m?$" + }, + "memory": { + "type": "string", + "pattern": "^[0-9]+(Ki|Mi|Gi|Ti)$" + } + } + }, + "limits": { + "type": "object", + "properties": { + "cpu": { + "type": "string", + "pattern": "^[0-9]+m?$" + }, + "memory": { + "type": "string", + "pattern": "^[0-9]+(Ki|Mi|Gi|Ti)$" + } + } + } + } + } + } + }, "imageRefresh": { "type": "object", "description": "Image-refresh CronJob (issue tracebloc/client#154). Polls Docker Hub for jobs-manager and pods-monitor digests under the floating CLIENT_ENV tag and rolls the deployment when a new image is published. Disable to keep the running image in place until manual restart.", diff --git a/client/values.yaml b/client/values.yaml index 22f3f37..71e315f 100644 --- a/client/values.yaml +++ b/client/values.yaml @@ -544,6 +544,43 @@ autoUpgrade: cpu: "500m" memory: "256Mi" +# -- One-time pre-cutover metadata backfill (data-ingestors #393, backend +# #1166/#1198). A post-upgrade/post-install Helm hook Job that runs the ingestor +# image's `tracebloc-backfill` entrypoint once per chart upgrade. It sweeps this +# client's already-registered datasets and brings their global-metadata up to +# the federated-alignment shape (enriched schema + feature_stats), recomputed +# from the persisted MySQL rows — so old competitions/experiments that predate +# the cutover keep working under the new backend alignment code. +# +# Safe to leave enabled: the sweep skips datasets already carrying new-shape +# metadata and the backend upsert is idempotent, so after the fleet has +# backfilled once it is a cheap no-op on every subsequent upgrade. Set +# `enabled: false` to opt a cluster out (e.g. one with no pre-cutover data). +metadataBackfill: + enabled: true + # Wall-clock bound the sweep gives ITSELF (`timeout` in the hook), after which + # it stops and exits 0 cleanly — the pod never fails, so the upgrade never + # fails. Remaining tables resume on the next upgrade (the sweep is idempotent + # and skips already-backfilled datasets). + # + # In-container sweep budget. Worst-case container runtime is this + the 30s + # SIGKILL grace (150s at the default). The Job's activeDeadlineSeconds is + # derived as this + 30s grace + 120s cold-pull/scheduling allowance (270s at + # the default) and is what bounds a stuck/Pending pod — see the hook template. + # Keep this small enough that (this + 30 + 120) stays under your Helm --timeout: + # comfortable under the auto-upgrade's 10m (the target path); on a MANUAL + # `helm install/upgrade` on a cold cluster pass `--timeout 10m` rather than + # relying on the 5m default. (bugbot) + backfillTimeoutSeconds: 120 + # Small: the sweep is I/O-bound on MySQL aggregates + backend calls. + resources: + requests: + cpu: "250m" + memory: "512Mi" + limits: + cpu: "1" + memory: "2Gi" + # -- Image-refresh CronJob (closes tracebloc/client#154). # Auto-restarts the jobs-manager Deployment when a new image digest is # published to Docker Hub under the floating CLIENT_ENV tag. Without