fix(installer): execute-gate installed tools so a broken binary fails Step 1 (#411)

scripts/install-k8s.ps1

@@ -176,6 +176,41 @@ function Invoke-WithRetry {
   }
 }
 
+# Execute-gate a freshly-installed tool (#411). The old post-install "check" was a
+# Log interpolation whose failure is non-terminating, so a corrupt or wrong-arch
+# binary (winget shims / partial installs skip the direct path's checksum verify)
+# still reached "System tools" and only died at cluster-create. Actually RUN the
+# tool's self-check; on failure Err with an arch-aware remedy so Step 1 fails
+# loudly. NOTE: kubectl uses `version --client` (NOT --short — removed in 1.28+);
+# helm uses bare `version` (--short may go the same way).
+#
+# -BinPath is our own install location. On failure we remove it ONLY when the
+# binary that actually ran resolves to it — so a broken copy WE placed (fresh or
+# left by a prior run) self-heals on re-run, while a winget/choco/pre-existing copy
+# elsewhere on PATH is never deleted (reviewer + Bugbot). Callers may pass -BinPath
+# on every path; the resolved-source guard sorts out ownership.
+function Assert-ToolRuns {
+  param(
+    [Parameter(Mandatory)][string]$Name,
+    [Parameter(Mandatory)][string[]]$VersionArgs,
+    [string]$BinPath
+  )
+  $ok = $false; $out = $null
+  try {
+    $out = (& $Name @VersionArgs 2>&1)
+    $ok  = ($LASTEXITCODE -eq 0)
+  } catch { $ok = $false }
+  if (-not $ok) {
+    if ($BinPath -and (Test-Path $BinPath)) {
+      $resolved = (Get-Command $Name -ErrorAction SilentlyContinue).Source
+      if ($resolved -and ($resolved -eq $BinPath)) { Remove-Item $BinPath -Force -ErrorAction SilentlyContinue }
+    }
+    $arch = Get-WindowsArch
+    Err "$Name was installed but won't run -- a corrupt or wrong-architecture binary (this machine is $arch). Re-run this script to re-download it; if it recurs, remove any $Name installed via a package manager (winget/choco) first, then re-run."
+  }
+  Log "$Name OK: $(($out | Select-Object -First 1))"
+}
+
 # Sanitize workspace name to comply with DNS-1123
 function ConvertTo-WorkspaceName {
   param([string]$Input_)
@@ -783,7 +818,9 @@ echo "NCT installed successfully."
 # =============================================================================
 
 function Install-Kubectl {
-  if (Has "kubectl") { Log "kubectl: $(cmd /c 'kubectl version --client 2>&1' | Select-Object -First 1)"; return }
+  # Execute-gate on both paths (#411): a present-but-broken kubectl is as fatal as
+  # a bad fresh install, and this runs in Step 1, before the cluster step.
+  if (Has "kubectl") { Assert-ToolRuns -Name "kubectl" -VersionArgs @("version","--client") -BinPath "$TOOL_DIR\kubectl.exe"; return }
 
   $arch = Get-WindowsArch
   $kVer = Invoke-WithRetry -Label "version check" -ScriptBlock {
@@ -806,6 +843,7 @@ function Install-Kubectl {
   }
   RefreshPath
   Log "kubectl $kVer installed."
+  Assert-ToolRuns -Name "kubectl" -VersionArgs @("version","--client") -BinPath $kubectlDest
 }
 
 # ── Pinned tool versions (#382 / #410) ──────────────────────────────────────
@@ -932,7 +970,7 @@ function Install-K3dAndHelm {
       RefreshPath
     }
   }
-  Log "k3d: $(k3d version | Select-Object -First 1)"
+  Assert-ToolRuns -Name "k3d" -VersionArgs @("version") -BinPath "$TOOL_DIR\k3d.exe"
 
   # -- Helm --
   if (-not (Has "helm")) {
@@ -969,7 +1007,7 @@ function Install-K3dAndHelm {
 
     if (-not (Has "helm")) { Err "Helm could not be installed. Install manually from https://helm.sh/docs/intro/install/ and re-run." }
   }
-  Log "helm: $(cmd /c 'helm version --short 2>&1')"
+  Assert-ToolRuns -Name "helm" -VersionArgs @("version") -BinPath "$TOOL_DIR\helm.exe"
 
   Ok "System tools"
 }

scripts/lib/common.sh

@@ -140,6 +140,41 @@ step_header()    { echo -e "  ${TB_HEADING}$1) $2${RESET}"; echo ""; }
 # ── Utility ──────────────────────────────────────────────────────────────────
 has() { command -v "$1" &>/dev/null; }
 
+# Execute-gate a freshly-installed tool (#411). The old post-install "check" was a
+# log-only interpolation (`... 2>/dev/null || echo present`) that masked failure,
+# so a corrupt or wrong-architecture binary — a partial pkg/brew install, or a
+# download no checksum path guarded — sat on PATH and failed only later, at
+# cluster-create, after a green "System tools". Actually RUN the tool's cheapest
+# self-check; on failure error() with an arch-aware remedy so the tool step fails
+# loudly instead. NOTE: kubectl is gated with `version --client` (NOT --short,
+# removed in kubectl 1.28+); helm with bare `version` (—short may go the same way).
+#
+# Removal is OPT-IN via a leading `--rm <path>`: pass it with the path where the
+# installer PLACES the binary (TB_TOOLS_DIR/<tool>). On failure we remove that path
+# ONLY when the binary that actually ran is that exact file (same inode, `-ef`).
+# So: a broken binary WE installed there (fresh OR left by a prior run) is cleared,
+# letting a re-run self-heal (Bugbot: otherwise `has` stays true → stuck loop);
+# but a brew/pkg-manager copy that lives elsewhere on PATH is never deleted — the
+# resolved binary won't match our path (reviewer). Callers may pass --rm on every
+# path; the `-ef` guard sorts out ownership. macOS/brew callers pass no --rm.
+# Usage: assert_tool_runs [--rm <placed-path>] <name> <version-arg>...
+assert_tool_runs() {
+  local rm_path=""
+  if [[ "${1:-}" == "--rm" ]]; then rm_path="$2"; shift 2; fi
+  local name="$1"; shift
+  local out
+  if out="$("$name" "$@" 2>&1)"; then
+    log "$name OK: $(printf '%s\n' "$out" | head -1)"
+    return 0
+  fi
+  # Remove only the file we placed AND only if it's the binary that just failed.
+  if [[ -n "$rm_path" && -f "$rm_path" ]]; then
+    local resolved; resolved="$(command -v "$name" 2>/dev/null || true)"
+    [[ -n "$resolved" && "$resolved" -ef "$rm_path" ]] && rm -f "$rm_path" 2>/dev/null || true
+  fi
+  error "$name was installed but won't run — a corrupt or wrong-architecture binary (this machine is ${ARCH:-$(uname -m)}). Re-run the installer to re-download it; if it recurs, remove ${rm_path:-the $name on your PATH} (and any package-manager copy) first."
+}
+
 # Sanitize a minutes-valued env override to a base-10 integer, else <default>.
 # The 10# base prefix matters: bash arithmetic reads a leading zero as octal,
 # so 08/09 would ABORT $(( … )) under set -e (mid-create, leaving a partial

scripts/lib/setup-linux.sh

@@ -401,9 +401,11 @@ install_kubectl() {
       || error "Couldn't resolve the kubectl version from dl.k8s.io/release/stable.txt — check network connectivity to dl.k8s.io and re-run."
     spin_cmd "Installing system tools…" _fetch_kubectl "$KUBE_VER" "$ARCH_DL"
     log "kubectl $KUBE_VER installed."
-  else
-    log "kubectl: $(kubectl version --client --short 2>/dev/null || echo present)"
   fi
+  # Gate on both paths (fresh + already-present). --rm removes our TB_TOOLS_DIR copy
+  # only if IT is the binary that failed (assert_tool_runs' -ef guard), so a broken
+  # installer-placed kubectl self-heals on re-run while a pkg copy elsewhere is safe.
+  assert_tool_runs --rm "$TB_TOOLS_DIR/kubectl" kubectl version --client
 }
 
 # ── k3d ──────────────────────────────────────────────────────────────────────
@@ -447,7 +449,7 @@ _fetch_k3d_release() {
 
 install_k3d() {
   if has k3d; then
-    log "k3d: $(k3d version | head -1)"
+    assert_tool_runs --rm "$TB_TOOLS_DIR/k3d" k3d version
     return 0
   fi
 
@@ -484,7 +486,7 @@ install_k3d() {
     error "System tool installation completed but not found on PATH."
   fi
 
-  log "k3d: $(k3d version | head -1)"
+  assert_tool_runs --rm "$TB_TOOLS_DIR/k3d" k3d version
 }
 
 # ── Helm ─────────────────────────────────────────────────────────────────────
@@ -646,7 +648,10 @@ install_helm() {
     fi
   fi
   _ensure_helm_executable
-  log "helm: $(helm version --short 2>/dev/null || echo installed)"
+  # bare `helm version` (not --short: it may be dropped like kubectl's was). --rm
+  # removes our TB_TOOLS_DIR copy only if IT is the binary that failed (-ef guard),
+  # never a pre-existing / pkg-managed helm elsewhere on PATH (#411 review).
+  assert_tool_runs --rm "$TB_TOOLS_DIR/helm" helm version
   success "System tools"
 }
 

scripts/lib/setup-macos.sh

@@ -253,20 +253,25 @@ install_docker_desktop() {
 }
 
 install_macos_cli_tools() {
+  # brew delivers these binaries with no checksum of our own (#429), so the
+  # execute-gate (#411) is the only thing standing between a partial/wrong-arch
+  # install and a green "System tools" that dies at cluster-create.
   if ! has kubectl; then
     spin_cmd "Installing system tools…" brew install kubectl
   fi
-  log "kubectl: $(kubectl version --client --short 2>/dev/null || echo installed)"
+  assert_tool_runs kubectl version --client
 
   if ! has k3d; then
     spin_cmd "Installing system tools…" brew install k3d
   fi
-  log "k3d: $(k3d version | head -1 2>/dev/null || echo installed)"
+  assert_tool_runs k3d version
 
   if ! has helm; then
     spin_cmd "Installing system tools…" brew install helm
   fi
-  log "helm: $(helm version --short 2>/dev/null || echo installed)"
+  # bare `helm version` (not --short: may be dropped like kubectl's). No --rm:
+  # brew owns these binaries; deleting a formula's symlink just wedges the re-run.
+  assert_tool_runs helm version
 
   success "System tools ready"
 }

scripts/manifest.sha256

@@ -1,11 +1,11 @@
 c29c4fafe6691bfbfb6f90d761aa0650d79d0a5962a599926284d9e3e6a10006  scripts/install-k8s.sh
-944e980e796852f152601e929ea8391f7208b104eadbb34de7ab47841e77d7a2  scripts/lib/common.sh
+c64c2bcf00cbaffc00ae54a9149436a375ee5052e3e882fd689bc19a7b4a1620  scripts/lib/common.sh
 5ecbc741b61bfdbbdae14123d12328d786aa1a7d4e16bc3c31ce256eb4a01933  scripts/lib/preflight.sh
 19be2771df0e1a41b4fa9678e1cf6a77492304f66f73cf705a2ae42b1dac2ba3  scripts/lib/detect-gpu.sh
 7977ef12a16fc6aee1c2824cae13bd1450f7fe1c1345323292e4a14decbfe83f  scripts/lib/gpu-nvidia.sh
 b569eec2d8ffb9673da287a2a59d249a7dbc7236c98ab6a5062136bcc69a942c  scripts/lib/gpu-amd.sh
-cdc27b7929c42713e84006010aee06585d95102fa34798ee5a80ff7f05e33799  scripts/lib/setup-macos.sh
-74c9228617baaafa407a5f03876412c3fd7bf9858cb11239a083e5e6675d84f0  scripts/lib/setup-linux.sh
+76f0d4230d4aff510114968a477ef60e28860d1827c8ff36853a3f59d30be617  scripts/lib/setup-macos.sh
+bf4f299ad501a90184dd3dc9dadee9f82f839351c44f0b2542c2020b9c0cf0b7  scripts/lib/setup-linux.sh
 0d864b857e81d0bfde0c0ed9cbf663ca71741e1db0de4ccfbaee774164876e0b  scripts/lib/cluster.sh
 045caf6efeb583e5005d881d9281edae6a6aedf8f318b0a4021964b3b4b29cc5  scripts/lib/gpu-plugins.sh
 e673941dd9d63b2fcbb2051a1c3a86ae9a7ef6b780b9f52b34f37bfdefd66948  scripts/lib/install-client-helm.sh
@@ -15,4 +15,4 @@ e2ea63d844e6649f1d3aaae9fd4733845a1a39df37d68abbaeda00330f9e1c7e  scripts/lib/as
 b6b903a97872925ad2c0189292a81b7ebf8f0c6d4e93ac7b1e103e15afd5df68  scripts/lib/probe.sh
 cf095d9a92d6f4099ad19c8783d908a5952c7cbe7851043c90279724f7767dff  scripts/lib/summary.sh
 77e03332ebfab1ef759c6148a57afcf479c02c5dc6cc7b0e0e680f58e20cd364  scripts/lib/diagnose.sh
-1a23655ce08c4db959347dd815c093a8d5dfc867a4f61264e0729528c028de93  scripts/install-k8s.ps1
+3a01d1f17ea4f257dfa37c31dde0166bbdf1f9a020b027f8f1bf4362645fba86  scripts/install-k8s.ps1

scripts/tests/check-drift.bats

@@ -89,7 +89,38 @@ YAML
   _drift=0; _drift_workload_names >/dev/null 2>&1; [ "$_drift" -eq 0 ]
 }
 
-# ── Check 4: preflight download-host parity (#416) ───────────────────────────
+# ── Check 4: tool execute-gate parity (#411) ─────────────────────────────────
+@test "execute-gates: all installers gate all tools -> no drift (#411)" {
+  printf 'assert_tool_runs kubectl version --client\nassert_tool_runs k3d version\nassert_tool_runs helm version --short\n' > "$ROOT/scripts/lib/setup-linux.sh"
+  printf 'assert_tool_runs kubectl version --client\nassert_tool_runs k3d version\nassert_tool_runs helm version --short\n' > "$ROOT/scripts/lib/setup-macos.sh"
+  printf 'Assert-ToolRuns -Name "kubectl" -VersionArgs @("version","--client")\nAssert-ToolRuns -Name "k3d" -VersionArgs @("version")\nAssert-ToolRuns -Name "helm" -VersionArgs @("version","--short")\n' >> "$ROOT/scripts/install-k8s.ps1"
+  _drift=0; _drift_execute_gates >/dev/null; [ "$_drift" -eq 0 ]
+}
+
+@test "execute-gates: an installer missing a tool gate -> drift (#411)" {
+  printf 'assert_tool_runs kubectl version --client\nassert_tool_runs k3d version\nassert_tool_runs helm version\n' > "$ROOT/scripts/lib/setup-linux.sh"
+  printf 'assert_tool_runs kubectl version --client\nassert_tool_runs k3d version\n' > "$ROOT/scripts/lib/setup-macos.sh"   # no helm gate
+  printf 'Assert-ToolRuns -Name "kubectl" -VersionArgs @("version","--client")\nAssert-ToolRuns -Name "k3d" -VersionArgs @("version")\nAssert-ToolRuns -Name "helm" -VersionArgs @("version")\n' >> "$ROOT/scripts/install-k8s.ps1"
+  _drift=0; _drift_execute_gates >/dev/null 2>&1; [ "$_drift" -ge 1 ]
+}
+
+@test "execute-gates: the --rm <path> form is recognized as a gate (#411 review)" {
+  printf 'assert_tool_runs --rm "$TB_TOOLS_DIR/kubectl" kubectl version --client\nassert_tool_runs --rm "$TB_TOOLS_DIR/k3d" k3d version\nassert_tool_runs helm version\n' > "$ROOT/scripts/lib/setup-linux.sh"
+  printf 'assert_tool_runs kubectl version --client\nassert_tool_runs k3d version\nassert_tool_runs helm version\n' > "$ROOT/scripts/lib/setup-macos.sh"
+  printf 'Assert-ToolRuns -Name "kubectl" -VersionArgs @("version","--client")\nAssert-ToolRuns -Name "k3d" -VersionArgs @("version")\nAssert-ToolRuns -Name "helm" -VersionArgs @("version")\n' >> "$ROOT/scripts/install-k8s.ps1"
+  _drift=0; _drift_execute_gates >/dev/null; [ "$_drift" -eq 0 ]
+}
+
+@test "execute-gates: a COMMENTED-OUT gate does NOT count -> drift (Bugbot #411)" {
+  # A gate that lingers only in a comment must not satisfy the check (structured
+  # call extraction, not whole-file grep).
+  printf 'assert_tool_runs kubectl version --client\nassert_tool_runs k3d version\n# assert_tool_runs helm version\n' > "$ROOT/scripts/lib/setup-linux.sh"
+  printf 'assert_tool_runs kubectl version --client\nassert_tool_runs k3d version\nassert_tool_runs helm version\n' > "$ROOT/scripts/lib/setup-macos.sh"
+  printf 'Assert-ToolRuns -Name "kubectl" -VersionArgs @("version","--client")\nAssert-ToolRuns -Name "k3d" -VersionArgs @("version")\n# Assert-ToolRuns -Name "helm" -VersionArgs @("version")\n' >> "$ROOT/scripts/install-k8s.ps1"
+  _drift=0; _drift_execute_gates >/dev/null 2>&1; [ "$_drift" -ge 1 ]
+}
+
+# ── Check 5: preflight download-host parity (#416) ───────────────────────────
 # The check extracts hosts from PROBE ENTRIES only — bash "label|https://host/…"
 # and ps1 @{ label = "…"; url = "https://host/…" } (label REQUIRED on the ps1 line,
 # so an unrelated $url = "https://…" download line can't count). Fixtures write

scripts/tests/check-drift.sh

@@ -186,7 +186,44 @@ _drift_cli_contract() {
   fi
 }
 
-# ── Check 4: preflight probes the same download hosts on both installers ─────
+# ── Check 4: every installer execute-gates every tool (#411) ─────────────────
+# The post-install "check" used to be a log-only interpolation that masked a
+# corrupt/wrong-arch binary until cluster-create. All installers now RUN each tool
+# (assert_tool_runs on bash, Assert-ToolRuns on PowerShell). If an installer drops
+# the gate for a tool, the "green System tools then dies in Step 2" bug reopens on
+# that OS — catch it here rather than in the field.
+# Match the actual gate CALL, never a whole-file grep: strip comment lines first,
+# then require the tool's own version-check invocation — `assert_tool_runs … <t>
+# version` (the optional `--rm <path>` sits between and is ignored) on bash,
+# `Assert-ToolRuns -Name "<t>"` on ps1. A commented-out or merely-mentioned gate no
+# longer satisfies the check (reviewer: parity checks must extract structured calls).
+# NOTE: no `grep -q` — under main()'s `set -o pipefail` a -q that closes the pipe
+# early makes the upstream grep exit on SIGPIPE (141), failing the pipeline even on
+# a match. Read to EOF and redirect instead.
+_drift_gate_bash() { grep -vE '^[[:space:]]*#' "$1" 2>/dev/null | grep -F assert_tool_runs | grep -E "(^| )$2 version" >/dev/null 2>&1; }
+_drift_gate_ps1()  { grep -vE '^[[:space:]]*#' "$1" 2>/dev/null | grep -E "Assert-ToolRuns -Name \"$2\"" >/dev/null 2>&1; }
+_drift_execute_gates() {
+  echo "▸ Tool execute-gate parity (setup-linux.sh · setup-macos.sh · install-k8s.ps1)"
+  local before=$_drift f t
+  local bash_files=( scripts/lib/setup-linux.sh scripts/lib/setup-macos.sh )
+  for f in "${bash_files[@]}"; do
+    if [[ ! -f "$DRIFT_ROOT/$f" ]]; then _note "$f is missing"; continue; fi
+    for t in kubectl k3d helm; do
+      _drift_gate_bash "$DRIFT_ROOT/$f" "$t" || \
+        _note "$f: no execute-gate call for '$t' (assert_tool_runs … $t version) — #411"
+    done
+  done
+  local ps1="scripts/install-k8s.ps1"
+  if [[ ! -f "$DRIFT_ROOT/$ps1" ]]; then _note "$ps1 is missing"; fi
+  for t in kubectl k3d helm; do
+    _drift_gate_ps1 "$DRIFT_ROOT/$ps1" "$t" || \
+      _note "$ps1: no execute-gate call for '$t' (Assert-ToolRuns -Name \"$t\") — #411"
+  done
+  if [[ "$_drift" -eq "$before" ]]; then _ok "all installers execute-gate kubectl / k3d / helm"; fi
+  return 0
+}
+
+# ── Check 5: preflight probes the same download hosts on both installers ─────
 # preflight.sh (Linux/macOS) and install-k8s.ps1 (Windows) each probe the hosts
 # the install fetches from (#416). A host added to one installer but not the other
 # silently reopens the "green preflight, blocked download 30s later" gap on
@@ -234,6 +271,7 @@ main() {
   _drift_backend_hosts
   _drift_workload_names
   _drift_cli_contract
+  _drift_execute_gates
   _drift_preflight_hosts
   echo "─────────────────────────────────────────────────────────────"
   if [[ "$_drift" -gt 0 ]]; then