fix(release): classify App-token 404s when creating nightly tags

.github/workflows/nightly.yml

@@ -293,16 +293,8 @@ jobs:
         shell: bash
         env:
           GH_TOKEN: ${{ steps.app-token.outputs.token }}
-        run: |
-          if tag_lookup="$(gh api --include "repos/${{ github.repository }}/git/ref/tags/$TAG" 2>&1)"; then
-            echo "nightly tag appeared before creation: $TAG" >&2
-            exit 1
-          fi
-          grep -Eq '^HTTP/[0-9.]+ 404([[:space:]]|$)' <<<"$tag_lookup" || { echo 'unable to prove nightly tag absence with the mutation token' >&2; printf '%s\n' "$tag_lookup" >&2; exit 1; }
-          response="$(gh api --method POST "repos/${{ github.repository }}/git/refs" -f ref="refs/tags/$TAG" -f sha="$SOURCE_SHA")"
-          [ "$(jq -r '.object.sha' <<<"$response")" = "$SOURCE_SHA" ]
-          [ "$(gh api "repos/${{ github.repository }}/git/ref/tags/$TAG" --jq '.object.sha')" = "$SOURCE_SHA" ]
-          jq -n --arg tag "$TAG" --arg sha "$SOURCE_SHA" --arg intent "$(sha256sum nightly-intent.json | awk '{print $1}')" --arg workflow "${{ github.workflow_sha }}" --argjson run "${{ github.run_id }}" --argjson attempt "${{ github.run_attempt }}" '{schemaVersion:1,tag:$tag,peeledSha:$sha,intentSha256:$intent,workflowSha:$workflow,runId:$run,runAttempt:$attempt,apiStatus:201}' >nightly-tag-receipt.json
+          WORKFLOW_SHA: ${{ github.workflow_sha }}
+        run: bash scripts/create-nightly-tag.sh
       - id: receipt
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with: {name: nightly-tag-creation-receipt, path: nightly-tag-receipt.json, retention-days: 90, if-no-files-found: error}

scripts/create-nightly-tag-harness.sh

@@ -0,0 +1,151 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+root="$(git rev-parse --show-toplevel)"
+tmp="$(mktemp -d)"
+trap 'rm -rf "$tmp"' EXIT
+log="$tmp/gh.log"
+peel_remaining_file="$tmp/peel-remaining"
+lookup_mode_file="$tmp/lookup-mode"
+create_status_file="$tmp/create-status"
+printf '1\n' >"$peel_remaining_file"
+printf 'missing-headers\n' >"$lookup_mode_file"
+printf '201\n' >"$create_status_file"
+
+gh() {
+  local endpoint='' method=GET
+  while [ "$#" -gt 0 ]; do
+    case "$1" in
+      --include) shift ;;
+      --method) method="$2"; shift 2 ;;
+      repos/*) endpoint="$1"; shift ;;
+      -f|-F) shift 2 ;;
+      *) shift ;;
+    esac
+  done
+  printf '%s %s\n' "$method" "$endpoint" >>"$NIGHTLY_GH_LOG"
+  case "$endpoint" in
+    */git/ref/tags/*)
+      if [ "$method" != GET ]; then
+        echo "create-nightly-tag-harness: unexpected tag method: $method" >&2
+        return 2
+      fi
+      if [ ! -s "$NIGHTLY_TAG_STATE" ]; then
+        if [ "$(cat "$NIGHTLY_LOOKUP_MODE")" = missing-headers ]; then
+          printf 'gh: Not Found (HTTP 404)\n' >&2
+        else
+          printf 'HTTP/2.0 404 Not Found\n\n{"message":"Not Found","status":"404"}\n' >&2
+        fi
+        return 1
+      fi
+      remaining="$(cat "$NIGHTLY_PEEL_REMAINING")"
+      if [ "$remaining" -gt 0 ]; then
+        printf '%s\n' "$((remaining - 1))" >"$NIGHTLY_PEEL_REMAINING"
+        printf 'gh: Not Found (HTTP 404)\n' >&2
+        return 1
+      fi
+      printf 'HTTP/2.0 200 OK\n\n{"object":{"sha":"%s"}}\n' "$(cat "$NIGHTLY_TAG_STATE")"
+      ;;
+    */git/refs)
+      [ "$method" = POST ]
+      status="$(cat "$NIGHTLY_CREATE_STATUS")"
+      if [ "$status" = 422 ]; then
+        printf 'HTTP/2.0 422 Unprocessable Entity\n\n{"message":"Reference already exists"}\n' >&2
+        return 1
+      fi
+      if [ "$status" != 201 ]; then
+        printf 'gh: Not Found (HTTP 404)\n' >&2
+        return 1
+      fi
+      printf '%s\n' "$SOURCE_SHA" >"$NIGHTLY_TAG_STATE"
+      printf 'HTTP/2.0 201 Created\n\n{"object":{"sha":"%s"}}\n' "$SOURCE_SHA"
+      ;;
+    *)
+      echo "create-nightly-tag-harness: unsupported gh endpoint: $endpoint" >&2
+      return 2
+      ;;
+  esac
+}
+export -f gh
+export NIGHTLY_GH_LOG="$log"
+export NIGHTLY_TAG_STATE="$tmp/tag-state"
+export NIGHTLY_PEEL_REMAINING="$peel_remaining_file"
+export NIGHTLY_LOOKUP_MODE="$lookup_mode_file"
+export NIGHTLY_CREATE_STATUS="$create_status_file"
+export GITHUB_REPOSITORY='jatmn/Codex-warp'
+export SOURCE_SHA='1111111111111111111111111111111111111111'
+export TAG='nightly-20260913-111111111111'
+export WORKFLOW_SHA='2222222222222222222222222222222222222222'
+export GITHUB_RUN_ID=34762668596
+export GITHUB_RUN_ATTEMPT=1
+export NIGHTLY_INTENT_FILE="$tmp/nightly-intent.json"
+export NIGHTLY_RECEIPT_FILE="$tmp/nightly-tag-receipt.json"
+export NIGHTLY_TAG_PEEL_ATTEMPTS=3
+export NIGHTLY_TAG_PEEL_SLEEP=0
+printf '{}\n' >"$NIGHTLY_INTENT_FILE"
+: >"$NIGHTLY_TAG_STATE"
+: >"$log"
+
+run_create() {
+  bash "$root/scripts/create-nightly-tag.sh"
+}
+
+# gh 2.100-style 404 with no HTTP status line, then a delayed peel.
+run_create
+jq -e --arg tag "$TAG" --arg sha "$SOURCE_SHA" --arg workflow "$WORKFLOW_SHA" \
+  --argjson run "$GITHUB_RUN_ID" --argjson attempt "$GITHUB_RUN_ATTEMPT" \
+  '.tag==$tag and .peeledSha==$sha and .workflowSha==$workflow and .runId==$run and .runAttempt==$attempt and .apiStatus==201 and (.intentSha256|test("^[0-9a-f]{64}$"))' \
+  "$NIGHTLY_RECEIPT_FILE" >/dev/null
+grep -c '^GET .*/git/ref/tags/' "$log" | grep -Fx 3 >/dev/null
+grep -E '^POST .*/git/refs$' "$log" >/dev/null
+
+# Header-bearing 404 still proves absence.
+: >"$NIGHTLY_TAG_STATE"
+: >"$log"
+printf 'headers\n' >"$lookup_mode_file"
+printf '0\n' >"$peel_remaining_file"
+rm -f "$NIGHTLY_RECEIPT_FILE"
+run_create
+jq -e '.apiStatus==201' "$NIGHTLY_RECEIPT_FILE" >/dev/null
+
+# Existing tag fails closed before POST.
+: >"$log"
+printf '%s\n' "$SOURCE_SHA" >"$NIGHTLY_TAG_STATE"
+printf '0\n' >"$peel_remaining_file"
+appeared=0
+run_create >/dev/null 2>"$tmp/appeared.err" || appeared=$?
+[ "$appeared" -ne 0 ]
+grep -F "nightly tag appeared before creation: $TAG" "$tmp/appeared.err" >/dev/null
+if grep -E '^POST ' "$log" >/dev/null; then
+  echo 'create-nightly-tag-harness: existing tag reached POST' >&2
+  exit 1
+fi
+
+# 422 after absence proof fails closed without a receipt.
+: >"$NIGHTLY_TAG_STATE"
+: >"$log"
+printf 'missing-headers\n' >"$lookup_mode_file"
+printf '201\n' >"$create_status_file"
+printf '422\n' >"$create_status_file"
+rm -f "$NIGHTLY_RECEIPT_FILE"
+raced=0
+run_create >/dev/null 2>"$tmp/race.err" || raced=$?
+[ "$raced" -ne 0 ]
+[ ! -f "$NIGHTLY_RECEIPT_FILE" ]
+grep -F 'tag create raced with 422' "$tmp/race.err" >/dev/null
+
+# Unclassifiable lookup is not treated as absence.
+: >"$NIGHTLY_TAG_STATE"
+: >"$log"
+gh() {
+  printf 'GET %s\n' "$1" >>"$NIGHTLY_GH_LOG"
+  printf 'gh: Gateway Timeout\n' >&2
+  return 1
+}
+export -f gh
+unclassified=0
+run_create >/dev/null 2>"$tmp/unclassified.err" || unclassified=$?
+[ "$unclassified" -ne 0 ]
+grep -F 'unable to prove nightly tag absence with the mutation token' "$tmp/unclassified.err" >/dev/null
+
+echo 'create-nightly-tag-harness: ok'

scripts/create-nightly-tag.sh

@@ -0,0 +1,116 @@
+#!/usr/bin/env bash
+# Create an immutable nightly tag with App-token HTTP classification and a
+# retried post-create peel before writing the tag-creation receipt.
+set -euo pipefail
+
+: "${GITHUB_REPOSITORY:?}"
+: "${SOURCE_SHA:?}"
+: "${TAG:?}"
+: "${WORKFLOW_SHA:?}"
+: "${GITHUB_RUN_ID:?}"
+: "${GITHUB_RUN_ATTEMPT:?}"
+[[ "$SOURCE_SHA" =~ ^[0-9a-f]{40}$ ]]
+[[ "$TAG" =~ ^nightly-[0-9]{8}-[0-9a-f]{12}$ ]]
+[[ "$WORKFLOW_SHA" =~ ^[0-9a-f]{40}$ ]]
+[[ "$GITHUB_RUN_ID" =~ ^[1-9][0-9]*$ ]]
+[[ "$GITHUB_RUN_ATTEMPT" =~ ^[1-9][0-9]*$ ]]
+
+root="$(git rev-parse --show-toplevel)"
+intent="${NIGHTLY_INTENT_FILE:-nightly-intent.json}"
+receipt="${NIGHTLY_RECEIPT_FILE:-nightly-tag-receipt.json}"
+peel_attempts="${NIGHTLY_TAG_PEEL_ATTEMPTS:-8}"
+peel_sleep="${NIGHTLY_TAG_PEEL_SLEEP:-1}"
+[[ "$peel_attempts" =~ ^[1-9][0-9]*$ ]]
+[[ "$peel_sleep" =~ ^[0-9]+([.][0-9]+)?$ ]]
+[ -f "$intent" ] || { echo "create-nightly-tag: missing intent file: $intent" >&2; exit 1; }
+
+api_body() {
+  sed -n '/^{/,$p'
+}
+
+http_code() {
+  local raw="$1"
+  if [[ "$raw" =~ HTTP/[0-9.]+[[:space:]]+([0-9]{3}) ]]; then
+    printf '%s\n' "${BASH_REMATCH[1]}"
+    return 0
+  fi
+  if [[ "$raw" =~ \(HTTP[[:space:]]+([0-9]{3})\) ]]; then
+    printf '%s\n' "${BASH_REMATCH[1]}"
+    return 0
+  fi
+  return 1
+}
+
+lookup=''
+if lookup="$(gh api --include "repos/$GITHUB_REPOSITORY/git/ref/tags/$TAG" 2>&1)"; then
+  [ "$(http_code "$lookup" || true)" = 200 ] || {
+    echo 'create-nightly-tag: tag lookup returned an unexpected success status' >&2
+    printf '%s\n' "$lookup" >&2
+    exit 1
+  }
+  echo "nightly tag appeared before creation: $TAG" >&2
+  exit 1
+fi
+[ "$(http_code "$lookup" || true)" = 404 ] || {
+  echo 'unable to prove nightly tag absence with the mutation token' >&2
+  printf '%s\n' "$lookup" >&2
+  exit 1
+}
+
+created=''
+if created="$(gh api --include --method POST "repos/$GITHUB_REPOSITORY/git/refs" -f ref="refs/tags/$TAG" -f sha="$SOURCE_SHA" 2>&1)"; then
+  [ "$(http_code "$created" || true)" = 201 ] || {
+    echo 'create-nightly-tag: create returned an unexpected success status' >&2
+    printf '%s\n' "$created" >&2
+    exit 1
+  }
+  [ "$(api_body <<<"$created" | jq -er '.object.sha')" = "$SOURCE_SHA" ]
+else
+  [ "$(http_code "$created" || true)" = 422 ] || {
+    echo 'create-nightly-tag: create failed without a classifiable race' >&2
+    printf '%s\n' "$created" >&2
+    exit 1
+  }
+  echo "create-nightly-tag: tag create raced with 422: $TAG" >&2
+  printf '%s\n' "$created" >&2
+  exit 1
+fi
+
+peeled=''
+peeled_sha=''
+attempt=0
+while [ "$attempt" -lt "$peel_attempts" ]; do
+  attempt=$((attempt + 1))
+  if peeled="$(gh api --include "repos/$GITHUB_REPOSITORY/git/ref/tags/$TAG" 2>&1)"; then
+    [ "$(http_code "$peeled" || true)" = 200 ] || {
+      echo 'create-nightly-tag: peel returned an unexpected success status' >&2
+      printf '%s\n' "$peeled" >&2
+      exit 1
+    }
+    peeled_sha="$(api_body <<<"$peeled" | jq -er '.object.sha')"
+    [ "$peeled_sha" = "$SOURCE_SHA" ]
+    break
+  fi
+  [ "$(http_code "$peeled" || true)" = 404 ] || {
+    echo 'create-nightly-tag: unable to peel nightly tag after creation' >&2
+    printf '%s\n' "$peeled" >&2
+    exit 1
+  }
+  [ "$attempt" -lt "$peel_attempts" ] || {
+    echo 'create-nightly-tag: peel kept returning 404 after tag create' >&2
+    printf '%s\n' "$peeled" >&2
+    exit 1
+  }
+  sleep "$peel_sleep"
+done
+[ -n "$peeled_sha" ]
+
+jq -n \
+  --arg tag "$TAG" \
+  --arg sha "$SOURCE_SHA" \
+  --arg intent "$(bash "$root/scripts/sha256-file.sh" "$intent")" \
+  --arg workflow "$WORKFLOW_SHA" \
+  --argjson run "$GITHUB_RUN_ID" \
+  --argjson attempt "$GITHUB_RUN_ATTEMPT" \
+  '{schemaVersion:1,tag:$tag,peeledSha:$sha,intentSha256:$intent,workflowSha:$workflow,runId:$run,runAttempt:$attempt,apiStatus:201}' \
+  >"$receipt"

scripts/source-checks.sh

@@ -123,6 +123,10 @@ if command -v node >/dev/null 2>&1; then
   if ! bash scripts/advance-nightly-branch-harness.sh; then
     fail=1
   fi
+
+  if ! bash scripts/create-nightly-tag-harness.sh; then
+    fail=1
+  fi
 else
   echo 'source-checks: Node is required for release policy validation' >&2
   fail=1

tools/release-please-policy/validate-workflows.mjs

@@ -262,7 +262,13 @@ assert.ok(!read('scripts/check-prior-official-releases.sh').includes('gh attesta
 assert.equal(nightly.jobs['repair-branch'].steps.find(step => step.uses?.startsWith('actions/checkout@')).with.ref, '${{ github.workflow_sha }}');
 assert.ok(nightly.jobs['repair-branch'].steps.some(step => step.run?.includes('scripts/advance-nightly-branch.sh')),
   'nightly repair must use the exact API branch race protocol');
-assert.ok(read('.github/workflows/nightly.yml').includes('unable to prove nightly tag absence with the mutation token'));
+assert.ok(nightly.jobs.publish.steps.some(step => step.name === 'Create immutable nightly tag and receipt' &&
+  step.run === 'bash scripts/create-nightly-tag.sh' &&
+  step.env.WORKFLOW_SHA === '${{ github.workflow_sha }}'),
+  'nightly publication must create the immutable tag through the classified App-token helper');
+assert.ok(read('scripts/create-nightly-tag.sh').includes('unable to prove nightly tag absence with the mutation token'));
+assert.ok(read('scripts/create-nightly-tag.sh').includes('NIGHTLY_TAG_PEEL_ATTEMPTS'),
+  'nightly tag creation must retry the post-create peel before writing the receipt');
 
 const release = parse('.github/workflows/release.yml');
 const releaseSource = read('.github/workflows/release.yml');