From 98da74680dd9c78a7cd9597eaf9e32264bbfb99f Mon Sep 17 00:00:00 2001 From: Akhila Illa Date: Wed, 30 Sep 2026 23:52:28 -0500 Subject: [PATCH 01/16] Integrate ARM semantic review with auto-signoff Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/arm-api-review.lock.yml | 192 ++++++- .github/workflows/arm-api-review.md | 190 ++++++- .../workflows/arm-universal-auto-signoff.yaml | 31 +- .../arm-semantic-review-workflow.ts | 331 ++++++++++++ .../arm-auto-signoff/arm-semantic-review.ts | 272 ++++++++++ .../arm-universal-auto-signoff.ts | 78 ++- .github/workflows/src/context.ts | 167 ++++-- .github/workflows/src/update-labels.ts | 19 + .../test/arm-api-review-workflow.test.ts | 58 +- .../arm-semantic-review.test.ts | 505 ++++++++++++++++++ .../arm-universal-auto-signoff.test.ts | 149 +++++- .github/workflows/test/context.test.ts | 162 ++++++ .github/workflows/test/mocks.ts | 1 + .github/workflows/test/update-labels.test.ts | 44 +- 14 files changed, 2065 insertions(+), 134 deletions(-) create mode 100644 .github/workflows/src/arm-auto-signoff/arm-semantic-review-workflow.ts create mode 100644 .github/workflows/src/arm-auto-signoff/arm-semantic-review.ts create mode 100644 .github/workflows/test/arm-auto-signoff/arm-semantic-review.test.ts diff --git a/.github/workflows/arm-api-review.lock.yml b/.github/workflows/arm-api-review.lock.yml index 22ce65e33f50..f41c5e0f7e72 100644 --- a/.github/workflows/arm-api-review.lock.yml +++ b/.github/workflows/arm-api-review.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"dbc4a4bcf44913026b742f3374995a32c7f79172f0192676338a68451c749308","body_hash":"4c92627abe9eab5cd6657b8a78c79ade836a9af4f97cf2dc3e33dc9cb7eec358","compiler_version":"v0.86.2","strict":true,"agent_id":"copilot","agent_model":"gpt-5.6-sol?effort=high","detection_agent_id":"copilot","detection_agent_model":"gpt-5.6-sol?effort=high","engine_versions":{"copilot":"1.0.79"}} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"9a949b6ca0721e43b0e2820f862298b806ab68bdeb2816804f272957d6ac3d64","body_hash":"7dabe4a0f4780c2250806fb5c092b7277760ceeb55d3840824c9a17615d8da9d","compiler_version":"v0.86.2","strict":true,"agent_id":"copilot","agent_model":"gpt-5.6-sol?effort=high","detection_agent_id":"copilot","detection_agent_model":"gpt-5.6-sol?effort=high","engine_versions":{"copilot":"1.0.79"}} # gh-aw-manifest: {"version":1,"secrets":["GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/cache/restore","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/cache/save","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"373c709c69115d41ff229c7e5df9f8788daa9553","version":"v9"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/github-script","sha":"d746ffe35508b1917358783b479e04febd2b8f71","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"820762786026740c76f36085b0efc47a31fe5020","version":"v7.0.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"},{"repo":"github/gh-aw-actions/setup","sha":"6aab9e5b5c91c615506061f09bedd81a23babe3c","version":"v0.86.2"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.27.44","digest":"sha256:0d727725c737b58c7bdf51f640cffb928385ec46517e0917c7f1a02f1bada8b4","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.27.44@sha256:0d727725c737b58c7bdf51f640cffb928385ec46517e0917c7f1a02f1bada8b4"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.44","digest":"sha256:b50fbadba138f6e9aba94aca09711335c489bb3b15861220cb66f6092e042dc7","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.44@sha256:b50fbadba138f6e9aba94aca09711335c489bb3b15861220cb66f6092e042dc7"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.27.44","digest":"sha256:83e48bbe12c634be8c228a576832fe45f66c529ac3659db92bddbcf2eeb6d627","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.27.44@sha256:83e48bbe12c634be8c228a576832fe45f66c529ac3659db92bddbcf2eeb6d627"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.9","digest":"sha256:e5a1569aeaf41820fa7bdee3e94468cae448133cdbf00119ad24f5b74db1ab9f","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.9@sha256:e5a1569aeaf41820fa7bdee3e94468cae448133cdbf00119ad24f5b74db1ab9f"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:0d9f1fb5fd6610c0ac1f5194a38e45a8a1e81f8a390d5142d8e4e6f26a4b3196","pinned_image":"ghcr.io/github/gh-aw-node@sha256:0d9f1fb5fd6610c0ac1f5194a38e45a8a1e81f8a390d5142d8e4e6f26a4b3196"},{"image":"ghcr.io/github/github-mcp-server:v1.9.0","digest":"sha256:881b53d6f75f69bdbc1b5b10fc2f1361717c19054143b3a8529fb5c32061a50e","pinned_image":"ghcr.io/github/github-mcp-server:v1.9.0@sha256:881b53d6f75f69bdbc1b5b10fc2f1361717c19054143b3a8529fb5c32061a50e"}],"has_pull_request_target":true} # This file was automatically generated by gh-aw (v0.86.2). DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # @@ -44,12 +44,14 @@ # - actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 # - actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 # - actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 +# - actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 (source v7) # - actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 # - actions/github-script@373c709c69115d41ff229c7e5df9f8788daa9553 # v9 # - actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 # - actions/github-script@d746ffe35508b1917358783b479e04febd2b8f71 # v9.0.0 # - actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 # - actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 +# - actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 (source v7) # - github/gh-aw-actions/setup@6aab9e5b5c91c615506061f09bedd81a23babe3c # v0.86.2 # # Container images used: @@ -69,6 +71,7 @@ on: - created # permissions: # Permissions applied to pre-activation job # pull-requests: read + # statuses: write pull_request_target: forks: - "*" @@ -88,7 +91,34 @@ on: # name: Resolve target pull request # uses: actions/github-script@v9 # with: - # script: "const value = process.env.TARGET_PR_NUMBER ?? \"\";\nif (!/^[1-9]\\d*$/.test(value)) {\n throw new Error(`Invalid or missing pull request number: ${JSON.stringify(value)}`);\n}\n\nif (context.eventName === \"issue_comment\" && !context.payload.issue?.pull_request) {\n throw new Error(`Issue #${value} is not a pull request`);\n}\n\nconst pullNumber = Number(value);\nif (!Number.isSafeInteger(pullNumber)) {\n throw new Error(`Pull request number is outside the safe integer range: ${value}`);\n}\n\nconst { data: pull } = await github.rest.pulls.get({\n ...context.repo,\n pull_number: pullNumber,\n});\ncore.setOutput(\"target_pr_number\", String(pull.number));\n" + # script: "const value = process.env.TARGET_PR_NUMBER ?? \"\";\nif (!/^[1-9]\\d*$/.test(value)) {\n throw new Error(`Invalid or missing pull request number: ${JSON.stringify(value)}`);\n}\n\nif (context.eventName === \"issue_comment\" && !context.payload.issue?.pull_request) {\n throw new Error(`Issue #${value} is not a pull request`);\n}\n\nconst pullNumber = Number(value);\nif (!Number.isSafeInteger(pullNumber)) {\n throw new Error(`Pull request number is outside the safe integer range: ${value}`);\n}\n\nconst { data: pull } = await github.rest.pulls.get({\n ...context.repo,\n pull_number: pullNumber,\n});\nif (!/^[0-9a-f]{40}$/i.test(pull.head.sha)) {\n throw new Error(`Invalid pull request head SHA: ${JSON.stringify(pull.head.sha)}`);\n}\ncore.setOutput(\"target_pr_number\", String(pull.number));\ncore.setOutput(\"target_head_sha\", pull.head.sha);\n" + # - if: steps.check_membership.outputs.is_team_member == 'true' + # name: Prepare ARM semantic review correlation artifact + # run: "mkdir -p \"$RUNNER_TEMP/arm-semantic-review\"\n: > \"$RUNNER_TEMP/arm-semantic-review/empty.txt\"\n" + # shell: bash + # - if: steps.check_membership.outputs.is_team_member == 'true' + # name: Upload ARM semantic review head SHA + # uses: actions/upload-artifact@v7 + # with: + # if-no-files-found: error + # name: head-sha=${{ steps.resolve_target_pr.outputs.target_head_sha }} + # overwrite: true + # path: ${{ runner.temp }}/arm-semantic-review/empty.txt + # - if: steps.check_membership.outputs.is_team_member == 'true' + # name: Upload ARM semantic review issue number + # uses: actions/upload-artifact@v7 + # with: + # if-no-files-found: error + # name: issue-number=${{ steps.resolve_target_pr.outputs.target_pr_number }} + # overwrite: true + # path: ${{ runner.temp }}/arm-semantic-review/empty.txt + # - env: + # TARGET_HEAD_SHA: ${{ steps.resolve_target_pr.outputs.target_head_sha }} + # if: steps.check_membership.outputs.is_team_member == 'true' + # name: Set ARM semantic review pending + # uses: actions/github-script@v9 + # with: + # script: "const runAttempt = process.env.GITHUB_RUN_ATTEMPT ?? \"\";\nif (!/^[1-9]\\d*$/.test(runAttempt)) {\n throw new Error(`Invalid workflow run attempt: ${JSON.stringify(runAttempt)}`);\n}\nawait github.rest.repos.createCommitStatus({\n ...context.repo,\n sha: process.env.TARGET_HEAD_SHA,\n state: \"pending\",\n context: \"ARM Semantic Review\",\n description: \"ARM API semantic review is pending\",\n target_url:\n `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/` +\n `${context.runId}/attempts/${runAttempt}`,\n});\n" workflow_dispatch: inputs: aw_context: @@ -332,9 +362,10 @@ jobs: GH_AW_GITHUB_REPOSITORY: ${{ github.repository }} GH_AW_GITHUB_RUN_ID: ${{ github.run_id }} GH_AW_GITHUB_WORKSPACE: ${{ github.workspace }} + GH_AW_NEEDS_PRE_ACTIVATION_OUTPUTS_TARGET_HEAD_SHA: ${{ needs.pre_activation.outputs.target_head_sha }} GH_AW_NEEDS_PRE_ACTIVATION_OUTPUTS_TARGET_PR_NUMBER: ${{ needs.pre_activation.outputs.target_pr_number }} GH_AW_PROMPT_CONTENT_0000: "\n" - GH_AW_PROMPT_CONTENT_0001: "\nTools: add_comment(max:4), create_pull_request_review_comment(max:50), submit_pull_request_review, reply_to_pull_request_review_comment(max:50), resolve_pull_request_review_thread(max:50), add_labels(max:3), remove_labels(max:3), missing_tool, missing_data, noop\n" + GH_AW_PROMPT_CONTENT_0001: "\nTools: add_comment(max:4), create_pull_request_review_comment(max:50), submit_pull_request_review, reply_to_pull_request_review_comment(max:50), resolve_pull_request_review_thread(max:50), add_labels(max:3), remove_labels(max:3), missing_tool, missing_data, noop, record_arm_semantic_review\n" GH_AW_PROMPT_CONTENT_0002: "\n" GH_AW_PROMPT_CONTENT_0003: "\nThe following GitHub context information is available for this workflow:\n{{#if github.actor}}\n- **actor**: __GH_AW_GITHUB_ACTOR__\n{{/if}}\n{{#if github.repository}}\n- **repository**: __GH_AW_GITHUB_REPOSITORY__\n{{/if}}\n{{#if github.workspace}}\n- **workspace**: __GH_AW_GITHUB_WORKSPACE__\n{{/if}}\n{{#if github.event.issue.number || (github.aw.context.item_type == 'issue' && github.aw.context.item_number)}}\n- **issue-number**: #__GH_AW_EXPR_802A9F6A__\n{{/if}}\n{{#if github.event.discussion.number || (github.aw.context.item_type == 'discussion' && github.aw.context.item_number)}}\n- **discussion-number**: #__GH_AW_EXPR_1A3A194A__\n{{/if}}\n{{#if github.event.pull_request.number || (github.aw.context.item_type == 'pull_request' && github.aw.context.item_number)}}\n- **pull-request-number**: #__GH_AW_EXPR_463A214A__\n{{/if}}\n{{#if github.event.comment.id || github.aw.context.comment_id}}\n- **comment-id**: __GH_AW_EXPR_FF1D34CE__\n{{/if}}\n{{#if github.run_id}}\n- **workflow-run-id**: __GH_AW_GITHUB_RUN_ID__\n{{/if}}\n\n\n" GH_AW_PROMPT_CONTENT_0004: "\n" @@ -359,6 +390,7 @@ jobs: GH_AW_ENGINE_ID: "copilot" GH_AW_GITHUB_EVENT_NAME: ${{ github.event_name }} GH_AW_GITHUB_REPOSITORY: ${{ github.repository }} + GH_AW_NEEDS_PRE_ACTIVATION_OUTPUTS_TARGET_HEAD_SHA: ${{ needs.pre_activation.outputs.target_head_sha }} GH_AW_NEEDS_PRE_ACTIVATION_OUTPUTS_TARGET_PR_NUMBER: ${{ needs.pre_activation.outputs.target_pr_number }} with: script: | @@ -381,6 +413,7 @@ jobs: GH_AW_GITHUB_WORKSPACE: ${{ github.workspace }} GH_AW_MCP_CLI_SERVERS_LIST: "- `github` β€” run `github --help` to see available tools\n- `safeoutputs` β€” run `safeoutputs --help` to see available tools" GH_AW_NEEDS_PRE_ACTIVATION_OUTPUTS_ACTIVATED: ${{ needs.pre_activation.outputs.activated }} + GH_AW_NEEDS_PRE_ACTIVATION_OUTPUTS_TARGET_HEAD_SHA: ${{ needs.pre_activation.outputs.target_head_sha }} GH_AW_NEEDS_PRE_ACTIVATION_OUTPUTS_TARGET_PR_NUMBER: ${{ needs.pre_activation.outputs.target_pr_number }} with: script: | @@ -404,6 +437,7 @@ jobs: GH_AW_GITHUB_WORKSPACE: process.env.GH_AW_GITHUB_WORKSPACE, GH_AW_MCP_CLI_SERVERS_LIST: process.env.GH_AW_MCP_CLI_SERVERS_LIST, GH_AW_NEEDS_PRE_ACTIVATION_OUTPUTS_ACTIVATED: process.env.GH_AW_NEEDS_PRE_ACTIVATION_OUTPUTS_ACTIVATED, + GH_AW_NEEDS_PRE_ACTIVATION_OUTPUTS_TARGET_HEAD_SHA: process.env.GH_AW_NEEDS_PRE_ACTIVATION_OUTPUTS_TARGET_HEAD_SHA, GH_AW_NEEDS_PRE_ACTIVATION_OUTPUTS_TARGET_PR_NUMBER: process.env.GH_AW_NEEDS_PRE_ACTIVATION_OUTPUTS_TARGET_PR_NUMBER } }); @@ -555,9 +589,9 @@ jobs: mkdir -p "${RUNNER_TEMP}/gh-aw/safeoutputs" mkdir -p /tmp/gh-aw/safeoutputs mkdir -p /tmp/gh-aw/mcp-logs/safeoutputs - cat > "${RUNNER_TEMP}/gh-aw/safeoutputs/config.json" << 'GH_AW_SAFE_OUTPUTS_CONFIG_193a4741548b2c28_EOF' - {"add_comment":{"max":4,"target":"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}"},"add_labels":{"max":3,"target":"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}"},"create_pull_request_review_comment":{"max":50,"side":"RIGHT","target":"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}"},"create_report_incomplete_issue":{},"missing_data":{},"missing_tool":{},"noop":{"max":1,"report-as-issue":"true"},"remove_labels":{"max":3,"target":"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}"},"reply_to_pull_request_review_comment":{"max":50,"target":"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}"},"report_incomplete":{},"resolve_pull_request_review_thread":{"max":50},"submit_pull_request_review":{"footer":"if-body","max":1,"target":"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}"}} - GH_AW_SAFE_OUTPUTS_CONFIG_193a4741548b2c28_EOF + cat > "${RUNNER_TEMP}/gh-aw/safeoutputs/config.json" << 'GH_AW_SAFE_OUTPUTS_CONFIG_114ad090dfc07ef3_EOF' + {"add_comment":{"max":4,"target":"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}"},"add_labels":{"max":3,"target":"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}"},"create_pull_request_review_comment":{"max":50,"side":"RIGHT","target":"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}"},"create_report_incomplete_issue":{},"missing_data":{},"missing_tool":{},"noop":{"max":1,"report-as-issue":"true"},"record-arm-semantic-review":{"description":"Validate and record the ARM semantic review result for auto-signoff","inputs":{"blocking_count":{"default":null,"description":"Number of verified, currently applicable Blocking findings after reconciliation","required":true,"type":"string"},"completeness":{"default":null,"description":"Whether the intended review completed reliably","options":["complete","incomplete","degraded"],"required":true,"type":"choice"},"head_sha":{"default":null,"description":"Full pull request head SHA that was reviewed","required":true,"type":"string"},"issue_number":{"default":null,"description":"Target pull request number","required":true,"type":"string"},"scope":{"default":null,"description":"Whether the full PR or only a scoped subset was reviewed","options":["full","scoped"],"required":true,"type":"choice"}},"output":"ARM semantic review result recorded"},"remove_labels":{"max":3,"target":"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}"},"reply_to_pull_request_review_comment":{"max":50,"target":"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}"},"report_incomplete":{},"resolve_pull_request_review_thread":{"max":50},"submit_pull_request_review":{"footer":"if-body","max":1,"target":"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}"}} + GH_AW_SAFE_OUTPUTS_CONFIG_114ad090dfc07ef3_EOF - name: Generate Safe Outputs Tools env: GH_AW_TOOLS_META_JSON: | @@ -572,7 +606,54 @@ jobs: "submit_pull_request_review": " CONSTRAINTS: Maximum 1 review(s) can be submitted. Target: ${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}." }, "repo_params": {}, - "dynamic_tools": [] + "dynamic_tools": [ + { + "description": "Validate and record the ARM semantic review result for auto-signoff", + "inputSchema": { + "additionalProperties": false, + "properties": { + "blocking_count": { + "description": "Number of verified, currently applicable Blocking findings after reconciliation", + "type": "string" + }, + "completeness": { + "description": "Whether the intended review completed reliably", + "enum": [ + "complete", + "incomplete", + "degraded" + ], + "type": "string" + }, + "head_sha": { + "description": "Full pull request head SHA that was reviewed", + "type": "string" + }, + "issue_number": { + "description": "Target pull request number", + "type": "string" + }, + "scope": { + "description": "Whether the full PR or only a scoped subset was reviewed", + "enum": [ + "full", + "scoped" + ], + "type": "string" + } + }, + "required": [ + "blocking_count", + "completeness", + "head_sha", + "issue_number", + "scope" + ], + "type": "object" + }, + "name": "record_arm_semantic_review" + } + ] } GH_AW_VALIDATION_JSON: | { @@ -1169,6 +1250,7 @@ jobs: - activation - agent - detection + - record_arm_semantic_review - safe_outputs if: > always() && (needs.agent.result != 'skipped' || needs.activation.outputs.lockdown_check_failed == 'true' || @@ -1435,6 +1517,7 @@ jobs: GH_AW_DETECTION_CONCLUSION: ${{ needs.detection.outputs.detection_conclusion }} GH_AW_DETECTION_REASON: ${{ needs.detection.outputs.detection_reason }} GH_AW_SAFE_OUTPUT_MESSAGES: "{\"footer\":\"\\u003e πŸ” *ARM API review by [{workflow_name}]({run_url})*\",\"runStarted\":\"πŸ” [{workflow_name}]({run_url}) is reviewing this PR for ARM API compliance…\",\"runSuccess\":\"πŸ” [{workflow_name}]({run_url}) completed the ARM API review. βœ…\",\"runFailure\":\"πŸ” [{workflow_name}]({run_url}) {status}. ❌\"}" + GH_AW_SAFE_OUTPUT_JOBS: "{\"record_arm_semantic_review\":\"\"}" with: github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} script: | @@ -1729,6 +1812,7 @@ jobs: runs-on: ubuntu-slim permissions: pull-requests: read + statuses: write env: GH_AW_RUNTIME_FEATURES: ${{ vars.GH_AW_RUNTIME_FEATURES }} outputs: @@ -1738,6 +1822,7 @@ jobs: setup-parent-span-id: ${{ steps.setup.outputs.parent-span-id || steps.setup.outputs.span-id }} setup-span-id: ${{ steps.setup.outputs.span-id }} setup-trace-id: ${{ steps.setup.outputs.trace-id }} + target_head_sha: ${{ steps.resolve_target_pr.outputs.target_head_sha }} target_pr_number: ${{ steps.resolve_target_pr.outputs.target_pr_number }} steps: - name: Setup Scripts @@ -1790,7 +1875,99 @@ jobs: ...context.repo, pull_number: pullNumber, }); + if (!/^[0-9a-f]{40}$/i.test(pull.head.sha)) { + throw new Error(`Invalid pull request head SHA: ${JSON.stringify(pull.head.sha)}`); + } core.setOutput("target_pr_number", String(pull.number)); + core.setOutput("target_head_sha", pull.head.sha); + - name: Prepare ARM semantic review correlation artifact + if: steps.check_membership.outputs.is_team_member == 'true' + run: | + mkdir -p "$RUNNER_TEMP/arm-semantic-review" + : > "$RUNNER_TEMP/arm-semantic-review/empty.txt" + shell: bash + - name: Upload ARM semantic review head SHA + if: steps.check_membership.outputs.is_team_member == 'true' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 (source v7) + with: + if-no-files-found: error + name: head-sha=${{ steps.resolve_target_pr.outputs.target_head_sha }} + overwrite: true + path: ${{ runner.temp }}/arm-semantic-review/empty.txt + - name: Upload ARM semantic review issue number + if: steps.check_membership.outputs.is_team_member == 'true' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 (source v7) + with: + if-no-files-found: error + name: issue-number=${{ steps.resolve_target_pr.outputs.target_pr_number }} + overwrite: true + path: ${{ runner.temp }}/arm-semantic-review/empty.txt + - name: Set ARM semantic review pending + if: steps.check_membership.outputs.is_team_member == 'true' + uses: actions/github-script@373c709c69115d41ff229c7e5df9f8788daa9553 # v9 + env: + TARGET_HEAD_SHA: ${{ steps.resolve_target_pr.outputs.target_head_sha }} + with: + script: | + const runAttempt = process.env.GITHUB_RUN_ATTEMPT ?? ""; + if (!/^[1-9]\d*$/.test(runAttempt)) { + throw new Error(`Invalid workflow run attempt: ${JSON.stringify(runAttempt)}`); + } + await github.rest.repos.createCommitStatus({ + ...context.repo, + sha: process.env.TARGET_HEAD_SHA, + state: "pending", + context: "ARM Semantic Review", + description: "ARM API semantic review is pending", + target_url: + `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/` + + `${context.runId}/attempts/${runAttempt}`, + }); + + record_arm_semantic_review: + needs: + - agent + - detection + if: > + (!cancelled()) && needs.agent.result != 'skipped' && contains(needs.agent.outputs.output_types, 'record_arm_semantic_review') && + (needs.detection.outputs.detection_success == 'true') + runs-on: ubuntu-slim + permissions: + contents: read + pull-requests: read + steps: + - name: Download agent output artifact + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: agent + path: ${{ runner.temp }}/gh-aw/safe-jobs/ + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 (source v7) + env: + GH_AW_AGENT_OUTPUT: ${{ runner.temp }}/gh-aw/safe-jobs/agent_output.json + TARGET_PR_NUMBER: ${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }} + with: + sparse-checkout: | + .github + - name: Validate ARM semantic review + id: validate + uses: actions/github-script@d746ffe35508b1917358783b479e04febd2b8f71 # v9.0.0 + env: + GH_AW_AGENT_OUTPUT: ${{ runner.temp }}/gh-aw/safe-jobs/agent_output.json + TARGET_PR_NUMBER: ${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }} + with: + script: | + const { default: validateArmSemanticReview } = + await import('${{ github.workspace }}/.github/workflows/src/arm-auto-signoff/arm-semantic-review-workflow.ts'); + return await validateArmSemanticReview({ github, context, core }); + - name: Upload ARM semantic review receipt + uses: ./.github/actions/add-empty-artifact + env: + GH_AW_AGENT_OUTPUT: ${{ runner.temp }}/gh-aw/safe-jobs/agent_output.json + TARGET_PR_NUMBER: ${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }} + with: + name: arm-semantic-review + value: ${{ fromJson(steps.validate.outputs.result).artifactValue }} safe_outputs: needs: @@ -1884,6 +2061,7 @@ jobs: GH_AW_ALLOWED_DOMAINS: "api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,github.com,host.docker.internal,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,ppa.launchpad.net,raw.githubusercontent.com,registry.npmjs.org,s.symcb.com,s.symcd.com,security.ubuntu.com,telemetry.enterprise.githubcopilot.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com,www.googleapis.com" GITHUB_SERVER_URL: ${{ github.server_url }} GITHUB_API_URL: ${{ github.api_url }} + GH_AW_SAFE_OUTPUT_JOBS: "{\"record_arm_semantic_review\":\"\"}" GH_AW_SAFE_OUTPUTS_HANDLER_CONFIG: "{\"add_comment\":{\"max\":4,\"target\":\"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}\"},\"add_labels\":{\"max\":3,\"target\":\"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}\"},\"create_pull_request_review_comment\":{\"max\":50,\"side\":\"RIGHT\",\"target\":\"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}\"},\"create_report_incomplete_issue\":{},\"missing_data\":{},\"missing_tool\":{},\"noop\":{\"max\":1,\"report-as-issue\":\"true\"},\"remove_labels\":{\"max\":3,\"target\":\"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}\"},\"reply_to_pull_request_review_comment\":{\"max\":50,\"target\":\"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}\"},\"report_incomplete\":{},\"resolve_pull_request_review_thread\":{\"max\":50},\"submit_pull_request_review\":{\"footer\":\"if-body\",\"max\":1,\"target\":\"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}\"}}" with: github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/arm-api-review.md b/.github/workflows/arm-api-review.md index 1317b7100912..754fa41e758f 100644 --- a/.github/workflows/arm-api-review.md +++ b/.github/workflows/arm-api-review.md @@ -34,6 +34,7 @@ on: type: string permissions: pull-requests: read + statuses: write steps: - name: Resolve target pull request id: resolve_target_pr @@ -60,7 +61,54 @@ on: ...context.repo, pull_number: pullNumber, }); + if (!/^[0-9a-f]{40}$/i.test(pull.head.sha)) { + throw new Error(`Invalid pull request head SHA: ${JSON.stringify(pull.head.sha)}`); + } core.setOutput("target_pr_number", String(pull.number)); + core.setOutput("target_head_sha", pull.head.sha); + - name: Prepare ARM semantic review correlation artifact + if: steps.check_membership.outputs.is_team_member == 'true' + shell: bash + run: | + mkdir -p "$RUNNER_TEMP/arm-semantic-review" + : > "$RUNNER_TEMP/arm-semantic-review/empty.txt" + - name: Upload ARM semantic review head SHA + if: steps.check_membership.outputs.is_team_member == 'true' + uses: actions/upload-artifact@v7 + with: + name: "head-sha=${{ steps.resolve_target_pr.outputs.target_head_sha }}" + path: "${{ runner.temp }}/arm-semantic-review/empty.txt" + if-no-files-found: error + overwrite: true + - name: Upload ARM semantic review issue number + if: steps.check_membership.outputs.is_team_member == 'true' + uses: actions/upload-artifact@v7 + with: + name: "issue-number=${{ steps.resolve_target_pr.outputs.target_pr_number }}" + path: "${{ runner.temp }}/arm-semantic-review/empty.txt" + if-no-files-found: error + overwrite: true + - name: Set ARM semantic review pending + if: steps.check_membership.outputs.is_team_member == 'true' + uses: actions/github-script@v9 + env: + TARGET_HEAD_SHA: ${{ steps.resolve_target_pr.outputs.target_head_sha }} + with: + script: | + const runAttempt = process.env.GITHUB_RUN_ATTEMPT ?? ""; + if (!/^[1-9]\d*$/.test(runAttempt)) { + throw new Error(`Invalid workflow run attempt: ${JSON.stringify(runAttempt)}`); + } + await github.rest.repos.createCommitStatus({ + ...context.repo, + sha: process.env.TARGET_HEAD_SHA, + state: "pending", + context: "ARM Semantic Review", + description: "ARM API semantic review is pending", + target_url: + `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/` + + `${context.runId}/attempts/${runAttempt}`, + }); # Only users with write access (or above) may trigger the workflow. With # `forks: ["*"]` enabled above, this `roles` gate is the primary guard that # keeps externally-authored fork PRs from auto-triggering a review and blocks @@ -80,6 +128,7 @@ on: jobs: pre-activation: outputs: + target_head_sha: ${{ steps.resolve_target_pr.outputs.target_head_sha }} target_pr_number: ${{ steps.resolve_target_pr.outputs.target_pr_number }} # Gate at the trigger level so the expensive agent job never starts for # ineligible events. Label / draft / comment gating that used to live in a @@ -209,6 +258,58 @@ safe-outputs: remove-labels: max: 3 target: "${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}" + jobs: + record-arm-semantic-review: + description: "Validate and record the ARM semantic review result for auto-signoff" + runs-on: ubuntu-slim + if: needs.detection.outputs.detection_success == 'true' + permissions: + contents: read + pull-requests: read + env: + TARGET_PR_NUMBER: ${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }} + inputs: + issue_number: + description: "Target pull request number" + required: true + type: string + head_sha: + description: "Full pull request head SHA that was reviewed" + required: true + type: string + scope: + description: "Whether the full PR or only a scoped subset was reviewed" + required: true + type: choice + options: ["full", "scoped"] + completeness: + description: "Whether the intended review completed reliably" + required: true + type: choice + options: ["complete", "incomplete", "degraded"] + blocking_count: + description: "Number of verified, currently applicable Blocking findings after reconciliation" + required: true + type: string + steps: + - uses: actions/checkout@v7 + with: + sparse-checkout: | + .github + - id: validate + name: Validate ARM semantic review + uses: actions/github-script@v9.0.0 + with: + script: | + const { default: validateArmSemanticReview } = + await import('${{ github.workspace }}/.github/workflows/src/arm-auto-signoff/arm-semantic-review-workflow.ts'); + return await validateArmSemanticReview({ github, context, core }); + - name: Upload ARM semantic review receipt + uses: ./.github/actions/add-empty-artifact + with: + name: "arm-semantic-review" + value: "${{ fromJson(steps.validate.outputs.result).artifactValue }}" + output: "ARM semantic review result recorded" noop: # Threat detection uses the same pinned model as the primary review so all ARM # API Reviewer experiences move together. Keep both literals identical; see @@ -239,6 +340,7 @@ reconciliation throughout. - Repository: `${{ github.repository }}` - Trigger event: `${{ github.event_name }}` - **Authoritative target pull request:** `#${{ needs.pre_activation.outputs.target_pr_number }}` +- **Authoritative target head SHA:** `${{ needs.pre_activation.outputs.target_head_sha }}` The target above was parsed, validated as a pull request, and canonicalized via the GitHub Pull Requests API before agent execution. Use it for every GitHub @@ -291,11 +393,12 @@ Exactly **three** things differ, by design: reviewer has selected in VS Code, and pinning one would simply fail for anyone without access to it. Expect wording and emphasis to vary between the two paths. The rules above are what keep the substance the same. -- **Human queue advancement after a clean review.** This unattended workflow is - advisory and leaves `WaitForARMFeedback` unchanged when it publishes no - Blocking finding. A human-approved interactive review removes - `WaitForARMFeedback` because a human reviewer explicitly completed the review. - Both paths remove it when they add `ARMChangesRequested`. +- **Human queue advancement after a clean review.** This unattended workflow + leaves `WaitForARMFeedback` unchanged after a clean review, but removes a + stale `ARMChangesRequested` label when the full review completes reliably. + A human-approved interactive review removes `WaitForARMFeedback` because a + human reviewer explicitly completed the review. Both paths remove + `WaitForARMFeedback` when they add `ARMChangesRequested`. **Repository coverage.** This workflow exists in **both** repositories, `Azure/azure-rest-api-specs` and `Azure/azure-rest-api-specs-pr`, and the two @@ -391,7 +494,8 @@ read-only `github` toolset. If any check fails, act as directed and stop. `pull_request_read(method: "get")`, call `report_incomplete` and stop. Do not call `noop` for target-resolution or infrastructure failures. Pin the returned `head.sha` immediately and use that session SHA for all subsequent PR-head - file reads. + file reads. The pinned SHA must equal the authoritative target head SHA above. + If it does not, call `report_incomplete` and stop. 2. **`skip-arm-review` label** β€” call `pull_request_read(method: "get")` and inspect the labels. If the PR carries `skip-arm-review`, call `noop` and stop (opt-out). From the same response, capture the exact label names matching @@ -462,6 +566,21 @@ read-only `github` toolset. If any check fails, act as directed and stop. Only when checks 1–3 pass should you proceed to the Review Workflow below. Check 4 sets the review scope; it never stops the review. +Track two values for the final semantic result: + +- `scope` is `full` unless the size cap or file-list truncation limits coverage; + otherwise it is `scoped`. +- `completeness` is `complete` only when the intended review and Critic + verification finish normally. Use `degraded` when the Critic is unavailable + and `incomplete` when required files, discussions, or other evidence cannot be + fetched. + +Before any `report_incomplete` stop after the target PR and head SHA are known, +also call `record_arm_semantic_review` with that PR, head SHA, the current +scope, `completeness: incomplete`, and the number of verified Blocking findings +known to remain applicable at that point. Use `"0"` if the failure occurs before +finding reconciliation. + ## Review Workflow ### Step 1: Fetch PR Metadata and Changed Files @@ -1000,17 +1119,21 @@ and never emit a marker that names neither the finding nor a degradation reason. ### Step 7: Update Labels -After queuing the reconciled posting set, apply label changes based on outputs -that will actually be published: +After reconciling the current findings with existing discussion, apply label +changes based on the verified findings that still apply to the current head: -- **At least one Blocking `POST-NEW` or Blocking `RESOLVE-AND-REPOST` queued - _and_ the Critic returned a verdict** +- **The review is scoped, incomplete, or degraded** β†’ leave + `WaitForARMFeedback`, `ARMChangesRequested`, and `ARMSignedOff` unchanged, + even when Blocking findings were identified. The PR remains in its existing + queue state for manual review or a retry. +- **The review is full and complete, and at least one verified, currently + applicable Blocking finding remains** (including `SKIP-COVERED`, `POST-NEW`, + and `RESOLVE-AND-REPOST`) β†’ add `ARMChangesRequested`, remove `WaitForARMFeedback` (if present). -- **No Blocking finding queued for publication** (clean, covered, - clarification-only, Critic-dropped, or overflow-only Blocking candidate) β†’ - leave `WaitForARMFeedback`, - `ARMChangesRequested`, and `ARMSignedOff` unchanged. The automated review is - advisory and must not advance or sign off the human ARM review queue. +- **No verified, currently applicable Blocking finding remains, and the review + is full and complete** β†’ remove + `ARMChangesRequested` if present; leave `WaitForARMFeedback` and + `ARMSignedOff` unchanged. - **Critic unavailable** (every dispatch attempt failed) β†’ leave all three labels unchanged, **even when Blocking findings were queued**. Those findings still publish at their original severity, but nothing independently verified @@ -1020,13 +1143,12 @@ that will actually be published: Use the `add-labels` and `remove-labels` safe outputs for label changes. -These three rules are **exhaustive**. Do not invent additional exceptions from PR +These rules are **exhaustive**. Do not invent additional exceptions from PR metadata: draft status, a `[Test]` or `[Do-Not-Merge]` title, a revert, a bot-authored PR, or the author's stated intent not to merge are **not** grounds -to skip a label change. There are exactly **two** inputs to this decision: -whether a Blocking finding was queued for publication, and whether the Critic -verified it. Nothing else, and in particular nothing read from PR metadata, may -change the outcome. +to skip a label change. The decision uses only the verified, currently +applicable Blocking findings after reconciliation, review scope, and review +completeness. Nothing read from unrelated PR metadata may change the outcome. ### Step 8: Summary Comment @@ -1131,6 +1253,34 @@ If no reviewable issues were found, post a brief "No issues found" summary rather than calling `noop`. This confirms the review ran and found the PR compliant. +### Step 9: Record Semantic Result + +Call `record_arm_semantic_review` exactly once after queuing the summary: + +- `issue_number`: the authoritative target pull request number; +- `head_sha`: the pinned full session SHA; +- `scope`: `full` or `scoped`, as recorded during Trigger Validation; +- `completeness`: `complete`, `incomplete`, or `degraded`; and +- `blocking_count`: the number of verified Blocking findings that remain + applicable after reconciliation. Include an unresolved finding classified as + `SKIP-COVERED`; do not count fixed, resolved, Critic-dropped, or + overflow-only candidates. + +Use `blocking_count: "0"` after a clean re-review. Do not claim `full` or +`complete` when any required evidence was unavailable. The trusted publisher +validates the PR and head SHA and records an attempt-specific receipt. Universal +Auto-Signoff publishes the final `ARM Semantic Review` status only after the +entire reviewer workflow completes. + +The final status is: + +- **Passed** for a full, complete review with no verified Blocking findings; +- **Changes requested** for a full, complete review with verified Blocking + findings; +- **Manual review required** when scope is `scoped`, including oversized or + truncated PRs; or +- **Review incomplete** when a full review is `incomplete` or `degraded`. + ## What to Review vs. Skip **Review:** `specification/**/*.json`, `specification/**/*.tsp`, diff --git a/.github/workflows/arm-universal-auto-signoff.yaml b/.github/workflows/arm-universal-auto-signoff.yaml index 52932b37bac7..292fcf0f75ad 100644 --- a/.github/workflows/arm-universal-auto-signoff.yaml +++ b/.github/workflows/arm-universal-auto-signoff.yaml @@ -18,7 +18,12 @@ on: - unlabeled workflow_run: # Re-evaluate as each mandatory status producer completes. - workflows: ["Swagger Avocado - Set Status", "Swagger LintDiff - Set Status"] + workflows: + [ + "ARM API Review: Automated Workflow", + "Swagger Avocado - Set Status", + "Swagger LintDiff - Set Status", + ] types: [completed] permissions: @@ -31,8 +36,8 @@ permissions: issues: read # issues.listLabelsOnIssue pull-requests: read - # repos.listCommitStatusesForRef - statuses: read + # repos.listCommitStatusesForRef, repos.createCommitStatus + statuses: write jobs: arm-universal-auto-signoff: @@ -71,6 +76,17 @@ jobs: sparse-checkout: | .github + - if: | + github.event_name == 'workflow_run' && + github.event.workflow_run.name == 'ARM API Review: Automated Workflow' + name: Finalize ARM Semantic Review + uses: actions/github-script@v9.0.0 + with: + script: | + const { finalizeArmSemanticReview } = + await import('${{ github.workspace }}/.github/workflows/src/arm-auto-signoff/arm-semantic-review-workflow.ts'); + await finalizeArmSemanticReview({ github, context, core }); + - id: get-label-action name: ARM Universal Auto SignOff uses: actions/github-script@v9.0.0 @@ -88,10 +104,16 @@ jobs: name: "ARMAutoSignedOff-Test" value: "${{ fromJson(steps.get-label-action.outputs.result).labelActions['ARMAutoSignedOff-Test'] == 'add' }}" + - if: fromJson(steps.get-label-action.outputs.result).labelActions['ARMManualSignoffRequired'] == 'add' + name: Upload artifact for ARMManualSignoffRequired label + uses: ./.github/actions/add-label-artifact + with: + name: "ARMManualSignoffRequired" + value: "true" + # The label updater uses these trusted artifacts to associate this workflow run with its PR. - if: | always() && - github.event_name == 'workflow_run' && fromJson(steps.get-label-action.outputs.result).headSha name: Upload artifact with head SHA uses: ./.github/actions/add-empty-artifact @@ -101,7 +123,6 @@ jobs: - if: | always() && - github.event_name == 'workflow_run' && fromJson(steps.get-label-action.outputs.result).issueNumber > 0 name: Upload artifact with issue number uses: ./.github/actions/add-empty-artifact diff --git a/.github/workflows/src/arm-auto-signoff/arm-semantic-review-workflow.ts b/.github/workflows/src/arm-auto-signoff/arm-semantic-review-workflow.ts new file mode 100644 index 000000000000..5d2233f2621e --- /dev/null +++ b/.github/workflows/src/arm-auto-signoff/arm-semantic-review-workflow.ts @@ -0,0 +1,331 @@ +import { readFile } from "node:fs/promises"; +import { isFullGitSha } from "../../../shared/src/git.ts"; +import { CommitStatusState, PER_PAGE_MAX } from "../../../shared/src/github.ts"; +import { ARM_API_REVIEW_WORKFLOW_NAME, extractInputs } from "../context.ts"; +import type { Core, GitHub, GitHubScriptArgs, WebhookEvent } from "../github.ts"; +import { + ARM_SEMANTIC_REVIEW_STATUS, + encodeSemanticReviewReceipt, + evaluateAutomatedReviewCoverage, + evaluateSemanticReview, + getLatestSemanticReviewStatus, + getSemanticReviewOutcome, + manualReviewRequiredStatus, + parseSemanticReviewReceipt, + parseSemanticReviewResult, + reviewIncompleteStatus, + SemanticReviewOutcome, + type CommitStatus, + type EvaluatedSemanticReview, + type SemanticReviewResult, +} from "./arm-semantic-review.ts"; + +type ChangedFile = { + filename: string; + additions: number; + deletions: number; +}; + +async function createSemanticReviewStatus({ + owner, + repo, + headSha, + targetUrl, + status, + github, + core, +}: { + owner: string; + repo: string; + headSha: string; + targetUrl: string; + status: Pick; + github: GitHub; + core: Core; +}): Promise { + core.info(`${ARM_SEMANTIC_REVIEW_STATUS}: ${status.state} - ${status.description}`); + await github.rest.repos.createCommitStatus({ + owner, + repo, + sha: headSha, + state: status.state, + context: ARM_SEMANTIC_REVIEW_STATUS, + description: status.description, + target_url: targetUrl, + }); +} + +export async function validateSemanticReviewResult({ + owner, + repo, + result, + github, +}: { + owner: string; + repo: string; + result: SemanticReviewResult; + github: GitHub; +}): Promise { + const { data: pullRequest } = await github.rest.pulls.get({ + owner, + repo, + pull_number: result.issueNumber, + }); + if (pullRequest.state !== "open" || pullRequest.head.sha !== result.headSha) { + throw new Error("The reviewed pull request head is no longer current"); + } +} + +function parseWorkflowRunIdentity( + targetUrl: string | null, +): { runId: number; runAttempt: number } | undefined { + const match = /\/actions\/runs\/([1-9]\d*)\/attempts\/([1-9]\d*)(?:\/|$)/.exec(targetUrl ?? ""); + if (!match) { + return undefined; + } + const runId = Number(match[1]); + const runAttempt = Number(match[2]); + if (!Number.isSafeInteger(runId) || !Number.isSafeInteger(runAttempt)) { + return undefined; + } + return { runId, runAttempt }; +} + +export async function finalizeSemanticReviewWorkflow({ + owner, + repo, + issueNumber, + headSha, + runId, + runAttempt, + workflowConclusion, + reviewerExecuted = true, + targetUrl, + artifactNames, + github, + core, +}: { + owner: string; + repo: string; + issueNumber: number; + headSha: string; + runId: number; + runAttempt: number; + workflowConclusion: string | null; + reviewerExecuted?: boolean; + targetUrl: string; + artifactNames: string[]; + github: GitHub; + core: Core; +}): Promise { + if (!reviewerExecuted) { + core.info("The ARM API Reviewer did not execute; semantic status is unchanged"); + return false; + } + if (!Number.isSafeInteger(issueNumber) || issueNumber <= 0 || !isFullGitSha(headSha)) { + core.info("ARM API review completion is missing a valid PR number or head SHA"); + return false; + } + + const { data: pullRequest } = await github.rest.pulls.get({ + owner, + repo, + pull_number: issueNumber, + }); + if (pullRequest.state !== "open" || pullRequest.head.sha !== headSha) { + core.info("Ignoring ARM API review completion for a closed PR or stale head SHA"); + return false; + } + + const statuses: CommitStatus[] = await github.paginate( + github.rest.repos.listCommitStatusesForRef, + { + owner, + repo, + ref: headSha, + per_page: PER_PAGE_MAX, + }, + ); + const latestStatus = getLatestSemanticReviewStatus(statuses); + if (latestStatus && latestStatus.target_url !== targetUrl) { + const latestIdentity = parseWorkflowRunIdentity(latestStatus.target_url); + if ( + latestIdentity && + (latestIdentity.runId > runId || + (latestIdentity.runId === runId && latestIdentity.runAttempt > runAttempt)) + ) { + core.info("A newer ARM API review run superseded this completion"); + return false; + } + } + + if (workflowConclusion === "cancelled") { + if (latestStatus && latestStatus.state !== CommitStatusState.PENDING) { + await createSemanticReviewStatus({ + owner, + repo, + headSha, + targetUrl, + status: { + state: CommitStatusState.PENDING, + description: "Review canceled; rerun required", + }, + github, + core, + }); + return true; + } + core.info("Canceled ARM API review did not publish a semantic result"); + return false; + } + + const changedFiles: ChangedFile[] = await github.paginate(github.rest.pulls.listFiles, { + owner, + repo, + pull_number: issueNumber, + per_page: PER_PAGE_MAX, + }); + const coverage = evaluateAutomatedReviewCoverage(pullRequest.changed_files, changedFiles); + + if (workflowConclusion === "success" && latestStatus?.state !== CommitStatusState.PENDING) { + const existingOutcome = getSemanticReviewOutcome(latestStatus); + if ( + !coverage.manualReviewRequired || + existingOutcome === SemanticReviewOutcome.ManualReviewRequired + ) { + core.info("ARM semantic review result is already finalized"); + return false; + } + } + + let receipt: SemanticReviewResult | undefined; + let receiptError: string | undefined; + try { + receipt = parseSemanticReviewReceipt(artifactNames, runAttempt); + } catch (error) { + receiptError = error instanceof Error ? error.message : "Invalid ARM semantic review receipt"; + core.warning(receiptError); + } + + if ( + receipt && + (receipt.issueNumber !== issueNumber || receipt.headSha.toLowerCase() !== headSha.toLowerCase()) + ) { + receiptError = "ARM semantic review receipt does not match its workflow run"; + core.warning(receiptError); + receipt = undefined; + } + + const evaluatedReceipt = receipt ? evaluateSemanticReview(receipt) : undefined; + let status: EvaluatedSemanticReview; + if (coverage.manualReviewRequired) { + status = manualReviewRequiredStatus(coverage.reason ?? "automated review coverage was limited"); + } else if (workflowConclusion === "success" && evaluatedReceipt) { + status = evaluatedReceipt; + } else if ( + workflowConclusion !== "success" && + evaluatedReceipt?.outcome === SemanticReviewOutcome.ManualReviewRequired + ) { + status = evaluatedReceipt; + } else { + const reason = receiptError + ? "semantic result was invalid" + : workflowConclusion === "success" + ? "semantic result was missing" + : `workflow concluded with ${workflowConclusion ?? "an unknown result"}`; + status = reviewIncompleteStatus(reason); + } + + await createSemanticReviewStatus({ + owner, + repo, + headSha, + targetUrl, + status, + github, + core, + }); + return true; +} + +/* v8 ignore start */ +export default async function validateArmSemanticReview({ + github, + context, +}: GitHubScriptArgs): Promise<{ + artifactValue: string; +}> { + const outputPath = process.env.GH_AW_AGENT_OUTPUT; + const issueNumberText = process.env.TARGET_PR_NUMBER; + if (!outputPath) { + throw new Error("GH_AW_AGENT_OUTPUT is unavailable"); + } + if (!issueNumberText || !/^[1-9]\d*$/.test(issueNumberText)) { + throw new Error(`Invalid target PR number: '${issueNumberText ?? ""}'`); + } + + const runAttemptText = process.env.GITHUB_RUN_ATTEMPT; + if (!runAttemptText || !/^[1-9]\d*$/.test(runAttemptText)) { + throw new Error(`Invalid workflow run attempt: '${runAttemptText ?? ""}'`); + } + + const agentOutput = JSON.parse(await readFile(outputPath, "utf8")) as unknown; + const result = parseSemanticReviewResult(agentOutput, Number(issueNumberText)); + await validateSemanticReviewResult({ + ...context.repo, + result, + github, + }); + return { + artifactValue: encodeSemanticReviewReceipt(result, Number(runAttemptText)), + }; +} + +export async function finalizeArmSemanticReview({ + github, + context, + core, +}: GitHubScriptArgs): Promise { + const payload = context.payload as WebhookEvent<"workflow-run", "completed">; + const workflowRun = payload.workflow_run; + if (workflowRun.name !== ARM_API_REVIEW_WORKFLOW_NAME) { + return; + } + + const repositoryOwner = workflowRun.repository.owner?.login; + if (!repositoryOwner) { + throw new Error("ARM API review workflow run is missing its repository owner"); + } + const artifacts = await github.paginate(github.rest.actions.listWorkflowRunArtifacts, { + owner: repositoryOwner, + repo: workflowRun.repository.name, + run_id: workflowRun.id, + per_page: PER_PAGE_MAX, + }); + const artifactNames = artifacts.map((artifact) => artifact.name); + const reviewerExecuted = + workflowRun.conclusion !== "success" || + (artifactNames.some((name) => name.startsWith("head-sha=")) && + artifactNames.some((name) => name.startsWith("issue-number="))); + if (!reviewerExecuted) { + core.info("The ARM API Reviewer did not execute; semantic status is unchanged"); + return; + } + + const { owner, repo, head_sha, issue_number } = await extractInputs(github, context, core); + await finalizeSemanticReviewWorkflow({ + owner, + repo, + issueNumber: issue_number, + headSha: head_sha, + runId: workflowRun.id, + runAttempt: workflowRun.run_attempt, + workflowConclusion: workflowRun.conclusion, + reviewerExecuted, + targetUrl: `${workflowRun.html_url}/attempts/${workflowRun.run_attempt}`, + artifactNames, + github, + core, + }); +} +/* v8 ignore stop */ diff --git a/.github/workflows/src/arm-auto-signoff/arm-semantic-review.ts b/.github/workflows/src/arm-auto-signoff/arm-semantic-review.ts new file mode 100644 index 000000000000..5447bb0b980f --- /dev/null +++ b/.github/workflows/src/arm-auto-signoff/arm-semantic-review.ts @@ -0,0 +1,272 @@ +import { isFullGitSha } from "../../../shared/src/git.ts"; +import { CommitStatusState } from "../../../shared/src/github.ts"; +import { byDate, invert } from "../../../shared/src/sort.ts"; + +export const ARM_SEMANTIC_REVIEW_STATUS = "ARM Semantic Review"; + +export const SemanticReviewOutcome = Object.freeze({ + Passed: "passed", + ChangesRequested: "changes-requested", + ManualReviewRequired: "manual-review-required", + ReviewIncomplete: "review-incomplete", + Pending: "pending", +}); +export type SemanticReviewOutcome = + (typeof SemanticReviewOutcome)[keyof typeof SemanticReviewOutcome]; + +export type SemanticReviewScope = "full" | "scoped"; +export type SemanticReviewCompleteness = "complete" | "incomplete" | "degraded"; + +export type SemanticReviewResult = { + issueNumber: number; + headSha: string; + scope: SemanticReviewScope; + completeness: SemanticReviewCompleteness; + blockingCount: number; +}; + +export type CommitStatus = { + context: string; + state: string; + description?: string | null; + target_url: string | null; + updated_at: string; +}; + +export type EvaluatedSemanticReview = { + outcome: Exclude; + state: CommitStatusState; + description: string; +}; + +const RECEIPT_PATTERN = + /^([1-9]\d*)\.([1-9]\d*)\.([0-9a-f]{40})\.(full|scoped)\.(complete|incomplete|degraded)\.(0|[1-9]\d*)$/i; +const MANUAL_REVIEW_REQUIRED_PREFIX = "Manual review required:"; +const REVIEW_INCOMPLETE_PREFIX = "Review incomplete:"; +const MAX_SPECIFICATION_FILES = 50; +const MAX_SPECIFICATION_LINES = 5_000; + +type ChangedFile = { + filename: string; + additions: number; + deletions: number; +}; + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function isSemanticReviewOutput(value: unknown): value is Record { + return isRecord(value) && value.type === "record_arm_semantic_review"; +} + +export function parseSemanticReviewResult( + agentOutput: unknown, + expectedIssueNumber: number, +): SemanticReviewResult { + if (!isRecord(agentOutput) || !Array.isArray(agentOutput.items)) { + throw new Error("ARM API Reviewer output is missing its items array"); + } + + const items: unknown[] = agentOutput.items; + const results = items.filter(isSemanticReviewOutput); + if (results.length !== 1) { + throw new Error(`Expected one ARM semantic review result, found ${results.length}`); + } + + const result = results[0]; + const issueNumber = Number(result.issue_number); + const blockingCount = Number(result.blocking_count); + if ( + !Number.isSafeInteger(issueNumber) || + issueNumber <= 0 || + issueNumber !== expectedIssueNumber + ) { + throw new Error(`Invalid ARM semantic review PR number: '${String(result.issue_number)}'`); + } + if (typeof result.head_sha !== "string" || !isFullGitSha(result.head_sha)) { + throw new Error(`Invalid ARM semantic review head SHA: '${String(result.head_sha)}'`); + } + if (result.scope !== "full" && result.scope !== "scoped") { + throw new Error(`Invalid ARM semantic review scope: '${String(result.scope)}'`); + } + if ( + result.completeness !== "complete" && + result.completeness !== "incomplete" && + result.completeness !== "degraded" + ) { + throw new Error(`Invalid ARM semantic review completeness: '${String(result.completeness)}'`); + } + if ( + typeof result.blocking_count !== "string" || + !/^(0|[1-9]\d*)$/.test(result.blocking_count) || + !Number.isSafeInteger(blockingCount) + ) { + throw new Error( + `Invalid ARM semantic review Blocking count: '${String(result.blocking_count)}'`, + ); + } + + return { + issueNumber, + headSha: result.head_sha, + scope: result.scope, + completeness: result.completeness, + blockingCount, + }; +} + +export function encodeSemanticReviewReceipt( + result: SemanticReviewResult, + runAttempt: number, +): string { + if (!Number.isSafeInteger(runAttempt) || runAttempt <= 0) { + throw new Error(`Invalid workflow run attempt: '${runAttempt}'`); + } + return [ + runAttempt, + result.issueNumber, + result.headSha, + result.scope, + result.completeness, + result.blockingCount, + ].join("."); +} + +export function parseSemanticReviewReceipt( + artifactNames: string[], + runAttempt: number, +): SemanticReviewResult | undefined { + const prefix = `arm-semantic-review=${runAttempt}.`; + const receiptNames = artifactNames.filter((name) => name.startsWith(prefix)); + if (receiptNames.length === 0) { + return undefined; + } + if (receiptNames.length !== 1) { + throw new Error(`Expected one ARM semantic review receipt, found ${receiptNames.length}`); + } + + const value = receiptNames[0].substring("arm-semantic-review=".length); + const match = RECEIPT_PATTERN.exec(value); + if (!match || Number(match[1]) !== runAttempt) { + throw new Error(`Invalid ARM semantic review receipt: '${receiptNames[0]}'`); + } + + const issueNumber = Number(match[2]); + const blockingCount = Number(match[6]); + if (!Number.isSafeInteger(issueNumber) || !Number.isSafeInteger(blockingCount)) { + throw new Error(`Invalid ARM semantic review receipt: '${receiptNames[0]}'`); + } + + return { + issueNumber, + headSha: match[3], + scope: match[4] as SemanticReviewScope, + completeness: match[5] as SemanticReviewCompleteness, + blockingCount, + }; +} + +export function manualReviewRequiredStatus(reason: string): EvaluatedSemanticReview { + return { + outcome: SemanticReviewOutcome.ManualReviewRequired, + state: CommitStatusState.ERROR, + description: `${MANUAL_REVIEW_REQUIRED_PREFIX} ${reason}`, + }; +} + +export function reviewIncompleteStatus(reason: string): EvaluatedSemanticReview { + return { + outcome: SemanticReviewOutcome.ReviewIncomplete, + state: CommitStatusState.ERROR, + description: `${REVIEW_INCOMPLETE_PREFIX} ${reason}`, + }; +} + +export function evaluateSemanticReview(result: SemanticReviewResult): EvaluatedSemanticReview { + if (result.scope === "scoped") { + return manualReviewRequiredStatus("automated review was scoped"); + } + if (result.completeness !== "complete") { + return reviewIncompleteStatus(`automated review was ${result.completeness}`); + } + if (result.blockingCount > 0) { + return { + outcome: SemanticReviewOutcome.ChangesRequested, + state: CommitStatusState.FAILURE, + description: `Changes requested: ${result.blockingCount} Blocking finding(s)`, + }; + } + return { + outcome: SemanticReviewOutcome.Passed, + state: CommitStatusState.SUCCESS, + description: "Passed: full review completed with no Blocking findings", + }; +} + +export function evaluateAutomatedReviewCoverage( + changedFileCount: number, + changedFiles: ChangedFile[], +): { manualReviewRequired: boolean; reason?: string } { + const fileListTruncated = + changedFiles.length >= 3_000 || + (changedFileCount === 0 && changedFiles.length > 0) || + changedFileCount > changedFiles.length; + if (fileListTruncated) { + return { + manualReviewRequired: true, + reason: "changed-file list was truncated", + }; + } + + const specificationFiles = changedFiles.filter((file) => + file.filename.startsWith("specification/"), + ); + const specificationLines = specificationFiles.reduce( + (total, file) => total + file.additions + file.deletions, + 0, + ); + if ( + specificationFiles.length > MAX_SPECIFICATION_FILES || + specificationLines > MAX_SPECIFICATION_LINES + ) { + return { + manualReviewRequired: true, + reason: "PR exceeds automated review size limits", + }; + } + return { manualReviewRequired: false }; +} + +export function getSemanticReviewOutcome( + status: Pick | undefined, +): SemanticReviewOutcome | undefined { + if (!status) { + return undefined; + } + if (status.state === CommitStatusState.PENDING) { + return SemanticReviewOutcome.Pending; + } + if (status.state === CommitStatusState.SUCCESS) { + return SemanticReviewOutcome.Passed; + } + if (status.state === CommitStatusState.FAILURE) { + return SemanticReviewOutcome.ChangesRequested; + } + if ( + status.state === CommitStatusState.ERROR && + status.description?.startsWith(MANUAL_REVIEW_REQUIRED_PREFIX) + ) { + return SemanticReviewOutcome.ManualReviewRequired; + } + if (status.state === CommitStatusState.ERROR) { + return SemanticReviewOutcome.ReviewIncomplete; + } + return undefined; +} + +export function getLatestSemanticReviewStatus(statuses: CommitStatus[]): CommitStatus | undefined { + return statuses + .filter((status) => status.context.toLowerCase() === ARM_SEMANTIC_REVIEW_STATUS.toLowerCase()) + .sort(invert(byDate((status) => status.updated_at)))[0]; +} diff --git a/.github/workflows/src/arm-auto-signoff/arm-universal-auto-signoff.ts b/.github/workflows/src/arm-auto-signoff/arm-universal-auto-signoff.ts index 32c477f34948..e33c162875fd 100644 --- a/.github/workflows/src/arm-auto-signoff/arm-universal-auto-signoff.ts +++ b/.github/workflows/src/arm-auto-signoff/arm-universal-auto-signoff.ts @@ -5,16 +5,23 @@ import { extractInputs } from "../context.ts"; import type { Core, GitHubScriptArgs } from "../github.ts"; import { LabelAction } from "../label.ts"; import { ArmAutoSignoffLabel } from "./arm-auto-signoff-labels.ts"; +import { + getLatestSemanticReviewStatus, + getSemanticReviewOutcome, + SemanticReviewOutcome, +} from "./arm-semantic-review.ts"; const requiredStatusNames = ["Swagger LintDiff", "Swagger Avocado"]; export type ManagedLabelActions = { "ARMAutoSignedOff-Test": LabelAction; + ARMManualSignoffRequired: LabelAction; }; function createNoneLabelActions(): ManagedLabelActions { return { [ArmAutoSignoffLabel.ArmAutoSignedOffTest]: LabelAction.None, + [ArmAutoSignoffLabel.ArmManualSignoffRequired]: LabelAction.None, }; } @@ -88,7 +95,7 @@ export async function getLabelActionImpl({ ).map((label) => label.name); const hasAutoSignoff = labelNames.includes(ArmAutoSignoffLabel.ArmAutoSignedOffTest); - const desiredAction = await getDesiredLabelAction({ + const labelActions = await getDesiredLabelActions({ owner, repo, head_sha, @@ -100,13 +107,11 @@ export async function getLabelActionImpl({ return { ...noneResult, - labelActions: { - [ArmAutoSignoffLabel.ArmAutoSignedOffTest]: desiredAction, - }, + labelActions, }; } -async function getDesiredLabelAction({ +async function getDesiredLabelActions({ owner, repo, head_sha, @@ -125,17 +130,17 @@ async function getDesiredLabelAction({ paginate: import("@octokit/plugin-paginate-rest").PaginateInterface; }; core: Core; -}): Promise { - const labelsAllowSignoff = - labelNames.includes("ARMReview") && - !labelNames.includes("NotReadyForARMReview") && - !labelNames.includes(ArmAutoSignoffLabel.ArmManualSignoffRequired) && - (!labelNames.includes("SuppressionReviewRequired") || - labelNames.includes("Approved-Suppression")); - - if (!labelsAllowSignoff) { - core.info("Labels do not meet requirements for universal auto-signoff"); - return hasAutoSignoff ? LabelAction.Remove : LabelAction.None; +}): Promise { + const labelActions = createNoneLabelActions(); + const isReadyForArmReview = + labelNames.includes("ARMReview") && !labelNames.includes("NotReadyForARMReview"); + + if (!isReadyForArmReview) { + core.info("Pull request is not ready for ARM review"); + labelActions[ArmAutoSignoffLabel.ArmAutoSignedOffTest] = hasAutoSignoff + ? LabelAction.Remove + : LabelAction.None; + return labelActions; } const statuses: import("@octokit/plugin-rest-endpoint-methods").RestEndpointMethodTypes["repos"]["listCommitStatusesForRef"]["response"]["data"] = @@ -146,6 +151,37 @@ async function getDesiredLabelAction({ per_page: PER_PAGE_MAX, }); + const semanticReviewOutcome = getSemanticReviewOutcome(getLatestSemanticReviewStatus(statuses)); + core.info(`ARM Semantic Review: ${semanticReviewOutcome ?? "missing"}`); + + if (semanticReviewOutcome !== SemanticReviewOutcome.Passed) { + if (semanticReviewOutcome === SemanticReviewOutcome.ManualReviewRequired) { + core.info("ARM semantic review requires manual signoff"); + if (!labelNames.includes(ArmAutoSignoffLabel.ArmManualSignoffRequired)) { + labelActions[ArmAutoSignoffLabel.ArmManualSignoffRequired] = LabelAction.Add; + } + } else { + core.info("ARM semantic review has not passed"); + } + labelActions[ArmAutoSignoffLabel.ArmAutoSignedOffTest] = hasAutoSignoff + ? LabelAction.Remove + : LabelAction.None; + return labelActions; + } + + const labelsAllowSignoff = + !labelNames.includes("ARMChangesRequested") && + !labelNames.includes(ArmAutoSignoffLabel.ArmManualSignoffRequired) && + (!labelNames.includes("SuppressionReviewRequired") || + labelNames.includes("Approved-Suppression")); + if (!labelsAllowSignoff) { + core.info("Labels do not meet requirements for universal auto-signoff"); + labelActions[ArmAutoSignoffLabel.ArmAutoSignedOffTest] = hasAutoSignoff + ? LabelAction.Remove + : LabelAction.None; + return labelActions; + } + for (const statusName of requiredStatusNames) { // A status context may appear more than once; only the most recently updated result applies. const matchingStatuses = statuses @@ -156,10 +192,16 @@ async function getDesiredLabelAction({ core.info(`${statusName}: ${latestStatus?.state ?? "missing"}`); if (latestStatus?.state !== CommitStatusState.SUCCESS) { core.info(`Required status '${statusName}' did not succeed`); - return hasAutoSignoff ? LabelAction.Remove : LabelAction.None; + labelActions[ArmAutoSignoffLabel.ArmAutoSignedOffTest] = hasAutoSignoff + ? LabelAction.Remove + : LabelAction.None; + return labelActions; } } core.info(`Universal auto-signoff pilot requirements met: ${inspect(requiredStatusNames)}`); - return hasAutoSignoff ? LabelAction.None : LabelAction.Add; + labelActions[ArmAutoSignoffLabel.ArmAutoSignedOffTest] = hasAutoSignoff + ? LabelAction.None + : LabelAction.Add; + return labelActions; } diff --git a/.github/workflows/src/context.ts b/.github/workflows/src/context.ts index 98b71d637d1d..1031ef1bf636 100644 --- a/.github/workflows/src/context.ts +++ b/.github/workflows/src/context.ts @@ -12,6 +12,8 @@ export type PullRequest = export type RestEndpointMethodTypes = import("@octokit/plugin-rest-endpoint-methods").RestEndpointMethodTypes; +export const ARM_API_REVIEW_WORKFLOW_NAME = "ARM API Review: Automated Workflow"; + /** * Extracts inputs from context based on event name and properties. * run_id is only defined for "workflow_run:completed" events. @@ -115,8 +117,8 @@ export async function extractInputs( } else if (context.eventName === "workflow_run" && context.payload.action === "completed") { const payload = context.payload as WebhookEvent<"workflow-run", "completed">; - let issue_number = NaN; - let head_sha = ""; + let issue_number: number; + let head_sha: string; if ( payload.workflow_run.event === "pull_request" || @@ -131,9 +133,8 @@ export async function extractInputs( // For pull_request, do NOT attempt to extract the issue number from an artifact, since this could be modified // in a fork PR. // - // For pull_request_target, it might be safe to extract the issue number from an artifact, since the workflow runs - // on the target branch and can be trusted. But it should also be unnecessary, since we should be able extract - // the issue number from the payload itself, just like pull_request. + // For pull_request_target, trusted artifacts carry the PR head SHA because workflow_run.head_sha identifies + // the base branch commit. The issue number is also read from the artifacts and checked against the event. const pullRequest = payload.workflow_run.pull_requests?.find((pr) => pr !== null); if (pullRequest) { @@ -207,57 +208,43 @@ export async function extractInputs( ); } } + + if (payload.workflow_run.event === "pull_request_target") { + const artifactInputs = await extractWorkflowRunArtifactInputs({ + github, + core, + repository: payload.workflow_run.repository, + runId: payload.workflow_run.id, + }); + if (artifactInputs.headSha) { + head_sha = artifactInputs.headSha; + } else if (payload.workflow_run.name === ARM_API_REVIEW_WORKFLOW_NAME) { + head_sha = ""; + } + if (artifactInputs.issueNumber) { + if (issue_number && issue_number !== artifactInputs.issueNumber) { + throw new Error( + `Workflow artifacts reference PR ${artifactInputs.issueNumber}, but the event references PR ${issue_number}`, + ); + } + issue_number = artifactInputs.issueNumber; + } + } } else if ( payload.workflow_run.event === "issue_comment" || payload.workflow_run.event == "workflow_run" || - payload.workflow_run.event == "check_run" + payload.workflow_run.event == "check_run" || + payload.workflow_run.event == "workflow_dispatch" ) { - // Attempt to extract issue number from artifact. This can be trusted, because it was uploaded from a workflow that is trusted, - // because "issue_comment" and "workflow_run" only trigger on workflows in the default branch. - const artifacts = await github.paginate(github.rest.actions.listWorkflowRunArtifacts, { - ...getRepositoryInfo(payload.workflow_run.repository), - run_id: payload.workflow_run.id, - per_page: PER_PAGE_MAX, + // These artifacts can be trusted because these event types run workflows from the default branch. + const artifactInputs = await extractWorkflowRunArtifactInputs({ + github, + core, + repository: payload.workflow_run.repository, + runId: payload.workflow_run.id, }); - - const artifactNames = artifacts.map((a) => a.name); - - core.info(`artifactNames: ${JSON.stringify(artifactNames)}`); - - for (const artifactName of artifactNames) { - // If artifactName has format "head-sha=valid-full-sha", set head_sha=value - // Else, if artifactName has format "head-sha=other-string", warn and set head_sha="" - // Else, if artifactName has format "issue-number=positive-integer", set issue_number=value - // Else, if artifactName has format "issue-number=other-string", warn and set issue_number=NaN - // - Workflows should probably only set "issue-number" to positive integers, but sometimes set it to "null" - // Else, if artifactName does not start with "head-sha=" or "issue-number=", ignore it - const firstEquals = artifactName.indexOf("="); - if (firstEquals !== -1) { - const key = artifactName.substring(0, firstEquals); - if (key === "head-sha") { - const value = artifactName.substring(firstEquals + 1); - if (isFullGitSha(value)) { - head_sha = value; - } else { - // Producers must ensure they only set head-sha to valid full git SHA - throw new Error(`head-sha is not a valid full git SHA: '${value}'`); - } - continue; - } else if (key === "issue-number") { - const value = artifactName.substring(firstEquals + 1); - const parsedValue = Number.parseInt(value); - if (parsedValue > 0) { - issue_number = parsedValue; - } else { - // TODO: Consider throwing instead of warning. May need to handle `issue-number=null|undefined`, - // but invalid integers should throw. - core.info(`Invalid issue-number: '${value}' parsed to '${parsedValue}'`); - issue_number = NaN; - } - continue; - } - } - } + head_sha = artifactInputs.headSha; + issue_number = artifactInputs.issueNumber; if (!head_sha) { core.info( `Could not find 'head-sha' artifact, which is required to associate the triggering workflow run with the head SHA of a PR`, @@ -274,6 +261,28 @@ export async function extractInputs( ); } + if ( + payload.workflow_run.name === ARM_API_REVIEW_WORKFLOW_NAME && + (!Number.isSafeInteger(issue_number) || issue_number <= 0 || !isFullGitSha(head_sha)) + ) { + const match = /^ARM API Review #([1-9]\d*) \(/.exec(payload.workflow_run.display_title ?? ""); + const fallbackIssueNumber = match ? Number(match[1]) : NaN; + const resolvedIssueNumber = + Number.isSafeInteger(issue_number) && issue_number > 0 ? issue_number : fallbackIssueNumber; + if (Number.isSafeInteger(resolvedIssueNumber) && resolvedIssueNumber > 0) { + const { owner, repo } = getRepositoryInfo(payload.workflow_run.repository); + const { data: pullRequest } = await github.rest.pulls.get({ + owner, + repo, + pull_number: resolvedIssueNumber, + }); + issue_number = resolvedIssueNumber; + if (!isFullGitSha(head_sha)) { + head_sha = pullRequest.head.sha; + } + } + } + inputs = { ...getRepositoryInfo(payload.workflow_run.repository), head_sha, @@ -316,6 +325,62 @@ export async function extractInputs( return inputs; } +async function extractWorkflowRunArtifactInputs({ + github, + core, + repository, + runId, +}: { + github: GitHub; + core: Core; + repository: Parameters[0]; + runId: number; +}): Promise<{ headSha: string; issueNumber: number }> { + const artifacts = await github.paginate(github.rest.actions.listWorkflowRunArtifacts, { + ...getRepositoryInfo(repository), + run_id: runId, + per_page: PER_PAGE_MAX, + }); + const artifactNames = artifacts.map((artifact) => artifact.name); + core.info(`artifactNames: ${JSON.stringify(artifactNames)}`); + + let headSha = ""; + let issueNumber = NaN; + for (const artifactName of artifactNames) { + const firstEquals = artifactName.indexOf("="); + if (firstEquals === -1) { + continue; + } + + const key = artifactName.substring(0, firstEquals); + const value = artifactName.substring(firstEquals + 1); + if (key === "head-sha") { + if (!isFullGitSha(value)) { + throw new Error(`head-sha is not a valid full git SHA: '${value}'`); + } + if (headSha && headSha.toLowerCase() !== value.toLowerCase()) { + throw new Error(`Conflicting head-sha artifacts: '${headSha}' and '${value}'`); + } + headSha = value; + } else if (key === "issue-number") { + const parsedValue = /^[1-9]\d*$/.test(value) ? Number(value) : NaN; + if (Number.isSafeInteger(parsedValue)) { + if (Number.isSafeInteger(issueNumber) && issueNumber !== parsedValue) { + throw new Error( + `Conflicting issue-number artifacts: '${issueNumber}' and '${parsedValue}'`, + ); + } + issueNumber = parsedValue; + } else { + core.info(`Invalid issue-number artifact: '${value}'`); + issueNumber = NaN; + } + } + } + + return { headSha, issueNumber }; +} + function getRepositoryInfo( repository: { name?: string; owner?: { login?: string } | null } | null | undefined, ): { diff --git a/.github/workflows/src/update-labels.ts b/.github/workflows/src/update-labels.ts index e5934d05b87d..7e6f4c005e03 100644 --- a/.github/workflows/src/update-labels.ts +++ b/.github/workflows/src/update-labels.ts @@ -59,6 +59,7 @@ export async function updateLabelsImpl({ }); const artifactNames: string[] = artifacts.map((a) => a.name); + const hasHeadShaArtifact = artifactNames.some((name) => name.startsWith("head-sha=")); core.info(`artifactNames: ${JSON.stringify(artifactNames)}`); @@ -104,6 +105,24 @@ export async function updateLabelsImpl({ ); } + if ( + (labelsToAdd.length > 0 || labelsToRemove.length > 0) && + hasHeadShaArtifact && + isFullGitSha(head_sha) && + Number.isInteger(issue_number) && + issue_number > 0 + ) { + const { data: pullRequest } = await github.rest.pulls.get({ + owner, + repo, + pull_number: issue_number, + }); + if (pullRequest.state !== "open" || pullRequest.head.sha !== head_sha) { + core.info("Skipping label update because the pull request is closed or its head changed"); + return; + } + } + const pullRequestUrl = `https://github.com/${owner}/${repo}/pull/${issue_number}`; core.info(`pull request url: ${pullRequestUrl}`); diff --git a/.github/workflows/test/arm-api-review-workflow.test.ts b/.github/workflows/test/arm-api-review-workflow.test.ts index 58abc519308b..34f86a6060c4 100644 --- a/.github/workflows/test/arm-api-review-workflow.test.ts +++ b/.github/workflows/test/arm-api-review-workflow.test.ts @@ -3,6 +3,7 @@ import { load } from "js-yaml"; import { join } from "node:path"; import { beforeAll, describe, expect, it, vi } from "vitest"; import { runInNewContext } from "node:vm"; +import { fullGitSha } from "../../shared/test/examples.ts"; // cspell:ignore REPOST vally @@ -209,7 +210,7 @@ describe("ARM API review workflow", () => { describe("target PR resolver", () => { it("resolves a workflow-dispatch target and publishes the canonical PR number", async () => { const harness = createResolverHarness(resolverScript, { value: "44499" }); - harness.get.mockResolvedValue({ data: { number: 44499 } }); + harness.get.mockResolvedValue({ data: { number: 44499, head: { sha: fullGitSha } } }); await harness.run(); @@ -220,6 +221,7 @@ describe("ARM API review workflow", () => { pull_number: 44499, }); expect(harness.setOutput).toHaveBeenCalledWith("target_pr_number", "44499"); + expect(harness.setOutput).toHaveBeenCalledWith("target_head_sha", fullGitSha); }); it("accepts an issue-shaped pull request comment payload", async () => { @@ -228,7 +230,7 @@ describe("ARM API review workflow", () => { eventName: "issue_comment", payload: { issue: { pull_request: { url: "https://api.github.com/pulls/44499" } } }, }); - harness.get.mockResolvedValue({ data: { number: 44499 } }); + harness.get.mockResolvedValue({ data: { number: 44499, head: { sha: fullGitSha } } }); await harness.run(); @@ -444,15 +446,21 @@ describe("ARM API review workflow", () => { it("keeps the agent read-only and preserves the human queue after a clean review", async () => { const [source, compiled] = await readWorkflowFiles(); + const collapsed = collapseWhitespace(source); - expect(source).toContain("permissions:\n pull-requests: read\n steps:"); expect(source).toContain( - "**At least one Blocking `POST-NEW` or Blocking `RESOLVE-AND-REPOST` queued", + "permissions:\n pull-requests: read\n statuses: write\n steps:", + ); + expect(source).toContain("**The review is scoped, incomplete, or degraded**"); + expect(source).toContain( + "**The review is full and complete, and at least one verified, currently", ); - expect(source).toContain("_and_ the Critic returned a verdict**"); - expect(source).toContain("**No Blocking finding queued for publication**"); + expect(collapsed).toContain("including `SKIP-COVERED`, `POST-NEW`, and `RESOLVE-AND-REPOST`"); expect(source).toContain( - "clean, covered,\n clarification-only, Critic-dropped, or overflow-only", + "**No verified, currently applicable Blocking finding remains, and the review", + ); + expect(collapsed).toContain( + "remove `ARMChangesRequested` if present; leave `WaitForARMFeedback`", ); expect(source).not.toContain("**Blocking findings found**"); @@ -478,6 +486,7 @@ describe("ARM API review workflow", () => { expect(agentJob).not.toContain("issues: write"); expect(agentJob).not.toContain("pull-requests: write"); expect(preActivationJob).toContain("pull-requests: read"); + expect(preActivationJob).toContain("statuses: write"); expect(preActivationJob).not.toContain("issues: write"); expect(safeOutputsJob).toContain("issues: write"); expect(safeOutputsJob).toContain("pull-requests: write"); @@ -533,18 +542,32 @@ describe("ARM API review posting reliability", () => { it("leaves no metadata-driven exception to the ARMChangesRequested label rule", async () => { const source = collapseWhitespace(await readFile(join(ROOT, SOURCE_FILE), "utf8")); - expect(source).toContain("These three rules are **exhaustive**."); + expect(source).toContain("These rules are **exhaustive**."); expect(source).toContain( "draft status, a `[Test]` or `[Do-Not-Merge]` title, a revert, a bot-authored PR, or the author's stated intent not to merge are **not** grounds to skip a label change", ); - // The decision now has exactly two inputs, not one: a Blocking finding must - // be queued AND the Critic must have verified it. PR metadata still may not - // influence the outcome. expect(source).toContain( - "There are exactly **two** inputs to this decision: whether a Blocking finding was queued for publication, and whether the Critic verified it.", + "The decision uses only the verified, currently applicable Blocking findings after reconciliation, review scope, and review completeness.", ); - expect(source).toContain( - "Nothing else, and in particular nothing read from PR metadata, may change the outcome.", + expect(source).toContain("Nothing read from unrelated PR metadata may change the outcome."); + }); + + it("records one head-bound semantic review result for completion-time finalization", async () => { + const [source, compiled] = await readWorkflowFiles(); + const collapsed = collapseWhitespace(source); + + expect(source).toContain("name: Set ARM semantic review pending"); + expect(source).toContain('context: "ARM Semantic Review"'); + expect(source).toContain("record-arm-semantic-review:"); + expect(source).toContain("### Step 9: Record Semantic Result"); + expect(collapsed).toContain( + "Call `record_arm_semantic_review` exactly once after queuing the summary", + ); + expect(compiled).toContain("record_arm_semantic_review"); + expect(compiled).toContain("ARM Semantic Review"); + expect(compiled).toContain("Upload ARM semantic review receipt"); + expect(collapsed).toContain( + "Universal Auto-Signoff publishes the final `ARM Semantic Review` status only after the entire reviewer workflow completes.", ); }); @@ -749,11 +772,10 @@ describe("ARM API review consistency and hardening", () => { it("gives the label rules a single unambiguous outcome per case", async () => { const source = collapseWhitespace(await readFile(join(ROOT, SOURCE_FILE), "utf8")); - // Rule 1 and the Critic-unavailable rule both matched the case - // (Blocking queued AND Critic unavailable) with opposite outcomes and no - // stated precedence. Rule 1 now carries the Critic condition itself. + // Unverified findings from a Critic-unavailable run do not satisfy the + // full-and-complete rule, so the explicit Critic-unavailable rule remains authoritative. expect(source).toContain( - "**At least one Blocking `POST-NEW` or Blocking `RESOLVE-AND-REPOST` queued _and_ the Critic returned a verdict**", + "**The review is full and complete, and at least one verified, currently applicable Blocking finding remains**", ); }); diff --git a/.github/workflows/test/arm-auto-signoff/arm-semantic-review.test.ts b/.github/workflows/test/arm-auto-signoff/arm-semantic-review.test.ts new file mode 100644 index 000000000000..e7cb35d72114 --- /dev/null +++ b/.github/workflows/test/arm-auto-signoff/arm-semantic-review.test.ts @@ -0,0 +1,505 @@ +import { describe, expect, it } from "vitest"; +import { CommitStatusState, PER_PAGE_MAX } from "../../../shared/src/github.ts"; +import { + ARM_SEMANTIC_REVIEW_STATUS, + encodeSemanticReviewReceipt, + evaluateAutomatedReviewCoverage, + evaluateSemanticReview, + getSemanticReviewOutcome, + parseSemanticReviewReceipt, + parseSemanticReviewResult, + SemanticReviewOutcome, + type SemanticReviewResult, +} from "../../src/arm-auto-signoff/arm-semantic-review.ts"; +import { + finalizeSemanticReviewWorkflow, + validateSemanticReviewResult, +} from "../../src/arm-auto-signoff/arm-semantic-review-workflow.ts"; +import { createMockCore, createMockGithub } from "../mocks.ts"; + +const owner = "Azure"; +const repo = "azure-rest-api-specs"; +const issueNumber = 123; +const headSha = "0123456789abcdef0123456789abcdef01234567"; +const targetUrl = "https://github.com/Azure/azure-rest-api-specs/actions/runs/456/attempts/1"; +const runId = 456; +const runAttempt = 1; + +const passedResult: SemanticReviewResult = { + issueNumber, + headSha, + scope: "full", + completeness: "complete", + blockingCount: 0, +}; + +function receiptName(result: SemanticReviewResult = passedResult): string { + return `arm-semantic-review=${encodeSemanticReviewReceipt(result, runAttempt)}`; +} + +function createFinalizeGithub({ + currentHeadSha = headSha, + changedFiles = [ + { + filename: "specification/test/resource-manager/test.json", + additions: 10, + deletions: 5, + }, + ], + statuses = [ + { + context: ARM_SEMANTIC_REVIEW_STATUS, + state: CommitStatusState.PENDING, + description: "ARM API semantic review is pending", + target_url: targetUrl, + updated_at: "2026-01-01T00:00:00Z", + }, + ], +}: { + currentHeadSha?: string; + changedFiles?: { filename: string; additions: number; deletions: number }[]; + statuses?: { + context: string; + state: string; + description: string; + target_url: string; + updated_at: string; + }[]; +} = {}) { + const github = createMockGithub(); + github.rest.pulls.get.mockResolvedValue({ + data: { + state: "open", + head: { sha: currentHeadSha }, + changed_files: changedFiles.length, + }, + }); + github.rest.pulls.listFiles.mockResolvedValue({ data: changedFiles }); + github.rest.repos.listCommitStatusesForRef.mockResolvedValue({ data: statuses }); + return github; +} + +function finalize({ + artifactNames = [receiptName()], + workflowConclusion = "success", + reviewerExecuted = true, + github = createFinalizeGithub(), +}: { + artifactNames?: string[]; + workflowConclusion?: string | null; + reviewerExecuted?: boolean; + github?: ReturnType; +} = {}) { + return finalizeSemanticReviewWorkflow({ + owner, + repo, + issueNumber, + headSha, + runId, + runAttempt, + workflowConclusion, + reviewerExecuted, + targetUrl, + artifactNames, + github, + core: createMockCore(), + }); +} + +describe("parseSemanticReviewResult", () => { + it("parses one valid semantic result", () => { + expect( + parseSemanticReviewResult( + { + items: [ + { + type: "record_arm_semantic_review", + issue_number: String(issueNumber), + head_sha: headSha, + scope: "full", + completeness: "complete", + blocking_count: "0", + }, + ], + }, + issueNumber, + ), + ).toEqual(passedResult); + }); + + it("rejects missing, duplicate, or untrusted correlation values", () => { + expect(() => parseSemanticReviewResult({ items: [] }, issueNumber)).toThrow( + "Expected one ARM semantic review result, found 0", + ); + expect(() => + parseSemanticReviewResult( + { + items: [{ type: "record_arm_semantic_review" }, { type: "record_arm_semantic_review" }], + }, + issueNumber, + ), + ).toThrow("Expected one ARM semantic review result, found 2"); + expect(() => + parseSemanticReviewResult( + { + items: [ + { + type: "record_arm_semantic_review", + issue_number: "456", + head_sha: headSha, + scope: "full", + completeness: "complete", + blocking_count: "0", + }, + ], + }, + issueNumber, + ), + ).toThrow("Invalid ARM semantic review PR number"); + }); +}); + +describe("semantic review receipts", () => { + it("round-trips an attempt-specific receipt", () => { + const value = encodeSemanticReviewReceipt(passedResult, runAttempt); + + expect(parseSemanticReviewReceipt([`arm-semantic-review=${value}`], runAttempt)).toEqual( + passedResult, + ); + }); + + it("ignores receipts from other attempts and rejects ambiguity", () => { + expect( + parseSemanticReviewReceipt( + [`arm-semantic-review=2.${issueNumber}.${headSha}.full.complete.0`], + 1, + ), + ).toBeUndefined(); + expect(() => parseSemanticReviewReceipt([receiptName(), receiptName()], runAttempt)).toThrow( + "Expected one ARM semantic review receipt, found 2", + ); + }); +}); + +describe("evaluateSemanticReview", () => { + it("distinguishes every semantic outcome", () => { + expect(evaluateSemanticReview(passedResult).outcome).toBe(SemanticReviewOutcome.Passed); + expect(evaluateSemanticReview({ ...passedResult, blockingCount: 2 }).outcome).toBe( + SemanticReviewOutcome.ChangesRequested, + ); + expect(evaluateSemanticReview({ ...passedResult, scope: "scoped" }).outcome).toBe( + SemanticReviewOutcome.ManualReviewRequired, + ); + expect(evaluateSemanticReview({ ...passedResult, completeness: "incomplete" }).outcome).toBe( + SemanticReviewOutcome.ReviewIncomplete, + ); + expect(evaluateSemanticReview({ ...passedResult, completeness: "degraded" }).outcome).toBe( + SemanticReviewOutcome.ReviewIncomplete, + ); + }); + + describe("evaluateAutomatedReviewCoverage", () => { + it("requires manual review for oversized or truncated changes", () => { + const tooManyFiles = Array.from({ length: 51 }, (_, index) => ({ + filename: `specification/test/resource-manager/file-${index}.json`, + additions: 1, + deletions: 0, + })); + expect(evaluateAutomatedReviewCoverage(tooManyFiles.length, tooManyFiles)).toEqual({ + manualReviewRequired: true, + reason: "PR exceeds automated review size limits", + }); + expect( + evaluateAutomatedReviewCoverage(1, [ + { + filename: "specification/test/resource-manager/test.json", + additions: 5_001, + deletions: 0, + }, + ]), + ).toEqual({ + manualReviewRequired: true, + reason: "PR exceeds automated review size limits", + }); + expect(evaluateAutomatedReviewCoverage(3_100, tooManyFiles)).toEqual({ + manualReviewRequired: true, + reason: "changed-file list was truncated", + }); + }); + + it("allows a complete review within the trusted limits", () => { + expect( + evaluateAutomatedReviewCoverage(1, [ + { + filename: "specification/test/resource-manager/test.json", + additions: 100, + deletions: 20, + }, + ]), + ).toEqual({ manualReviewRequired: false }); + }); + }); + + it("routes scoped reviews to manual review even when execution was degraded", () => { + expect( + evaluateSemanticReview({ + ...passedResult, + scope: "scoped", + completeness: "degraded", + }).outcome, + ).toBe(SemanticReviewOutcome.ManualReviewRequired); + }); + + it("reads outcomes from commit statuses", () => { + expect(getSemanticReviewOutcome({ state: "pending" })).toBe(SemanticReviewOutcome.Pending); + expect(getSemanticReviewOutcome({ state: "success" })).toBe(SemanticReviewOutcome.Passed); + expect(getSemanticReviewOutcome({ state: "failure" })).toBe( + SemanticReviewOutcome.ChangesRequested, + ); + expect( + getSemanticReviewOutcome({ + state: "error", + description: "Manual review required: automated review was scoped", + }), + ).toBe(SemanticReviewOutcome.ManualReviewRequired); + expect( + getSemanticReviewOutcome({ + state: "error", + description: "Review incomplete: semantic result was missing", + }), + ).toBe(SemanticReviewOutcome.ReviewIncomplete); + }); +}); + +describe("validateSemanticReviewResult", () => { + it("accepts only the current open PR head", async () => { + const github = createFinalizeGithub(); + await expect( + validateSemanticReviewResult({ owner, repo, result: passedResult, github }), + ).resolves.toBeUndefined(); + + github.rest.pulls.get.mockResolvedValue({ + data: { state: "open", head: { sha: "fedcba9876543210fedcba9876543210fedcba98" } }, + }); + await expect( + validateSemanticReviewResult({ owner, repo, result: passedResult, github }), + ).rejects.toThrow("The reviewed pull request head is no longer current"); + }); +}); + +describe("finalizeSemanticReviewWorkflow", () => { + it("publishes Passed only after the reviewer workflow succeeds", async () => { + const github = createFinalizeGithub(); + + await expect(finalize({ github })).resolves.toBe(true); + expect(github.rest.repos.createCommitStatus).toHaveBeenCalledWith({ + owner, + repo, + sha: headSha, + state: CommitStatusState.SUCCESS, + context: ARM_SEMANTIC_REVIEW_STATUS, + description: "Passed: full review completed with no Blocking findings", + target_url: targetUrl, + }); + }); + + it("publishes Manual review required for scoped coverage", async () => { + const github = createFinalizeGithub(); + const result = { ...passedResult, scope: "scoped" as const }; + + await finalize({ github, artifactNames: [receiptName(result)] }); + expect(github.rest.repos.createCommitStatus).toHaveBeenCalledWith( + expect.objectContaining({ + state: CommitStatusState.ERROR, + description: "Manual review required: automated review was scoped", + }), + ); + }); + + it("requires manual review when trusted size checks find limited coverage", async () => { + const changedFiles = Array.from({ length: 51 }, (_, index) => ({ + filename: `specification/test/resource-manager/file-${index}.json`, + additions: 1, + deletions: 0, + })); + const github = createFinalizeGithub({ changedFiles }); + + await finalize({ github }); + expect(github.rest.repos.createCommitStatus).toHaveBeenCalledWith( + expect.objectContaining({ + state: CommitStatusState.ERROR, + description: "Manual review required: PR exceeds automated review size limits", + }), + ); + + const failedGithub = createFinalizeGithub({ changedFiles }); + await finalize({ + github: failedGithub, + artifactNames: [], + workflowConclusion: "failure", + }); + expect(failedGithub.rest.repos.createCommitStatus).toHaveBeenCalledWith( + expect.objectContaining({ + description: "Manual review required: PR exceeds automated review size limits", + }), + ); + }); + + it("publishes Review incomplete for missing or failed review execution", async () => { + const missingGithub = createFinalizeGithub(); + await finalize({ github: missingGithub, artifactNames: [] }); + expect(missingGithub.rest.repos.createCommitStatus).toHaveBeenCalledWith( + expect.objectContaining({ + state: CommitStatusState.ERROR, + description: "Review incomplete: semantic result was missing", + }), + ); + + const failedGithub = createFinalizeGithub(); + await finalize({ github: failedGithub, workflowConclusion: "failure" }); + expect(failedGithub.rest.repos.createCommitStatus).toHaveBeenCalledWith( + expect.objectContaining({ + state: CommitStatusState.ERROR, + description: "Review incomplete: workflow concluded with failure", + }), + ); + + const malformedGithub = createFinalizeGithub(); + await finalize({ + github: malformedGithub, + artifactNames: [`arm-semantic-review=${runAttempt}.malformed`], + }); + expect(malformedGithub.rest.repos.createCommitStatus).toHaveBeenCalledWith( + expect.objectContaining({ + description: "Review incomplete: semantic result was invalid", + }), + ); + + const mismatchedGithub = createFinalizeGithub(); + const mismatchedReceipt = { + ...passedResult, + issueNumber: 456, + }; + await finalize({ + github: mismatchedGithub, + artifactNames: [receiptName(mismatchedReceipt)], + }); + expect(mismatchedGithub.rest.repos.createCommitStatus).toHaveBeenCalledWith( + expect.objectContaining({ + description: "Review incomplete: semantic result was invalid", + }), + ); + }); + + it("does not finalize a canceled pending review", async () => { + const github = createFinalizeGithub(); + + await expect(finalize({ github, workflowConclusion: "cancelled" })).resolves.toBe(false); + expect(github.rest.repos.createCommitStatus).not.toHaveBeenCalled(); + }); + + it("ignores a successful workflow where the reviewer did not execute", async () => { + const github = createFinalizeGithub({ + statuses: [ + { + context: ARM_SEMANTIC_REVIEW_STATUS, + state: CommitStatusState.SUCCESS, + description: "Passed: full review completed with no Blocking findings", + target_url: "https://github.com/Azure/azure-rest-api-specs/actions/runs/123/attempts/1", + updated_at: "2026-01-01T00:00:00Z", + }, + ], + }); + + await expect( + finalize({ + github, + artifactNames: [], + reviewerExecuted: false, + }), + ).resolves.toBe(false); + expect(github.rest.repos.createCommitStatus).not.toHaveBeenCalled(); + }); + + it("invalidates a final result if its workflow is later canceled", async () => { + const github = createFinalizeGithub({ + statuses: [ + { + context: ARM_SEMANTIC_REVIEW_STATUS, + state: CommitStatusState.SUCCESS, + description: "Passed: full review completed with no Blocking findings", + target_url: targetUrl, + updated_at: "2026-01-01T00:00:00Z", + }, + ], + }); + + await expect(finalize({ github, workflowConclusion: "cancelled" })).resolves.toBe(true); + expect(github.rest.repos.createCommitStatus).toHaveBeenCalledWith( + expect.objectContaining({ + state: CommitStatusState.PENDING, + description: "Review canceled; rerun required", + }), + ); + }); + + it("ignores stale heads and superseded runs", async () => { + const staleGithub = createFinalizeGithub({ + currentHeadSha: "fedcba9876543210fedcba9876543210fedcba98", + }); + await expect(finalize({ github: staleGithub })).resolves.toBe(false); + expect(staleGithub.rest.repos.createCommitStatus).not.toHaveBeenCalled(); + + const supersededGithub = createFinalizeGithub({ + statuses: [ + { + context: ARM_SEMANTIC_REVIEW_STATUS, + state: CommitStatusState.PENDING, + description: "ARM API semantic review is pending", + target_url: "https://github.com/Azure/azure-rest-api-specs/actions/runs/789/attempts/1", + updated_at: "2026-01-02T00:00:00Z", + }, + ], + }); + await expect(finalize({ github: supersededGithub })).resolves.toBe(false); + expect(supersededGithub.rest.repos.createCommitStatus).not.toHaveBeenCalled(); + }); + + it("overrides an older Passed status when a newer review fails before setting pending", async () => { + const github = createFinalizeGithub({ + statuses: [ + { + context: ARM_SEMANTIC_REVIEW_STATUS, + state: CommitStatusState.SUCCESS, + description: "Passed: full review completed with no Blocking findings", + target_url: "https://github.com/Azure/azure-rest-api-specs/actions/runs/123/attempts/1", + updated_at: "2026-01-01T00:00:00Z", + }, + ], + }); + + await finalize({ + github, + artifactNames: [], + workflowConclusion: "failure", + }); + expect(github.rest.repos.createCommitStatus).toHaveBeenCalledWith( + expect.objectContaining({ + state: CommitStatusState.ERROR, + description: "Review incomplete: workflow concluded with failure", + }), + ); + }); + + it("lists commit statuses for the exact reviewed head", async () => { + const github = createFinalizeGithub(); + + await finalize({ github }); + expect(github.rest.repos.listCommitStatusesForRef).toHaveBeenCalledWith({ + owner, + repo, + ref: headSha, + per_page: PER_PAGE_MAX, + }); + }); +}); diff --git a/.github/workflows/test/arm-auto-signoff/arm-universal-auto-signoff.test.ts b/.github/workflows/test/arm-auto-signoff/arm-universal-auto-signoff.test.ts index fd8135fb89ec..d8f6f777ca73 100644 --- a/.github/workflows/test/arm-auto-signoff/arm-universal-auto-signoff.test.ts +++ b/.github/workflows/test/arm-auto-signoff/arm-universal-auto-signoff.test.ts @@ -1,3 +1,5 @@ +import { readFile } from "node:fs/promises"; +import { join } from "node:path"; import { describe, expect, it } from "vitest"; import { CommitStatusState } from "../../../shared/src/github.ts"; import { ArmAutoSignoffLabel } from "../../src/arm-auto-signoff/arm-auto-signoff-labels.ts"; @@ -10,11 +12,19 @@ const owner = "TestOwner"; const repo = "TestRepo"; const issueNumber = 123; const headSha = "abc123"; +const GITHUB_ROOT = join(import.meta.dirname, "..", "..", ".."); -const successfulStatuses = [ +const semanticPassed = { + context: "ARM Semantic Review", + state: CommitStatusState.SUCCESS, + description: "Passed: full review completed with no Blocking findings", + updated_at: "2026-01-01", +}; +const deterministicStatuses = [ { context: "Swagger LintDiff", state: CommitStatusState.SUCCESS, updated_at: "2026-01-01" }, { context: "Swagger Avocado", state: CommitStatusState.SUCCESS, updated_at: "2026-01-01" }, ]; +const successfulStatuses = [semanticPassed, ...deterministicStatuses]; function createMockGithub({ labelNames = [], @@ -23,7 +33,7 @@ function createMockGithub({ pullRequestState = "open", }: { labelNames?: string[]; - statuses?: { context: string; state: string; updated_at: string }[]; + statuses?: { context: string; state: string; description?: string; updated_at: string }[]; currentHeadSha?: string; pullRequestState?: "open" | "closed"; } = {}) { @@ -38,7 +48,6 @@ function createMockGithub({ data: labelNames.map((name) => ({ name })), }); github.rest.repos.listCommitStatusesForRef.mockResolvedValue({ data: statuses }); - return github; } @@ -54,7 +63,7 @@ function run(github: ReturnType) { } describe("getLabelActionImpl", () => { - it("adds the pilot label when the universal requirements pass", async () => { + it("adds the pilot label when all universal requirements pass", async () => { const github = createMockGithub({ labelNames: ["ARMReview"] }); await expect(run(github)).resolves.toEqual({ @@ -62,6 +71,7 @@ describe("getLabelActionImpl", () => { issueNumber, labelActions: { [ArmAutoSignoffLabel.ArmAutoSignedOffTest]: LabelAction.Add, + [ArmAutoSignoffLabel.ArmManualSignoffRequired]: LabelAction.None, }, }); }); @@ -75,7 +85,7 @@ describe("getLabelActionImpl", () => { expect(result.labelActions[ArmAutoSignoffLabel.ArmAutoSignedOffTest]).toBe(LabelAction.None); }); - it("removes the pilot label when manual signoff is required", async () => { + it("treats a manual signoff label as a hard stop without removing it", async () => { const github = createMockGithub({ labelNames: [ "ARMReview", @@ -85,13 +95,136 @@ describe("getLabelActionImpl", () => { }); const result = await run(github); - expect(result.labelActions[ArmAutoSignoffLabel.ArmAutoSignedOffTest]).toBe(LabelAction.Remove); + expect(result.labelActions).toEqual({ + [ArmAutoSignoffLabel.ArmAutoSignedOffTest]: LabelAction.Remove, + [ArmAutoSignoffLabel.ArmManualSignoffRequired]: LabelAction.None, + }); + }); + + describe("auto-signoff workflow correlation", () => { + it("always publishes live-head correlation artifacts", async () => { + const workflow = await readFile( + join(GITHUB_ROOT, "workflows", "arm-universal-auto-signoff.yaml"), + "utf8", + ); + const headArtifact = workflow.indexOf("name: Upload artifact with head SHA"); + const issueArtifact = workflow.indexOf("name: Upload artifact with issue number"); + + expect(headArtifact).toBeGreaterThan(-1); + expect(issueArtifact).toBeGreaterThan(headArtifact); + expect(workflow.slice(headArtifact - 180, headArtifact)).not.toContain( + "github.event_name ==", + ); + expect(workflow.slice(issueArtifact - 180, issueArtifact)).not.toContain( + "github.event_name ==", + ); + }); + }); + + it("does not sign off while ARMChangesRequested is present", async () => { + const github = createMockGithub({ + labelNames: ["ARMReview", "ARMChangesRequested", ArmAutoSignoffLabel.ArmAutoSignedOffTest], + }); + + const result = await run(github); + expect(result.labelActions).toEqual({ + [ArmAutoSignoffLabel.ArmAutoSignedOffTest]: LabelAction.Remove, + [ArmAutoSignoffLabel.ArmManualSignoffRequired]: LabelAction.None, + }); + }); + + it.each([ + { + name: "missing", + status: undefined, + }, + { + name: "pending", + status: { + context: "ARM Semantic Review", + state: CommitStatusState.PENDING, + description: "ARM API semantic review is pending", + updated_at: "2026-01-01", + }, + }, + { + name: "changes requested", + status: { + context: "ARM Semantic Review", + state: CommitStatusState.FAILURE, + description: "Changes requested: 1 Blocking finding(s)", + updated_at: "2026-01-01", + }, + }, + { + name: "review incomplete", + status: { + context: "ARM Semantic Review", + state: CommitStatusState.ERROR, + description: "Review incomplete: workflow concluded with failure", + updated_at: "2026-01-01", + }, + }, + ])( + "does not sign off or require manual signoff when semantic review is $name", + async ({ status }) => { + const github = createMockGithub({ + labelNames: ["ARMReview"], + statuses: status ? [status, ...deterministicStatuses] : deterministicStatuses, + }); + + const result = await run(github); + expect(result.labelActions).toEqual({ + [ArmAutoSignoffLabel.ArmAutoSignedOffTest]: LabelAction.None, + [ArmAutoSignoffLabel.ArmManualSignoffRequired]: LabelAction.None, + }); + }, + ); + + it("adds manual signoff only when automated coverage was scoped", async () => { + const github = createMockGithub({ + labelNames: ["ARMReview"], + statuses: [ + { + context: "ARM Semantic Review", + state: CommitStatusState.ERROR, + description: "Manual review required: automated review was scoped", + updated_at: "2026-01-01", + }, + ...deterministicStatuses, + ], + }); + + const result = await run(github); + expect(result.labelActions).toEqual({ + [ArmAutoSignoffLabel.ArmAutoSignedOffTest]: LabelAction.None, + [ArmAutoSignoffLabel.ArmManualSignoffRequired]: LabelAction.Add, + }); + }); + + it("adds manual signoff for scoped coverage even when changes were requested", async () => { + const github = createMockGithub({ + labelNames: ["ARMReview", "ARMChangesRequested"], + statuses: [ + { + context: "ARM Semantic Review", + state: CommitStatusState.ERROR, + description: "Manual review required: automated review was scoped", + updated_at: "2026-01-01", + }, + ...deterministicStatuses, + ], + }); + + const result = await run(github); + expect(result.labelActions[ArmAutoSignoffLabel.ArmManualSignoffRequired]).toBe(LabelAction.Add); }); - it("removes the pilot label while a mandatory status is pending", async () => { + it("removes the pilot label while a deterministic status is pending", async () => { const github = createMockGithub({ labelNames: ["ARMReview", ArmAutoSignoffLabel.ArmAutoSignedOffTest], statuses: [ + semanticPassed, { context: "Swagger LintDiff", state: CommitStatusState.SUCCESS, updated_at: "2026-01-01" }, { context: "Swagger Avocado", state: CommitStatusState.PENDING, updated_at: "2026-01-01" }, ], @@ -101,7 +234,7 @@ describe("getLabelActionImpl", () => { expect(result.labelActions[ArmAutoSignoffLabel.ArmAutoSignedOffTest]).toBe(LabelAction.Remove); }); - it("removes the pilot label when a suppression lacks approval", async () => { + it("removes the pilot label when suppression approval is missing", async () => { const github = createMockGithub({ labelNames: [ "ARMReview", diff --git a/.github/workflows/test/context.test.ts b/.github/workflows/test/context.test.ts index df38e667edbe..e4b9cf7f88cd 100644 --- a/.github/workflows/test/context.test.ts +++ b/.github/workflows/test/context.test.ts @@ -221,6 +221,168 @@ describe("extractInputs", () => { }, ); + it("workflow_run:completed:pull_request_target uses trusted correlation artifacts", async () => { + const github = createMockGithub(); + github.rest.actions.listWorkflowRunArtifacts.mockResolvedValue({ + data: { + artifacts: [{ name: `head-sha=${fullGitSha}` }, { name: "issue-number=123" }], + }, + }); + const context = { + eventName: "workflow_run", + payload: { + action: "completed", + workflow_run: { + event: "pull_request_target", + head_sha: "base-branch-sha", + id: 456, + repository: { + name: "TestRepoName", + owner: { + login: "TestRepoOwnerLogin", + }, + }, + pull_requests: [{ number: 123 }], + }, + }, + }; + + await expect(extractInputs(github, context, createMockCore())).resolves.toEqual({ + owner: "TestRepoOwnerLogin", + repo: "TestRepoName", + head_sha: fullGitSha, + issue_number: 123, + run_id: 456, + }); + expect(github.rest.actions.listWorkflowRunArtifacts).toHaveBeenCalledWith({ + owner: "TestRepoOwnerLogin", + repo: "TestRepoName", + run_id: 456, + per_page: PER_PAGE_MAX, + }); + }); + + it("workflow_run:completed:workflow_dispatch uses trusted correlation artifacts", async () => { + const github = createMockGithub(); + github.rest.actions.listWorkflowRunArtifacts.mockResolvedValue({ + data: { + artifacts: [{ name: `head-sha=${fullGitSha}` }, { name: "issue-number=123" }], + }, + }); + const context = { + eventName: "workflow_run", + payload: { + action: "completed", + workflow_run: { + event: "workflow_dispatch", + head_sha: "default-branch-sha", + id: 456, + repository: { + name: "TestRepoName", + owner: { + login: "TestRepoOwnerLogin", + }, + }, + pull_requests: [], + }, + }, + }; + + await expect(extractInputs(github, context, createMockCore())).resolves.toEqual({ + owner: "TestRepoOwnerLogin", + repo: "TestRepoName", + head_sha: fullGitSha, + issue_number: 123, + run_id: 456, + }); + }); + + it("recovers ARM API review correlation when pre-activation artifacts are missing", async () => { + const github = createMockGithub(); + github.rest.actions.listWorkflowRunArtifacts.mockResolvedValue({ + data: { artifacts: [] }, + }); + github.rest.pulls.get.mockResolvedValue({ + data: { head: { sha: fullGitSha } }, + }); + const context = { + eventName: "workflow_run", + payload: { + action: "completed", + workflow_run: { + name: "ARM API Review: Automated Workflow", + display_title: "ARM API Review #123 (workflow_dispatch)", + event: "workflow_dispatch", + head_sha: "default-branch-sha", + id: 456, + repository: { + name: "TestRepoName", + owner: { + login: "TestRepoOwnerLogin", + }, + }, + pull_requests: [], + }, + }, + }; + + await expect(extractInputs(github, context, createMockCore())).resolves.toEqual({ + owner: "TestRepoOwnerLogin", + repo: "TestRepoName", + head_sha: fullGitSha, + issue_number: 123, + run_id: 456, + }); + expect(github.rest.pulls.get).toHaveBeenCalledWith({ + owner: "TestRepoOwnerLogin", + repo: "TestRepoName", + pull_number: 123, + }); + }); + + it("rejects conflicting workflow correlation artifacts", async () => { + const github = createMockGithub(); + const context = { + eventName: "workflow_run", + payload: { + action: "completed", + workflow_run: { + event: "workflow_dispatch", + head_sha: "default-branch-sha", + id: 456, + repository: { + name: "TestRepoName", + owner: { + login: "TestRepoOwnerLogin", + }, + }, + pull_requests: [], + }, + }, + }; + + github.rest.actions.listWorkflowRunArtifacts.mockResolvedValue({ + data: { + artifacts: [ + { name: `head-sha=${fullGitSha}` }, + { name: "head-sha=abcdefabcdefabcdefabcdefabcdefabcdefabcd" }, + ], + }, + }); + await expect(extractInputs(github, context, createMockCore())).rejects.toThrow( + "Conflicting head-sha artifacts", + ); + + github.rest.actions.listWorkflowRunArtifacts.mockResolvedValue({ + data: { + artifacts: [{ name: "issue-number=123" }, { name: "issue-number=456" }], + }, + }); + await expect(extractInputs(github, context, createMockCore())).rejects.toThrow( + "Conflicting issue-number artifacts", + ); + }); + it.each(["repository", "head_repository"])( "rejects a workflow run with a null %s owner", async (missingOwner) => { diff --git a/.github/workflows/test/mocks.ts b/.github/workflows/test/mocks.ts index ea08d797907a..50fd3a2a019f 100644 --- a/.github/workflows/test/mocks.ts +++ b/.github/workflows/test/mocks.ts @@ -51,6 +51,7 @@ function createMockGithubImpl() { }, pulls: { get: vi.fn(), + listFiles: vi.fn().mockResolvedValue({ data: [] }), }, repos: { createCommitStatus: vi.fn(), diff --git a/.github/workflows/test/update-labels.test.ts b/.github/workflows/test/update-labels.test.ts index ce82eeb279d4..a9cdb8acddba 100644 --- a/.github/workflows/test/update-labels.test.ts +++ b/.github/workflows/test/update-labels.test.ts @@ -5,6 +5,14 @@ import { fullGitSha } from "../../shared/test/examples.ts"; import updateLabelsSrc, { updateLabelsImpl } from "../src/update-labels.ts"; import { createMockCore, createMockGithub, createMockRequestError } from "./mocks.ts"; +function createLabelMockGithub(currentHeadSha = fullGitSha) { + const github = createMockGithub(); + github.rest.pulls.get.mockResolvedValue({ + data: { state: "open", head: { sha: currentHeadSha } }, + }); + return github; +} + function updateLabels(asyncFunctionArgs: unknown) { return updateLabelsSrc(asyncFunctionArgs as GitHubScriptArgs); } @@ -13,7 +21,7 @@ describe("updateLabels", () => { it("loads inputs from context", async () => { const core = createMockCore(); - const github = createMockGithub(); + const github = createLabelMockGithub(); github.rest.actions.listWorkflowRunArtifacts.mockResolvedValue({ data: { artifacts: [{ name: "label-foo=true" }], @@ -62,7 +70,7 @@ describe("updateLabels", () => { describe("updateLabelsImpl", () => { it("throws if no run_id", async () => { - const github = createMockGithub(); + const github = createLabelMockGithub(); await expect( updateLabelsImpl({ @@ -83,7 +91,7 @@ describe("updateLabelsImpl", () => { }); it("handles missing issue_number", async () => { - const github = createMockGithub(); + const github = createLabelMockGithub(); github.rest.actions.listWorkflowRunArtifacts.mockResolvedValue({ data: { @@ -128,7 +136,7 @@ describe("updateLabelsImpl", () => { }); it("adds and removes labels for artifacts", async () => { - const github = createMockGithub(); + const github = createLabelMockGithub(); github.rest.actions.listWorkflowRunArtifacts.mockResolvedValue({ data: { artifacts: [ @@ -183,7 +191,7 @@ describe("updateLabelsImpl", () => { }); it("throws for invalid label value", async () => { - const github = createMockGithub(); + const github = createLabelMockGithub(); github.rest.actions.listWorkflowRunArtifacts.mockResolvedValue({ data: { artifacts: [ @@ -221,7 +229,7 @@ describe("updateLabelsImpl", () => { }); it("throws for invalid label name", async () => { - const github = createMockGithub(); + const github = createLabelMockGithub(); github.rest.actions.listWorkflowRunArtifacts.mockResolvedValue({ data: { artifacts: [ @@ -257,7 +265,7 @@ describe("updateLabelsImpl", () => { }); it.each([404, 500, 501])("handles error removing label (%s)", async (status) => { - const github = createMockGithub(); + const github = createLabelMockGithub(); github.rest.actions.listWorkflowRunArtifacts.mockResolvedValue({ data: { artifacts: [{ name: "label-foo=false" }], @@ -295,4 +303,26 @@ describe("updateLabelsImpl", () => { name: "foo", }); }); + + it("does not apply labels for a stale pull request head", async () => { + const github = createLabelMockGithub("fedcba9876543210fedcba9876543210fedcba98"); + github.rest.actions.listWorkflowRunArtifacts.mockResolvedValue({ + data: { + artifacts: [{ name: "label-foo=true" }, { name: `head-sha=${fullGitSha}` }], + }, + }); + + await updateLabelsImpl({ + owner: "owner", + repo: "repo", + head_sha: fullGitSha, + issue_number: 123, + run_id: 456, + github, + core: createMockCore(), + }); + + expect(github.rest.issues.addLabels).not.toHaveBeenCalled(); + expect(github.rest.issues.removeLabel).not.toHaveBeenCalled(); + }); }); From a620337de0059a29afdfcf0083dc8fd09a7561b0 Mon Sep 17 00:00:00 2001 From: Akhila Illa Date: Mon, 5 Oct 2026 10:05:27 -0500 Subject: [PATCH 02/16] Isolate ARM semantic status publication Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../workflows/arm-semantic-review-status.yaml | 50 + .../workflows/arm-universal-auto-signoff.yaml | 17 +- .../arm-semantic-review-workflow.ts | 22 +- ...rm-semantic-review-status-workflow.test.ts | 106 ++ ...ntic-review-auto-signoff-implementation.md | 1083 +++++++++++++++++ 5 files changed, 1260 insertions(+), 18 deletions(-) create mode 100644 .github/workflows/arm-semantic-review-status.yaml create mode 100644 .github/workflows/test/arm-auto-signoff/arm-semantic-review-status-workflow.test.ts create mode 100644 documentation/arm-semantic-review-auto-signoff-implementation.md diff --git a/.github/workflows/arm-semantic-review-status.yaml b/.github/workflows/arm-semantic-review-status.yaml new file mode 100644 index 000000000000..bf93ab041605 --- /dev/null +++ b/.github/workflows/arm-semantic-review-status.yaml @@ -0,0 +1,50 @@ +name: ARM Semantic Review - Set Status + +on: + workflow_run: + workflows: ["ARM API Review: Automated Workflow"] + types: [completed] + +permissions: + # actions.listWorkflowRunArtifacts + actions: read + # actions/checkout + contents: read + # pulls.get, pulls.listFiles + pull-requests: read + # repos.listCommitStatusesForRef, repos.createCommitStatus + statuses: write + +jobs: + arm-semantic-review-status: + name: ARM Semantic Review - Set Status + runs-on: ubuntu-slim + + steps: + - uses: actions/checkout@v7 + with: + sparse-checkout: | + .github + + - id: finalize + name: Finalize ARM Semantic Review + uses: actions/github-script@v9.0.0 + with: + script: | + const { finalizeArmSemanticReview } = + await import('${{ github.workspace }}/.github/workflows/src/arm-auto-signoff/arm-semantic-review-workflow.ts'); + return await finalizeArmSemanticReview({ github, context, core }); + + - if: fromJson(steps.finalize.outputs.result).headSha + name: Upload artifact with head SHA + uses: ./.github/actions/add-empty-artifact + with: + name: "head-sha" + value: "${{ fromJson(steps.finalize.outputs.result).headSha }}" + + - if: fromJson(steps.finalize.outputs.result).issueNumber > 0 + name: Upload artifact with issue number + uses: ./.github/actions/add-empty-artifact + with: + name: "issue-number" + value: "${{ fromJson(steps.finalize.outputs.result).issueNumber }}" diff --git a/.github/workflows/arm-universal-auto-signoff.yaml b/.github/workflows/arm-universal-auto-signoff.yaml index 292fcf0f75ad..e31cda444e3c 100644 --- a/.github/workflows/arm-universal-auto-signoff.yaml +++ b/.github/workflows/arm-universal-auto-signoff.yaml @@ -20,7 +20,7 @@ on: # Re-evaluate as each mandatory status producer completes. workflows: [ - "ARM API Review: Automated Workflow", + "ARM Semantic Review - Set Status", "Swagger Avocado - Set Status", "Swagger LintDiff - Set Status", ] @@ -36,8 +36,8 @@ permissions: issues: read # issues.listLabelsOnIssue pull-requests: read - # repos.listCommitStatusesForRef, repos.createCommitStatus - statuses: write + # repos.listCommitStatusesForRef + statuses: read jobs: arm-universal-auto-signoff: @@ -76,17 +76,6 @@ jobs: sparse-checkout: | .github - - if: | - github.event_name == 'workflow_run' && - github.event.workflow_run.name == 'ARM API Review: Automated Workflow' - name: Finalize ARM Semantic Review - uses: actions/github-script@v9.0.0 - with: - script: | - const { finalizeArmSemanticReview } = - await import('${{ github.workspace }}/.github/workflows/src/arm-auto-signoff/arm-semantic-review-workflow.ts'); - await finalizeArmSemanticReview({ github, context, core }); - - id: get-label-action name: ARM Universal Auto SignOff uses: actions/github-script@v9.0.0 diff --git a/.github/workflows/src/arm-auto-signoff/arm-semantic-review-workflow.ts b/.github/workflows/src/arm-auto-signoff/arm-semantic-review-workflow.ts index 5d2233f2621e..326ceeacc8fb 100644 --- a/.github/workflows/src/arm-auto-signoff/arm-semantic-review-workflow.ts +++ b/.github/workflows/src/arm-auto-signoff/arm-semantic-review-workflow.ts @@ -285,11 +285,20 @@ export async function finalizeArmSemanticReview({ github, context, core, -}: GitHubScriptArgs): Promise { +}: GitHubScriptArgs): Promise<{ + headSha: string; + issueNumber: number; + statusPublished: boolean; +}> { + const emptyResult = { + headSha: "", + issueNumber: 0, + statusPublished: false, + }; const payload = context.payload as WebhookEvent<"workflow-run", "completed">; const workflowRun = payload.workflow_run; if (workflowRun.name !== ARM_API_REVIEW_WORKFLOW_NAME) { - return; + return emptyResult; } const repositoryOwner = workflowRun.repository.owner?.login; @@ -309,11 +318,11 @@ export async function finalizeArmSemanticReview({ artifactNames.some((name) => name.startsWith("issue-number="))); if (!reviewerExecuted) { core.info("The ARM API Reviewer did not execute; semantic status is unchanged"); - return; + return emptyResult; } const { owner, repo, head_sha, issue_number } = await extractInputs(github, context, core); - await finalizeSemanticReviewWorkflow({ + const statusPublished = await finalizeSemanticReviewWorkflow({ owner, repo, issueNumber: issue_number, @@ -327,5 +336,10 @@ export async function finalizeArmSemanticReview({ github, core, }); + return { + headSha: isFullGitSha(head_sha) ? head_sha : "", + issueNumber: Number.isSafeInteger(issue_number) && issue_number > 0 ? issue_number : 0, + statusPublished, + }; } /* v8 ignore stop */ diff --git a/.github/workflows/test/arm-auto-signoff/arm-semantic-review-status-workflow.test.ts b/.github/workflows/test/arm-auto-signoff/arm-semantic-review-status-workflow.test.ts new file mode 100644 index 000000000000..2071e4d0e4cb --- /dev/null +++ b/.github/workflows/test/arm-auto-signoff/arm-semantic-review-status-workflow.test.ts @@ -0,0 +1,106 @@ +import { readFile } from "node:fs/promises"; +import { load } from "js-yaml"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; + +const GITHUB_ROOT = join(import.meta.dirname, "..", "..", ".."); + +type Workflow = { + name?: string; + on?: { + workflow_run?: { + workflows?: string[]; + types?: string[]; + }; + }; + permissions?: Record; + jobs?: Record< + string, + { + steps?: Array<{ + id?: string; + name?: string; + if?: string; + with?: { + script?: string; + name?: string; + value?: string; + }; + }>; + } + >; +}; + +async function readWorkflow(name: string): Promise<{ + source: string; + workflow: Workflow; +}> { + const source = await readFile(join(GITHUB_ROOT, "workflows", name), "utf8"); + return { + source, + workflow: load(source) as Workflow, + }; +} + +describe("ARM Semantic Review - Set Status workflow", () => { + it("finalizes only completed ARM API Reviewer runs", async () => { + const { workflow } = await readWorkflow("arm-semantic-review-status.yaml"); + + expect(workflow.name).toBe("ARM Semantic Review - Set Status"); + expect(workflow.on?.workflow_run).toEqual({ + workflows: ["ARM API Review: Automated Workflow"], + types: ["completed"], + }); + }); + + it("isolates status-write permission from Universal Auto-Signoff", async () => { + const [{ workflow: semanticStatus }, { workflow: universal }] = await Promise.all([ + readWorkflow("arm-semantic-review-status.yaml"), + readWorkflow("arm-universal-auto-signoff.yaml"), + ]); + + expect(semanticStatus.permissions).toEqual({ + actions: "read", + contents: "read", + "pull-requests": "read", + statuses: "write", + }); + expect(universal.permissions?.statuses).toBe("read"); + }); + + it("publishes PR and SHA correlation for Universal Auto-Signoff", async () => { + const { workflow } = await readWorkflow("arm-semantic-review-status.yaml"); + const steps = workflow.jobs?.["arm-semantic-review-status"]?.steps ?? []; + const finalize = steps.find((step) => step.id === "finalize"); + + expect(finalize?.with?.script).toContain("finalizeArmSemanticReview"); + expect(steps).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + name: "Upload artifact with head SHA", + with: { + name: "head-sha", + value: "${{ fromJson(steps.finalize.outputs.result).headSha }}", + }, + }), + expect.objectContaining({ + name: "Upload artifact with issue number", + with: { + name: "issue-number", + value: "${{ fromJson(steps.finalize.outputs.result).issueNumber }}", + }, + }), + ]), + ); + }); + + it("makes Universal depend on semantic status completion, not reviewer completion", async () => { + const { source, workflow } = await readWorkflow("arm-universal-auto-signoff.yaml"); + const workflows = workflow.on?.workflow_run?.workflows ?? []; + + expect(workflows).toContain("ARM Semantic Review - Set Status"); + expect(workflows).not.toContain("ARM API Review: Automated Workflow"); + expect(source).not.toContain("Finalize ARM Semantic Review"); + expect(source).not.toContain("finalizeArmSemanticReview"); + }); +}); diff --git a/documentation/arm-semantic-review-auto-signoff-implementation.md b/documentation/arm-semantic-review-auto-signoff-implementation.md new file mode 100644 index 000000000000..895823476875 --- /dev/null +++ b/documentation/arm-semantic-review-auto-signoff-implementation.md @@ -0,0 +1,1083 @@ +# ARM Semantic Review and Universal Auto-Signoff + +## Purpose + +This document explains how the ARM API Reviewer integrates with Universal ARM +Auto-Signoff, why the implementation uses both workflow artifacts and commit +statuses, which alternatives were considered, and what risks remain. + +It is also a reading guide for someone who is new to GitHub Actions, workflow +coordination, and automated signoff policy. + +The implementation is currently a pilot. Universal Auto-Signoff manages +`ARMAutoSignedOff-Test`; it does not yet replace the production +`ARMSignedOff` process. + +## Start with this mental model + +The whole design can be reduced to one sentence: + +> The ARM API Reviewer produces evidence for one workflow run, trusted code +> converts that evidence into a result for one commit SHA, and Universal +> Auto-Signoff signs off only when every required result for the current SHA +> passes. + +There are two representations of the semantic result: + +| Representation | Mental model | Lifetime | +| ----------------- | ---------------------------------------------------- | -------------------------------- | +| Workflow artifact | "This reviewer run observed this result." | Associated with one workflow run | +| Commit status | "This is the current semantic state of this commit." | Associated with one commit SHA | + +The artifact is an intermediate handoff. The commit status is the durable input +that every later auto-signoff evaluation reads. + +## Vocabulary + +### PR head SHA + +The immutable Git commit at the tip of the pull request. + +All evidence used for one auto-signoff decision must refer to the same head SHA. +A result for an older SHA cannot authorize a newer SHA. + +### Workflow run + +One execution of a GitHub Actions workflow. A rerun has the same run ID and a +new run attempt. + +### Workflow artifact + +Data associated with one workflow run. In this design, artifacts carry: + +- the PR number; +- the reviewed head SHA; and +- the semantic-review receipt. + +Artifacts are not the long-lived auto-signoff state. + +### Commit status + +A GitHub status attached to a commit SHA and named by a context, for example: + +```text +ARM Semantic Review = success +Swagger LintDiff = success +Swagger Avocado = success +``` + +GitHub retains status history. When the same context is published more than +once for one SHA, the newest entry is treated as current. + +### ARM queue labels + +The existing human-review state includes: + +- `NotReadyForARMReview` +- `WaitForARMFeedback` +- `ARMChangesRequested` +- `ARMSignedOff` + +`ARMManualSignoffRequired` is the proposed hard stop that prevents automatic +signoff when a human decision is required. + +## Components + +### ARM API Reviewer + +Source: +[arm-api-review.md](../.github/workflows/arm-api-review.md) + +Generated workflow: +[arm-api-review.lock.yml](../.github/workflows/arm-api-review.lock.yml) + +Responsibilities: + +- review the cumulative PR diff at an exact head SHA; +- use the Critic to verify findings; +- reconcile findings with existing PR discussion; +- publish comments and update ARM queue labels; +- record a structured semantic result. + +### Semantic state model + +[arm-semantic-review.ts](../.github/workflows/src/arm-auto-signoff/arm-semantic-review.ts) + +Responsibilities: + +- parse and validate model output; +- encode and parse semantic receipts; +- map evidence to semantic outcomes; +- evaluate trusted file-count and line-count coverage; +- interpret the latest semantic commit status. + +This module contains pure state and parsing logic. It does not orchestrate +GitHub workflows. + +### Semantic workflow orchestration + +[arm-semantic-review-workflow.ts](../.github/workflows/src/arm-auto-signoff/arm-semantic-review-workflow.ts) + +Responsibilities: + +- validate the agent's result against the live PR; +- finalize a completed reviewer workflow; +- reject stale and superseded runs; +- independently verify automated-review coverage; +- publish the final `ARM Semantic Review` commit status. + +### Semantic status workflow + +[arm-semantic-review-status.yaml](../.github/workflows/arm-semantic-review-status.yaml) + +Responsibilities: + +- trigger directly from the completed ARM API Reviewer run; +- invoke semantic finalization with `statuses: write`; +- upload trusted PR and SHA correlation artifacts; +- provide the completion event that triggers Universal Auto-Signoff. + +### Universal Auto-Signoff + +Workflow: +[arm-universal-auto-signoff.yaml](../.github/workflows/arm-universal-auto-signoff.yaml) + +Decision logic: +[arm-universal-auto-signoff.ts](../.github/workflows/src/arm-auto-signoff/arm-universal-auto-signoff.ts) + +Responsibilities: + +- read statuses for the current head SHA; +- evaluate semantic review, LintDiff, Avocado, labels, and approvals; +- upload label-action artifacts. + +Universal Auto-Signoff has `statuses: read`. Status publication is isolated in +the semantic status workflow. + +### Label application + +Workflow: +[update-labels.yaml](../.github/workflows/update-labels.yaml) + +Implementation: +[update-labels.ts](../.github/workflows/src/update-labels.ts) + +Responsibilities: + +- consume trusted label artifacts; +- recheck the live PR head when the producer supplied a head-SHA artifact; +- apply label additions and removals. + +## End-to-end flow + +```mermaid +flowchart TD + A[Eligible PR trigger or /arm-review] --> B[ARM API Reviewer starts] + B --> C[Resolve PR and exact head SHA] + C --> D[Publish ARM Semantic Review = Pending] + D --> E[Agent reviews cumulative PR diff] + E --> F[Critic verifies findings] + F --> G[Reconcile existing comments] + G --> H[Queue comments and ARM label changes] + G --> I[Record validated semantic receipt] + + H --> J[Reviewer workflow completes] + I --> J + + J --> K[Semantic Set Status workflow_run trigger] + K --> L[Read exact reviewer run artifacts] + L --> M[Validate PR, SHA, attempt, scope, and coverage] + M --> N[Publish final ARM Semantic Review status] + + N --> O[Universal Auto-Signoff workflow_run trigger] + O --> P[Read all statuses for current head SHA] + Y[LintDiff, Avocado, PR, or label event] --> P + + P --> Q{Semantic status is Passed?} + Q -- No --> R[Do not auto-sign off] + Q -- Manual review required --> S[Add ARMManualSignoffRequired] + Q -- Yes --> T{LintDiff, Avocado,\nlabels, and approvals pass?} + T -- No --> R + T -- Yes --> U[Add ARMAutoSignedOff-Test] + + U --> V[Update Labels workflow] + S --> V + V --> W[Recheck live PR head] + W --> X[Apply labels if still current] +``` + +## Detailed workflow + +### 1. Start the reviewer + +The reviewer can start from: + +- an eligible PR open, synchronize, or ready-for-review event; +- adding `WaitForARMFeedback`; +- an authorized `/arm-review` comment; or +- manual workflow dispatch. + +The reviewer has one concurrency group per PR with: + +```yaml +cancel-in-progress: true +``` + +A newer eligible trigger replaces an active reviewer run for the same PR. +Unrelated comments receive run-specific concurrency groups and do not cancel an +active review. + +### 2. Correlate the PR and set Pending + +Before agent execution, trusted pre-activation code: + +1. validates the target PR number; +2. reads the exact current head SHA; +3. uploads `issue-number=` and `head-sha=` artifacts; +4. publishes: + +```text +SHA: +Context: ARM Semantic Review +State: pending +Description: ARM API semantic review is pending +Target URL: exact reviewer run and attempt +``` + +Publishing Pending is important when the same SHA is reviewed again. It prevents +an older Passed status from continuing to authorize signoff while the new review +is in progress. + +### 3. Perform semantic review + +The reviewer evaluates: + +```text +base SHA -> current head SHA +``` + +It reviews the cumulative PR, not only the most recent commit. + +The reviewer: + +- identifies semantic API-design findings; +- asks the Critic to verify findings; +- reads existing workflow-owned and human-authored discussion; +- reconciles current findings with prior comments; +- posts only net-new or relocated findings; +- resolves fixed workflow-owned findings. + +An unresolved verified Blocking finding still counts when its reconciliation +action is `SKIP-COVERED`. "No new comment was posted" does not mean the PR is +clean. + +### 4. Update the ARM queue state + +The reviewer applies these rules: + +| Review result | Queue behavior | +| ---------------------------------------------------------------- | --------------------------------------------------------------- | +| Scoped, incomplete, or degraded | Preserve existing queue labels | +| Full and complete, with applicable verified Blocking findings | Add `ARMChangesRequested`; remove `WaitForARMFeedback` | +| Full and complete, with no applicable verified Blocking findings | Remove stale `ARMChangesRequested`; retain `WaitForARMFeedback` | +| Critic unavailable | Preserve existing queue labels | + +Partial reviews preserve the human queue even when they identify Blocking +findings. They cannot make a complete queue-state decision. + +### 5. Record the semantic receipt + +After reconciliation and summary generation, the agent calls one structured +safe output with: + +```text +PR number +reviewed head SHA +scope: full | scoped +completeness: complete | incomplete | degraded +number of verified Blocking findings still applicable +``` + +Example artifact name: + +```text +arm-semantic-review=1.123.abc123abc123abc123abc123abc123abc123abcd.full.complete.0 +``` + +Fields: + +```text +1 workflow attempt +123 PR number +abc123... reviewed head SHA +full review scope +complete review completeness +0 currently applicable verified Blocking findings +``` + +A trusted safe-output job validates the receipt and uploads it. The agent does +not directly publish a final Passed status. + +### 6. Wait for the whole reviewer workflow + +The generated reviewer workflow includes: + +- agent execution; +- threat detection; +- custom semantic receipt validation; +- comment and label safe-output processing; and +- a final conclusion job. + +The dedicated Semantic Set Status workflow finalizes the result only after the +reviewer emits `workflow_run: completed`. + +This boundary is intended to prevent Passed from becoming visible while the +reviewer is still publishing comments or labels. + +### 7. Finalize the semantic result + +`ARM Semantic Review - Set Status` is triggered with the exact completed ARM +reviewer run. It calls `finalizeArmSemanticReview`. + +It verifies: + +- the reviewer actually executed; +- the PR number and head SHA are valid; +- the PR is still open; +- the reviewed SHA is still the current head; +- a newer run or attempt did not supersede this one; +- the receipt belongs to this PR, SHA, and run attempt; +- the changed-file inventory is not truncated; +- no more than 50 specification files changed; +- no more than 5,000 specification lines changed. + +Trusted code independently evaluates size and truncation. It does not rely only +on the model's `scope` value. + +### 8. Publish the final commit status + +| Semantic outcome | GitHub state | Description pattern | +| ---------------------- | ------------ | ---------------------------------- | +| Passed | `success` | `Passed: ...` | +| Changes requested | `failure` | `Changes requested: ...` | +| Manual review required | `error` | `Manual review required: ...` | +| Review incomplete | `error` | `Review incomplete: ...` | +| Pending | `pending` | Review is running or must be rerun | + +`Manual review required` is used for deterministic coverage limits such as: + +- oversized PRs; +- scoped review; +- truncated changed-file inventory. + +`Review incomplete` is used for retryable or technical failures such as: + +- reviewer workflow failure; +- missing receipt; +- malformed receipt; +- degraded full review. + +After finalization, the status workflow uploads the validated PR number and head +SHA as correlation artifacts. Universal Auto-Signoff is triggered by completion +of `ARM Semantic Review - Set Status`. + +### 9. Evaluate auto-signoff + +The normal auto-signoff decision runs on every Universal Auto-Signoff event: + +- semantic status completion; +- LintDiff completion; +- Avocado completion; +- PR update; +- relevant label addition or removal. + +It fetches all commit statuses for the current head SHA, then selects the newest +status for each required context: + +- `ARM Semantic Review` +- `Swagger LintDiff` +- `Swagger Avocado` + +Effective policy: + +```text +if semantic outcome is Manual review required: + add ARMManualSignoffRequired + do not sign off + +if semantic outcome is not Passed: + do not sign off + +if ARMChangesRequested is present: + do not sign off + +if ARMManualSignoffRequired is present: + do not sign off + +if LintDiff or Avocado is not successful: + do not sign off + +if required approval is missing: + do not sign off + +otherwise: + add ARMAutoSignedOff-Test +``` + +Only Passed is positive authorization. Missing, Pending, Failure, and Review +incomplete all fail closed. + +### 10. Apply labels safely + +Universal Auto-Signoff uploads label artifacts instead of directly mutating +labels. + +When the producer includes a trusted head-SHA artifact, Update Labels reads the +live PR again. It discards the label action if: + +- the PR is closed; or +- the PR head moved. + +This prevents a decision calculated for one commit from changing labels on a +later commit. + +## Event-order examples + +### Reviewer completes before Avocado + +```text +Reviewer starts -> Semantic = Pending +Reviewer completes -> Semantic Set Status publishes Passed +Auto-Signoff evaluates -> Avocado still pending, no signoff +Avocado completes -> Auto-Signoff rereads all current-SHA statuses +All pass -> Add pilot auto-signoff label +``` + +### Avocado completes before reviewer + +```text +Reviewer starts -> Semantic = Pending +Avocado completes -> Auto-Signoff sees Pending, no signoff +Reviewer completes -> Semantic Set Status publishes Passed +Auto-Signoff rereads Avocado and LintDiff +All pass -> Add pilot auto-signoff label +``` + +### New commit arrives during review + +```text +Reviewer is reviewing SHA A +PR moves to SHA B +Finalizer sees current head != reviewed SHA +SHA A result is ignored +SHA B requires its own semantic review +``` + +### Same SHA is retriggered + +```text +Old result for SHA A = Passed +New review starts for SHA A +New Pending entry becomes the newest semantic status +Auto-Signoff waits +New final result replaces Pending as the newest entry +``` + +### Oversized PR + +```text +Trusted code counts > 50 specification files or > 5,000 changed lines +Semantic result = Manual review required +Auto-Signoff does not proceed +ARMManualSignoffRequired is added +``` + +## Why this design was chosen + +### Independent workflows complete in any order + +The reviewer, LintDiff, and Avocado are independent workflows. A durable +SHA-bound status lets any completion event reevaluate the same current state. + +### Labels are not SHA-bound + +The absence of `ARMChangesRequested` does not prove that: + +- the reviewer ran; +- the reviewer finished; +- the review covered the whole PR; or +- the result belongs to the current commit. + +### Workflow conclusion is not the semantic result + +A reviewer workflow can finish successfully while: + +- finding Blocking issues; +- reviewing only part of a PR; +- producing incomplete semantic evidence. + +### Model output should not directly authorize signoff + +The model emits evidence. Trusted code verifies correlation and deterministic +coverage before publishing the durable status. + +### Finalization should occur after reviewer completion + +Publishing Passed inside the agent job can expose success before comment and +label publishing completes. `workflow_run: completed` provides a later +coordination boundary. + +## Alternative designs and tradeoffs + +| Design | Advantages | Tradeoffs | +| -------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------- | +| **Current: receipt, dedicated Set Status, then commit status** | SHA-bound, reusable by every trigger, fail-closed, least-privilege status writer, matches LintDiff and Avocado | Adds one workflow hop and another workflow to maintain | +| Finalize inside Universal Auto-Signoff | Fewer workflow files | Gives Universal `statuses: write` on every trigger and mixes publication with policy evaluation | +| One combined workflow | Strong ordering and direct outputs | Tightly couples AI review, linters, labels, retries, and every trigger | +| Label-only handshake | Simple and visible | Labels are mutable PR state, not SHA-bound; bot-label events can be unreliable | +| Artifact-only lookup | No commit status needed | Every event must search historical runs and artifacts; expiration and supersession are complex | +| Publish final status inside reviewer | Fewer workflow steps | Risks exposing Passed before comments and labels finish | +| GitHub Check Run | Rich UI and potentially stronger producer identity | More API and permission complexity; may require a dedicated GitHub App | +| Reusable `workflow_call` pipeline | Typed outputs and strong ordering | Difficult to retrofit around independent triggers and manual review commands | + +## Trust boundaries + +### Untrusted or partially trusted + +- PR content; +- specification files; +- PR description and comments; +- model reasoning; +- model-supplied semantic fields. + +### Trusted + +- workflow source from the base/default branch; +- pre-activation PR and SHA lookup; +- workflow artifacts uploaded by privileged/default-branch workflows; +- semantic receipt validation; +- trusted file-count and line-count evaluation; +- commit-status publication; +- final live-head check before correlated label application. + +## Risk assessment + +### Rollout blocker: `ARMManualSignoffRequired` does not exist + +At the time of writing, GitHub returns 404 for the repository label +`ARMManualSignoffRequired`, and it is not defined in checked-in label +configuration. + +Expected failure: + +```text +Semantic outcome = Manual review required +Universal uploads label artifact +Update Labels attempts to add nonexistent label +Label application fails +``` + +Semantic safety is preserved because the result is not Passed, but the intended +manual-review routing does not appear. + +Before rollout: + +1. Create `ARMManualSignoffRequired` in both spec repositories. +2. Decide whether it should be listed in + [protected-labels.yml](../.github/protected-labels.yml). +3. Document who may add and remove it. + +### High before production: commit-status provenance is not authenticated + +Auto-signoff trusts the newest status whose context is `ARM Semantic Review`. +It does not verify: + +- the status creator; +- the target workflow URL; +- the workflow name; +- that the run belongs to this repository; +- that the run has a valid semantic receipt. + +Any workflow or identity with `statuses: write` could publish a competing +success status. + +This is lower impact while the pilot controls only `ARMAutoSignedOff-Test`, but +it should be hardened before controlling production `ARMSignedOff`. + +Possible mitigations: + +- validate the status creator; +- require a target URL matching a trusted Actions run; +- resolve the run and verify its workflow name; +- publish through a dedicated GitHub App or Check Run identity. + +### High or medium: semantic truth still comes from model output + +Trusted code validates shape, correlation, and coverage. It does not +independently prove: + +- that the review was complete; +- that `blocking_count` matches the findings; +- that a Blocking finding was not omitted; +- that the receipt agrees with queued label changes. + +The live `ARMChangesRequested` check provides defense in depth, but inconsistent +model output plus failed label output could theoretically produce Passed. + +Possible mitigations: + +- derive Blocking count from structured finding outputs; +- cross-check the receipt against safe-output items; +- require a Passed receipt to agree with the queued label transition; +- reject Passed when safe-output processing reports failed items. + +### Medium: manual override removal is not protected + +The current protected-label workflow watches only `labeled`, not `unlabeled`. + +An unauthorized user with label permissions could remove a manually applied +`ARMManualSignoffRequired`. If semantic status is Passed, automation may then +proceed. + +Possible mitigations: + +- enforce authorization on `unlabeled`; +- record the hold in a trusted status or review record; +- require an authorized actor to clear the hold. + +### Medium: stale reviewer runs can mutate PR-level queue labels + +Semantic statuses are SHA-bound. Reviewer safe outputs for +`ARMChangesRequested` and `WaitForARMFeedback` are direct PR-level mutations. + +A narrow race remains: + +```text +Old run verifies head SHA +New commit arrives +Old run applies queue label mutation +``` + +The semantic gate still prevents unsafe auto-signoff, but the human queue can +become confusing. + +Possible mitigation: route reviewer queue-label changes through a correlated +artifact workflow with a final head-SHA check. + +### Medium, pending confirmation: partial safe-output failure + +The generated safe-output job exposes: + +- `items_failed`; +- `items_succeeded`; +- `process_safe_outputs_status`. + +The finalizer does not inspect those outputs. + +Confirm whether gh-aw can complete the workflow successfully when individual +comment or label items fail. If it can, reject Passed whenever safe-output +processing reports failed or canceled items. + +### Medium operational risk: private-repository parity + +The ARM reviewer source states that public and private repository copies must +remain byte-identical. + +Rollout must update `Azure/azure-rest-api-specs-pr` in coordination with this +repository. + +### Medium operational constraint: workflow-run chain depth + +The success path is: + +```text +ARM API Reviewer + -> ARM Semantic Review - Set Status + -> ARM Universal Auto-Signoff + -> Update Labels +``` + +This uses the same three-level `workflow_run` chain pattern as LintDiff and +Avocado and reaches GitHub's supported chaining limit. Do not insert another +downstream `workflow_run` workflow after Update Labels; it may not run. + +### Low or medium: fallback can block a newer head + +When trusted correlation artifacts are missing after a failed reviewer run, +correlation falls back to the PR number in the workflow title and fetches the +current head. + +An old failed run could therefore mark a newer head Review incomplete. + +This fails closed; it cannot incorrectly authorize signoff. It can cause a false +block and require another review. + +### Low: timestamp ordering + +The latest status is selected by `updated_at`. + +If two entries have identical timestamp resolution, ordering relies on GitHub's +return order and stable sorting. A status ID or run-attempt comparison would be +a stronger tiebreaker. + +### Low: canceled-run comment duplication + +Cancellation is not transactional. A canceled safe-output publisher may have +already posted some comments. The replacement run reconciles marked findings, +which reduces but cannot eliminate duplication. + +This is primarily a PR-noise risk, not an auto-signoff safety risk. + +### Low: manual-review label persists + +`ARMManualSignoffRequired` is intentionally never removed automatically. + +If an oversized PR is later reduced and receives a clean full review, a human +must clear the label before auto-signoff can proceed. + +This is safe, but should be documented for authors and reviewers. + +## How to learn this design from zero + +Do not start by reading the generated workflow or every test. Build the model in +layers. + +### Layer 1: learn the policy + +Write this decision rule on paper: + +```text +Semantic Passed +AND LintDiff Passed +AND Avocado Passed +AND required labels and approvals allow signoff += auto-signoff +``` + +Everything else is implementation detail supporting this rule. + +Questions to answer: + +- What is a positive authorization? +- Which states merely block? +- Which states require human routing? +- Which result must be tied to a commit? + +### Layer 2: learn the identities + +Track four IDs: + +```text +PR number +head SHA +workflow run ID +workflow attempt +``` + +Use this table: + +| Identity | What it distinguishes | +| ----------- | -------------------------------------- | +| PR number | The review conversation and labels | +| Head SHA | The exact code/specification snapshot | +| Run ID | One workflow execution | +| Run attempt | A rerun of the same workflow execution | + +Most race conditions are an accidental mismatch among these identities. + +### Layer 3: learn durable versus transient state + +Classify each signal: + +| Signal | Scope | +| ---------------- | ---------------------------- | +| Receipt artifact | One workflow run and attempt | +| Commit status | One commit SHA | +| Label | Entire PR until changed | +| Comment | Entire PR discussion | + +Then ask: + +> Is this state being used outside the scope where it is valid? + +That question explains why labels alone are insufficient and why old artifacts +must not authorize new commits. + +### Layer 4: trace only the happy path + +Read these files in order: + +1. [arm-api-review.md](../.github/workflows/arm-api-review.md), focusing only on: + - trigger and concurrency; + - Pending status; + - Step 7; + - Step 9. +2. [arm-semantic-review-status.yaml](../.github/workflows/arm-semantic-review-status.yaml) +3. [arm-semantic-review.ts](../.github/workflows/src/arm-auto-signoff/arm-semantic-review.ts) +4. [arm-semantic-review-workflow.ts](../.github/workflows/src/arm-auto-signoff/arm-semantic-review-workflow.ts) +5. [arm-universal-auto-signoff.yaml](../.github/workflows/arm-universal-auto-signoff.yaml) +6. [arm-universal-auto-signoff.ts](../.github/workflows/src/arm-auto-signoff/arm-universal-auto-signoff.ts) + +Ignore the generated lock file at first. + +While reading, follow one example: + +```text +Full review, no findings, LintDiff and Avocado pass +``` + +Write down every state transition. + +### Layer 5: add one failure at a time + +Use this order: + +1. Blocking finding. +2. Avocado finishes before reviewer. +3. New commit during review. +4. Same-SHA retrigger. +5. Oversized PR. +6. Critic unavailable. +7. Missing receipt. +8. Canceled run. +9. Stale label artifact. + +For each scenario, answer: + +```text +What is the current head SHA? +What is the latest semantic status for that SHA? +What labels are present? +Can auto-signoff proceed? +Who or what triggers reevaluation? +``` + +### Layer 6: read tests as executable examples + +After the state model is clear, read: + +- [arm-semantic-review.test.ts](../.github/workflows/test/arm-auto-signoff/arm-semantic-review.test.ts) +- [arm-universal-auto-signoff.test.ts](../.github/workflows/test/arm-auto-signoff/arm-universal-auto-signoff.test.ts) +- [context.test.ts](../.github/workflows/test/context.test.ts) +- [update-labels.test.ts](../.github/workflows/test/update-labels.test.ts) + +Read test names first. They provide a scenario inventory. + +### Layer 7: inspect the generated workflow last + +Use [arm-api-review.lock.yml](../.github/workflows/arm-api-review.lock.yml) only +to answer compiler-specific questions: + +- Which jobs depend on which jobs? +- Which permissions does each generated job receive? +- When does the safe-output publisher run? +- Which failures are `continue-on-error`? + +Do not use the generated YAML as the first source for understanding policy. + +## How to reason about tradeoffs with no prior knowledge + +### Step 1: state the non-negotiable invariants + +For this design: + +```text +Never auto-sign off without Semantic Passed for the current SHA. +Never reuse an older SHA's Passed result. +Never treat a partial review as a full review. +Never let unvalidated model output directly authorize signoff. +Human override must win over automation. +``` + +Tradeoffs may change implementation complexity. They must not violate these +invariants. + +### Step 2: list failure modes before solutions + +Examples: + +- workflows finish in a different order; +- a new commit arrives mid-review; +- a run is canceled; +- a label event does not trigger another workflow; +- an artifact is missing; +- a status is stale; +- the agent reports inconsistent fields; +- label application is delayed; +- the PR is too large to review fully. + +Design from failure cases, not only from the happy path. + +### Step 3: choose fail-open or fail-closed + +For signoff and security decisions, prefer fail-closed: + +```text +Missing semantic result -> no signoff +Malformed receipt -> no signoff +Stale SHA -> ignore result +Partial review -> manual review +``` + +For user experience, distinguish retryable from permanent/manual cases so +fail-closed does not automatically mean "send everything to a human." + +### Step 4: identify the authority for every field + +Use a table: + +| Field | Source | Can it be trusted? | Verification | +| ------------------- | -------------- | ------------------ | ----------------------------- | +| PR number | Event plus API | Mostly | Re-fetch PR | +| Head SHA | PR API | Yes in trusted job | Compare before publication | +| Scope | Agent | No by itself | Recompute trusted size limits | +| Blocking count | Agent | Partially | Cross-check recommended | +| Workflow conclusion | GitHub | Yes | Read `workflow_run` | +| Manual hold | Label | Mutable | Protect actor and removal | + +If you cannot name the authority, the design has a trust gap. + +### Step 5: separate evidence, decision, and effect + +```text +Evidence: reviewer receipt, statuses, labels, approvals +Decision: auto-signoff policy function +Effect: label mutation +``` + +Keeping these separate makes retry, auditing, and stale-result protection +easier. + +### Step 6: reason about time + +Draw a timeline: + +```text +T1 reviewer starts +T2 Avocado finishes +T3 new commit arrives +T4 reviewer finishes +T5 label updater runs +``` + +At every point ask: + +> Which SHA does this operation refer to? + +This catches most distributed-workflow races. + +### Step 7: evaluate alternatives on explicit axes + +Score each design from 1 to 5: + +| Axis | Question | +| ---------------- | ------------------------------------------------------------- | +| Correctness | Can stale or partial evidence authorize signoff? | +| Security | Can untrusted input or a broad token affect the decision? | +| Ordering | Does the design work regardless of workflow completion order? | +| Observability | Can a reviewer understand why signoff did or did not occur? | +| Recovery | Can a failed run be retried safely? | +| Complexity | How many workflows, data forms, and APIs must be understood? | +| Maintainability | Is policy centralized and tested? | +| Operational cost | What must be provisioned and kept in sync? | + +Do not choose "simplest" without specifying which axis is being simplified. + +### Step 8: document accepted residual risk + +A design does not need zero risk. It needs explicit risk ownership. + +Example: + +```text +Accepted for pilot: + occasional duplicate comment after cancellation + +Not accepted for production: + unauthenticated producer of a Passed status +``` + +### Step 9: use a decision worksheet + +For every unresolved choice, fill in: + +```text +Decision: +Options: +Invariant affected: +Failure mode: +Security boundary: +Operational impact: +Recommended option: +Why: +What would make us revisit it: +``` + +This lets you make defensible tradeoffs without already being an expert. + +## Suggested exercises + +### Exercise 1: trace a clean PR + +Create a table with: + +```text +Event +Current SHA +Semantic status +LintDiff +Avocado +Labels +Decision +``` + +Fill it from reviewer start through pilot signoff. + +### Exercise 2: break the ordering + +Repeat the table with Avocado finishing first. The final decision should be the +same. + +### Exercise 3: change the SHA + +Insert a new commit before reviewer completion. Verify every older result is +ignored. + +### Exercise 4: challenge each trust boundary + +Ask: + +- What if the model lies about scope? +- What if another workflow publishes `ARM Semantic Review = success`? +- What if a user removes the manual hold? +- What if a label artifact arrives late? + +If the answer is unclear, add a check or document the residual risk. + +## Review checklist + +Before enabling production signoff: + +- [ ] Create `ARMManualSignoffRequired` in both repositories. +- [ ] Decide and enforce who may add and remove the manual hold. +- [ ] Mirror the ARM reviewer workflow to `azure-rest-api-specs-pr`. +- [ ] Authenticate the semantic status producer. +- [ ] Confirm safe-output partial-failure semantics. +- [ ] Cross-check semantic receipt against actual published findings and labels. +- [ ] Test same-SHA reruns, new-SHA races, cancellation, and artifact loss in CI. +- [ ] Document how authors return a PR from manual review to automation. +- [ ] Replace the pilot label only after observing stable behavior. + +## Bottom line + +The design is intentionally fail-closed and is suitable for a pilot: + +- semantic evidence is tied to an exact SHA; +- only Passed allows auto-signoff; +- stale and superseded results are ignored; +- partial coverage routes to manual review; +- delayed correlated label changes recheck the live head. + +Before controlling production `ARMSignedOff`, address the missing manual-review +label, producer provenance, safe-output consistency, and private-repository +parity. From a4acd7d3608126c5980bc73d240711e64105d8bf Mon Sep 17 00:00:00 2001 From: Akhila Illa Date: Mon, 5 Oct 2026 10:24:03 -0500 Subject: [PATCH 03/16] Address semantic review feedback Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../arm-semantic-review-workflow.ts | 16 +++--- .github/workflows/src/context.ts | 28 +++------- .../arm-semantic-review.test.ts | 51 +++++++++++++++++++ .github/workflows/test/context.test.ts | 15 ++---- ...ntic-review-auto-signoff-implementation.md | 16 +++--- 5 files changed, 76 insertions(+), 50 deletions(-) diff --git a/.github/workflows/src/arm-auto-signoff/arm-semantic-review-workflow.ts b/.github/workflows/src/arm-auto-signoff/arm-semantic-review-workflow.ts index 326ceeacc8fb..2ee9adc23db9 100644 --- a/.github/workflows/src/arm-auto-signoff/arm-semantic-review-workflow.ts +++ b/.github/workflows/src/arm-auto-signoff/arm-semantic-review-workflow.ts @@ -1,7 +1,7 @@ import { readFile } from "node:fs/promises"; import { isFullGitSha } from "../../../shared/src/git.ts"; import { CommitStatusState, PER_PAGE_MAX } from "../../../shared/src/github.ts"; -import { ARM_API_REVIEW_WORKFLOW_NAME, extractInputs } from "../context.ts"; +import { ARM_API_REVIEW_WORKFLOW_NAME, parseWorkflowRunArtifactInputs } from "../context.ts"; import type { Core, GitHub, GitHubScriptArgs, WebhookEvent } from "../github.ts"; import { ARM_SEMANTIC_REVIEW_STATUS, @@ -321,12 +321,12 @@ export async function finalizeArmSemanticReview({ return emptyResult; } - const { owner, repo, head_sha, issue_number } = await extractInputs(github, context, core); + const { headSha, issueNumber } = parseWorkflowRunArtifactInputs(artifactNames, core); const statusPublished = await finalizeSemanticReviewWorkflow({ - owner, - repo, - issueNumber: issue_number, - headSha: head_sha, + owner: repositoryOwner, + repo: workflowRun.repository.name, + issueNumber, + headSha, runId: workflowRun.id, runAttempt: workflowRun.run_attempt, workflowConclusion: workflowRun.conclusion, @@ -337,8 +337,8 @@ export async function finalizeArmSemanticReview({ core, }); return { - headSha: isFullGitSha(head_sha) ? head_sha : "", - issueNumber: Number.isSafeInteger(issue_number) && issue_number > 0 ? issue_number : 0, + headSha: isFullGitSha(headSha) ? headSha : "", + issueNumber: Number.isSafeInteger(issueNumber) && issueNumber > 0 ? issueNumber : 0, statusPublished, }; } diff --git a/.github/workflows/src/context.ts b/.github/workflows/src/context.ts index 1031ef1bf636..7cfd21d52000 100644 --- a/.github/workflows/src/context.ts +++ b/.github/workflows/src/context.ts @@ -261,28 +261,6 @@ export async function extractInputs( ); } - if ( - payload.workflow_run.name === ARM_API_REVIEW_WORKFLOW_NAME && - (!Number.isSafeInteger(issue_number) || issue_number <= 0 || !isFullGitSha(head_sha)) - ) { - const match = /^ARM API Review #([1-9]\d*) \(/.exec(payload.workflow_run.display_title ?? ""); - const fallbackIssueNumber = match ? Number(match[1]) : NaN; - const resolvedIssueNumber = - Number.isSafeInteger(issue_number) && issue_number > 0 ? issue_number : fallbackIssueNumber; - if (Number.isSafeInteger(resolvedIssueNumber) && resolvedIssueNumber > 0) { - const { owner, repo } = getRepositoryInfo(payload.workflow_run.repository); - const { data: pullRequest } = await github.rest.pulls.get({ - owner, - repo, - pull_number: resolvedIssueNumber, - }); - issue_number = resolvedIssueNumber; - if (!isFullGitSha(head_sha)) { - head_sha = pullRequest.head.sha; - } - } - } - inputs = { ...getRepositoryInfo(payload.workflow_run.repository), head_sha, @@ -343,7 +321,13 @@ async function extractWorkflowRunArtifactInputs({ }); const artifactNames = artifacts.map((artifact) => artifact.name); core.info(`artifactNames: ${JSON.stringify(artifactNames)}`); + return parseWorkflowRunArtifactInputs(artifactNames, core); +} +export function parseWorkflowRunArtifactInputs( + artifactNames: string[], + core: Core, +): { headSha: string; issueNumber: number } { let headSha = ""; let issueNumber = NaN; for (const artifactName of artifactNames) { diff --git a/.github/workflows/test/arm-auto-signoff/arm-semantic-review.test.ts b/.github/workflows/test/arm-auto-signoff/arm-semantic-review.test.ts index e7cb35d72114..01ffd149f70f 100644 --- a/.github/workflows/test/arm-auto-signoff/arm-semantic-review.test.ts +++ b/.github/workflows/test/arm-auto-signoff/arm-semantic-review.test.ts @@ -12,9 +12,11 @@ import { type SemanticReviewResult, } from "../../src/arm-auto-signoff/arm-semantic-review.ts"; import { + finalizeArmSemanticReview, finalizeSemanticReviewWorkflow, validateSemanticReviewResult, } from "../../src/arm-auto-signoff/arm-semantic-review-workflow.ts"; +import type { GitHubScriptArgs } from "../../src/github.ts"; import { createMockCore, createMockGithub } from "../mocks.ts"; const owner = "Azure"; @@ -316,6 +318,55 @@ describe("finalizeSemanticReviewWorkflow", () => { ); }); + describe("finalizeArmSemanticReview", () => { + it("reuses the exact reviewer artifact listing for correlation and finalization", async () => { + const github = createFinalizeGithub(); + github.rest.actions.listWorkflowRunArtifacts.mockResolvedValue({ + data: { + artifacts: [ + { name: `head-sha=${headSha}` }, + { name: `issue-number=${issueNumber}` }, + { name: receiptName() }, + ], + }, + }); + const context = { + payload: { + workflow_run: { + name: "ARM API Review: Automated Workflow", + conclusion: "success", + id: runId, + run_attempt: runAttempt, + html_url: "https://github.com/Azure/azure-rest-api-specs/actions/runs/456", + repository: { + name: repo, + owner: { login: owner }, + }, + }, + }, + }; + + await expect( + finalizeArmSemanticReview({ + github, + context, + core: createMockCore(), + } as unknown as GitHubScriptArgs), + ).resolves.toEqual({ + headSha, + issueNumber, + statusPublished: true, + }); + expect(github.rest.actions.listWorkflowRunArtifacts).toHaveBeenCalledOnce(); + expect(github.rest.actions.listWorkflowRunArtifacts).toHaveBeenCalledWith({ + owner, + repo, + run_id: runId, + per_page: PER_PAGE_MAX, + }); + }); + }); + it("requires manual review when trusted size checks find limited coverage", async () => { const changedFiles = Array.from({ length: 51 }, (_, index) => ({ filename: `specification/test/resource-manager/file-${index}.json`, diff --git a/.github/workflows/test/context.test.ts b/.github/workflows/test/context.test.ts index e4b9cf7f88cd..73b32cb0e095 100644 --- a/.github/workflows/test/context.test.ts +++ b/.github/workflows/test/context.test.ts @@ -297,14 +297,11 @@ describe("extractInputs", () => { }); }); - it("recovers ARM API review correlation when pre-activation artifacts are missing", async () => { + it("does not substitute the current head when ARM API review artifacts are missing", async () => { const github = createMockGithub(); github.rest.actions.listWorkflowRunArtifacts.mockResolvedValue({ data: { artifacts: [] }, }); - github.rest.pulls.get.mockResolvedValue({ - data: { head: { sha: fullGitSha } }, - }); const context = { eventName: "workflow_run", payload: { @@ -329,15 +326,11 @@ describe("extractInputs", () => { await expect(extractInputs(github, context, createMockCore())).resolves.toEqual({ owner: "TestRepoOwnerLogin", repo: "TestRepoName", - head_sha: fullGitSha, - issue_number: 123, + head_sha: "", + issue_number: NaN, run_id: 456, }); - expect(github.rest.pulls.get).toHaveBeenCalledWith({ - owner: "TestRepoOwnerLogin", - repo: "TestRepoName", - pull_number: 123, - }); + expect(github.rest.pulls.get).not.toHaveBeenCalled(); }); it("rejects conflicting workflow correlation artifacts", async () => { diff --git a/documentation/arm-semantic-review-auto-signoff-implementation.md b/documentation/arm-semantic-review-auto-signoff-implementation.md index 895823476875..fd651bb23b8c 100644 --- a/documentation/arm-semantic-review-auto-signoff-implementation.md +++ b/documentation/arm-semantic-review-auto-signoff-implementation.md @@ -341,6 +341,8 @@ reviewer run. It calls `finalizeArmSemanticReview`. It verifies: +- the reviewer run's artifacts are listed once and reused for both correlation + and receipt finalization; - the reviewer actually executed; - the PR number and head SHA are valid; - the PR is still open; @@ -701,16 +703,12 @@ This uses the same three-level `workflow_run` chain pattern as LintDiff and Avocado and reaches GitHub's supported chaining limit. Do not insert another downstream `workflow_run` workflow after Update Labels; it may not run. -### Low or medium: fallback can block a newer head +### Missing correlation artifacts -When trusted correlation artifacts are missing after a failed reviewer run, -correlation falls back to the PR number in the workflow title and fetches the -current head. - -An old failed run could therefore mark a newer head Review incomplete. - -This fails closed; it cannot incorrectly authorize signoff. It can cause a false -block and require another review. +If the reviewer run does not contain trusted PR and head-SHA artifacts, the +status workflow leaves semantic status unchanged. It does not substitute the +PR's current head because that could attach an older run's result to a newer +commit. ### Low: timestamp ordering From 1e42a2da043460949f6e7ec683d1bd951da75724 Mon Sep 17 00:00:00 2001 From: Akhila Illa Date: Tue, 6 Oct 2026 10:42:31 -0500 Subject: [PATCH 04/16] implmentation cleanup, optimizations --- .github/workflows/arm-api-review.lock.yml | 28 +- .github/workflows/arm-api-review.md | 37 +- .../workflows/arm-semantic-review-status.yaml | 32 +- .../arm-semantic-review-status.ts | 184 +++ .../arm-semantic-review-workflow.ts | 304 +---- .../arm-auto-signoff/arm-semantic-review.ts | 204 ++- .../arm-universal-auto-signoff.ts | 2 +- .../test/arm-api-review-workflow.test.ts | 28 +- ...rm-semantic-review-status-workflow.test.ts | 42 + .../arm-semantic-review.test.ts | 753 +++++------ .../arm-universal-auto-signoff.test.ts | 32 +- ...ntic-review-auto-signoff-implementation.md | 1195 ++++------------- 12 files changed, 1093 insertions(+), 1748 deletions(-) create mode 100644 .github/workflows/src/arm-auto-signoff/arm-semantic-review-status.ts diff --git a/.github/workflows/arm-api-review.lock.yml b/.github/workflows/arm-api-review.lock.yml index f41c5e0f7e72..b26a305a4bd7 100644 --- a/.github/workflows/arm-api-review.lock.yml +++ b/.github/workflows/arm-api-review.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"9a949b6ca0721e43b0e2820f862298b806ab68bdeb2816804f272957d6ac3d64","body_hash":"7dabe4a0f4780c2250806fb5c092b7277760ceeb55d3840824c9a17615d8da9d","compiler_version":"v0.86.2","strict":true,"agent_id":"copilot","agent_model":"gpt-5.6-sol?effort=high","detection_agent_id":"copilot","detection_agent_model":"gpt-5.6-sol?effort=high","engine_versions":{"copilot":"1.0.79"}} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"d0e25907942ce2a5d5fb164345a82abe46a78f3b12af3d1335b5283ab10064d4","body_hash":"b20da3b07a7c27740234fc9915ee5c9bde85495698024bbb1cc6c015a4bf0a47","compiler_version":"v0.86.2","strict":true,"agent_id":"copilot","agent_model":"gpt-5.6-sol?effort=high","detection_agent_id":"copilot","detection_agent_model":"gpt-5.6-sol?effort=high","engine_versions":{"copilot":"1.0.79"}} # gh-aw-manifest: {"version":1,"secrets":["GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/cache/restore","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/cache/save","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"373c709c69115d41ff229c7e5df9f8788daa9553","version":"v9"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/github-script","sha":"d746ffe35508b1917358783b479e04febd2b8f71","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"820762786026740c76f36085b0efc47a31fe5020","version":"v7.0.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"},{"repo":"github/gh-aw-actions/setup","sha":"6aab9e5b5c91c615506061f09bedd81a23babe3c","version":"v0.86.2"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.27.44","digest":"sha256:0d727725c737b58c7bdf51f640cffb928385ec46517e0917c7f1a02f1bada8b4","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.27.44@sha256:0d727725c737b58c7bdf51f640cffb928385ec46517e0917c7f1a02f1bada8b4"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.44","digest":"sha256:b50fbadba138f6e9aba94aca09711335c489bb3b15861220cb66f6092e042dc7","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.44@sha256:b50fbadba138f6e9aba94aca09711335c489bb3b15861220cb66f6092e042dc7"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.27.44","digest":"sha256:83e48bbe12c634be8c228a576832fe45f66c529ac3659db92bddbcf2eeb6d627","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.27.44@sha256:83e48bbe12c634be8c228a576832fe45f66c529ac3659db92bddbcf2eeb6d627"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.9","digest":"sha256:e5a1569aeaf41820fa7bdee3e94468cae448133cdbf00119ad24f5b74db1ab9f","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.9@sha256:e5a1569aeaf41820fa7bdee3e94468cae448133cdbf00119ad24f5b74db1ab9f"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:0d9f1fb5fd6610c0ac1f5194a38e45a8a1e81f8a390d5142d8e4e6f26a4b3196","pinned_image":"ghcr.io/github/gh-aw-node@sha256:0d9f1fb5fd6610c0ac1f5194a38e45a8a1e81f8a390d5142d8e4e6f26a4b3196"},{"image":"ghcr.io/github/github-mcp-server:v1.9.0","digest":"sha256:881b53d6f75f69bdbc1b5b10fc2f1361717c19054143b3a8529fb5c32061a50e","pinned_image":"ghcr.io/github/github-mcp-server:v1.9.0@sha256:881b53d6f75f69bdbc1b5b10fc2f1361717c19054143b3a8529fb5c32061a50e"}],"has_pull_request_target":true} # This file was automatically generated by gh-aw (v0.86.2). DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # @@ -78,6 +78,7 @@ on: types: - opened - synchronize + - reopened - labeled - ready_for_review # roles: # Roles processed as role check in pre-activation job @@ -91,7 +92,7 @@ on: # name: Resolve target pull request # uses: actions/github-script@v9 # with: - # script: "const value = process.env.TARGET_PR_NUMBER ?? \"\";\nif (!/^[1-9]\\d*$/.test(value)) {\n throw new Error(`Invalid or missing pull request number: ${JSON.stringify(value)}`);\n}\n\nif (context.eventName === \"issue_comment\" && !context.payload.issue?.pull_request) {\n throw new Error(`Issue #${value} is not a pull request`);\n}\n\nconst pullNumber = Number(value);\nif (!Number.isSafeInteger(pullNumber)) {\n throw new Error(`Pull request number is outside the safe integer range: ${value}`);\n}\n\nconst { data: pull } = await github.rest.pulls.get({\n ...context.repo,\n pull_number: pullNumber,\n});\nif (!/^[0-9a-f]{40}$/i.test(pull.head.sha)) {\n throw new Error(`Invalid pull request head SHA: ${JSON.stringify(pull.head.sha)}`);\n}\ncore.setOutput(\"target_pr_number\", String(pull.number));\ncore.setOutput(\"target_head_sha\", pull.head.sha);\n" + # script: "const value = process.env.TARGET_PR_NUMBER ?? \"\";\nif (!/^[1-9]\\d*$/.test(value)) {\n throw new Error(`Invalid or missing pull request number: ${JSON.stringify(value)}`);\n}\n\nif (context.eventName === \"issue_comment\" && !context.payload.issue?.pull_request) {\n throw new Error(`Issue #${value} is not a pull request`);\n}\n\nconst pullNumber = Number(value);\nif (!Number.isSafeInteger(pullNumber)) {\n throw new Error(`Pull request number is outside the safe integer range: ${value}`);\n}\n\nconst { data: pull } = await github.rest.pulls.get({\n ...context.repo,\n pull_number: pullNumber,\n});\nif (!/^[0-9a-f]{40}$/i.test(pull.head.sha)) {\n throw new Error(`Invalid pull request head SHA: ${JSON.stringify(pull.head.sha)}`);\n}\nconst runAttempt = process.env.GITHUB_RUN_ATTEMPT ?? \"\";\nif (!/^[1-9]\\d*$/.test(runAttempt)) {\n throw new Error(`Invalid workflow run attempt: ${JSON.stringify(runAttempt)}`);\n}\ncore.setOutput(\"target_pr_number\", String(pull.number));\ncore.setOutput(\"target_head_sha\", pull.head.sha);\ncore.setOutput(\"run_attempt\", runAttempt);\n" # - if: steps.check_membership.outputs.is_team_member == 'true' # name: Prepare ARM semantic review correlation artifact # run: "mkdir -p \"$RUNNER_TEMP/arm-semantic-review\"\n: > \"$RUNNER_TEMP/arm-semantic-review/empty.txt\"\n" @@ -146,6 +147,7 @@ jobs: needs.pre_activation.outputs.activated == 'true' && (github.event_name == 'workflow_dispatch' || (github.event_name == 'pull_request_target' && (github.event.action == 'opened' || github.event.action == 'synchronize' || + github.event.action == 'reopened' || github.event.action == 'ready_for_review') && github.event.pull_request.draft == false && contains(github.event.pull_request.labels.*.name, 'WaitForARMFeedback')) || @@ -362,6 +364,7 @@ jobs: GH_AW_GITHUB_REPOSITORY: ${{ github.repository }} GH_AW_GITHUB_RUN_ID: ${{ github.run_id }} GH_AW_GITHUB_WORKSPACE: ${{ github.workspace }} + GH_AW_NEEDS_PRE_ACTIVATION_OUTPUTS_RUN_ATTEMPT: ${{ needs.pre_activation.outputs.run_attempt }} GH_AW_NEEDS_PRE_ACTIVATION_OUTPUTS_TARGET_HEAD_SHA: ${{ needs.pre_activation.outputs.target_head_sha }} GH_AW_NEEDS_PRE_ACTIVATION_OUTPUTS_TARGET_PR_NUMBER: ${{ needs.pre_activation.outputs.target_pr_number }} GH_AW_PROMPT_CONTENT_0000: "\n" @@ -390,6 +393,7 @@ jobs: GH_AW_ENGINE_ID: "copilot" GH_AW_GITHUB_EVENT_NAME: ${{ github.event_name }} GH_AW_GITHUB_REPOSITORY: ${{ github.repository }} + GH_AW_NEEDS_PRE_ACTIVATION_OUTPUTS_RUN_ATTEMPT: ${{ needs.pre_activation.outputs.run_attempt }} GH_AW_NEEDS_PRE_ACTIVATION_OUTPUTS_TARGET_HEAD_SHA: ${{ needs.pre_activation.outputs.target_head_sha }} GH_AW_NEEDS_PRE_ACTIVATION_OUTPUTS_TARGET_PR_NUMBER: ${{ needs.pre_activation.outputs.target_pr_number }} with: @@ -413,6 +417,7 @@ jobs: GH_AW_GITHUB_WORKSPACE: ${{ github.workspace }} GH_AW_MCP_CLI_SERVERS_LIST: "- `github` β€” run `github --help` to see available tools\n- `safeoutputs` β€” run `safeoutputs --help` to see available tools" GH_AW_NEEDS_PRE_ACTIVATION_OUTPUTS_ACTIVATED: ${{ needs.pre_activation.outputs.activated }} + GH_AW_NEEDS_PRE_ACTIVATION_OUTPUTS_RUN_ATTEMPT: ${{ needs.pre_activation.outputs.run_attempt }} GH_AW_NEEDS_PRE_ACTIVATION_OUTPUTS_TARGET_HEAD_SHA: ${{ needs.pre_activation.outputs.target_head_sha }} GH_AW_NEEDS_PRE_ACTIVATION_OUTPUTS_TARGET_PR_NUMBER: ${{ needs.pre_activation.outputs.target_pr_number }} with: @@ -437,6 +442,7 @@ jobs: GH_AW_GITHUB_WORKSPACE: process.env.GH_AW_GITHUB_WORKSPACE, GH_AW_MCP_CLI_SERVERS_LIST: process.env.GH_AW_MCP_CLI_SERVERS_LIST, GH_AW_NEEDS_PRE_ACTIVATION_OUTPUTS_ACTIVATED: process.env.GH_AW_NEEDS_PRE_ACTIVATION_OUTPUTS_ACTIVATED, + GH_AW_NEEDS_PRE_ACTIVATION_OUTPUTS_RUN_ATTEMPT: process.env.GH_AW_NEEDS_PRE_ACTIVATION_OUTPUTS_RUN_ATTEMPT, GH_AW_NEEDS_PRE_ACTIVATION_OUTPUTS_TARGET_HEAD_SHA: process.env.GH_AW_NEEDS_PRE_ACTIVATION_OUTPUTS_TARGET_HEAD_SHA, GH_AW_NEEDS_PRE_ACTIVATION_OUTPUTS_TARGET_PR_NUMBER: process.env.GH_AW_NEEDS_PRE_ACTIVATION_OUTPUTS_TARGET_PR_NUMBER } @@ -589,9 +595,9 @@ jobs: mkdir -p "${RUNNER_TEMP}/gh-aw/safeoutputs" mkdir -p /tmp/gh-aw/safeoutputs mkdir -p /tmp/gh-aw/mcp-logs/safeoutputs - cat > "${RUNNER_TEMP}/gh-aw/safeoutputs/config.json" << 'GH_AW_SAFE_OUTPUTS_CONFIG_114ad090dfc07ef3_EOF' - {"add_comment":{"max":4,"target":"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}"},"add_labels":{"max":3,"target":"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}"},"create_pull_request_review_comment":{"max":50,"side":"RIGHT","target":"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}"},"create_report_incomplete_issue":{},"missing_data":{},"missing_tool":{},"noop":{"max":1,"report-as-issue":"true"},"record-arm-semantic-review":{"description":"Validate and record the ARM semantic review result for auto-signoff","inputs":{"blocking_count":{"default":null,"description":"Number of verified, currently applicable Blocking findings after reconciliation","required":true,"type":"string"},"completeness":{"default":null,"description":"Whether the intended review completed reliably","options":["complete","incomplete","degraded"],"required":true,"type":"choice"},"head_sha":{"default":null,"description":"Full pull request head SHA that was reviewed","required":true,"type":"string"},"issue_number":{"default":null,"description":"Target pull request number","required":true,"type":"string"},"scope":{"default":null,"description":"Whether the full PR or only a scoped subset was reviewed","options":["full","scoped"],"required":true,"type":"choice"}},"output":"ARM semantic review result recorded"},"remove_labels":{"max":3,"target":"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}"},"reply_to_pull_request_review_comment":{"max":50,"target":"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}"},"report_incomplete":{},"resolve_pull_request_review_thread":{"max":50},"submit_pull_request_review":{"footer":"if-body","max":1,"target":"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}"}} - GH_AW_SAFE_OUTPUTS_CONFIG_114ad090dfc07ef3_EOF + cat > "${RUNNER_TEMP}/gh-aw/safeoutputs/config.json" << 'GH_AW_SAFE_OUTPUTS_CONFIG_511927e73593d4c5_EOF' + {"add_comment":{"max":4,"target":"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}"},"add_labels":{"max":3,"target":"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}"},"create_pull_request_review_comment":{"max":50,"side":"RIGHT","target":"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}"},"create_report_incomplete_issue":{},"missing_data":{},"missing_tool":{},"noop":{"max":1,"report-as-issue":"true"},"record-arm-semantic-review":{"description":"Validate and record the ARM semantic review result for auto-signoff","inputs":{"blocking_count":{"default":null,"description":"Number of verified, currently applicable Blocking findings after reconciliation","required":true,"type":"string"},"completeness":{"default":null,"description":"Whether the intended review completed reliably","options":["complete","incomplete","degraded"],"required":true,"type":"choice"},"head_sha":{"default":null,"description":"Full pull request head SHA that was reviewed","required":true,"type":"string"},"issue_number":{"default":null,"description":"Target pull request number","required":true,"type":"string"},"run_attempt":{"default":null,"description":"GitHub Actions attempt that produced this result","required":true,"type":"string"},"scope":{"default":null,"description":"Whether the full PR or only a scoped subset was reviewed","options":["full","scoped"],"required":true,"type":"choice"}},"output":"ARM semantic review result recorded"},"remove_labels":{"max":3,"target":"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}"},"reply_to_pull_request_review_comment":{"max":50,"target":"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}"},"report_incomplete":{},"resolve_pull_request_review_thread":{"max":50},"submit_pull_request_review":{"footer":"if-body","max":1,"target":"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}"}} + GH_AW_SAFE_OUTPUTS_CONFIG_511927e73593d4c5_EOF - name: Generate Safe Outputs Tools env: GH_AW_TOOLS_META_JSON: | @@ -633,6 +639,10 @@ jobs: "description": "Target pull request number", "type": "string" }, + "run_attempt": { + "description": "GitHub Actions attempt that produced this result", + "type": "string" + }, "scope": { "description": "Whether the full PR or only a scoped subset was reviewed", "enum": [ @@ -647,6 +657,7 @@ jobs: "completeness", "head_sha", "issue_number", + "run_attempt", "scope" ], "type": "object" @@ -1798,6 +1809,7 @@ jobs: github.event_name == 'workflow_dispatch' || (github.event_name == 'pull_request_target' && (github.event.action == 'opened' || github.event.action == 'synchronize' || + github.event.action == 'reopened' || github.event.action == 'ready_for_review') && github.event.pull_request.draft == false && contains(github.event.pull_request.labels.*.name, 'WaitForARMFeedback')) || @@ -1819,6 +1831,7 @@ jobs: activated: ${{ steps.check_membership.outputs.is_team_member == 'true' }} matched_command: '' resolve_target_pr_result: ${{ steps.resolve_target_pr.outcome }} + run_attempt: ${{ steps.resolve_target_pr.outputs.run_attempt }} setup-parent-span-id: ${{ steps.setup.outputs.parent-span-id || steps.setup.outputs.span-id }} setup-span-id: ${{ steps.setup.outputs.span-id }} setup-trace-id: ${{ steps.setup.outputs.trace-id }} @@ -1878,8 +1891,13 @@ jobs: if (!/^[0-9a-f]{40}$/i.test(pull.head.sha)) { throw new Error(`Invalid pull request head SHA: ${JSON.stringify(pull.head.sha)}`); } + const runAttempt = process.env.GITHUB_RUN_ATTEMPT ?? ""; + if (!/^[1-9]\d*$/.test(runAttempt)) { + throw new Error(`Invalid workflow run attempt: ${JSON.stringify(runAttempt)}`); + } core.setOutput("target_pr_number", String(pull.number)); core.setOutput("target_head_sha", pull.head.sha); + core.setOutput("run_attempt", runAttempt); - name: Prepare ARM semantic review correlation artifact if: steps.check_membership.outputs.is_team_member == 'true' run: | diff --git a/.github/workflows/arm-api-review.md b/.github/workflows/arm-api-review.md index 754fa41e758f..1dbe9a452b5b 100644 --- a/.github/workflows/arm-api-review.md +++ b/.github/workflows/arm-api-review.md @@ -22,7 +22,7 @@ on: # fork PRs are reviewed only after a maintainer applies the label or runs # `/arm-review`. pull_request_target: - types: [opened, synchronize, labeled, ready_for_review] + types: [opened, synchronize, reopened, labeled, ready_for_review] forks: ["*"] issue_comment: types: [created] @@ -64,8 +64,13 @@ on: if (!/^[0-9a-f]{40}$/i.test(pull.head.sha)) { throw new Error(`Invalid pull request head SHA: ${JSON.stringify(pull.head.sha)}`); } + const runAttempt = process.env.GITHUB_RUN_ATTEMPT ?? ""; + if (!/^[1-9]\d*$/.test(runAttempt)) { + throw new Error(`Invalid workflow run attempt: ${JSON.stringify(runAttempt)}`); + } core.setOutput("target_pr_number", String(pull.number)); core.setOutput("target_head_sha", pull.head.sha); + core.setOutput("run_attempt", runAttempt); - name: Prepare ARM semantic review correlation artifact if: steps.check_membership.outputs.is_team_member == 'true' shell: bash @@ -128,6 +133,7 @@ on: jobs: pre-activation: outputs: + run_attempt: ${{ steps.resolve_target_pr.outputs.run_attempt }} target_head_sha: ${{ steps.resolve_target_pr.outputs.target_head_sha }} target_pr_number: ${{ steps.resolve_target_pr.outputs.target_pr_number }} # Gate at the trigger level so the expensive agent job never starts for @@ -140,6 +146,7 @@ if: > (github.event_name == 'pull_request_target' && (github.event.action == 'opened' || github.event.action == 'synchronize' || + github.event.action == 'reopened' || github.event.action == 'ready_for_review') && github.event.pull_request.draft == false && contains(github.event.pull_request.labels.*.name, 'WaitForARMFeedback')) || @@ -291,6 +298,10 @@ safe-outputs: description: "Number of verified, currently applicable Blocking findings after reconciliation" required: true type: string + run_attempt: + description: "GitHub Actions attempt that produced this result" + required: true + type: string steps: - uses: actions/checkout@v7 with: @@ -339,6 +350,7 @@ reconciliation throughout. - Repository: `${{ github.repository }}` - Trigger event: `${{ github.event_name }}` +- **Authoritative workflow attempt:** `${{ needs.pre_activation.outputs.run_attempt }}` - **Authoritative target pull request:** `#${{ needs.pre_activation.outputs.target_pr_number }}` - **Authoritative target head SHA:** `${{ needs.pre_activation.outputs.target_head_sha }}` @@ -469,8 +481,8 @@ already guaranteed, before this agent starts, that: or is the explicitly allowlisted `github-actions[bot]` repository automation. This replaces any manual collaborator check. Do **not** re-verify permissions. -- The event is eligible: an automated `opened` / `synchronize` run only reaches - the agent when the PR is not a draft and already carries the +- The event is eligible: an automated `opened` / `synchronize` / `reopened` run + only reaches the agent when the PR is not a draft and already carries the `WaitForARMFeedback` label; `ready_for_review` follows the same label gate; a `labeled` run only fires when that exact label is added to a non-draft PR; an `issue_comment` run only fires for a PR comment whose body is exactly @@ -577,9 +589,9 @@ Track two values for the final semantic result: Before any `report_incomplete` stop after the target PR and head SHA are known, also call `record_arm_semantic_review` with that PR, head SHA, the current -scope, `completeness: incomplete`, and the number of verified Blocking findings -known to remain applicable at that point. Use `"0"` if the failure occurs before -finding reconciliation. +scope, `completeness: incomplete`, the number of verified Blocking findings +known to remain applicable at that point, and the workflow attempt. Use `"0"` +if the failure occurs before finding reconciliation. ## Review Workflow @@ -1260,17 +1272,18 @@ Call `record_arm_semantic_review` exactly once after queuing the summary: - `issue_number`: the authoritative target pull request number; - `head_sha`: the pinned full session SHA; - `scope`: `full` or `scoped`, as recorded during Trigger Validation; -- `completeness`: `complete`, `incomplete`, or `degraded`; and +- `completeness`: `complete`, `incomplete`, or `degraded`; - `blocking_count`: the number of verified Blocking findings that remain applicable after reconciliation. Include an unresolved finding classified as `SKIP-COVERED`; do not count fixed, resolved, Critic-dropped, or - overflow-only candidates. + overflow-only candidates; and +- `run_attempt`: the authoritative workflow attempt above. Use `blocking_count: "0"` after a clean re-review. Do not claim `full` or -`complete` when any required evidence was unavailable. The trusted publisher -validates the PR and head SHA and records an attempt-specific receipt. Universal -Auto-Signoff publishes the final `ARM Semantic Review` status only after the -entire reviewer workflow completes. +`complete` when any required evidence was unavailable. `ARM Semantic Review - +Set Status` consumes the exact completed run's agent output, validates its +correlation and shape, and publishes the final status only after the entire +reviewer workflow completes. The final status is: diff --git a/.github/workflows/arm-semantic-review-status.yaml b/.github/workflows/arm-semantic-review-status.yaml index bf93ab041605..16abb9583519 100644 --- a/.github/workflows/arm-semantic-review-status.yaml +++ b/.github/workflows/arm-semantic-review-status.yaml @@ -1,10 +1,17 @@ +# Converts one completed ARM API Reviewer run into the durable, head-bound +# `ARM Semantic Review` commit status consumed by Universal Auto-Signoff. This +# workflow validates existing reviewer evidence; it does not run another AI review. name: ARM Semantic Review - Set Status on: workflow_run: + # workflow_run supplies the exact reviewer run ID, so finalization can read + # that run's artifacts directly instead of searching reviewer-run history. workflows: ["ARM API Review: Automated Workflow"] types: [completed] +# Keep status publication isolated here. Universal Auto-Signoff needs only +# `statuses: read` when it evaluates the finalized result. permissions: # actions.listWorkflowRunArtifacts actions: read @@ -21,20 +28,41 @@ jobs: runs-on: ubuntu-slim steps: + # workflow_run executes trusted default-branch automation. Check out only + # .github because finalization never needs files from the untrusted PR head. - uses: actions/checkout@v7 with: sparse-checkout: | .github + # Download the model's structured safe outputs from the exact completed + # reviewer run; never search for a "latest" reviewer run. + - name: Download ARM API Reviewer output + continue-on-error: true + uses: actions/download-artifact@v8.0.1 + with: + name: "agent" + path: "${{ runner.temp }}/arm-semantic-review" + github-token: "${{ github.token }}" + repository: "${{ github.repository }}" + run-id: "${{ github.event.workflow_run.id }}" + + # Read the same reviewer run's trusted correlation artifacts, validate its + # structured semantic item, reject stale or superseded results, independently + # verify coverage limits, and publish the final status for the reviewed SHA. - id: finalize name: Finalize ARM Semantic Review uses: actions/github-script@v9.0.0 + env: + GH_AW_AGENT_OUTPUT: "${{ runner.temp }}/arm-semantic-review/agent_output.json" with: script: | const { finalizeArmSemanticReview } = - await import('${{ github.workspace }}/.github/workflows/src/arm-auto-signoff/arm-semantic-review-workflow.ts'); + await import('${{ github.workspace }}/.github/workflows/src/arm-auto-signoff/arm-semantic-review-status.ts'); return await finalizeArmSemanticReview({ github, context, core }); + # Universal Auto-Signoff is triggered by this workflow's completion. These + # artifacts let it identify the exact PR and SHA without searching prior runs. - if: fromJson(steps.finalize.outputs.result).headSha name: Upload artifact with head SHA uses: ./.github/actions/add-empty-artifact @@ -42,6 +70,8 @@ jobs: name: "head-sha" value: "${{ fromJson(steps.finalize.outputs.result).headSha }}" + # Keep PR correlation separate from the SHA artifact to match the existing + # Set Status workflow contract used by LintDiff and Avocado. - if: fromJson(steps.finalize.outputs.result).issueNumber > 0 name: Upload artifact with issue number uses: ./.github/actions/add-empty-artifact diff --git a/.github/workflows/src/arm-auto-signoff/arm-semantic-review-status.ts b/.github/workflows/src/arm-auto-signoff/arm-semantic-review-status.ts new file mode 100644 index 000000000000..27891ae5b7a3 --- /dev/null +++ b/.github/workflows/src/arm-auto-signoff/arm-semantic-review-status.ts @@ -0,0 +1,184 @@ +import { readFile } from "node:fs/promises"; +import { isFullGitSha } from "../../../shared/src/git.ts"; +import { CommitStatusState, PER_PAGE_MAX } from "../../../shared/src/github.ts"; +import { ARM_API_REVIEW_WORKFLOW_NAME, parseWorkflowRunArtifactInputs } from "../context.ts"; +import type { GitHubScriptArgs, WebhookEvent } from "../github.ts"; +import { + ARM_SEMANTIC_REVIEW_STATUS, + evaluateAutomatedReviewCoverage, + evaluateSemanticReview, + getLatestSemanticReviewStatus, + parseSemanticReviewResult, + type ChangedFile, + type CommitStatus, + type EvaluatedSemanticReview, +} from "./arm-semantic-review.ts"; + +const MAX_STATUS_DESCRIPTION = 140; + +const EMPTY_RESULT = { + headSha: "", + issueNumber: 0, + statusPublished: false, +}; + +async function publishStatus( + { github, core }: Pick, + owner: string, + repo: string, + headSha: string, + targetUrl: string, + status: EvaluatedSemanticReview, +): Promise { + core.info(`${ARM_SEMANTIC_REVIEW_STATUS}: ${status.state} - ${status.description}`); + await github.rest.repos.createCommitStatus({ + owner, + repo, + sha: headSha, + state: status.state, + context: ARM_SEMANTIC_REVIEW_STATUS, + description: status.description, + target_url: targetUrl, + }); +} + +function isSuperseded( + latestStatus: CommitStatus | undefined, + runId: number, + runAttempt: number, +): boolean { + const match = /\/actions\/runs\/([1-9]\d*)\/attempts\/([1-9]\d*)(?:\/|$)/.exec( + latestStatus?.target_url ?? "", + ); + if (!match) { + return false; + } + const latestRunId = Number(match[1]); + const latestAttempt = Number(match[2]); + return latestRunId > runId || (latestRunId === runId && latestAttempt > runAttempt); +} + +/** + * Converts the exact completed ARM API Reviewer run into one head-bound commit status. + * Missing or malformed evidence publishes `error`; stale SHAs are harmless because + * Universal Auto-Signoff reads statuses only for the PR's current SHA. + */ +export async function finalizeArmSemanticReview({ + github, + context, + core, +}: GitHubScriptArgs): Promise<{ + headSha: string; + issueNumber: number; + statusPublished: boolean; +}> { + const workflowRun = (context.payload as WebhookEvent<"workflow-run", "completed">).workflow_run; + if (workflowRun.name !== ARM_API_REVIEW_WORKFLOW_NAME) { + return EMPTY_RESULT; + } + + const owner = workflowRun.repository.owner?.login; + if (!owner) { + throw new Error("ARM API review workflow run is missing its repository owner"); + } + const repo = workflowRun.repository.name; + const artifactNames = ( + await github.paginate(github.rest.actions.listWorkflowRunArtifacts, { + owner, + repo, + run_id: workflowRun.id, + per_page: PER_PAGE_MAX, + }) + ).map((artifact) => artifact.name); + const { headSha, issueNumber } = parseWorkflowRunArtifactInputs(artifactNames, core); + if (!isFullGitSha(headSha) || !Number.isSafeInteger(issueNumber) || issueNumber <= 0) { + core.info("The reviewer run has no trusted PR/SHA correlation; status is unchanged"); + return EMPTY_RESULT; + } + + if (workflowRun.conclusion === "cancelled") { + core.info("Canceled reviewer run leaves its Pending status unchanged"); + return { headSha, issueNumber, statusPublished: false }; + } + + const statuses: CommitStatus[] = await github.paginate( + github.rest.repos.listCommitStatusesForRef, + { + owner, + repo, + ref: headSha, + per_page: PER_PAGE_MAX, + }, + ); + if ( + isSuperseded(getLatestSemanticReviewStatus(statuses), workflowRun.id, workflowRun.run_attempt) + ) { + core.info("A newer ARM API review run superseded this completion"); + return { headSha, issueNumber, statusPublished: false }; + } + + let status: EvaluatedSemanticReview; + try { + if (workflowRun.conclusion !== "success") { + throw new Error(`workflow concluded with ${workflowRun.conclusion ?? "an unknown result"}`); + } + const outputPath = process.env.GH_AW_AGENT_OUTPUT; + if (!outputPath) { + throw new Error("agent output is unavailable"); + } + const agentOutput = JSON.parse(await readFile(outputPath, "utf8")) as unknown; + const result = parseSemanticReviewResult(agentOutput, issueNumber, workflowRun.run_attempt); + if (result.headSha.toLowerCase() !== headSha.toLowerCase()) { + throw new Error("semantic result SHA does not match reviewer correlation"); + } + + // Independently verify that the PR is within automated-review coverage limits. + // The trusted file list from pulls.listFiles takes precedence over the model-reported + // scope/completeness so that an oversized or truncated PR cannot be auto-signed-off. + const { data: pullRequest } = await github.rest.pulls.get({ + owner, + repo, + pull_number: issueNumber, + }); + if (pullRequest.state !== "open" || pullRequest.head.sha !== headSha) { + core.info("Ignoring ARM API review completion for a closed PR or stale head SHA"); + return { headSha, issueNumber, statusPublished: false }; + } + const changedFiles = (await github.paginate(github.rest.pulls.listFiles, { + owner, + repo, + pull_number: issueNumber, + per_page: PER_PAGE_MAX, + })) as ChangedFile[]; + const coverage = evaluateAutomatedReviewCoverage(pullRequest.changed_files, changedFiles); + if (coverage.manualReviewRequired) { + status = { + state: CommitStatusState.ERROR, + description: `Review incomplete: ${coverage.reason ?? "automated review coverage was limited"}`, + }; + } else { + status = evaluateSemanticReview(result); + } + } catch (error) { + const fullReason = error instanceof Error ? error.message : "invalid semantic result"; + core.warning(fullReason); + const prefix = "Review incomplete: "; + const available = MAX_STATUS_DESCRIPTION - prefix.length; + const truncatedReason = + fullReason.length > available ? `${fullReason.slice(0, available - 1)}\u2026` : fullReason; + status = { + state: CommitStatusState.ERROR, + description: `${prefix}${truncatedReason}`, + }; + } + + await publishStatus( + { github, core }, + owner, + repo, + headSha, + `${workflowRun.html_url}/attempts/${workflowRun.run_attempt}`, + status, + ); + return { headSha, issueNumber, statusPublished: true }; +} diff --git a/.github/workflows/src/arm-auto-signoff/arm-semantic-review-workflow.ts b/.github/workflows/src/arm-auto-signoff/arm-semantic-review-workflow.ts index 2ee9adc23db9..884d9f552470 100644 --- a/.github/workflows/src/arm-auto-signoff/arm-semantic-review-workflow.ts +++ b/.github/workflows/src/arm-auto-signoff/arm-semantic-review-workflow.ts @@ -1,59 +1,6 @@ import { readFile } from "node:fs/promises"; -import { isFullGitSha } from "../../../shared/src/git.ts"; -import { CommitStatusState, PER_PAGE_MAX } from "../../../shared/src/github.ts"; -import { ARM_API_REVIEW_WORKFLOW_NAME, parseWorkflowRunArtifactInputs } from "../context.ts"; -import type { Core, GitHub, GitHubScriptArgs, WebhookEvent } from "../github.ts"; -import { - ARM_SEMANTIC_REVIEW_STATUS, - encodeSemanticReviewReceipt, - evaluateAutomatedReviewCoverage, - evaluateSemanticReview, - getLatestSemanticReviewStatus, - getSemanticReviewOutcome, - manualReviewRequiredStatus, - parseSemanticReviewReceipt, - parseSemanticReviewResult, - reviewIncompleteStatus, - SemanticReviewOutcome, - type CommitStatus, - type EvaluatedSemanticReview, - type SemanticReviewResult, -} from "./arm-semantic-review.ts"; - -type ChangedFile = { - filename: string; - additions: number; - deletions: number; -}; - -async function createSemanticReviewStatus({ - owner, - repo, - headSha, - targetUrl, - status, - github, - core, -}: { - owner: string; - repo: string; - headSha: string; - targetUrl: string; - status: Pick; - github: GitHub; - core: Core; -}): Promise { - core.info(`${ARM_SEMANTIC_REVIEW_STATUS}: ${status.state} - ${status.description}`); - await github.rest.repos.createCommitStatus({ - owner, - repo, - sha: headSha, - state: status.state, - context: ARM_SEMANTIC_REVIEW_STATUS, - description: status.description, - target_url: targetUrl, - }); -} +import type { GitHub, GitHubScriptArgs } from "../github.ts"; +import { parseSemanticReviewResult, type SemanticReviewResult } from "./arm-semantic-review.ts"; export async function validateSemanticReviewResult({ owner, @@ -76,176 +23,15 @@ export async function validateSemanticReviewResult({ } } -function parseWorkflowRunIdentity( - targetUrl: string | null, -): { runId: number; runAttempt: number } | undefined { - const match = /\/actions\/runs\/([1-9]\d*)\/attempts\/([1-9]\d*)(?:\/|$)/.exec(targetUrl ?? ""); - if (!match) { - return undefined; - } - const runId = Number(match[1]); - const runAttempt = Number(match[2]); - if (!Number.isSafeInteger(runId) || !Number.isSafeInteger(runAttempt)) { - return undefined; - } - return { runId, runAttempt }; -} - -export async function finalizeSemanticReviewWorkflow({ - owner, - repo, - issueNumber, - headSha, - runId, - runAttempt, - workflowConclusion, - reviewerExecuted = true, - targetUrl, - artifactNames, - github, - core, -}: { - owner: string; - repo: string; - issueNumber: number; - headSha: string; - runId: number; - runAttempt: number; - workflowConclusion: string | null; - reviewerExecuted?: boolean; - targetUrl: string; - artifactNames: string[]; - github: GitHub; - core: Core; -}): Promise { - if (!reviewerExecuted) { - core.info("The ARM API Reviewer did not execute; semantic status is unchanged"); - return false; - } - if (!Number.isSafeInteger(issueNumber) || issueNumber <= 0 || !isFullGitSha(headSha)) { - core.info("ARM API review completion is missing a valid PR number or head SHA"); - return false; - } - - const { data: pullRequest } = await github.rest.pulls.get({ - owner, - repo, - pull_number: issueNumber, - }); - if (pullRequest.state !== "open" || pullRequest.head.sha !== headSha) { - core.info("Ignoring ARM API review completion for a closed PR or stale head SHA"); - return false; - } - - const statuses: CommitStatus[] = await github.paginate( - github.rest.repos.listCommitStatusesForRef, - { - owner, - repo, - ref: headSha, - per_page: PER_PAGE_MAX, - }, - ); - const latestStatus = getLatestSemanticReviewStatus(statuses); - if (latestStatus && latestStatus.target_url !== targetUrl) { - const latestIdentity = parseWorkflowRunIdentity(latestStatus.target_url); - if ( - latestIdentity && - (latestIdentity.runId > runId || - (latestIdentity.runId === runId && latestIdentity.runAttempt > runAttempt)) - ) { - core.info("A newer ARM API review run superseded this completion"); - return false; - } - } - - if (workflowConclusion === "cancelled") { - if (latestStatus && latestStatus.state !== CommitStatusState.PENDING) { - await createSemanticReviewStatus({ - owner, - repo, - headSha, - targetUrl, - status: { - state: CommitStatusState.PENDING, - description: "Review canceled; rerun required", - }, - github, - core, - }); - return true; - } - core.info("Canceled ARM API review did not publish a semantic result"); - return false; - } - - const changedFiles: ChangedFile[] = await github.paginate(github.rest.pulls.listFiles, { - owner, - repo, - pull_number: issueNumber, - per_page: PER_PAGE_MAX, - }); - const coverage = evaluateAutomatedReviewCoverage(pullRequest.changed_files, changedFiles); - - if (workflowConclusion === "success" && latestStatus?.state !== CommitStatusState.PENDING) { - const existingOutcome = getSemanticReviewOutcome(latestStatus); - if ( - !coverage.manualReviewRequired || - existingOutcome === SemanticReviewOutcome.ManualReviewRequired - ) { - core.info("ARM semantic review result is already finalized"); - return false; - } - } - - let receipt: SemanticReviewResult | undefined; - let receiptError: string | undefined; - try { - receipt = parseSemanticReviewReceipt(artifactNames, runAttempt); - } catch (error) { - receiptError = error instanceof Error ? error.message : "Invalid ARM semantic review receipt"; - core.warning(receiptError); - } - - if ( - receipt && - (receipt.issueNumber !== issueNumber || receipt.headSha.toLowerCase() !== headSha.toLowerCase()) - ) { - receiptError = "ARM semantic review receipt does not match its workflow run"; - core.warning(receiptError); - receipt = undefined; - } - - const evaluatedReceipt = receipt ? evaluateSemanticReview(receipt) : undefined; - let status: EvaluatedSemanticReview; - if (coverage.manualReviewRequired) { - status = manualReviewRequiredStatus(coverage.reason ?? "automated review coverage was limited"); - } else if (workflowConclusion === "success" && evaluatedReceipt) { - status = evaluatedReceipt; - } else if ( - workflowConclusion !== "success" && - evaluatedReceipt?.outcome === SemanticReviewOutcome.ManualReviewRequired - ) { - status = evaluatedReceipt; - } else { - const reason = receiptError - ? "semantic result was invalid" - : workflowConclusion === "success" - ? "semantic result was missing" - : `workflow concluded with ${workflowConclusion ?? "an unknown result"}`; - status = reviewIncompleteStatus(reason); - } - - await createSemanticReviewStatus({ - owner, - repo, - headSha, - targetUrl, - status, - github, - core, - }); - return true; +function encodeSemanticReviewReceipt(result: SemanticReviewResult): string { + return [ + result.runAttempt, + result.issueNumber, + result.headSha, + result.reviewScope, + result.completion, + result.blockingCount, + ].join("."); } /* v8 ignore start */ @@ -269,77 +55,17 @@ export default async function validateArmSemanticReview({ throw new Error(`Invalid workflow run attempt: '${runAttemptText ?? ""}'`); } + const issueNumber = Number(issueNumberText); + const runAttempt = Number(runAttemptText); const agentOutput = JSON.parse(await readFile(outputPath, "utf8")) as unknown; - const result = parseSemanticReviewResult(agentOutput, Number(issueNumberText)); + const result = parseSemanticReviewResult(agentOutput, issueNumber, runAttempt); await validateSemanticReviewResult({ ...context.repo, result, github, }); return { - artifactValue: encodeSemanticReviewReceipt(result, Number(runAttemptText)), - }; -} - -export async function finalizeArmSemanticReview({ - github, - context, - core, -}: GitHubScriptArgs): Promise<{ - headSha: string; - issueNumber: number; - statusPublished: boolean; -}> { - const emptyResult = { - headSha: "", - issueNumber: 0, - statusPublished: false, - }; - const payload = context.payload as WebhookEvent<"workflow-run", "completed">; - const workflowRun = payload.workflow_run; - if (workflowRun.name !== ARM_API_REVIEW_WORKFLOW_NAME) { - return emptyResult; - } - - const repositoryOwner = workflowRun.repository.owner?.login; - if (!repositoryOwner) { - throw new Error("ARM API review workflow run is missing its repository owner"); - } - const artifacts = await github.paginate(github.rest.actions.listWorkflowRunArtifacts, { - owner: repositoryOwner, - repo: workflowRun.repository.name, - run_id: workflowRun.id, - per_page: PER_PAGE_MAX, - }); - const artifactNames = artifacts.map((artifact) => artifact.name); - const reviewerExecuted = - workflowRun.conclusion !== "success" || - (artifactNames.some((name) => name.startsWith("head-sha=")) && - artifactNames.some((name) => name.startsWith("issue-number="))); - if (!reviewerExecuted) { - core.info("The ARM API Reviewer did not execute; semantic status is unchanged"); - return emptyResult; - } - - const { headSha, issueNumber } = parseWorkflowRunArtifactInputs(artifactNames, core); - const statusPublished = await finalizeSemanticReviewWorkflow({ - owner: repositoryOwner, - repo: workflowRun.repository.name, - issueNumber, - headSha, - runId: workflowRun.id, - runAttempt: workflowRun.run_attempt, - workflowConclusion: workflowRun.conclusion, - reviewerExecuted, - targetUrl: `${workflowRun.html_url}/attempts/${workflowRun.run_attempt}`, - artifactNames, - github, - core, - }); - return { - headSha: isFullGitSha(headSha) ? headSha : "", - issueNumber: Number.isSafeInteger(issueNumber) && issueNumber > 0 ? issueNumber : 0, - statusPublished, + artifactValue: encodeSemanticReviewReceipt(result), }; } /* v8 ignore stop */ diff --git a/.github/workflows/src/arm-auto-signoff/arm-semantic-review.ts b/.github/workflows/src/arm-auto-signoff/arm-semantic-review.ts index 5447bb0b980f..f907356e0810 100644 --- a/.github/workflows/src/arm-auto-signoff/arm-semantic-review.ts +++ b/.github/workflows/src/arm-auto-signoff/arm-semantic-review.ts @@ -1,85 +1,103 @@ -import { isFullGitSha } from "../../../shared/src/git.ts"; import { CommitStatusState } from "../../../shared/src/github.ts"; import { byDate, invert } from "../../../shared/src/sort.ts"; +import { isFullGitSha } from "../../../shared/src/git.ts"; + +const MAX_SPECIFICATION_FILES = 50; +const MAX_SPECIFICATION_LINES = 5_000; + +export type ChangedFile = { + filename: string; + additions: number; + deletions: number; +}; export const ARM_SEMANTIC_REVIEW_STATUS = "ARM Semantic Review"; export const SemanticReviewOutcome = Object.freeze({ Passed: "passed", ChangesRequested: "changes-requested", - ManualReviewRequired: "manual-review-required", ReviewIncomplete: "review-incomplete", Pending: "pending", }); export type SemanticReviewOutcome = (typeof SemanticReviewOutcome)[keyof typeof SemanticReviewOutcome]; -export type SemanticReviewScope = "full" | "scoped"; -export type SemanticReviewCompleteness = "complete" | "incomplete" | "degraded"; +export const SemanticReviewScope = Object.freeze({ + Full: "full", + Scoped: "scoped", +}); +export type SemanticReviewScope = (typeof SemanticReviewScope)[keyof typeof SemanticReviewScope]; + +export const SemanticReviewCompletion = Object.freeze({ + Complete: "complete", + Incomplete: "incomplete", + Degraded: "degraded", +}); +export type SemanticReviewCompletion = + (typeof SemanticReviewCompletion)[keyof typeof SemanticReviewCompletion]; export type SemanticReviewResult = { + runAttempt: number; issueNumber: number; headSha: string; - scope: SemanticReviewScope; - completeness: SemanticReviewCompleteness; blockingCount: number; + reviewScope: SemanticReviewScope; + completion: SemanticReviewCompletion; }; export type CommitStatus = { context: string; state: string; - description?: string | null; - target_url: string | null; + target_url?: string | null; updated_at: string; }; export type EvaluatedSemanticReview = { - outcome: Exclude; state: CommitStatusState; description: string; }; -const RECEIPT_PATTERN = - /^([1-9]\d*)\.([1-9]\d*)\.([0-9a-f]{40})\.(full|scoped)\.(complete|incomplete|degraded)\.(0|[1-9]\d*)$/i; -const MANUAL_REVIEW_REQUIRED_PREFIX = "Manual review required:"; -const REVIEW_INCOMPLETE_PREFIX = "Review incomplete:"; -const MAX_SPECIFICATION_FILES = 50; -const MAX_SPECIFICATION_LINES = 5_000; - -type ChangedFile = { - filename: string; - additions: number; - deletions: number; -}; - function isRecord(value: unknown): value is Record { return typeof value === "object" && value !== null && !Array.isArray(value); } -function isSemanticReviewOutput(value: unknown): value is Record { - return isRecord(value) && value.type === "record_arm_semantic_review"; -} - +/** + * Extracts exactly one semantic result from gh-aw's agent output and validates + * that it belongs to the completed reviewer run. + */ export function parseSemanticReviewResult( agentOutput: unknown, expectedIssueNumber: number, + expectedRunAttempt: number, ): SemanticReviewResult { if (!isRecord(agentOutput) || !Array.isArray(agentOutput.items)) { throw new Error("ARM API Reviewer output is missing its items array"); } - const items: unknown[] = agentOutput.items; - const results = items.filter(isSemanticReviewOutput); + const results = (agentOutput.items as unknown[]).filter( + (item): item is Record => + isRecord(item) && item.type === "record_arm_semantic_review", + ); if (results.length !== 1) { throw new Error(`Expected one ARM semantic review result, found ${results.length}`); } const result = results[0]; + const runAttempt = Number(result.run_attempt); const issueNumber = Number(result.issue_number); const blockingCount = Number(result.blocking_count); if ( + typeof result.run_attempt !== "string" || + !/^[1-9]\d*$/.test(result.run_attempt) || + !Number.isSafeInteger(runAttempt) || + runAttempt !== expectedRunAttempt + ) { + throw new Error(`Invalid ARM semantic review run attempt: '${String(result.run_attempt)}'`); + } + if ( + typeof result.issue_number !== "string" || + !/^[1-9]\d*$/.test(result.issue_number) || !Number.isSafeInteger(issueNumber) || - issueNumber <= 0 || issueNumber !== expectedIssueNumber ) { throw new Error(`Invalid ARM semantic review PR number: '${String(result.issue_number)}'`); @@ -87,16 +105,6 @@ export function parseSemanticReviewResult( if (typeof result.head_sha !== "string" || !isFullGitSha(result.head_sha)) { throw new Error(`Invalid ARM semantic review head SHA: '${String(result.head_sha)}'`); } - if (result.scope !== "full" && result.scope !== "scoped") { - throw new Error(`Invalid ARM semantic review scope: '${String(result.scope)}'`); - } - if ( - result.completeness !== "complete" && - result.completeness !== "incomplete" && - result.completeness !== "degraded" - ) { - throw new Error(`Invalid ARM semantic review completeness: '${String(result.completeness)}'`); - } if ( typeof result.blocking_count !== "string" || !/^(0|[1-9]\d*)$/.test(result.blocking_count) || @@ -106,104 +114,66 @@ export function parseSemanticReviewResult( `Invalid ARM semantic review Blocking count: '${String(result.blocking_count)}'`, ); } - - return { - issueNumber, - headSha: result.head_sha, - scope: result.scope, - completeness: result.completeness, - blockingCount, - }; -} - -export function encodeSemanticReviewReceipt( - result: SemanticReviewResult, - runAttempt: number, -): string { - if (!Number.isSafeInteger(runAttempt) || runAttempt <= 0) { - throw new Error(`Invalid workflow run attempt: '${runAttempt}'`); - } - return [ - runAttempt, - result.issueNumber, - result.headSha, - result.scope, - result.completeness, - result.blockingCount, - ].join("."); -} - -export function parseSemanticReviewReceipt( - artifactNames: string[], - runAttempt: number, -): SemanticReviewResult | undefined { - const prefix = `arm-semantic-review=${runAttempt}.`; - const receiptNames = artifactNames.filter((name) => name.startsWith(prefix)); - if (receiptNames.length === 0) { - return undefined; - } - if (receiptNames.length !== 1) { - throw new Error(`Expected one ARM semantic review receipt, found ${receiptNames.length}`); - } - - const value = receiptNames[0].substring("arm-semantic-review=".length); - const match = RECEIPT_PATTERN.exec(value); - if (!match || Number(match[1]) !== runAttempt) { - throw new Error(`Invalid ARM semantic review receipt: '${receiptNames[0]}'`); + if (result.scope !== SemanticReviewScope.Full && result.scope !== SemanticReviewScope.Scoped) { + throw new Error(`Invalid ARM semantic review scope: '${String(result.scope)}'`); } - - const issueNumber = Number(match[2]); - const blockingCount = Number(match[6]); - if (!Number.isSafeInteger(issueNumber) || !Number.isSafeInteger(blockingCount)) { - throw new Error(`Invalid ARM semantic review receipt: '${receiptNames[0]}'`); + if ( + result.completeness !== SemanticReviewCompletion.Complete && + result.completeness !== SemanticReviewCompletion.Incomplete && + result.completeness !== SemanticReviewCompletion.Degraded + ) { + throw new Error(`Invalid ARM semantic review completion: '${String(result.completeness)}'`); } return { + runAttempt, issueNumber, - headSha: match[3], - scope: match[4] as SemanticReviewScope, - completeness: match[5] as SemanticReviewCompleteness, + headSha: result.head_sha, blockingCount, - }; -} - -export function manualReviewRequiredStatus(reason: string): EvaluatedSemanticReview { - return { - outcome: SemanticReviewOutcome.ManualReviewRequired, - state: CommitStatusState.ERROR, - description: `${MANUAL_REVIEW_REQUIRED_PREFIX} ${reason}`, - }; -} - -export function reviewIncompleteStatus(reason: string): EvaluatedSemanticReview { - return { - outcome: SemanticReviewOutcome.ReviewIncomplete, - state: CommitStatusState.ERROR, - description: `${REVIEW_INCOMPLETE_PREFIX} ${reason}`, + reviewScope: result.scope, + completion: result.completeness, }; } export function evaluateSemanticReview(result: SemanticReviewResult): EvaluatedSemanticReview { - if (result.scope === "scoped") { - return manualReviewRequiredStatus("automated review was scoped"); + // Scope and completeness are checked first: Changes requested applies only to + // full, complete reviews. A scoped or incomplete review requires manual signoff + // regardless of the blocking count. + if (result.reviewScope === SemanticReviewScope.Scoped) { + return { + state: CommitStatusState.ERROR, + description: "Review incomplete: scoped review requires manual signoff", + }; } - if (result.completeness !== "complete") { - return reviewIncompleteStatus(`automated review was ${result.completeness}`); + if (result.completion === SemanticReviewCompletion.Incomplete) { + return { + state: CommitStatusState.ERROR, + description: "Review incomplete: reviewer did not complete", + }; + } + if (result.completion === SemanticReviewCompletion.Degraded) { + return { + state: CommitStatusState.ERROR, + description: "Review incomplete: reviewer completed in degraded mode", + }; } if (result.blockingCount > 0) { return { - outcome: SemanticReviewOutcome.ChangesRequested, state: CommitStatusState.FAILURE, description: `Changes requested: ${result.blockingCount} Blocking finding(s)`, }; } return { - outcome: SemanticReviewOutcome.Passed, state: CommitStatusState.SUCCESS, description: "Passed: full review completed with no Blocking findings", }; } +/** + * Independently verifies that the PR is within automated-review coverage limits. + * A full-and-complete agent result must still be rejected if the trusted file list + * shows the PR was too large or the list was truncated. + */ export function evaluateAutomatedReviewCoverage( changedFileCount: number, changedFiles: ChangedFile[], @@ -239,7 +209,7 @@ export function evaluateAutomatedReviewCoverage( } export function getSemanticReviewOutcome( - status: Pick | undefined, + status: Pick | undefined, ): SemanticReviewOutcome | undefined { if (!status) { return undefined; @@ -253,12 +223,6 @@ export function getSemanticReviewOutcome( if (status.state === CommitStatusState.FAILURE) { return SemanticReviewOutcome.ChangesRequested; } - if ( - status.state === CommitStatusState.ERROR && - status.description?.startsWith(MANUAL_REVIEW_REQUIRED_PREFIX) - ) { - return SemanticReviewOutcome.ManualReviewRequired; - } if (status.state === CommitStatusState.ERROR) { return SemanticReviewOutcome.ReviewIncomplete; } diff --git a/.github/workflows/src/arm-auto-signoff/arm-universal-auto-signoff.ts b/.github/workflows/src/arm-auto-signoff/arm-universal-auto-signoff.ts index e33c162875fd..6d830b59198a 100644 --- a/.github/workflows/src/arm-auto-signoff/arm-universal-auto-signoff.ts +++ b/.github/workflows/src/arm-auto-signoff/arm-universal-auto-signoff.ts @@ -155,7 +155,7 @@ async function getDesiredLabelActions({ core.info(`ARM Semantic Review: ${semanticReviewOutcome ?? "missing"}`); if (semanticReviewOutcome !== SemanticReviewOutcome.Passed) { - if (semanticReviewOutcome === SemanticReviewOutcome.ManualReviewRequired) { + if (semanticReviewOutcome === SemanticReviewOutcome.ReviewIncomplete) { core.info("ARM semantic review requires manual signoff"); if (!labelNames.includes(ArmAutoSignoffLabel.ArmManualSignoffRequired)) { labelActions[ArmAutoSignoffLabel.ArmManualSignoffRequired] = LabelAction.Add; diff --git a/.github/workflows/test/arm-api-review-workflow.test.ts b/.github/workflows/test/arm-api-review-workflow.test.ts index 34f86a6060c4..6263349393c4 100644 --- a/.github/workflows/test/arm-api-review-workflow.test.ts +++ b/.github/workflows/test/arm-api-review-workflow.test.ts @@ -60,7 +60,7 @@ function createResolverHarness( github: { rest: { pulls: { get } } }, context, core: { setOutput }, - process: { env: { TARGET_PR_NUMBER: value } }, + process: { env: { GITHUB_RUN_ATTEMPT: "1", TARGET_PR_NUMBER: value } }, }), }; } @@ -180,6 +180,7 @@ describe("ARM API review workflow", () => { expect(source).toContain(`TARGET_PR_NUMBER: ${TARGET_EXPRESSION}`); expect(source).toContain("const { data: pull } = await github.rest.pulls.get({"); expect(source).toContain('core.setOutput("target_pr_number", String(pull.number))'); + expect(source).toContain('core.setOutput("run_attempt", runAttempt)'); expect(source).toContain( "target_pr_number: ${{ steps.resolve_target_pr.outputs.target_pr_number }}", ); @@ -222,6 +223,7 @@ describe("ARM API review workflow", () => { }); expect(harness.setOutput).toHaveBeenCalledWith("target_pr_number", "44499"); expect(harness.setOutput).toHaveBeenCalledWith("target_head_sha", fullGitSha); + expect(harness.setOutput).toHaveBeenCalledWith("run_attempt", "1"); }); it("accepts an issue-shaped pull request comment payload", async () => { @@ -285,6 +287,8 @@ describe("ARM API review workflow", () => { expect(source).toContain("github.event.comment.body == '/arm-review'"); expect(source).not.toContain("contains(github.event.comment.body, '/arm-review')"); + expect(source).toContain("types: [opened, synchronize, reopened, labeled, ready_for_review]"); + expect(source).toContain("github.event.action == 'reopened'"); expect(source).toContain('bots: ["github-actions[bot]"]'); expect(source).toContain("status-comment: true"); expect(source).toContain(`run-name: "ARM API Review #${TARGET_EXPRESSION}`); @@ -292,6 +296,8 @@ describe("ARM API review workflow", () => { expect(compiled).toContain('GH_AW_ALLOWED_BOTS: "github-actions[bot]"'); expect(compiled).toContain("- name: Add comment with workflow run link"); expect(compiled).toContain("- name: Update reaction comment with completion status"); + expect(compiled).toContain(" - reopened"); + expect(compiled).toContain("github.event.action == 'reopened'"); expect(compiled).toContain(`run-name: "ARM API Review #${TARGET_EXPRESSION}`); const statusCommentStart = compiled.indexOf("- name: Add comment with workflow run link"); @@ -451,7 +457,6 @@ describe("ARM API review workflow", () => { expect(source).toContain( "permissions:\n pull-requests: read\n statuses: write\n steps:", ); - expect(source).toContain("**The review is scoped, incomplete, or degraded**"); expect(source).toContain( "**The review is full and complete, and at least one verified, currently", ); @@ -543,6 +548,12 @@ describe("ARM API review posting reliability", () => { const source = collapseWhitespace(await readFile(join(ROOT, SOURCE_FILE), "utf8")); expect(source).toContain("These rules are **exhaustive**."); + expect(source).toContain( + "The review is scoped, incomplete, or degraded** β†’ leave `WaitForARMFeedback`, `ARMChangesRequested`, and `ARMSignedOff` unchanged", + ); + expect(source).toContain( + "No verified, currently applicable Blocking finding remains, and the review is full and complete", + ); expect(source).toContain( "draft status, a `[Test]` or `[Do-Not-Merge]` title, a revert, a bot-authored PR, or the author's stated intent not to merge are **not** grounds to skip a label change", ); @@ -559,15 +570,24 @@ describe("ARM API review posting reliability", () => { expect(source).toContain("name: Set ARM semantic review pending"); expect(source).toContain('context: "ARM Semantic Review"'); expect(source).toContain("record-arm-semantic-review:"); + expect(source).toContain("scope:"); + expect(source).toContain("completeness:"); expect(source).toContain("### Step 9: Record Semantic Result"); expect(collapsed).toContain( "Call `record_arm_semantic_review` exactly once after queuing the summary", ); expect(compiled).toContain("record_arm_semantic_review"); expect(compiled).toContain("ARM Semantic Review"); + expect(source).toContain('run_attempt:\n description: "GitHub Actions attempt'); expect(compiled).toContain("Upload ARM semantic review receipt"); + const semanticJob = compiled.slice( + compiled.indexOf("\n record_arm_semantic_review:\n"), + compiled.indexOf("\n safe_outputs:\n"), + ); + expect(semanticJob).toContain("name: Validate ARM semantic review"); + expect(semanticJob).toContain("permissions:"); expect(collapsed).toContain( - "Universal Auto-Signoff publishes the final `ARM Semantic Review` status only after the entire reviewer workflow completes.", + "`ARM Semantic Review - Set Status` consumes the exact completed run's agent output, validates its correlation and shape, and publishes the final status only after the entire reviewer workflow completes.", ); }); @@ -773,7 +793,7 @@ describe("ARM API review consistency and hardening", () => { const source = collapseWhitespace(await readFile(join(ROOT, SOURCE_FILE), "utf8")); // Unverified findings from a Critic-unavailable run do not satisfy the - // full-and-complete rule, so the explicit Critic-unavailable rule remains authoritative. + // verified-finding rule, so the explicit Critic-unavailable rule remains authoritative. expect(source).toContain( "**The review is full and complete, and at least one verified, currently applicable Blocking finding remains**", ); diff --git a/.github/workflows/test/arm-auto-signoff/arm-semantic-review-status-workflow.test.ts b/.github/workflows/test/arm-auto-signoff/arm-semantic-review-status-workflow.test.ts index 2071e4d0e4cb..8c684436bee5 100644 --- a/.github/workflows/test/arm-auto-signoff/arm-semantic-review-status-workflow.test.ts +++ b/.github/workflows/test/arm-auto-signoff/arm-semantic-review-status-workflow.test.ts @@ -21,10 +21,17 @@ type Workflow = { id?: string; name?: string; if?: string; + uses?: string; + "continue-on-error"?: boolean; + env?: Record; with?: { script?: string; name?: string; value?: string; + path?: string; + "github-token"?: string; + repository?: string; + "run-id"?: string; }; }>; } @@ -94,6 +101,30 @@ describe("ARM Semantic Review - Set Status workflow", () => { ); }); + it("downloads agent output from the exact completed reviewer run", async () => { + const { workflow } = await readWorkflow("arm-semantic-review-status.yaml"); + const steps = workflow.jobs?.["arm-semantic-review-status"]?.steps ?? []; + const download = steps.find((step) => step.name === "Download ARM API Reviewer output"); + const finalize = steps.find((step) => step.id === "finalize"); + + expect(download).toEqual( + expect.objectContaining({ + "continue-on-error": true, + uses: "actions/download-artifact@v8.0.1", + with: { + name: "agent", + path: "${{ runner.temp }}/arm-semantic-review", + "github-token": "${{ github.token }}", + repository: "${{ github.repository }}", + "run-id": "${{ github.event.workflow_run.id }}", + }, + }), + ); + expect(finalize?.env?.GH_AW_AGENT_OUTPUT).toBe( + "${{ runner.temp }}/arm-semantic-review/agent_output.json", + ); + }); + it("makes Universal depend on semantic status completion, not reviewer completion", async () => { const { source, workflow } = await readWorkflow("arm-universal-auto-signoff.yaml"); const workflows = workflow.on?.workflow_run?.workflows ?? []; @@ -103,4 +134,15 @@ describe("ARM Semantic Review - Set Status workflow", () => { expect(source).not.toContain("Finalize ARM Semantic Review"); expect(source).not.toContain("finalizeArmSemanticReview"); }); + + it("publishes only pilot signoff and manual-review label artifacts", async () => { + const { source } = await readWorkflow("arm-universal-auto-signoff.yaml"); + + for (const label of ["ARMAutoSignedOff-Test", "ARMManualSignoffRequired"]) { + expect(source).toContain(`Upload artifact for ${label} label`); + } + for (const label of ["ARMSignedOff", "ARMAutoSignedOff", "ARMChangesRequested"]) { + expect(source).not.toContain(`Upload artifact for ${label} label`); + } + }); }); diff --git a/.github/workflows/test/arm-auto-signoff/arm-semantic-review.test.ts b/.github/workflows/test/arm-auto-signoff/arm-semantic-review.test.ts index 01ffd149f70f..0272f51caa99 100644 --- a/.github/workflows/test/arm-auto-signoff/arm-semantic-review.test.ts +++ b/.github/workflows/test/arm-auto-signoff/arm-semantic-review.test.ts @@ -1,21 +1,21 @@ +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; import { describe, expect, it } from "vitest"; import { CommitStatusState, PER_PAGE_MAX } from "../../../shared/src/github.ts"; import { ARM_SEMANTIC_REVIEW_STATUS, - encodeSemanticReviewReceipt, evaluateAutomatedReviewCoverage, evaluateSemanticReview, getSemanticReviewOutcome, - parseSemanticReviewReceipt, parseSemanticReviewResult, + SemanticReviewCompletion, SemanticReviewOutcome, + SemanticReviewScope, + type ChangedFile, type SemanticReviewResult, } from "../../src/arm-auto-signoff/arm-semantic-review.ts"; -import { - finalizeArmSemanticReview, - finalizeSemanticReviewWorkflow, - validateSemanticReviewResult, -} from "../../src/arm-auto-signoff/arm-semantic-review-workflow.ts"; +import { finalizeArmSemanticReview } from "../../src/arm-auto-signoff/arm-semantic-review-status.ts"; import type { GitHubScriptArgs } from "../../src/github.ts"; import { createMockCore, createMockGithub } from "../mocks.ts"; @@ -23,277 +23,297 @@ const owner = "Azure"; const repo = "azure-rest-api-specs"; const issueNumber = 123; const headSha = "0123456789abcdef0123456789abcdef01234567"; -const targetUrl = "https://github.com/Azure/azure-rest-api-specs/actions/runs/456/attempts/1"; const runId = 456; const runAttempt = 1; +const runUrl = "https://github.com/Azure/azure-rest-api-specs/actions/runs/456"; const passedResult: SemanticReviewResult = { + runAttempt, issueNumber, headSha, - scope: "full", - completeness: "complete", blockingCount: 0, + reviewScope: SemanticReviewScope.Full, + completion: SemanticReviewCompletion.Complete, }; -function receiptName(result: SemanticReviewResult = passedResult): string { - return `arm-semantic-review=${encodeSemanticReviewReceipt(result, runAttempt)}`; +function agentOutput(result: SemanticReviewResult = passedResult) { + return { + items: [ + { + type: "record_arm_semantic_review", + run_attempt: String(result.runAttempt), + issue_number: String(result.issueNumber), + head_sha: result.headSha, + blocking_count: String(result.blockingCount), + scope: result.reviewScope, + completeness: result.completion, + }, + ], + }; } -function createFinalizeGithub({ - currentHeadSha = headSha, - changedFiles = [ - { - filename: "specification/test/resource-manager/test.json", - additions: 10, - deletions: 5, - }, - ], - statuses = [ - { - context: ARM_SEMANTIC_REVIEW_STATUS, - state: CommitStatusState.PENDING, - description: "ARM API semantic review is pending", - target_url: targetUrl, - updated_at: "2026-01-01T00:00:00Z", - }, - ], -}: { - currentHeadSha?: string; - changedFiles?: { filename: string; additions: number; deletions: number }[]; - statuses?: { - context: string; - state: string; - description: string; - target_url: string; - updated_at: string; - }[]; -} = {}) { +const defaultChangedFiles: ChangedFile[] = [ + { filename: "specification/foo/foo.json", additions: 10, deletions: 5 }, +]; + +function createFinalizeGithub() { const github = createMockGithub(); - github.rest.pulls.get.mockResolvedValue({ + github.rest.actions.listWorkflowRunArtifacts.mockResolvedValue({ data: { - state: "open", - head: { sha: currentHeadSha }, - changed_files: changedFiles.length, + artifacts: [{ name: `head-sha=${headSha}` }, { name: `issue-number=${issueNumber}` }], }, }); - github.rest.pulls.listFiles.mockResolvedValue({ data: changedFiles }); - github.rest.repos.listCommitStatusesForRef.mockResolvedValue({ data: statuses }); + // Default: small PR well within coverage limits + github.rest.pulls.get.mockResolvedValue({ + data: { changed_files: 1, state: "open", head: { sha: headSha } }, + }); + github.rest.pulls.listFiles.mockResolvedValue({ data: defaultChangedFiles }); return github; } -function finalize({ - artifactNames = [receiptName()], - workflowConclusion = "success", - reviewerExecuted = true, +function createContext(conclusion: string | null = "success") { + return { + payload: { + workflow_run: { + name: "ARM API Review: Automated Workflow", + conclusion, + id: runId, + run_attempt: runAttempt, + html_url: runUrl, + repository: { + name: repo, + owner: { login: owner }, + }, + }, + }, + }; +} + +async function runFinalizer({ + conclusion = "success", + output = agentOutput(), + includeOutput = true, github = createFinalizeGithub(), }: { - artifactNames?: string[]; - workflowConclusion?: string | null; - reviewerExecuted?: boolean; + conclusion?: string | null; + output?: unknown; + includeOutput?: boolean; github?: ReturnType; } = {}) { - return finalizeSemanticReviewWorkflow({ - owner, - repo, - issueNumber, - headSha, - runId, - runAttempt, - workflowConclusion, - reviewerExecuted, - targetUrl, - artifactNames, - github, - core: createMockCore(), - }); + const directory = await mkdtemp(join(tmpdir(), "arm-semantic-review-")); + const outputPath = join(directory, "agent_output.json"); + const previousOutputPath = process.env.GH_AW_AGENT_OUTPUT; + try { + if (includeOutput) { + await writeFile(outputPath, JSON.stringify(output), "utf8"); + } + process.env.GH_AW_AGENT_OUTPUT = outputPath; + const result = await finalizeArmSemanticReview({ + github, + context: createContext(conclusion), + core: createMockCore(), + } as unknown as GitHubScriptArgs); + return { github, result }; + } finally { + if (previousOutputPath === undefined) { + delete process.env.GH_AW_AGENT_OUTPUT; + } else { + process.env.GH_AW_AGENT_OUTPUT = previousOutputPath; + } + await rm(directory, { recursive: true, force: true }); + } } describe("parseSemanticReviewResult", () => { - it("parses one valid semantic result", () => { - expect( + it("parses one valid result", () => { + expect(parseSemanticReviewResult(agentOutput(), issueNumber, runAttempt)).toEqual(passedResult); + }); + + it("rejects missing, duplicate, or mismatched results", () => { + expect(() => parseSemanticReviewResult({ items: [] }, issueNumber, runAttempt)).toThrow( + "Expected one ARM semantic review result, found 0", + ); + expect(() => parseSemanticReviewResult( { items: [ - { - type: "record_arm_semantic_review", - issue_number: String(issueNumber), - head_sha: headSha, - scope: "full", - completeness: "complete", - blocking_count: "0", - }, + ...agentOutput().items, + ...agentOutput({ ...passedResult, blockingCount: 1 }).items, ], }, issueNumber, + runAttempt, ), - ).toEqual(passedResult); - }); - - it("rejects missing, duplicate, or untrusted correlation values", () => { - expect(() => parseSemanticReviewResult({ items: [] }, issueNumber)).toThrow( - "Expected one ARM semantic review result, found 0", - ); + ).toThrow("Expected one ARM semantic review result, found 2"); + expect(() => + parseSemanticReviewResult( + agentOutput({ ...passedResult, runAttempt: 2 }), + issueNumber, + runAttempt, + ), + ).toThrow("Invalid ARM semantic review run attempt"); + expect(() => + parseSemanticReviewResult( + agentOutput({ ...passedResult, issueNumber: 456 }), + issueNumber, + runAttempt, + ), + ).toThrow("Invalid ARM semantic review PR number"); expect(() => parseSemanticReviewResult( { - items: [{ type: "record_arm_semantic_review" }, { type: "record_arm_semantic_review" }], + items: [{ ...agentOutput().items[0], scope: "partial" }], }, issueNumber, + runAttempt, ), - ).toThrow("Expected one ARM semantic review result, found 2"); + ).toThrow("Invalid ARM semantic review scope"); expect(() => parseSemanticReviewResult( { - items: [ - { - type: "record_arm_semantic_review", - issue_number: "456", - head_sha: headSha, - scope: "full", - completeness: "complete", - blocking_count: "0", - }, - ], + items: [{ ...agentOutput().items[0], completeness: "unknown" }], }, issueNumber, + runAttempt, ), - ).toThrow("Invalid ARM semantic review PR number"); - }); -}); - -describe("semantic review receipts", () => { - it("round-trips an attempt-specific receipt", () => { - const value = encodeSemanticReviewReceipt(passedResult, runAttempt); - - expect(parseSemanticReviewReceipt([`arm-semantic-review=${value}`], runAttempt)).toEqual( - passedResult, - ); - }); - - it("ignores receipts from other attempts and rejects ambiguity", () => { - expect( - parseSemanticReviewReceipt( - [`arm-semantic-review=2.${issueNumber}.${headSha}.full.complete.0`], - 1, - ), - ).toBeUndefined(); - expect(() => parseSemanticReviewReceipt([receiptName(), receiptName()], runAttempt)).toThrow( - "Expected one ARM semantic review receipt, found 2", - ); + ).toThrow("Invalid ARM semantic review completion"); }); }); describe("evaluateSemanticReview", () => { - it("distinguishes every semantic outcome", () => { - expect(evaluateSemanticReview(passedResult).outcome).toBe(SemanticReviewOutcome.Passed); - expect(evaluateSemanticReview({ ...passedResult, blockingCount: 2 }).outcome).toBe( - SemanticReviewOutcome.ChangesRequested, - ); - expect(evaluateSemanticReview({ ...passedResult, scope: "scoped" }).outcome).toBe( - SemanticReviewOutcome.ManualReviewRequired, - ); - expect(evaluateSemanticReview({ ...passedResult, completeness: "incomplete" }).outcome).toBe( - SemanticReviewOutcome.ReviewIncomplete, - ); - expect(evaluateSemanticReview({ ...passedResult, completeness: "degraded" }).outcome).toBe( - SemanticReviewOutcome.ReviewIncomplete, - ); + it("passes when a full, complete review has no Blocking findings", () => { + expect(evaluateSemanticReview(passedResult)).toEqual({ + state: CommitStatusState.SUCCESS, + description: "Passed: full review completed with no Blocking findings", + }); }); - describe("evaluateAutomatedReviewCoverage", () => { - it("requires manual review for oversized or truncated changes", () => { - const tooManyFiles = Array.from({ length: 51 }, (_, index) => ({ - filename: `specification/test/resource-manager/file-${index}.json`, - additions: 1, - deletions: 0, - })); - expect(evaluateAutomatedReviewCoverage(tooManyFiles.length, tooManyFiles)).toEqual({ - manualReviewRequired: true, - reason: "PR exceeds automated review size limits", - }); - expect( - evaluateAutomatedReviewCoverage(1, [ - { - filename: "specification/test/resource-manager/test.json", - additions: 5_001, - deletions: 0, - }, - ]), - ).toEqual({ - manualReviewRequired: true, - reason: "PR exceeds automated review size limits", - }); - expect(evaluateAutomatedReviewCoverage(3_100, tooManyFiles)).toEqual({ - manualReviewRequired: true, - reason: "changed-file list was truncated", - }); + it("requests changes when Blocking findings remain", () => { + expect(evaluateSemanticReview({ ...passedResult, blockingCount: 2 })).toEqual({ + state: CommitStatusState.FAILURE, + description: "Changes requested: 2 Blocking finding(s)", }); + }); - it("allows a complete review within the trusted limits", () => { - expect( - evaluateAutomatedReviewCoverage(1, [ - { - filename: "specification/test/resource-manager/test.json", - additions: 100, - deletions: 20, - }, - ]), - ).toEqual({ manualReviewRequired: false }); + it.each([ + { + name: "scoped", + result: { ...passedResult, reviewScope: SemanticReviewScope.Scoped }, + description: "Review incomplete: scoped review requires manual signoff", + }, + { + name: "incomplete", + result: { ...passedResult, completion: SemanticReviewCompletion.Incomplete }, + description: "Review incomplete: reviewer did not complete", + }, + { + name: "degraded", + result: { ...passedResult, completion: SemanticReviewCompletion.Degraded }, + description: "Review incomplete: reviewer completed in degraded mode", + }, + ])("requires manual review when the semantic review is $name", ({ result, description }) => { + expect(evaluateSemanticReview(result)).toEqual({ + state: CommitStatusState.ERROR, + description, }); }); - it("routes scoped reviews to manual review even when execution was degraded", () => { - expect( - evaluateSemanticReview({ + it.each([ + { + name: "scoped with blockers", + result: { ...passedResult, reviewScope: SemanticReviewScope.Scoped, blockingCount: 2 }, + description: "Review incomplete: scoped review requires manual signoff", + }, + { + name: "incomplete with blockers", + result: { ...passedResult, - scope: "scoped", - completeness: "degraded", - }).outcome, - ).toBe(SemanticReviewOutcome.ManualReviewRequired); - }); + completion: SemanticReviewCompletion.Incomplete, + blockingCount: 1, + }, + description: "Review incomplete: reviewer did not complete", + }, + { + name: "degraded with blockers", + result: { + ...passedResult, + completion: SemanticReviewCompletion.Degraded, + blockingCount: 3, + }, + description: "Review incomplete: reviewer completed in degraded mode", + }, + ])( + "requires manual review rather than changes requested for $name", + ({ result, description }) => { + expect(evaluateSemanticReview(result)).toEqual({ + state: CommitStatusState.ERROR, + description, + }); + }, + ); - it("reads outcomes from commit statuses", () => { + it("maps commit status states back to semantic outcomes", () => { expect(getSemanticReviewOutcome({ state: "pending" })).toBe(SemanticReviewOutcome.Pending); expect(getSemanticReviewOutcome({ state: "success" })).toBe(SemanticReviewOutcome.Passed); expect(getSemanticReviewOutcome({ state: "failure" })).toBe( SemanticReviewOutcome.ChangesRequested, ); - expect( - getSemanticReviewOutcome({ - state: "error", - description: "Manual review required: automated review was scoped", - }), - ).toBe(SemanticReviewOutcome.ManualReviewRequired); - expect( - getSemanticReviewOutcome({ - state: "error", - description: "Review incomplete: semantic result was missing", - }), - ).toBe(SemanticReviewOutcome.ReviewIncomplete); + expect(getSemanticReviewOutcome({ state: "error" })).toBe( + SemanticReviewOutcome.ReviewIncomplete, + ); }); }); -describe("validateSemanticReviewResult", () => { - it("accepts only the current open PR head", async () => { - const github = createFinalizeGithub(); - await expect( - validateSemanticReviewResult({ owner, repo, result: passedResult, github }), - ).resolves.toBeUndefined(); +describe("evaluateAutomatedReviewCoverage", () => { + const smallFile: ChangedFile = { + filename: "specification/foo/foo.json", + additions: 10, + deletions: 5, + }; - github.rest.pulls.get.mockResolvedValue({ - data: { state: "open", head: { sha: "fedcba9876543210fedcba9876543210fedcba98" } }, + it("passes a small PR within limits", () => { + expect(evaluateAutomatedReviewCoverage(1, [smallFile])).toEqual({ + manualReviewRequired: false, + }); + }); + + it("requires manual review when the file list is truncated", () => { + expect(evaluateAutomatedReviewCoverage(10, [smallFile])).toEqual({ + manualReviewRequired: true, + reason: "changed-file list was truncated", + }); + }); + + it("requires manual review when specification file count exceeds limit", () => { + const manyFiles = Array.from({ length: 51 }, (_, i): ChangedFile => ({ + filename: `specification/foo/foo${i}.json`, + additions: 1, + deletions: 0, + })); + expect(evaluateAutomatedReviewCoverage(51, manyFiles)).toEqual({ + manualReviewRequired: true, + reason: "PR exceeds automated review size limits", + }); + }); + + it("requires manual review when specification line count exceeds limit", () => { + const bigFile: ChangedFile = { + filename: "specification/foo/foo.json", + additions: 4_000, + deletions: 2_000, + }; + expect(evaluateAutomatedReviewCoverage(1, [bigFile])).toEqual({ + manualReviewRequired: true, + reason: "PR exceeds automated review size limits", }); - await expect( - validateSemanticReviewResult({ owner, repo, result: passedResult, github }), - ).rejects.toThrow("The reviewed pull request head is no longer current"); }); }); -describe("finalizeSemanticReviewWorkflow", () => { - it("publishes Passed only after the reviewer workflow succeeds", async () => { - const github = createFinalizeGithub(); +describe("finalizeArmSemanticReview", () => { + it("publishes Passed for a clean reviewer result", async () => { + const { github, result } = await runFinalizer(); - await expect(finalize({ github })).resolves.toBe(true); + expect(result).toEqual({ headSha, issueNumber, statusPublished: true }); expect(github.rest.repos.createCommitStatus).toHaveBeenCalledWith({ owner, repo, @@ -301,256 +321,199 @@ describe("finalizeSemanticReviewWorkflow", () => { state: CommitStatusState.SUCCESS, context: ARM_SEMANTIC_REVIEW_STATUS, description: "Passed: full review completed with no Blocking findings", - target_url: targetUrl, + target_url: `${runUrl}/attempts/${runAttempt}`, + }); + expect(github.rest.actions.listWorkflowRunArtifacts).toHaveBeenCalledOnce(); + expect(github.rest.actions.listWorkflowRunArtifacts).toHaveBeenCalledWith({ + owner, + repo, + run_id: runId, + per_page: PER_PAGE_MAX, }); }); - it("publishes Manual review required for scoped coverage", async () => { - const github = createFinalizeGithub(); - const result = { ...passedResult, scope: "scoped" as const }; + it("publishes Changes requested for Blocking findings", async () => { + const { github } = await runFinalizer({ + output: agentOutput({ ...passedResult, blockingCount: 3 }), + }); - await finalize({ github, artifactNames: [receiptName(result)] }); expect(github.rest.repos.createCommitStatus).toHaveBeenCalledWith( expect.objectContaining({ - state: CommitStatusState.ERROR, - description: "Manual review required: automated review was scoped", + state: CommitStatusState.FAILURE, + description: "Changes requested: 3 Blocking finding(s)", }), ); }); - describe("finalizeArmSemanticReview", () => { - it("reuses the exact reviewer artifact listing for correlation and finalization", async () => { - const github = createFinalizeGithub(); - github.rest.actions.listWorkflowRunArtifacts.mockResolvedValue({ - data: { - artifacts: [ - { name: `head-sha=${headSha}` }, - { name: `issue-number=${issueNumber}` }, - { name: receiptName() }, - ], - }, - }); - const context = { - payload: { - workflow_run: { - name: "ARM API Review: Automated Workflow", - conclusion: "success", - id: runId, - run_attempt: runAttempt, - html_url: "https://github.com/Azure/azure-rest-api-specs/actions/runs/456", - repository: { - name: repo, - owner: { login: owner }, - }, - }, - }, - }; - - await expect( - finalizeArmSemanticReview({ - github, - context, - core: createMockCore(), - } as unknown as GitHubScriptArgs), - ).resolves.toEqual({ - headSha, - issueNumber, - statusPublished: true, - }); - expect(github.rest.actions.listWorkflowRunArtifacts).toHaveBeenCalledOnce(); - expect(github.rest.actions.listWorkflowRunArtifacts).toHaveBeenCalledWith({ - owner, - repo, - run_id: runId, - per_page: PER_PAGE_MAX, - }); + it("publishes Review incomplete for a scoped clean review", async () => { + const { github } = await runFinalizer({ + output: agentOutput({ ...passedResult, reviewScope: SemanticReviewScope.Scoped }), }); - }); - - it("requires manual review when trusted size checks find limited coverage", async () => { - const changedFiles = Array.from({ length: 51 }, (_, index) => ({ - filename: `specification/test/resource-manager/file-${index}.json`, - additions: 1, - deletions: 0, - })); - const github = createFinalizeGithub({ changedFiles }); - await finalize({ github }); expect(github.rest.repos.createCommitStatus).toHaveBeenCalledWith( expect.objectContaining({ state: CommitStatusState.ERROR, - description: "Manual review required: PR exceeds automated review size limits", - }), - ); - - const failedGithub = createFinalizeGithub({ changedFiles }); - await finalize({ - github: failedGithub, - artifactNames: [], - workflowConclusion: "failure", - }); - expect(failedGithub.rest.repos.createCommitStatus).toHaveBeenCalledWith( - expect.objectContaining({ - description: "Manual review required: PR exceeds automated review size limits", + description: "Review incomplete: scoped review requires manual signoff", }), ); }); - it("publishes Review incomplete for missing or failed review execution", async () => { - const missingGithub = createFinalizeGithub(); - await finalize({ github: missingGithub, artifactNames: [] }); - expect(missingGithub.rest.repos.createCommitStatus).toHaveBeenCalledWith( - expect.objectContaining({ - state: CommitStatusState.ERROR, - description: "Review incomplete: semantic result was missing", - }), - ); + it.each([ + { + name: "failed workflow", + options: { conclusion: "failure" }, + description: "Review incomplete: workflow concluded with failure", + }, + { + name: "missing agent output", + options: { includeOutput: false }, + description: "Review incomplete: ENOENT", + }, + { + name: "missing semantic item", + options: { output: { items: [] } }, + description: "Review incomplete: Expected one ARM semantic review result, found 0", + }, + { + name: "wrong SHA", + options: { output: agentOutput({ ...passedResult, headSha: "f".repeat(40) }) }, + description: "Review incomplete: semantic result SHA does not match reviewer correlation", + }, + ])("publishes Review incomplete for $name", async ({ options, description }) => { + const { github } = await runFinalizer(options); - const failedGithub = createFinalizeGithub(); - await finalize({ github: failedGithub, workflowConclusion: "failure" }); - expect(failedGithub.rest.repos.createCommitStatus).toHaveBeenCalledWith( + expect(github.rest.repos.createCommitStatus).toHaveBeenCalledWith( expect.objectContaining({ state: CommitStatusState.ERROR, - description: "Review incomplete: workflow concluded with failure", }), ); + const status = github.rest.repos.createCommitStatus.mock.calls[0]?.[0] as { + description: string; + }; + expect(status.description).toContain(description); + }); - const malformedGithub = createFinalizeGithub(); - await finalize({ - github: malformedGithub, - artifactNames: [`arm-semantic-review=${runAttempt}.malformed`], - }); - expect(malformedGithub.rest.repos.createCommitStatus).toHaveBeenCalledWith( - expect.objectContaining({ - description: "Review incomplete: semantic result was invalid", - }), - ); + it("leaves Pending unchanged when the reviewer run is canceled", async () => { + const { github, result } = await runFinalizer({ conclusion: "cancelled" }); - const mismatchedGithub = createFinalizeGithub(); - const mismatchedReceipt = { - ...passedResult, - issueNumber: 456, - }; - await finalize({ - github: mismatchedGithub, - artifactNames: [receiptName(mismatchedReceipt)], - }); - expect(mismatchedGithub.rest.repos.createCommitStatus).toHaveBeenCalledWith( - expect.objectContaining({ - description: "Review incomplete: semantic result was invalid", - }), - ); + expect(result).toEqual({ headSha, issueNumber, statusPublished: false }); + expect(github.rest.repos.createCommitStatus).not.toHaveBeenCalled(); }); - it("does not finalize a canceled pending review", async () => { + it("does nothing when trusted PR/SHA correlation is missing", async () => { const github = createFinalizeGithub(); + github.rest.actions.listWorkflowRunArtifacts.mockResolvedValue({ + data: { artifacts: [] }, + }); - await expect(finalize({ github, workflowConclusion: "cancelled" })).resolves.toBe(false); + const execution = await runFinalizer({ github }); + expect(execution.result).toEqual({ + headSha: "", + issueNumber: 0, + statusPublished: false, + }); expect(github.rest.repos.createCommitStatus).not.toHaveBeenCalled(); }); - it("ignores a successful workflow where the reviewer did not execute", async () => { - const github = createFinalizeGithub({ - statuses: [ - { - context: ARM_SEMANTIC_REVIEW_STATUS, - state: CommitStatusState.SUCCESS, - description: "Passed: full review completed with no Blocking findings", - target_url: "https://github.com/Azure/azure-rest-api-specs/actions/runs/123/attempts/1", - updated_at: "2026-01-01T00:00:00Z", - }, - ], - }); + it.each([ + { + name: "closed", + pullRequest: { changed_files: 1, state: "closed", head: { sha: headSha } }, + }, + { + name: "stale", + pullRequest: { changed_files: 1, state: "open", head: { sha: "f".repeat(40) } }, + }, + ])("does not publish a status when the pull request is $name", async ({ pullRequest }) => { + const github = createFinalizeGithub(); + github.rest.pulls.get.mockResolvedValue({ data: pullRequest }); - await expect( - finalize({ - github, - artifactNames: [], - reviewerExecuted: false, - }), - ).resolves.toBe(false); + const execution = await runFinalizer({ github }); + + expect(execution.result).toEqual({ + headSha, + issueNumber, + statusPublished: false, + }); + expect(github.rest.pulls.listFiles).not.toHaveBeenCalled(); expect(github.rest.repos.createCommitStatus).not.toHaveBeenCalled(); }); - it("invalidates a final result if its workflow is later canceled", async () => { - const github = createFinalizeGithub({ - statuses: [ - { - context: ARM_SEMANTIC_REVIEW_STATUS, - state: CommitStatusState.SUCCESS, - description: "Passed: full review completed with no Blocking findings", - target_url: targetUrl, - updated_at: "2026-01-01T00:00:00Z", - }, - ], + it("publishes Review incomplete when the PR exceeds automated review size limits", async () => { + const github = createFinalizeGithub(); + const oversizedFiles = Array.from({ length: 51 }, (_, i): ChangedFile => ({ + filename: `specification/foo/foo${i}.json`, + additions: 1, + deletions: 0, + })); + github.rest.pulls.get.mockResolvedValue({ + data: { changed_files: 51, state: "open", head: { sha: headSha } }, }); + github.rest.pulls.listFiles.mockResolvedValue({ data: oversizedFiles }); - await expect(finalize({ github, workflowConclusion: "cancelled" })).resolves.toBe(true); - expect(github.rest.repos.createCommitStatus).toHaveBeenCalledWith( + const { github: g } = await runFinalizer({ github }); + expect(g.rest.repos.createCommitStatus).toHaveBeenCalledWith( expect.objectContaining({ - state: CommitStatusState.PENDING, - description: "Review canceled; rerun required", + state: CommitStatusState.ERROR, + description: "Review incomplete: PR exceeds automated review size limits", }), ); }); - it("ignores stale heads and superseded runs", async () => { - const staleGithub = createFinalizeGithub({ - currentHeadSha: "fedcba9876543210fedcba9876543210fedcba98", + it("truncates an overly long error description to 140 characters", async () => { + const longValue = "x".repeat(200); + const { github } = await runFinalizer({ + output: { items: [{ ...agentOutput().items[0], run_attempt: longValue }] }, }); - await expect(finalize({ github: staleGithub })).resolves.toBe(false); - expect(staleGithub.rest.repos.createCommitStatus).not.toHaveBeenCalled(); - const supersededGithub = createFinalizeGithub({ - statuses: [ + const call = github.rest.repos.createCommitStatus.mock.calls[0]?.[0] as { + description: string; + }; + expect(call.description.length).toBeLessThanOrEqual(140); + expect(call.description).toMatch(/^Review incomplete:/); + }); + + it("does not overwrite a newer review of the same SHA", async () => { + const github = createFinalizeGithub(); + github.rest.repos.listCommitStatusesForRef.mockResolvedValue({ + data: [ { context: ARM_SEMANTIC_REVIEW_STATUS, state: CommitStatusState.PENDING, - description: "ARM API semantic review is pending", target_url: "https://github.com/Azure/azure-rest-api-specs/actions/runs/789/attempts/1", updated_at: "2026-01-02T00:00:00Z", }, ], }); - await expect(finalize({ github: supersededGithub })).resolves.toBe(false); - expect(supersededGithub.rest.repos.createCommitStatus).not.toHaveBeenCalled(); + + const execution = await runFinalizer({ github }); + expect(execution.result).toEqual({ + headSha, + issueNumber, + statusPublished: false, + }); + expect(github.rest.repos.createCommitStatus).not.toHaveBeenCalled(); }); - it("overrides an older Passed status when a newer review fails before setting pending", async () => { - const github = createFinalizeGithub({ - statuses: [ + it("does not overwrite a newer attempt of the same run", async () => { + const github = createFinalizeGithub(); + github.rest.repos.listCommitStatusesForRef.mockResolvedValue({ + data: [ { context: ARM_SEMANTIC_REVIEW_STATUS, - state: CommitStatusState.SUCCESS, - description: "Passed: full review completed with no Blocking findings", - target_url: "https://github.com/Azure/azure-rest-api-specs/actions/runs/123/attempts/1", - updated_at: "2026-01-01T00:00:00Z", + state: CommitStatusState.PENDING, + target_url: `https://github.com/Azure/azure-rest-api-specs/actions/runs/${runId}/attempts/2`, + updated_at: "2026-01-02T00:00:00Z", }, ], }); - await finalize({ - github, - artifactNames: [], - workflowConclusion: "failure", - }); - expect(github.rest.repos.createCommitStatus).toHaveBeenCalledWith( - expect.objectContaining({ - state: CommitStatusState.ERROR, - description: "Review incomplete: workflow concluded with failure", - }), - ); - }); - - it("lists commit statuses for the exact reviewed head", async () => { - const github = createFinalizeGithub(); - - await finalize({ github }); - expect(github.rest.repos.listCommitStatusesForRef).toHaveBeenCalledWith({ - owner, - repo, - ref: headSha, - per_page: PER_PAGE_MAX, + const execution = await runFinalizer({ github }); + expect(execution.result).toEqual({ + headSha, + issueNumber, + statusPublished: false, }); + expect(github.rest.repos.createCommitStatus).not.toHaveBeenCalled(); }); }); diff --git a/.github/workflows/test/arm-auto-signoff/arm-universal-auto-signoff.test.ts b/.github/workflows/test/arm-auto-signoff/arm-universal-auto-signoff.test.ts index d8f6f777ca73..b610f21e865d 100644 --- a/.github/workflows/test/arm-auto-signoff/arm-universal-auto-signoff.test.ts +++ b/.github/workflows/test/arm-auto-signoff/arm-universal-auto-signoff.test.ts @@ -156,15 +156,6 @@ describe("getLabelActionImpl", () => { updated_at: "2026-01-01", }, }, - { - name: "review incomplete", - status: { - context: "ARM Semantic Review", - state: CommitStatusState.ERROR, - description: "Review incomplete: workflow concluded with failure", - updated_at: "2026-01-01", - }, - }, ])( "does not sign off or require manual signoff when semantic review is $name", async ({ status }) => { @@ -181,7 +172,28 @@ describe("getLabelActionImpl", () => { }, ); - it("adds manual signoff only when automated coverage was scoped", async () => { + it("adds manual signoff when semantic review is incomplete", async () => { + const github = createMockGithub({ + labelNames: ["ARMReview"], + statuses: [ + { + context: "ARM Semantic Review", + state: CommitStatusState.ERROR, + description: "Review incomplete: workflow concluded with failure", + updated_at: "2026-01-01", + }, + ...deterministicStatuses, + ], + }); + + const result = await run(github); + expect(result.labelActions).toEqual({ + [ArmAutoSignoffLabel.ArmAutoSignedOffTest]: LabelAction.None, + [ArmAutoSignoffLabel.ArmManualSignoffRequired]: LabelAction.Add, + }); + }); + + it("adds manual signoff when automated coverage was scoped", async () => { const github = createMockGithub({ labelNames: ["ARMReview"], statuses: [ diff --git a/documentation/arm-semantic-review-auto-signoff-implementation.md b/documentation/arm-semantic-review-auto-signoff-implementation.md index fd651bb23b8c..e560af983ecd 100644 --- a/documentation/arm-semantic-review-auto-signoff-implementation.md +++ b/documentation/arm-semantic-review-auto-signoff-implementation.md @@ -2,84 +2,41 @@ ## Purpose -This document explains how the ARM API Reviewer integrates with Universal ARM -Auto-Signoff, why the implementation uses both workflow artifacts and commit -statuses, which alternatives were considered, and what risks remain. +This guide explains the pilot integration between the ARM API Reviewer and +Universal Auto-Signoff. The legacy ARM Auto SignOff workflows remain the +production signoff path while Universal runs in parallel. -It is also a reading guide for someone who is new to GitHub Actions, workflow -coordination, and automated signoff policy. +The policy is intentionally small: -The implementation is currently a pilot. Universal Auto-Signoff manages -`ARMAutoSignedOff-Test`; it does not yet replace the production -`ARMSignedOff` process. - -## Start with this mental model - -The whole design can be reduced to one sentence: - -> The ARM API Reviewer produces evidence for one workflow run, trusted code -> converts that evidence into a result for one commit SHA, and Universal -> Auto-Signoff signs off only when every required result for the current SHA -> passes. - -There are two representations of the semantic result: - -| Representation | Mental model | Lifetime | -| ----------------- | ---------------------------------------------------- | -------------------------------- | -| Workflow artifact | "This reviewer run observed this result." | Associated with one workflow run | -| Commit status | "This is the current semantic state of this commit." | Associated with one commit SHA | - -The artifact is an intermediate handoff. The commit status is the durable input -that every later auto-signoff evaluation reads. - -## Vocabulary - -### PR head SHA - -The immutable Git commit at the tip of the pull request. - -All evidence used for one auto-signoff decision must refer to the same head SHA. -A result for an older SHA cannot authorize a newer SHA. - -### Workflow run - -One execution of a GitHub Actions workflow. A rerun has the same run ID and a -new run attempt. - -### Workflow artifact - -Data associated with one workflow run. In this design, artifacts carry: - -- the PR number; -- the reviewed head SHA; and -- the semantic-review receipt. - -Artifacts are not the long-lived auto-signoff state. +```text +ARM Semantic Review passed for the current PR head +AND Swagger LintDiff passed for the current PR head +AND Swagger Avocado passed for the current PR head +AND current labels and approvals allow signoff += add ARMAutoSignedOff-Test +``` -### Commit status +Only `Passed` authorizes the pilot label. Missing, Pending, Changes requested, +and Review incomplete all block it. Universal never changes `ARMSignedOff`. -A GitHub status attached to a commit SHA and named by a context, for example: +## Mental model ```text -ARM Semantic Review = success -Swagger LintDiff = success -Swagger Avocado = success -``` - -GitHub retains status history. When the same context is published more than -once for one SHA, the newest entry is treated as current. +ARM API Reviewer + produces one structured semantic item in agent_output.json -### ARM queue labels +ARM Semantic Review - Set Status + validates the item and publishes a commit status on its SHA -The existing human-review state includes: +Universal Auto-Signoff + reads statuses for the current PR head and decides -- `NotReadyForARMReview` -- `WaitForARMFeedback` -- `ARMChangesRequested` -- `ARMSignedOff` +Update Labels + applies the decision if the PR head is still current +``` -`ARMManualSignoffRequired` is the proposed hard stop that prevents automatic -signoff when a human decision is required. +The status workflow may publish a result for an older SHA. That is safe: +Universal Auto-Signoff reads statuses only for the PR's current SHA. ## Components @@ -93,49 +50,44 @@ Generated workflow: Responsibilities: -- review the cumulative PR diff at an exact head SHA; +- review the cumulative PR diff at a pinned head SHA; - use the Critic to verify findings; -- reconcile findings with existing PR discussion; -- publish comments and update ARM queue labels; -- record a structured semantic result. +- reconcile current findings with existing PR discussion; +- publish comments and ARM queue-label changes; +- emit one `record_arm_semantic_review` item. -### Semantic state model +### Semantic policy [arm-semantic-review.ts](../.github/workflows/src/arm-auto-signoff/arm-semantic-review.ts) Responsibilities: -- parse and validate model output; -- encode and parse semantic receipts; -- map evidence to semantic outcomes; -- evaluate trusted file-count and line-count coverage; -- interpret the latest semantic commit status. +- parse exactly one structured semantic item; +- validate run attempt, PR number, SHA, Blocking count, scope, and completion; +- map the evidence to Passed, Changes requested, or Review incomplete; +- interpret GitHub commit-status states; +- select the newest semantic status for a SHA. -This module contains pure state and parsing logic. It does not orchestrate -GitHub workflows. +This file is pure policy and parsing. It makes no GitHub API calls. -### Semantic workflow orchestration - -[arm-semantic-review-workflow.ts](../.github/workflows/src/arm-auto-signoff/arm-semantic-review-workflow.ts) - -Responsibilities: - -- validate the agent's result against the live PR; -- finalize a completed reviewer workflow; -- reject stale and superseded runs; -- independently verify automated-review coverage; -- publish the final `ARM Semantic Review` commit status. - -### Semantic status workflow +### Semantic Set Status orchestration +Workflow: [arm-semantic-review-status.yaml](../.github/workflows/arm-semantic-review-status.yaml) +Implementation: +[arm-semantic-review-status.ts](../.github/workflows/src/arm-auto-signoff/arm-semantic-review-status.ts) + Responsibilities: -- trigger directly from the completed ARM API Reviewer run; -- invoke semantic finalization with `statuses: write`; -- upload trusted PR and SHA correlation artifacts; -- provide the completion event that triggers Universal Auto-Signoff. +- trigger from the exact completed reviewer run; +- download that run's `agent_output.json`; +- read trusted PR/SHA correlation artifacts; +- validate the semantic item once; +- publish `ARM Semantic Review` on the reviewed SHA; +- upload PR/SHA correlation for Universal Auto-Signoff. + +Only this workflow has `statuses: write`. ### Universal Auto-Signoff @@ -147,550 +99,302 @@ Decision logic: Responsibilities: -- read statuses for the current head SHA; -- evaluate semantic review, LintDiff, Avocado, labels, and approvals; -- upload label-action artifacts. +- resolve the current PR and current head SHA; +- read Semantic Review, LintDiff, and Avocado statuses for that SHA; +- evaluate current labels and approvals; +- manage the pilot `ARMAutoSignedOff-Test` label and add the manual-review label + when semantic evidence is incomplete. -Universal Auto-Signoff has `statuses: read`. Status publication is isolated in -the semantic status workflow. +Universal has only `statuses: read`. ### Label application -Workflow: -[update-labels.yaml](../.github/workflows/update-labels.yaml) - -Implementation: -[update-labels.ts](../.github/workflows/src/update-labels.ts) - -Responsibilities: - -- consume trusted label artifacts; -- recheck the live PR head when the producer supplied a head-SHA artifact; -- apply label additions and removals. +[update-labels.ts](../.github/workflows/src/update-labels.ts) applies label +artifacts. When a producer supplies a head-SHA artifact, it confirms that the PR +is still open and still has that head before mutating labels. ## End-to-end flow ```mermaid flowchart TD - A[Eligible PR trigger or /arm-review] --> B[ARM API Reviewer starts] - B --> C[Resolve PR and exact head SHA] - C --> D[Publish ARM Semantic Review = Pending] - D --> E[Agent reviews cumulative PR diff] - E --> F[Critic verifies findings] - F --> G[Reconcile existing comments] - G --> H[Queue comments and ARM label changes] - G --> I[Record validated semantic receipt] - - H --> J[Reviewer workflow completes] - I --> J - - J --> K[Semantic Set Status workflow_run trigger] - K --> L[Read exact reviewer run artifacts] - L --> M[Validate PR, SHA, attempt, scope, and coverage] - M --> N[Publish final ARM Semantic Review status] - - N --> O[Universal Auto-Signoff workflow_run trigger] - O --> P[Read all statuses for current head SHA] - Y[LintDiff, Avocado, PR, or label event] --> P - - P --> Q{Semantic status is Passed?} - Q -- No --> R[Do not auto-sign off] - Q -- Manual review required --> S[Add ARMManualSignoffRequired] - Q -- Yes --> T{LintDiff, Avocado,\nlabels, and approvals pass?} - T -- No --> R - T -- Yes --> U[Add ARMAutoSignedOff-Test] - - U --> V[Update Labels workflow] - S --> V - V --> W[Recheck live PR head] - W --> X[Apply labels if still current] + A[Eligible reviewer trigger] --> B[Set Semantic Review Pending on reviewed SHA] + B --> C[Agent and Critic review PR] + C --> D[Publish comments and queue-label changes] + C --> E[Emit record_arm_semantic_review item] + D --> F[Reviewer workflow completes] + E --> F + + F --> G[Semantic Set Status workflow] + G --> H[Download exact run agent_output.json] + H --> I[Validate attempt, PR, SHA, Blocking count, scope, and completion] + I --> J[Publish semantic status on reviewed SHA] + J --> K[Universal Auto-Signoff] + + L[LintDiff, Avocado, PR, or label event] --> K + K --> M[Read all statuses for current PR head] + M --> N{All requirements pass?} + N -- No --> O[Remove stale pilot signoff] + N -- Yes --> P[Add ARMAutoSignedOff-Test artifact] + P --> Q[Update Labels rechecks current head] + Q --> R[Apply label] ``` -## Detailed workflow - -### 1. Start the reviewer - -The reviewer can start from: +## Structured semantic item -- an eligible PR open, synchronize, or ready-for-review event; -- adding `WaitForARMFeedback`; -- an authorized `/arm-review` comment; or -- manual workflow dispatch. +The agent emits: -The reviewer has one concurrency group per PR with: - -```yaml -cancel-in-progress: true +```json +{ + "type": "record_arm_semantic_review", + "run_attempt": "1", + "issue_number": "123", + "head_sha": "abc123abc123abc123abc123abc123abc123abcd", + "blocking_count": "0", + "scope": "full", + "completeness": "complete" +} ``` -A newer eligible trigger replaces an active reviewer run for the same PR. -Unrelated comments receive run-specific concurrency groups and do not cancel an -active review. - -### 2. Correlate the PR and set Pending - -Before agent execution, trusted pre-activation code: - -1. validates the target PR number; -2. reads the exact current head SHA; -3. uploads `issue-number=` and `head-sha=` artifacts; -4. publishes: - -```text -SHA: -Context: ARM Semantic Review -State: pending -Description: ARM API semantic review is pending -Target URL: exact reviewer run and attempt -``` - -Publishing Pending is important when the same SHA is reviewed again. It prevents -an older Passed status from continuing to authorize signoff while the new review -is in progress. - -### 3. Perform semantic review - -The reviewer evaluates: - -```text -base SHA -> current head SHA -``` - -It reviews the cumulative PR, not only the most recent commit. +Fields: -The reviewer: +| Field | Meaning | +| ---------------- | ---------------------------------------------------------------- | +| `run_attempt` | Reviewer workflow attempt that produced the item | +| `issue_number` | Reviewed PR | +| `head_sha` | Reviewed commit | +| `blocking_count` | Verified Blocking findings still applicable after reconciliation | +| `scope` | Whether the complete PR or a size-limited subset was reviewed | +| `completeness` | Whether the review completed normally, incompletely, or degraded | -- identifies semantic API-design findings; -- asks the Critic to verify findings; -- reads existing workflow-owned and human-authored discussion; -- reconciles current findings with prior comments; -- posts only net-new or relocated findings; -- resolves fixed workflow-owned findings. +The reviewer custom safe-output job exists because gh-aw requires a job for a +typed custom output. It has no token permissions and performs no authoritative +validation. -An unresolved verified Blocking finding still counts when its reconciliation -action is `SKIP-COVERED`. "No new comment was posted" does not mean the PR is -clean. +## Semantic status publication -### 4. Update the ARM queue state +`ARM Semantic Review - Set Status` receives the exact reviewer run ID through +`workflow_run`. -The reviewer applies these rules: +It: -| Review result | Queue behavior | -| ---------------------------------------------------------------- | --------------------------------------------------------------- | -| Scoped, incomplete, or degraded | Preserve existing queue labels | -| Full and complete, with applicable verified Blocking findings | Add `ARMChangesRequested`; remove `WaitForARMFeedback` | -| Full and complete, with no applicable verified Blocking findings | Remove stale `ARMChangesRequested`; retain `WaitForARMFeedback` | -| Critic unavailable | Preserve existing queue labels | +1. lists that run's artifact names once; +2. reads trusted `head-sha` and `issue-number` artifacts; +3. downloads that run's `agent` artifact; +4. finds exactly one semantic item; +5. validates the attempt, PR, SHA, Blocking count, scope, and completion; +6. publishes a status on the semantic item's SHA. -Partial reviews preserve the human queue even when they identify Blocking -findings. They cannot make a complete queue-state decision. +Outcome mapping: -### 5. Record the semantic receipt +| Evidence | Status | +| --------------------------------------------------------------------------------------------- | ---------------------------- | +| Full, complete review with `blocking_count = 0` | `success`: Passed | +| Full, complete review with `blocking_count > 0` | `failure`: Changes requested | +| Scoped, incomplete, or degraded review, regardless of `blocking_count` | `error`: Review incomplete | +| Workflow failure, missing output, malformed output, duplicate item, or mismatched correlation | `error`: Review incomplete | +| Canceled reviewer run | Leave Pending unchanged | +| Missing trusted PR/SHA artifacts | Leave status unchanged | -After reconciliation and summary generation, the agent calls one structured -safe output with: +The status is head-bound: ```text -PR number -reviewed head SHA -scope: full | scoped -completeness: complete | incomplete | degraded -number of verified Blocking findings still applicable +SHA: reviewed SHA +Context: ARM Semantic Review +State: success | failure | error | pending +Target URL: exact reviewer run and attempt ``` -Example artifact name: +## Stale-result protection -```text -arm-semantic-review=1.123.abc123abc123abc123abc123abc123abc123abcd.full.complete.0 -``` +Set Status publishes only while the PR is open and its current head matches the +reviewed SHA. A completion for a closed PR or stale head leaves the existing +status unchanged. -Fields: +Universal independently reads statuses only for the current head SHA: ```text -1 workflow attempt -123 PR number -abc123... reviewed head SHA -full review scope -complete review completeness -0 currently applicable verified Blocking findings +Reviewer publishes Passed on SHA A +PR head is now SHA B +Universal reads statuses only for SHA B +No Semantic Passed exists on B +Universal does not sign off ``` -A trusted safe-output job validates the receipt and uploads it. The agent does -not directly publish a final Passed status. - -### 6. Wait for the whole reviewer workflow +## Universal decision -The generated reviewer workflow includes: +Universal fetches all commit statuses for the current head SHA and selects the +newest entry for: -- agent execution; -- threat detection; -- custom semantic receipt validation; -- comment and label safe-output processing; and -- a final conclusion job. +- `ARM Semantic Review`; +- `Swagger LintDiff`; +- `Swagger Avocado`. -The dedicated Semantic Set Status workflow finalizes the result only after the -reviewer emits `workflow_run: completed`. - -This boundary is intended to prevent Passed from becoming visible while the -reviewer is still publishing comments or labels. - -### 7. Finalize the semantic result - -`ARM Semantic Review - Set Status` is triggered with the exact completed ARM -reviewer run. It calls `finalizeArmSemanticReview`. - -It verifies: - -- the reviewer run's artifacts are listed once and reused for both correlation - and receipt finalization; -- the reviewer actually executed; -- the PR number and head SHA are valid; -- the PR is still open; -- the reviewed SHA is still the current head; -- a newer run or attempt did not supersede this one; -- the receipt belongs to this PR, SHA, and run attempt; -- the changed-file inventory is not truncated; -- no more than 50 specification files changed; -- no more than 5,000 specification lines changed. - -Trusted code independently evaluates size and truncation. It does not rely only -on the model's `scope` value. - -### 8. Publish the final commit status - -| Semantic outcome | GitHub state | Description pattern | -| ---------------------- | ------------ | ---------------------------------- | -| Passed | `success` | `Passed: ...` | -| Changes requested | `failure` | `Changes requested: ...` | -| Manual review required | `error` | `Manual review required: ...` | -| Review incomplete | `error` | `Review incomplete: ...` | -| Pending | `pending` | Review is running or must be rerun | - -`Manual review required` is used for deterministic coverage limits such as: - -- oversized PRs; -- scoped review; -- truncated changed-file inventory. - -`Review incomplete` is used for retryable or technical failures such as: - -- reviewer workflow failure; -- missing receipt; -- malformed receipt; -- degraded full review. - -After finalization, the status workflow uploads the validated PR number and head -SHA as correlation artifacts. Universal Auto-Signoff is triggered by completion -of `ARM Semantic Review - Set Status`. - -### 9. Evaluate auto-signoff - -The normal auto-signoff decision runs on every Universal Auto-Signoff event: - -- semantic status completion; -- LintDiff completion; -- Avocado completion; -- PR update; -- relevant label addition or removal. - -It fetches all commit statuses for the current head SHA, then selects the newest -status for each required context: - -- `ARM Semantic Review` -- `Swagger LintDiff` -- `Swagger Avocado` - -Effective policy: +Decision: ```text -if semantic outcome is Manual review required: - add ARMManualSignoffRequired - do not sign off +Semantic Review is missing or Pending + -> wait and remove stale pilot signoff -if semantic outcome is not Passed: - do not sign off +Semantic Review requested changes + -> remove pilot signoff -if ARMChangesRequested is present: - do not sign off +Semantic Review is incomplete + -> add ARMManualSignoffRequired + -> retain WaitForARMFeedback + -> remove pilot signoff -if ARMManualSignoffRequired is present: - do not sign off +ARMManualSignoffRequired is present + -> no pilot signoff -if LintDiff or Avocado is not successful: - do not sign off +LintDiff or Avocado is not successful + -> no pilot signoff -if required approval is missing: - do not sign off +required approval is missing + -> no pilot signoff -otherwise: - add ARMAutoSignedOff-Test +otherwise + -> add ARMAutoSignedOff-Test ``` -Only Passed is positive authorization. Missing, Pending, Failure, and Review -incomplete all fail closed. - -### 10. Apply labels safely - -Universal Auto-Signoff uploads label artifacts instead of directly mutating -labels. - -When the producer includes a trusted head-SHA artifact, Update Labels reads the -live PR again. It discards the label action if: - -- the PR is closed; or -- the PR head moved. - -This prevents a decision calculated for one commit from changing labels on a -later commit. +`ARMAutoSignedOff-Test` records the pilot decision without changing production +signoff. `ARMManualSignoffRequired` remains an explicit human veto and is never +removed automatically. ## Event-order examples -### Reviewer completes before Avocado +### Avocado completes first ```text -Reviewer starts -> Semantic = Pending -Reviewer completes -> Semantic Set Status publishes Passed -Auto-Signoff evaluates -> Avocado still pending, no signoff -Avocado completes -> Auto-Signoff rereads all current-SHA statuses -All pass -> Add pilot auto-signoff label +Semantic = Pending +Avocado = Passed +Universal runs +Semantic is not Passed +No signoff + +Reviewer completes +Set Status publishes Semantic = Passed +Universal runs again +All statuses pass +Signoff can proceed ``` -### Avocado completes before reviewer +### Reviewer completes first ```text -Reviewer starts -> Semantic = Pending -Avocado completes -> Auto-Signoff sees Pending, no signoff -Reviewer completes -> Semantic Set Status publishes Passed -Auto-Signoff rereads Avocado and LintDiff -All pass -> Add pilot auto-signoff label +Set Status publishes Semantic = Passed +Universal runs +Avocado is pending +No signoff + +Avocado completes +Universal runs again +All statuses pass +Signoff can proceed ``` -### New commit arrives during review +### New commit during review ```text -Reviewer is reviewing SHA A -PR moves to SHA B -Finalizer sees current head != reviewed SHA -SHA A result is ignored -SHA B requires its own semantic review +Reviewer publishes result on SHA A +Current PR head is SHA B +Universal reads B +No semantic status for B +No signoff ``` -### Same SHA is retriggered +### Same-SHA rerun ```text -Old result for SHA A = Passed -New review starts for SHA A -New Pending entry becomes the newest semantic status -Auto-Signoff waits -New final result replaces Pending as the newest entry +Old status on A = Passed +New review starts on A +New Pending entry becomes newest +Universal waits +Set Status publishes the new final result ``` -### Oversized PR +### Missing or malformed semantic output ```text -Trusted code counts > 50 specification files or > 5,000 changed lines -Semantic result = Manual review required -Auto-Signoff does not proceed -ARMManualSignoffRequired is added +Set Status publishes Review incomplete on reviewed SHA +Universal adds ARMManualSignoffRequired +No automatic signoff ``` -## Why this design was chosen - -### Independent workflows complete in any order - -The reviewer, LintDiff, and Avocado are independent workflows. A durable -SHA-bound status lets any completion event reevaluate the same current state. - -### Labels are not SHA-bound - -The absence of `ARMChangesRequested` does not prove that: - -- the reviewer ran; -- the reviewer finished; -- the review covered the whole PR; or -- the result belongs to the current commit. - -### Workflow conclusion is not the semantic result +## Why this design -A reviewer workflow can finish successfully while: +### Commit status instead of labels -- finding Blocking issues; -- reviewing only part of a PR; -- producing incomplete semantic evidence. +Commit statuses are SHA-bound. PR labels are not. -### Model output should not directly authorize signoff +### Commit status instead of artifact-only lookup -The model emits evidence. Trusted code verifies correlation and deterministic -coverage before publishing the durable status. +Any later event can read the current SHA's status without searching reviewer-run +history or downloading old artifacts. -### Finalization should occur after reviewer completion +### Dedicated Set Status workflow -Publishing Passed inside the agent job can expose success before comment and -label publishing completes. `workflow_run: completed` provides a later -coordination boundary. - -## Alternative designs and tradeoffs - -| Design | Advantages | Tradeoffs | -| -------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------- | -| **Current: receipt, dedicated Set Status, then commit status** | SHA-bound, reusable by every trigger, fail-closed, least-privilege status writer, matches LintDiff and Avocado | Adds one workflow hop and another workflow to maintain | -| Finalize inside Universal Auto-Signoff | Fewer workflow files | Gives Universal `statuses: write` on every trigger and mixes publication with policy evaluation | -| One combined workflow | Strong ordering and direct outputs | Tightly couples AI review, linters, labels, retries, and every trigger | -| Label-only handshake | Simple and visible | Labels are mutable PR state, not SHA-bound; bot-label events can be unreliable | -| Artifact-only lookup | No commit status needed | Every event must search historical runs and artifacts; expiration and supersession are complex | -| Publish final status inside reviewer | Fewer workflow steps | Risks exposing Passed before comments and labels finish | -| GitHub Check Run | Rich UI and potentially stronger producer identity | More API and permission complexity; may require a dedicated GitHub App | -| Reusable `workflow_call` pipeline | Typed outputs and strong ordering | Difficult to retrofit around independent triggers and manual review commands | - -## Trust boundaries - -### Untrusted or partially trusted - -- PR content; -- specification files; -- PR description and comments; -- model reasoning; -- model-supplied semantic fields. - -### Trusted - -- workflow source from the base/default branch; -- pre-activation PR and SHA lookup; -- workflow artifacts uploaded by privileged/default-branch workflows; -- semantic receipt validation; -- trusted file-count and line-count evaluation; -- commit-status publication; -- final live-head check before correlated label application. - -## Risk assessment - -### Rollout blocker: `ARMManualSignoffRequired` does not exist - -At the time of writing, GitHub returns 404 for the repository label -`ARMManualSignoffRequired`, and it is not defined in checked-in label -configuration. - -Expected failure: +It matches LintDiff and Avocado: ```text -Semantic outcome = Manual review required -Universal uploads label artifact -Update Labels attempts to add nonexistent label -Label application fails +producer workflow + -> Set Status workflow + -> Universal evaluator ``` -Semantic safety is preserved because the result is not Passed, but the intended -manual-review routing does not appear. - -Before rollout: - -1. Create `ARMManualSignoffRequired` in both spec repositories. -2. Decide whether it should be listed in - [protected-labels.yml](../.github/protected-labels.yml). -3. Document who may add and remove it. - -### High before production: commit-status provenance is not authenticated - -Auto-signoff trusts the newest status whose context is `ARM Semantic Review`. -It does not verify: - -- the status creator; -- the target workflow URL; -- the workflow name; -- that the run belongs to this repository; -- that the run has a valid semantic receipt. +It also keeps `statuses: write` out of Universal. -Any workflow or identity with `statuses: write` could publish a competing -success status. +### One authoritative validation -This is lower impact while the pilot controls only `ARMAutoSignedOff-Test`, but -it should be hardened before controlling production `ARMSignedOff`. +Validation occurs only in Set Status. The reviewer produces the typed item; Set +Status validates and publishes it. -Possible mitigations: +## Security and reliability properties -- validate the status creator; -- require a target URL matching a trusted Actions run; -- resolve the run and verify its workflow name; -- publish through a dedicated GitHub App or Check Run identity. +- no untrusted PR code is checked out; +- only write-access users or the trusted bot can trigger reviewer writes; +- Set Status consumes the exact completed reviewer run ID; +- the semantic item is bound to the workflow attempt, PR number, and SHA; +- malformed, missing, duplicate, mismatched, scoped, incomplete, or degraded + output fails closed; +- only `success` permits signoff; +- Universal always reads the current head SHA; +- delayed label artifacts are rechecked against the live PR head; +- Universal has `statuses: read`; +- only Set Status has `statuses: write`. -### High or medium: semantic truth still comes from model output +## Accepted tradeoffs -Trusted code validates shape, correlation, and coverage. It does not -independently prove: +### The model is the semantic source of truth -- that the review was complete; -- that `blocking_count` matches the findings; -- that a Blocking finding was not omitted; -- that the receipt agrees with queued label changes. +The same verified findings drive: -The live `ARMChangesRequested` check provides defense in depth, but inconsistent -model output plus failed label output could theoretically produce Passed. +- comments authors act on; +- `ARMChangesRequested`; +- semantic Blocking count. -Possible mitigations: +Set Status validates structure and correlation; it does not independently +re-review semantic correctness. -- derive Blocking count from structured finding outputs; -- cross-check the receipt against safe-output items; -- require a Passed receipt to agree with the queued label transition; -- reject Passed when safe-output processing reports failed items. +### Coupling to gh-aw agent output -### Medium: manual override removal is not protected - -The current protected-label workflow watches only `labeled`, not `unlabeled`. - -An unauthorized user with label permissions could remove a manually applied -`ARMManualSignoffRequired`. If semantic status is Passed, automation may then -proceed. - -Possible mitigations: - -- enforce authorization on `unlabeled`; -- record the hold in a trusted status or review record; -- require an authorized actor to clear the hold. - -### Medium: stale reviewer runs can mutate PR-level queue labels - -Semantic statuses are SHA-bound. Reviewer safe outputs for -`ARMChangesRequested` and `WaitForARMFeedback` are direct PR-level mutations. - -A narrow race remains: +Set Status expects: ```text -Old run verifies head SHA -New commit arrives -Old run applies queue label mutation +artifact name: agent +file: agent_output.json +items[] with type record_arm_semantic_review ``` -The semantic gate still prevents unsafe auto-signoff, but the human queue can -become confusing. - -Possible mitigation: route reviewer queue-label changes through a correlated -artifact workflow with a final head-SHA check. - -### Medium, pending confirmation: partial safe-output failure - -The generated safe-output job exposes: - -- `items_failed`; -- `items_succeeded`; -- `process_safe_outputs_status`. - -The finalizer does not inspect those outputs. - -Confirm whether gh-aw can complete the workflow successfully when individual -comment or label items fail. If it can, reject Passed whenever safe-output -processing reports failed or canceled items. - -### Medium operational risk: private-repository parity - -The ARM reviewer source states that public and private repository copies must -remain byte-identical. +A gh-aw format change can break extraction. That failure produces Review +incomplete and cannot authorize signoff. -Rollout must update `Azure/azure-rest-api-specs-pr` in coordination with this -repository. +### PR-level comment and label races -### Medium operational constraint: workflow-run chain depth +Comments and queue labels are not transactional. A canceled run can publish +some outputs before cancellation. Reconciliation reduces duplicate comments, +and the current-SHA semantic gate prevents stale auto-signoff. -The success path is: +### Workflow chain depth ```text ARM API Reviewer @@ -699,383 +403,52 @@ ARM API Reviewer -> Update Labels ``` -This uses the same three-level `workflow_run` chain pattern as LintDiff and -Avocado and reaches GitHub's supported chaining limit. Do not insert another -downstream `workflow_run` workflow after Update Labels; it may not run. +This reaches GitHub's supported `workflow_run` chain depth. Do not add another +downstream `workflow_run` workflow after Update Labels. -### Missing correlation artifacts +## Operational prerequisites -If the reviewer run does not contain trusted PR and head-SHA artifacts, the -status workflow leaves semantic status unchanged. It does not substitute the -PR's current head because that could attach an older run's result to a newer -commit. +- ensure `ARMAutoSignedOff-Test` and `ARMManualSignoffRequired` exist; +- mirror the ARM reviewer workflow to `azure-rest-api-specs-pr`; +- document who may remove manual hold labels; +- monitor incomplete/degraded rates and label transitions after rollout. -### Low: timestamp ordering +## How to understand the implementation -The latest status is selected by `updated_at`. +Read in this order: -If two entries have identical timestamp resolution, ordering relies on GitHub's -return order and stable sorting. A status ID or run-attempt comparison would be -a stronger tiebreaker. +1. [arm-semantic-review-status.yaml](../.github/workflows/arm-semantic-review-status.yaml) +2. [finalizeArmSemanticReview](../.github/workflows/src/arm-auto-signoff/arm-semantic-review-status.ts) +3. [parseSemanticReviewResult](../.github/workflows/src/arm-auto-signoff/arm-semantic-review.ts) +4. [arm-universal-auto-signoff.ts](../.github/workflows/src/arm-auto-signoff/arm-universal-auto-signoff.ts) +5. tests: + - [arm-semantic-review.test.ts](../.github/workflows/test/arm-auto-signoff/arm-semantic-review.test.ts) + - [arm-semantic-review-status-workflow.test.ts](../.github/workflows/test/arm-auto-signoff/arm-semantic-review-status-workflow.test.ts) + - [arm-universal-auto-signoff.test.ts](../.github/workflows/test/arm-auto-signoff/arm-universal-auto-signoff.test.ts) -### Low: canceled-run comment duplication - -Cancellation is not transactional. A canceled safe-output publisher may have -already posted some comments. The replacement run reconciles marked findings, -which reduces but cannot eliminate duplication. - -This is primarily a PR-noise risk, not an auto-signoff safety risk. - -### Low: manual-review label persists - -`ARMManualSignoffRequired` is intentionally never removed automatically. - -If an oversized PR is later reduced and receives a clean full review, a human -must clear the label before auto-signoff can proceed. - -This is safe, but should be documented for authors and reviewers. - -## How to learn this design from zero - -Do not start by reading the generated workflow or every test. Build the model in -layers. - -### Layer 1: learn the policy - -Write this decision rule on paper: - -```text -Semantic Passed -AND LintDiff Passed -AND Avocado Passed -AND required labels and approvals allow signoff -= auto-signoff -``` - -Everything else is implementation detail supporting this rule. - -Questions to answer: - -- What is a positive authorization? -- Which states merely block? -- Which states require human routing? -- Which result must be tied to a commit? - -### Layer 2: learn the identities - -Track four IDs: - -```text -PR number -head SHA -workflow run ID -workflow attempt -``` - -Use this table: - -| Identity | What it distinguishes | -| ----------- | -------------------------------------- | -| PR number | The review conversation and labels | -| Head SHA | The exact code/specification snapshot | -| Run ID | One workflow execution | -| Run attempt | A rerun of the same workflow execution | - -Most race conditions are an accidental mismatch among these identities. - -### Layer 3: learn durable versus transient state - -Classify each signal: - -| Signal | Scope | -| ---------------- | ---------------------------- | -| Receipt artifact | One workflow run and attempt | -| Commit status | One commit SHA | -| Label | Entire PR until changed | -| Comment | Entire PR discussion | - -Then ask: - -> Is this state being used outside the scope where it is valid? - -That question explains why labels alone are insufficient and why old artifacts -must not authorize new commits. - -### Layer 4: trace only the happy path - -Read these files in order: - -1. [arm-api-review.md](../.github/workflows/arm-api-review.md), focusing only on: - - trigger and concurrency; - - Pending status; - - Step 7; - - Step 9. -2. [arm-semantic-review-status.yaml](../.github/workflows/arm-semantic-review-status.yaml) -3. [arm-semantic-review.ts](../.github/workflows/src/arm-auto-signoff/arm-semantic-review.ts) -4. [arm-semantic-review-workflow.ts](../.github/workflows/src/arm-auto-signoff/arm-semantic-review-workflow.ts) -5. [arm-universal-auto-signoff.yaml](../.github/workflows/arm-universal-auto-signoff.yaml) -6. [arm-universal-auto-signoff.ts](../.github/workflows/src/arm-auto-signoff/arm-universal-auto-signoff.ts) - -Ignore the generated lock file at first. - -While reading, follow one example: - -```text -Full review, no findings, LintDiff and Avocado pass -``` - -Write down every state transition. - -### Layer 5: add one failure at a time - -Use this order: - -1. Blocking finding. -2. Avocado finishes before reviewer. -3. New commit during review. -4. Same-SHA retrigger. -5. Oversized PR. -6. Critic unavailable. -7. Missing receipt. -8. Canceled run. -9. Stale label artifact. - -For each scenario, answer: +For each scenario, ask: ```text -What is the current head SHA? -What is the latest semantic status for that SHA? -What labels are present? -Can auto-signoff proceed? -Who or what triggers reevaluation? +What SHA did the reviewer inspect? +What semantic status exists on that SHA? +What is the PR's current SHA? +What statuses does Universal see on the current SHA? +Can signoff proceed? ``` -### Layer 6: read tests as executable examples - -After the state model is clear, read: - -- [arm-semantic-review.test.ts](../.github/workflows/test/arm-auto-signoff/arm-semantic-review.test.ts) -- [arm-universal-auto-signoff.test.ts](../.github/workflows/test/arm-auto-signoff/arm-universal-auto-signoff.test.ts) -- [context.test.ts](../.github/workflows/test/context.test.ts) -- [update-labels.test.ts](../.github/workflows/test/update-labels.test.ts) - -Read test names first. They provide a scenario inventory. - -### Layer 7: inspect the generated workflow last - -Use [arm-api-review.lock.yml](../.github/workflows/arm-api-review.lock.yml) only -to answer compiler-specific questions: - -- Which jobs depend on which jobs? -- Which permissions does each generated job receive? -- When does the safe-output publisher run? -- Which failures are `continue-on-error`? - -Do not use the generated YAML as the first source for understanding policy. - -## How to reason about tradeoffs with no prior knowledge - -### Step 1: state the non-negotiable invariants - -For this design: - -```text -Never auto-sign off without Semantic Passed for the current SHA. -Never reuse an older SHA's Passed result. -Never treat a partial review as a full review. -Never let unvalidated model output directly authorize signoff. -Human override must win over automation. -``` - -Tradeoffs may change implementation complexity. They must not violate these -invariants. - -### Step 2: list failure modes before solutions - -Examples: - -- workflows finish in a different order; -- a new commit arrives mid-review; -- a run is canceled; -- a label event does not trigger another workflow; -- an artifact is missing; -- a status is stale; -- the agent reports inconsistent fields; -- label application is delayed; -- the PR is too large to review fully. - -Design from failure cases, not only from the happy path. - -### Step 3: choose fail-open or fail-closed - -For signoff and security decisions, prefer fail-closed: - -```text -Missing semantic result -> no signoff -Malformed receipt -> no signoff -Stale SHA -> ignore result -Partial review -> manual review -``` - -For user experience, distinguish retryable from permanent/manual cases so -fail-closed does not automatically mean "send everything to a human." - -### Step 4: identify the authority for every field - -Use a table: - -| Field | Source | Can it be trusted? | Verification | -| ------------------- | -------------- | ------------------ | ----------------------------- | -| PR number | Event plus API | Mostly | Re-fetch PR | -| Head SHA | PR API | Yes in trusted job | Compare before publication | -| Scope | Agent | No by itself | Recompute trusted size limits | -| Blocking count | Agent | Partially | Cross-check recommended | -| Workflow conclusion | GitHub | Yes | Read `workflow_run` | -| Manual hold | Label | Mutable | Protect actor and removal | - -If you cannot name the authority, the design has a trust gap. - -### Step 5: separate evidence, decision, and effect - -```text -Evidence: reviewer receipt, statuses, labels, approvals -Decision: auto-signoff policy function -Effect: label mutation -``` - -Keeping these separate makes retry, auditing, and stale-result protection -easier. - -### Step 6: reason about time - -Draw a timeline: - -```text -T1 reviewer starts -T2 Avocado finishes -T3 new commit arrives -T4 reviewer finishes -T5 label updater runs -``` - -At every point ask: - -> Which SHA does this operation refer to? - -This catches most distributed-workflow races. - -### Step 7: evaluate alternatives on explicit axes - -Score each design from 1 to 5: - -| Axis | Question | -| ---------------- | ------------------------------------------------------------- | -| Correctness | Can stale or partial evidence authorize signoff? | -| Security | Can untrusted input or a broad token affect the decision? | -| Ordering | Does the design work regardless of workflow completion order? | -| Observability | Can a reviewer understand why signoff did or did not occur? | -| Recovery | Can a failed run be retried safely? | -| Complexity | How many workflows, data forms, and APIs must be understood? | -| Maintainability | Is policy centralized and tested? | -| Operational cost | What must be provisioned and kept in sync? | - -Do not choose "simplest" without specifying which axis is being simplified. - -### Step 8: document accepted residual risk - -A design does not need zero risk. It needs explicit risk ownership. - -Example: - -```text -Accepted for pilot: - occasional duplicate comment after cancellation - -Not accepted for production: - unauthenticated producer of a Passed status -``` - -### Step 9: use a decision worksheet - -For every unresolved choice, fill in: - -```text -Decision: -Options: -Invariant affected: -Failure mode: -Security boundary: -Operational impact: -Recommended option: -Why: -What would make us revisit it: -``` - -This lets you make defensible tradeoffs without already being an expert. - -## Suggested exercises - -### Exercise 1: trace a clean PR - -Create a table with: - -```text -Event -Current SHA -Semantic status -LintDiff -Avocado -Labels -Decision -``` - -Fill it from reviewer start through pilot signoff. - -### Exercise 2: break the ordering - -Repeat the table with Avocado finishing first. The final decision should be the -same. - -### Exercise 3: change the SHA - -Insert a new commit before reviewer completion. Verify every older result is -ignored. - -### Exercise 4: challenge each trust boundary - -Ask: - -- What if the model lies about scope? -- What if another workflow publishes `ARM Semantic Review = success`? -- What if a user removes the manual hold? -- What if a label artifact arrives late? - -If the answer is unclear, add a check or document the residual risk. - ## Review checklist -Before enabling production signoff: - -- [ ] Create `ARMManualSignoffRequired` in both repositories. -- [ ] Decide and enforce who may add and remove the manual hold. -- [ ] Mirror the ARM reviewer workflow to `azure-rest-api-specs-pr`. -- [ ] Authenticate the semantic status producer. -- [ ] Confirm safe-output partial-failure semantics. -- [ ] Cross-check semantic receipt against actual published findings and labels. -- [ ] Test same-SHA reruns, new-SHA races, cancellation, and artifact loss in CI. -- [ ] Document how authors return a PR from manual review to automation. -- [ ] Replace the pilot label only after observing stable behavior. - -## Bottom line - -The design is intentionally fail-closed and is suitable for a pilot: - -- semantic evidence is tied to an exact SHA; -- only Passed allows auto-signoff; -- stale and superseded results are ignored; -- partial coverage routes to manual review; -- delayed correlated label changes recheck the live head. - -Before controlling production `ARMSignedOff`, address the missing manual-review -label, producer provenance, safe-output consistency, and private-repository -parity. +- [ ] Clean review publishes Passed. +- [ ] Blocking review publishes Changes requested. +- [ ] Scoped, incomplete, and degraded reviews publish Review incomplete. +- [ ] Missing or malformed item publishes Review incomplete. +- [ ] Wrong attempt, PR, or SHA publishes Review incomplete. +- [ ] Canceled review leaves Pending. +- [ ] New SHA cannot use old SHA's Passed result. +- [ ] Same-SHA rerun publishes Pending before reevaluation. +- [ ] Universal remains status-read-only. +- [ ] Delayed label application rechecks the live head. +- [ ] Passing evidence adds `ARMAutoSignedOff-Test` without changing `ARMSignedOff`. +- [ ] Later failures remove the pilot label. +- [ ] Review incomplete adds the manual-review label without changing ARM queue labels. +- [ ] Private repository copy is updated in coordination. From 4150092f888bfa224d6a36c27a60fc1e0ee9b9f8 Mon Sep 17 00:00:00 2001 From: Akhila Illa Date: Tue, 6 Oct 2026 11:58:22 -0500 Subject: [PATCH 05/16] Fix ARM semantic review workflow correlation Identify reviewer runs by their stable workflow path so dynamic run names do not suppress status artifacts. Fail visibly for unexpected workflow sources and cover the behavior with regression tests. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../arm-semantic-review-status.ts | 9 +++-- .github/workflows/src/context.ts | 4 +-- .../arm-semantic-review.test.ts | 26 ++++++++++++-- .github/workflows/test/context.test.ts | 35 +++++++++++++++++++ 4 files changed, 66 insertions(+), 8 deletions(-) diff --git a/.github/workflows/src/arm-auto-signoff/arm-semantic-review-status.ts b/.github/workflows/src/arm-auto-signoff/arm-semantic-review-status.ts index 27891ae5b7a3..f64e7b0d624e 100644 --- a/.github/workflows/src/arm-auto-signoff/arm-semantic-review-status.ts +++ b/.github/workflows/src/arm-auto-signoff/arm-semantic-review-status.ts @@ -1,7 +1,7 @@ import { readFile } from "node:fs/promises"; import { isFullGitSha } from "../../../shared/src/git.ts"; import { CommitStatusState, PER_PAGE_MAX } from "../../../shared/src/github.ts"; -import { ARM_API_REVIEW_WORKFLOW_NAME, parseWorkflowRunArtifactInputs } from "../context.ts"; +import { ARM_API_REVIEW_WORKFLOW_PATH, parseWorkflowRunArtifactInputs } from "../context.ts"; import type { GitHubScriptArgs, WebhookEvent } from "../github.ts"; import { ARM_SEMANTIC_REVIEW_STATUS, @@ -73,8 +73,11 @@ export async function finalizeArmSemanticReview({ statusPublished: boolean; }> { const workflowRun = (context.payload as WebhookEvent<"workflow-run", "completed">).workflow_run; - if (workflowRun.name !== ARM_API_REVIEW_WORKFLOW_NAME) { - return EMPTY_RESULT; + if (workflowRun.path !== ARM_API_REVIEW_WORKFLOW_PATH) { + throw new Error( + `Unexpected triggering workflow path: expected '${ARM_API_REVIEW_WORKFLOW_PATH}', ` + + `received '${workflowRun.path}'`, + ); } const owner = workflowRun.repository.owner?.login; diff --git a/.github/workflows/src/context.ts b/.github/workflows/src/context.ts index 7cfd21d52000..11a97c94a003 100644 --- a/.github/workflows/src/context.ts +++ b/.github/workflows/src/context.ts @@ -12,7 +12,7 @@ export type PullRequest = export type RestEndpointMethodTypes = import("@octokit/plugin-rest-endpoint-methods").RestEndpointMethodTypes; -export const ARM_API_REVIEW_WORKFLOW_NAME = "ARM API Review: Automated Workflow"; +export const ARM_API_REVIEW_WORKFLOW_PATH = ".github/workflows/arm-api-review.lock.yml"; /** * Extracts inputs from context based on event name and properties. @@ -218,7 +218,7 @@ export async function extractInputs( }); if (artifactInputs.headSha) { head_sha = artifactInputs.headSha; - } else if (payload.workflow_run.name === ARM_API_REVIEW_WORKFLOW_NAME) { + } else if (payload.workflow_run.path === ARM_API_REVIEW_WORKFLOW_PATH) { head_sha = ""; } if (artifactInputs.issueNumber) { diff --git a/.github/workflows/test/arm-auto-signoff/arm-semantic-review.test.ts b/.github/workflows/test/arm-auto-signoff/arm-semantic-review.test.ts index 0272f51caa99..030c470610db 100644 --- a/.github/workflows/test/arm-auto-signoff/arm-semantic-review.test.ts +++ b/.github/workflows/test/arm-auto-signoff/arm-semantic-review.test.ts @@ -26,6 +26,7 @@ const headSha = "0123456789abcdef0123456789abcdef01234567"; const runId = 456; const runAttempt = 1; const runUrl = "https://github.com/Azure/azure-rest-api-specs/actions/runs/456"; +const workflowPath = ".github/workflows/arm-api-review.lock.yml"; const passedResult: SemanticReviewResult = { runAttempt, @@ -71,11 +72,12 @@ function createFinalizeGithub() { return github; } -function createContext(conclusion: string | null = "success") { +function createContext(conclusion: string | null = "success", path = workflowPath) { return { payload: { workflow_run: { - name: "ARM API Review: Automated Workflow", + name: `ARM API Review #${issueNumber} (issue_comment)`, + path, conclusion, id: runId, run_attempt: runAttempt, @@ -94,11 +96,13 @@ async function runFinalizer({ output = agentOutput(), includeOutput = true, github = createFinalizeGithub(), + path = workflowPath, }: { conclusion?: string | null; output?: unknown; includeOutput?: boolean; github?: ReturnType; + path?: string; } = {}) { const directory = await mkdtemp(join(tmpdir(), "arm-semantic-review-")); const outputPath = join(directory, "agent_output.json"); @@ -110,7 +114,7 @@ async function runFinalizer({ process.env.GH_AW_AGENT_OUTPUT = outputPath; const result = await finalizeArmSemanticReview({ github, - context: createContext(conclusion), + context: createContext(conclusion, path), core: createMockCore(), } as unknown as GitHubScriptArgs); return { github, result }; @@ -332,6 +336,22 @@ describe("finalizeArmSemanticReview", () => { }); }); + it("rejects a completion from another workflow", async () => { + const github = createFinalizeGithub(); + await expect( + runFinalizer({ + github, + path: ".github/workflows/other-workflow.yaml", + }), + ).rejects.toThrow( + "Unexpected triggering workflow path: expected " + + "'.github/workflows/arm-api-review.lock.yml', " + + "received '.github/workflows/other-workflow.yaml'", + ); + expect(github.rest.actions.listWorkflowRunArtifacts).not.toHaveBeenCalled(); + expect(github.rest.repos.createCommitStatus).not.toHaveBeenCalled(); + }); + it("publishes Changes requested for Blocking findings", async () => { const { github } = await runFinalizer({ output: agentOutput({ ...passedResult, blockingCount: 3 }), diff --git a/.github/workflows/test/context.test.ts b/.github/workflows/test/context.test.ts index 73b32cb0e095..233a3a94c6f9 100644 --- a/.github/workflows/test/context.test.ts +++ b/.github/workflows/test/context.test.ts @@ -262,6 +262,41 @@ describe("extractInputs", () => { }); }); + it("does not substitute the base SHA when ARM review correlation is missing", async () => { + const github = createMockGithub(); + github.rest.actions.listWorkflowRunArtifacts.mockResolvedValue({ + data: { artifacts: [] }, + }); + const context = { + eventName: "workflow_run", + payload: { + action: "completed", + workflow_run: { + name: "ARM API Review #123 (pull_request_target)", + path: ".github/workflows/arm-api-review.lock.yml", + event: "pull_request_target", + head_sha: "base-branch-sha", + id: 456, + repository: { + name: "TestRepoName", + owner: { + login: "TestRepoOwnerLogin", + }, + }, + pull_requests: [{ number: 123 }], + }, + }, + }; + + await expect(extractInputs(github, context, createMockCore())).resolves.toEqual({ + owner: "TestRepoOwnerLogin", + repo: "TestRepoName", + head_sha: "", + issue_number: 123, + run_id: 456, + }); + }); + it("workflow_run:completed:workflow_dispatch uses trusted correlation artifacts", async () => { const github = createMockGithub(); github.rest.actions.listWorkflowRunArtifacts.mockResolvedValue({ From d0f96d5fcf67b30461be642a585dec35cde3287e Mon Sep 17 00:00:00 2001 From: Akhila Illa Date: Tue, 6 Oct 2026 12:41:33 -0500 Subject: [PATCH 06/16] Use trusted ARM review correlation metadata Source PR, head SHA, and run-attempt correlation from trusted workflow artifacts and environment data instead of model output. Keep the model result semantic-only and add regression coverage for malformed model identifiers. --- .github/workflows/arm-api-review.lock.yml | 27 +--- .github/workflows/arm-api-review.md | 38 ++---- .../arm-semantic-review-status.ts | 27 ++-- .../arm-semantic-review-workflow.ts | 19 ++- .../arm-auto-signoff/arm-semantic-review.ts | 46 +++---- .github/workflows/src/context.ts | 19 +-- .../test/arm-api-review-workflow.test.ts | 15 ++- .../arm-semantic-review.test.ts | 121 +++++++++++++----- 8 files changed, 175 insertions(+), 137 deletions(-) diff --git a/.github/workflows/arm-api-review.lock.yml b/.github/workflows/arm-api-review.lock.yml index b26a305a4bd7..21501515c3a3 100644 --- a/.github/workflows/arm-api-review.lock.yml +++ b/.github/workflows/arm-api-review.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"d0e25907942ce2a5d5fb164345a82abe46a78f3b12af3d1335b5283ab10064d4","body_hash":"b20da3b07a7c27740234fc9915ee5c9bde85495698024bbb1cc6c015a4bf0a47","compiler_version":"v0.86.2","strict":true,"agent_id":"copilot","agent_model":"gpt-5.6-sol?effort=high","detection_agent_id":"copilot","detection_agent_model":"gpt-5.6-sol?effort=high","engine_versions":{"copilot":"1.0.79"}} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"78da42ab16e548b086755f4a431586fa000a660b6de2e03dd84265d41a3bc2c2","body_hash":"adf12160a7fb3ac9331fa4d801bc8a212b8f44e7ee03095f32497ec8df56559d","compiler_version":"v0.86.2","strict":true,"agent_id":"copilot","agent_model":"gpt-5.6-sol?effort=high","detection_agent_id":"copilot","detection_agent_model":"gpt-5.6-sol?effort=high","engine_versions":{"copilot":"1.0.79"}} # gh-aw-manifest: {"version":1,"secrets":["GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/cache/restore","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/cache/save","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"373c709c69115d41ff229c7e5df9f8788daa9553","version":"v9"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/github-script","sha":"d746ffe35508b1917358783b479e04febd2b8f71","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"820762786026740c76f36085b0efc47a31fe5020","version":"v7.0.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"},{"repo":"github/gh-aw-actions/setup","sha":"6aab9e5b5c91c615506061f09bedd81a23babe3c","version":"v0.86.2"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.27.44","digest":"sha256:0d727725c737b58c7bdf51f640cffb928385ec46517e0917c7f1a02f1bada8b4","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.27.44@sha256:0d727725c737b58c7bdf51f640cffb928385ec46517e0917c7f1a02f1bada8b4"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.44","digest":"sha256:b50fbadba138f6e9aba94aca09711335c489bb3b15861220cb66f6092e042dc7","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.44@sha256:b50fbadba138f6e9aba94aca09711335c489bb3b15861220cb66f6092e042dc7"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.27.44","digest":"sha256:83e48bbe12c634be8c228a576832fe45f66c529ac3659db92bddbcf2eeb6d627","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.27.44@sha256:83e48bbe12c634be8c228a576832fe45f66c529ac3659db92bddbcf2eeb6d627"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.9","digest":"sha256:e5a1569aeaf41820fa7bdee3e94468cae448133cdbf00119ad24f5b74db1ab9f","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.9@sha256:e5a1569aeaf41820fa7bdee3e94468cae448133cdbf00119ad24f5b74db1ab9f"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:0d9f1fb5fd6610c0ac1f5194a38e45a8a1e81f8a390d5142d8e4e6f26a4b3196","pinned_image":"ghcr.io/github/gh-aw-node@sha256:0d9f1fb5fd6610c0ac1f5194a38e45a8a1e81f8a390d5142d8e4e6f26a4b3196"},{"image":"ghcr.io/github/github-mcp-server:v1.9.0","digest":"sha256:881b53d6f75f69bdbc1b5b10fc2f1361717c19054143b3a8529fb5c32061a50e","pinned_image":"ghcr.io/github/github-mcp-server:v1.9.0@sha256:881b53d6f75f69bdbc1b5b10fc2f1361717c19054143b3a8529fb5c32061a50e"}],"has_pull_request_target":true} # This file was automatically generated by gh-aw (v0.86.2). DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # @@ -595,9 +595,9 @@ jobs: mkdir -p "${RUNNER_TEMP}/gh-aw/safeoutputs" mkdir -p /tmp/gh-aw/safeoutputs mkdir -p /tmp/gh-aw/mcp-logs/safeoutputs - cat > "${RUNNER_TEMP}/gh-aw/safeoutputs/config.json" << 'GH_AW_SAFE_OUTPUTS_CONFIG_511927e73593d4c5_EOF' - {"add_comment":{"max":4,"target":"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}"},"add_labels":{"max":3,"target":"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}"},"create_pull_request_review_comment":{"max":50,"side":"RIGHT","target":"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}"},"create_report_incomplete_issue":{},"missing_data":{},"missing_tool":{},"noop":{"max":1,"report-as-issue":"true"},"record-arm-semantic-review":{"description":"Validate and record the ARM semantic review result for auto-signoff","inputs":{"blocking_count":{"default":null,"description":"Number of verified, currently applicable Blocking findings after reconciliation","required":true,"type":"string"},"completeness":{"default":null,"description":"Whether the intended review completed reliably","options":["complete","incomplete","degraded"],"required":true,"type":"choice"},"head_sha":{"default":null,"description":"Full pull request head SHA that was reviewed","required":true,"type":"string"},"issue_number":{"default":null,"description":"Target pull request number","required":true,"type":"string"},"run_attempt":{"default":null,"description":"GitHub Actions attempt that produced this result","required":true,"type":"string"},"scope":{"default":null,"description":"Whether the full PR or only a scoped subset was reviewed","options":["full","scoped"],"required":true,"type":"choice"}},"output":"ARM semantic review result recorded"},"remove_labels":{"max":3,"target":"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}"},"reply_to_pull_request_review_comment":{"max":50,"target":"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}"},"report_incomplete":{},"resolve_pull_request_review_thread":{"max":50},"submit_pull_request_review":{"footer":"if-body","max":1,"target":"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}"}} - GH_AW_SAFE_OUTPUTS_CONFIG_511927e73593d4c5_EOF + cat > "${RUNNER_TEMP}/gh-aw/safeoutputs/config.json" << 'GH_AW_SAFE_OUTPUTS_CONFIG_ea012e85f88bb73f_EOF' + {"add_comment":{"max":4,"target":"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}"},"add_labels":{"max":3,"target":"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}"},"create_pull_request_review_comment":{"max":50,"side":"RIGHT","target":"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}"},"create_report_incomplete_issue":{},"missing_data":{},"missing_tool":{},"noop":{"max":1,"report-as-issue":"true"},"record-arm-semantic-review":{"description":"Validate and record the ARM semantic review result for auto-signoff","inputs":{"blocking_count":{"default":null,"description":"Number of verified, currently applicable Blocking findings after reconciliation","required":true,"type":"string"},"completeness":{"default":null,"description":"Whether the intended review completed reliably","options":["complete","incomplete","degraded"],"required":true,"type":"choice"},"scope":{"default":null,"description":"Whether the full PR or only a scoped subset was reviewed","options":["full","scoped"],"required":true,"type":"choice"}},"output":"ARM semantic review result recorded"},"remove_labels":{"max":3,"target":"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}"},"reply_to_pull_request_review_comment":{"max":50,"target":"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}"},"report_incomplete":{},"resolve_pull_request_review_thread":{"max":50},"submit_pull_request_review":{"footer":"if-body","max":1,"target":"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}"}} + GH_AW_SAFE_OUTPUTS_CONFIG_ea012e85f88bb73f_EOF - name: Generate Safe Outputs Tools env: GH_AW_TOOLS_META_JSON: | @@ -631,18 +631,6 @@ jobs: ], "type": "string" }, - "head_sha": { - "description": "Full pull request head SHA that was reviewed", - "type": "string" - }, - "issue_number": { - "description": "Target pull request number", - "type": "string" - }, - "run_attempt": { - "description": "GitHub Actions attempt that produced this result", - "type": "string" - }, "scope": { "description": "Whether the full PR or only a scoped subset was reviewed", "enum": [ @@ -655,9 +643,6 @@ jobs: "required": [ "blocking_count", "completeness", - "head_sha", - "issue_number", - "run_attempt", "scope" ], "type": "object" @@ -1951,6 +1936,7 @@ jobs: (needs.detection.outputs.detection_success == 'true') runs-on: ubuntu-slim permissions: + actions: read contents: read pull-requests: read steps: @@ -1963,7 +1949,6 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 (source v7) env: GH_AW_AGENT_OUTPUT: ${{ runner.temp }}/gh-aw/safe-jobs/agent_output.json - TARGET_PR_NUMBER: ${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }} with: sparse-checkout: | .github @@ -1972,7 +1957,6 @@ jobs: uses: actions/github-script@d746ffe35508b1917358783b479e04febd2b8f71 # v9.0.0 env: GH_AW_AGENT_OUTPUT: ${{ runner.temp }}/gh-aw/safe-jobs/agent_output.json - TARGET_PR_NUMBER: ${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }} with: script: | const { default: validateArmSemanticReview } = @@ -1982,7 +1966,6 @@ jobs: uses: ./.github/actions/add-empty-artifact env: GH_AW_AGENT_OUTPUT: ${{ runner.temp }}/gh-aw/safe-jobs/agent_output.json - TARGET_PR_NUMBER: ${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }} with: name: arm-semantic-review value: ${{ fromJson(steps.validate.outputs.result).artifactValue }} diff --git a/.github/workflows/arm-api-review.md b/.github/workflows/arm-api-review.md index 1dbe9a452b5b..a316b58af941 100644 --- a/.github/workflows/arm-api-review.md +++ b/.github/workflows/arm-api-review.md @@ -271,19 +271,10 @@ safe-outputs: runs-on: ubuntu-slim if: needs.detection.outputs.detection_success == 'true' permissions: + actions: read contents: read pull-requests: read - env: - TARGET_PR_NUMBER: ${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }} inputs: - issue_number: - description: "Target pull request number" - required: true - type: string - head_sha: - description: "Full pull request head SHA that was reviewed" - required: true - type: string scope: description: "Whether the full PR or only a scoped subset was reviewed" required: true @@ -298,10 +289,6 @@ safe-outputs: description: "Number of verified, currently applicable Blocking findings after reconciliation" required: true type: string - run_attempt: - description: "GitHub Actions attempt that produced this result" - required: true - type: string steps: - uses: actions/checkout@v7 with: @@ -588,10 +575,10 @@ Track two values for the final semantic result: fetched. Before any `report_incomplete` stop after the target PR and head SHA are known, -also call `record_arm_semantic_review` with that PR, head SHA, the current -scope, `completeness: incomplete`, the number of verified Blocking findings -known to remain applicable at that point, and the workflow attempt. Use `"0"` -if the failure occurs before finding reconciliation. +also call `record_arm_semantic_review` with the current scope, +`completeness: incomplete`, and the number of verified Blocking findings known +to remain applicable at that point. Use `"0"` if the failure occurs before +finding reconciliation. ## Review Workflow @@ -1269,21 +1256,20 @@ compliant. Call `record_arm_semantic_review` exactly once after queuing the summary: -- `issue_number`: the authoritative target pull request number; -- `head_sha`: the pinned full session SHA; - `scope`: `full` or `scoped`, as recorded during Trigger Validation; - `completeness`: `complete`, `incomplete`, or `degraded`; - `blocking_count`: the number of verified Blocking findings that remain applicable after reconciliation. Include an unresolved finding classified as `SKIP-COVERED`; do not count fixed, resolved, Critic-dropped, or - overflow-only candidates; and -- `run_attempt`: the authoritative workflow attempt above. + overflow-only candidates. Use `blocking_count: "0"` after a clean re-review. Do not claim `full` or -`complete` when any required evidence was unavailable. `ARM Semantic Review - -Set Status` consumes the exact completed run's agent output, validates its -correlation and shape, and publishes the final status only after the entire -reviewer workflow completes. +`complete` when any required evidence was unavailable. Trusted workflow +artifacts attach the pull request, head SHA, and run attempt; never reproduce +those correlation values in this model-generated result. `ARM Semantic Review - +Set Status` consumes the exact completed run's agent output and trusted +correlation artifacts, validates their shape, and publishes the final status +only after the entire reviewer workflow completes. The final status is: diff --git a/.github/workflows/src/arm-auto-signoff/arm-semantic-review-status.ts b/.github/workflows/src/arm-auto-signoff/arm-semantic-review-status.ts index f64e7b0d624e..a291161cda57 100644 --- a/.github/workflows/src/arm-auto-signoff/arm-semantic-review-status.ts +++ b/.github/workflows/src/arm-auto-signoff/arm-semantic-review-status.ts @@ -1,7 +1,7 @@ import { readFile } from "node:fs/promises"; import { isFullGitSha } from "../../../shared/src/git.ts"; import { CommitStatusState, PER_PAGE_MAX } from "../../../shared/src/github.ts"; -import { ARM_API_REVIEW_WORKFLOW_PATH, parseWorkflowRunArtifactInputs } from "../context.ts"; +import { ARM_API_REVIEW_WORKFLOW_PATH, getWorkflowRunArtifactInputs } from "../context.ts"; import type { GitHubScriptArgs, WebhookEvent } from "../github.ts"; import { ARM_SEMANTIC_REVIEW_STATUS, @@ -85,15 +85,13 @@ export async function finalizeArmSemanticReview({ throw new Error("ARM API review workflow run is missing its repository owner"); } const repo = workflowRun.repository.name; - const artifactNames = ( - await github.paginate(github.rest.actions.listWorkflowRunArtifacts, { - owner, - repo, - run_id: workflowRun.id, - per_page: PER_PAGE_MAX, - }) - ).map((artifact) => artifact.name); - const { headSha, issueNumber } = parseWorkflowRunArtifactInputs(artifactNames, core); + const { headSha, issueNumber } = await getWorkflowRunArtifactInputs({ + github, + core, + owner, + repo, + runId: workflowRun.id, + }); if (!isFullGitSha(headSha) || !Number.isSafeInteger(issueNumber) || issueNumber <= 0) { core.info("The reviewer run has no trusted PR/SHA correlation; status is unchanged"); return EMPTY_RESULT; @@ -130,10 +128,11 @@ export async function finalizeArmSemanticReview({ throw new Error("agent output is unavailable"); } const agentOutput = JSON.parse(await readFile(outputPath, "utf8")) as unknown; - const result = parseSemanticReviewResult(agentOutput, issueNumber, workflowRun.run_attempt); - if (result.headSha.toLowerCase() !== headSha.toLowerCase()) { - throw new Error("semantic result SHA does not match reviewer correlation"); - } + const result = parseSemanticReviewResult(agentOutput, { + headSha, + issueNumber, + runAttempt: workflowRun.run_attempt, + }); // Independently verify that the PR is within automated-review coverage limits. // The trusted file list from pulls.listFiles takes precedence over the model-reported diff --git a/.github/workflows/src/arm-auto-signoff/arm-semantic-review-workflow.ts b/.github/workflows/src/arm-auto-signoff/arm-semantic-review-workflow.ts index 884d9f552470..c5b42bb22912 100644 --- a/.github/workflows/src/arm-auto-signoff/arm-semantic-review-workflow.ts +++ b/.github/workflows/src/arm-auto-signoff/arm-semantic-review-workflow.ts @@ -1,4 +1,5 @@ import { readFile } from "node:fs/promises"; +import { getWorkflowRunArtifactInputs } from "../context.ts"; import type { GitHub, GitHubScriptArgs } from "../github.ts"; import { parseSemanticReviewResult, type SemanticReviewResult } from "./arm-semantic-review.ts"; @@ -38,27 +39,33 @@ function encodeSemanticReviewReceipt(result: SemanticReviewResult): string { export default async function validateArmSemanticReview({ github, context, + core, }: GitHubScriptArgs): Promise<{ artifactValue: string; }> { const outputPath = process.env.GH_AW_AGENT_OUTPUT; - const issueNumberText = process.env.TARGET_PR_NUMBER; if (!outputPath) { throw new Error("GH_AW_AGENT_OUTPUT is unavailable"); } - if (!issueNumberText || !/^[1-9]\d*$/.test(issueNumberText)) { - throw new Error(`Invalid target PR number: '${issueNumberText ?? ""}'`); - } const runAttemptText = process.env.GITHUB_RUN_ATTEMPT; if (!runAttemptText || !/^[1-9]\d*$/.test(runAttemptText)) { throw new Error(`Invalid workflow run attempt: '${runAttemptText ?? ""}'`); } - const issueNumber = Number(issueNumberText); const runAttempt = Number(runAttemptText); + const { headSha, issueNumber } = await getWorkflowRunArtifactInputs({ + github, + core, + ...context.repo, + runId: context.runId, + }); const agentOutput = JSON.parse(await readFile(outputPath, "utf8")) as unknown; - const result = parseSemanticReviewResult(agentOutput, issueNumber, runAttempt); + const result = parseSemanticReviewResult(agentOutput, { + headSha, + issueNumber, + runAttempt, + }); await validateSemanticReviewResult({ ...context.repo, result, diff --git a/.github/workflows/src/arm-auto-signoff/arm-semantic-review.ts b/.github/workflows/src/arm-auto-signoff/arm-semantic-review.ts index f907356e0810..4458e03e2897 100644 --- a/.github/workflows/src/arm-auto-signoff/arm-semantic-review.ts +++ b/.github/workflows/src/arm-auto-signoff/arm-semantic-review.ts @@ -45,6 +45,11 @@ export type SemanticReviewResult = { completion: SemanticReviewCompletion; }; +export type SemanticReviewCorrelation = Pick< + SemanticReviewResult, + "runAttempt" | "issueNumber" | "headSha" +>; + export type CommitStatus = { context: string; state: string; @@ -62,14 +67,22 @@ function isRecord(value: unknown): value is Record { } /** - * Extracts exactly one semantic result from gh-aw's agent output and validates - * that it belongs to the completed reviewer run. + * Combines exactly one semantic result from gh-aw's agent output with trusted + * workflow correlation. */ export function parseSemanticReviewResult( agentOutput: unknown, - expectedIssueNumber: number, - expectedRunAttempt: number, + correlation: SemanticReviewCorrelation, ): SemanticReviewResult { + if ( + !Number.isSafeInteger(correlation.runAttempt) || + correlation.runAttempt <= 0 || + !Number.isSafeInteger(correlation.issueNumber) || + correlation.issueNumber <= 0 || + !isFullGitSha(correlation.headSha) + ) { + throw new Error("ARM semantic review correlation is missing or invalid"); + } if (!isRecord(agentOutput) || !Array.isArray(agentOutput.items)) { throw new Error("ARM API Reviewer output is missing its items array"); } @@ -83,28 +96,7 @@ export function parseSemanticReviewResult( } const result = results[0]; - const runAttempt = Number(result.run_attempt); - const issueNumber = Number(result.issue_number); const blockingCount = Number(result.blocking_count); - if ( - typeof result.run_attempt !== "string" || - !/^[1-9]\d*$/.test(result.run_attempt) || - !Number.isSafeInteger(runAttempt) || - runAttempt !== expectedRunAttempt - ) { - throw new Error(`Invalid ARM semantic review run attempt: '${String(result.run_attempt)}'`); - } - if ( - typeof result.issue_number !== "string" || - !/^[1-9]\d*$/.test(result.issue_number) || - !Number.isSafeInteger(issueNumber) || - issueNumber !== expectedIssueNumber - ) { - throw new Error(`Invalid ARM semantic review PR number: '${String(result.issue_number)}'`); - } - if (typeof result.head_sha !== "string" || !isFullGitSha(result.head_sha)) { - throw new Error(`Invalid ARM semantic review head SHA: '${String(result.head_sha)}'`); - } if ( typeof result.blocking_count !== "string" || !/^(0|[1-9]\d*)$/.test(result.blocking_count) || @@ -126,9 +118,7 @@ export function parseSemanticReviewResult( } return { - runAttempt, - issueNumber, - headSha: result.head_sha, + ...correlation, blockingCount, reviewScope: result.scope, completion: result.completeness, diff --git a/.github/workflows/src/context.ts b/.github/workflows/src/context.ts index 11a97c94a003..31b29496c025 100644 --- a/.github/workflows/src/context.ts +++ b/.github/workflows/src/context.ts @@ -210,10 +210,10 @@ export async function extractInputs( } if (payload.workflow_run.event === "pull_request_target") { - const artifactInputs = await extractWorkflowRunArtifactInputs({ + const artifactInputs = await getWorkflowRunArtifactInputs({ github, core, - repository: payload.workflow_run.repository, + ...getRepositoryInfo(payload.workflow_run.repository), runId: payload.workflow_run.id, }); if (artifactInputs.headSha) { @@ -237,10 +237,10 @@ export async function extractInputs( payload.workflow_run.event == "workflow_dispatch" ) { // These artifacts can be trusted because these event types run workflows from the default branch. - const artifactInputs = await extractWorkflowRunArtifactInputs({ + const artifactInputs = await getWorkflowRunArtifactInputs({ github, core, - repository: payload.workflow_run.repository, + ...getRepositoryInfo(payload.workflow_run.repository), runId: payload.workflow_run.id, }); head_sha = artifactInputs.headSha; @@ -303,19 +303,22 @@ export async function extractInputs( return inputs; } -async function extractWorkflowRunArtifactInputs({ +export async function getWorkflowRunArtifactInputs({ github, core, - repository, + owner, + repo, runId, }: { github: GitHub; core: Core; - repository: Parameters[0]; + owner: string; + repo: string; runId: number; }): Promise<{ headSha: string; issueNumber: number }> { const artifacts = await github.paginate(github.rest.actions.listWorkflowRunArtifacts, { - ...getRepositoryInfo(repository), + owner, + repo, run_id: runId, per_page: PER_PAGE_MAX, }); diff --git a/.github/workflows/test/arm-api-review-workflow.test.ts b/.github/workflows/test/arm-api-review-workflow.test.ts index 6263349393c4..82e871fe0813 100644 --- a/.github/workflows/test/arm-api-review-workflow.test.ts +++ b/.github/workflows/test/arm-api-review-workflow.test.ts @@ -578,16 +578,27 @@ describe("ARM API review posting reliability", () => { ); expect(compiled).toContain("record_arm_semantic_review"); expect(compiled).toContain("ARM Semantic Review"); - expect(source).toContain('run_attempt:\n description: "GitHub Actions attempt'); expect(compiled).toContain("Upload ARM semantic review receipt"); const semanticJob = compiled.slice( compiled.indexOf("\n record_arm_semantic_review:\n"), compiled.indexOf("\n safe_outputs:\n"), ); + const semanticConfig = source.slice( + source.indexOf(" record-arm-semantic-review:\n"), + source.indexOf(" noop:\n"), + ); expect(semanticJob).toContain("name: Validate ARM semantic review"); + expect(semanticJob).toContain("actions: read"); + expect(semanticJob).not.toContain("TARGET_PR_NUMBER"); expect(semanticJob).toContain("permissions:"); + expect(semanticConfig).not.toContain("issue_number:"); + expect(semanticConfig).not.toContain("head_sha:"); + expect(semanticConfig).not.toContain("run_attempt:"); + expect(collapsed).toContain( + "Trusted workflow artifacts attach the pull request, head SHA, and run attempt", + ); expect(collapsed).toContain( - "`ARM Semantic Review - Set Status` consumes the exact completed run's agent output, validates its correlation and shape, and publishes the final status only after the entire reviewer workflow completes.", + "`ARM Semantic Review - Set Status` consumes the exact completed run's agent output and trusted correlation artifacts", ); }); diff --git a/.github/workflows/test/arm-auto-signoff/arm-semantic-review.test.ts b/.github/workflows/test/arm-auto-signoff/arm-semantic-review.test.ts index 030c470610db..14603f2ce879 100644 --- a/.github/workflows/test/arm-auto-signoff/arm-semantic-review.test.ts +++ b/.github/workflows/test/arm-auto-signoff/arm-semantic-review.test.ts @@ -16,6 +16,7 @@ import { type SemanticReviewResult, } from "../../src/arm-auto-signoff/arm-semantic-review.ts"; import { finalizeArmSemanticReview } from "../../src/arm-auto-signoff/arm-semantic-review-status.ts"; +import validateArmSemanticReview from "../../src/arm-auto-signoff/arm-semantic-review-workflow.ts"; import type { GitHubScriptArgs } from "../../src/github.ts"; import { createMockCore, createMockGithub } from "../mocks.ts"; @@ -42,9 +43,6 @@ function agentOutput(result: SemanticReviewResult = passedResult) { items: [ { type: "record_arm_semantic_review", - run_attempt: String(result.runAttempt), - issue_number: String(result.issueNumber), - head_sha: result.headSha, blocking_count: String(result.blockingCount), scope: result.reviewScope, completeness: result.completion, @@ -130,13 +128,28 @@ async function runFinalizer({ describe("parseSemanticReviewResult", () => { it("parses one valid result", () => { - expect(parseSemanticReviewResult(agentOutput(), issueNumber, runAttempt)).toEqual(passedResult); + expect(parseSemanticReviewResult(agentOutput(), { headSha, issueNumber, runAttempt })).toEqual( + passedResult, + ); }); - it("rejects missing, duplicate, or mismatched results", () => { - expect(() => parseSemanticReviewResult({ items: [] }, issueNumber, runAttempt)).toThrow( - "Expected one ARM semantic review result, found 0", + it("uses trusted correlation instead of model-supplied identifiers", () => { + const output = agentOutput(); + Object.assign(output.items[0], { + head_sha: "675718552ce00736684ed7a184bab", + issue_number: "999", + run_attempt: "99", + }); + + expect(parseSemanticReviewResult(output, { headSha, issueNumber, runAttempt })).toEqual( + passedResult, ); + }); + + it("rejects missing, duplicate, or invalid semantic results", () => { + expect(() => + parseSemanticReviewResult({ items: [] }, { headSha, issueNumber, runAttempt }), + ).toThrow("Expected one ARM semantic review result, found 0"); expect(() => parseSemanticReviewResult( { @@ -145,31 +158,15 @@ describe("parseSemanticReviewResult", () => { ...agentOutput({ ...passedResult, blockingCount: 1 }).items, ], }, - issueNumber, - runAttempt, + { headSha, issueNumber, runAttempt }, ), ).toThrow("Expected one ARM semantic review result, found 2"); - expect(() => - parseSemanticReviewResult( - agentOutput({ ...passedResult, runAttempt: 2 }), - issueNumber, - runAttempt, - ), - ).toThrow("Invalid ARM semantic review run attempt"); - expect(() => - parseSemanticReviewResult( - agentOutput({ ...passedResult, issueNumber: 456 }), - issueNumber, - runAttempt, - ), - ).toThrow("Invalid ARM semantic review PR number"); expect(() => parseSemanticReviewResult( { items: [{ ...agentOutput().items[0], scope: "partial" }], }, - issueNumber, - runAttempt, + { headSha, issueNumber, runAttempt }, ), ).toThrow("Invalid ARM semantic review scope"); expect(() => @@ -177,11 +174,69 @@ describe("parseSemanticReviewResult", () => { { items: [{ ...agentOutput().items[0], completeness: "unknown" }], }, - issueNumber, - runAttempt, + { headSha, issueNumber, runAttempt }, ), ).toThrow("Invalid ARM semantic review completion"); }); + + it("rejects invalid trusted correlation", () => { + expect(() => + parseSemanticReviewResult(agentOutput(), { + headSha: "675718552ce00736684ed7a184bab", + issueNumber, + runAttempt, + }), + ).toThrow("ARM semantic review correlation is missing or invalid"); + }); +}); + +describe("validateArmSemanticReview", () => { + it("records trusted correlation when model-supplied identifiers are malformed", async () => { + const directory = await mkdtemp(join(tmpdir(), "arm-semantic-review-receipt-")); + const outputPath = join(directory, "agent_output.json"); + const output = agentOutput(); + Object.assign(output.items[0], { + head_sha: "675718552ce00736684ed7a184bab", + issue_number: "999", + run_attempt: "99", + }); + const previousOutputPath = process.env.GH_AW_AGENT_OUTPUT; + const previousRunAttempt = process.env.GITHUB_RUN_ATTEMPT; + try { + await writeFile(outputPath, JSON.stringify(output), "utf8"); + process.env.GH_AW_AGENT_OUTPUT = outputPath; + process.env.GITHUB_RUN_ATTEMPT = String(runAttempt); + const github = createFinalizeGithub(); + + const result = await validateArmSemanticReview({ + github, + context: { repo: { owner, repo }, runId }, + core: createMockCore(), + } as unknown as GitHubScriptArgs); + + expect(result).toEqual({ + artifactValue: `${runAttempt}.${issueNumber}.${headSha}.full.complete.0`, + }); + expect(github.rest.actions.listWorkflowRunArtifacts).toHaveBeenCalledWith({ + owner, + repo, + run_id: runId, + per_page: PER_PAGE_MAX, + }); + } finally { + if (previousOutputPath === undefined) { + delete process.env.GH_AW_AGENT_OUTPUT; + } else { + process.env.GH_AW_AGENT_OUTPUT = previousOutputPath; + } + if (previousRunAttempt === undefined) { + delete process.env.GITHUB_RUN_ATTEMPT; + } else { + process.env.GITHUB_RUN_ATTEMPT = previousRunAttempt; + } + await rm(directory, { recursive: true, force: true }); + } + }); }); describe("evaluateSemanticReview", () => { @@ -395,9 +450,13 @@ describe("finalizeArmSemanticReview", () => { description: "Review incomplete: Expected one ARM semantic review result, found 0", }, { - name: "wrong SHA", - options: { output: agentOutput({ ...passedResult, headSha: "f".repeat(40) }) }, - description: "Review incomplete: semantic result SHA does not match reviewer correlation", + name: "invalid Blocking count", + options: { + output: { + items: [{ ...agentOutput().items[0], blocking_count: "not-a-number" }], + }, + }, + description: "Review incomplete: Invalid ARM semantic review Blocking count", }, ])("publishes Review incomplete for $name", async ({ options, description }) => { const { github } = await runFinalizer(options); @@ -483,7 +542,7 @@ describe("finalizeArmSemanticReview", () => { it("truncates an overly long error description to 140 characters", async () => { const longValue = "x".repeat(200); const { github } = await runFinalizer({ - output: { items: [{ ...agentOutput().items[0], run_attempt: longValue }] }, + output: { items: [{ ...agentOutput().items[0], scope: longValue }] }, }); const call = github.rest.repos.createCommitStatus.mock.calls[0]?.[0] as { From 4ae87595dbe9bc12c9159e479b91f0bf1d3e7500 Mon Sep 17 00:00:00 2001 From: Akhila Illa Date: Tue, 6 Oct 2026 14:17:57 -0500 Subject: [PATCH 07/16] Package ARM critic runtime and improve diagnostics Embed the Critic instructions, protocol, and input template into the checkout-free runtime agent. Scope incomplete-review issue creation to the canonical repository and add structured semantic and auto-signoff decision logging with regression coverage. --- .github/workflows/arm-api-review.lock.yml | 13 ++- .github/workflows/arm-api-review.md | 28 ++++- .../arm-semantic-review-status.ts | 15 ++- .../arm-auto-signoff/arm-semantic-review.ts | 1 + .../arm-universal-auto-signoff.ts | 104 +++++++++++++++--- .github/workflows/src/context.ts | 6 + .../test/arm-api-review-workflow.test.ts | 28 ++++- .../arm-universal-auto-signoff.test.ts | 70 ++++++++++++ 8 files changed, 239 insertions(+), 26 deletions(-) diff --git a/.github/workflows/arm-api-review.lock.yml b/.github/workflows/arm-api-review.lock.yml index 21501515c3a3..ecfacf7eec4d 100644 --- a/.github/workflows/arm-api-review.lock.yml +++ b/.github/workflows/arm-api-review.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"78da42ab16e548b086755f4a431586fa000a660b6de2e03dd84265d41a3bc2c2","body_hash":"adf12160a7fb3ac9331fa4d801bc8a212b8f44e7ee03095f32497ec8df56559d","compiler_version":"v0.86.2","strict":true,"agent_id":"copilot","agent_model":"gpt-5.6-sol?effort=high","detection_agent_id":"copilot","detection_agent_model":"gpt-5.6-sol?effort=high","engine_versions":{"copilot":"1.0.79"}} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"997ed2e4774694fcb4708e93d1eb79667b6af6caf2ea4e18dd99e7e992892634","body_hash":"44117918e9efc77b0ebc76874b572ac48492669d88e865e9f45e6ddfd53691b3","compiler_version":"v0.86.2","strict":true,"agent_id":"copilot","agent_model":"gpt-5.6-sol?effort=high","detection_agent_id":"copilot","detection_agent_model":"gpt-5.6-sol?effort=high","engine_versions":{"copilot":"1.0.79"}} # gh-aw-manifest: {"version":1,"secrets":["GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/cache/restore","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/cache/save","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"373c709c69115d41ff229c7e5df9f8788daa9553","version":"v9"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/github-script","sha":"d746ffe35508b1917358783b479e04febd2b8f71","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"820762786026740c76f36085b0efc47a31fe5020","version":"v7.0.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"},{"repo":"github/gh-aw-actions/setup","sha":"6aab9e5b5c91c615506061f09bedd81a23babe3c","version":"v0.86.2"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.27.44","digest":"sha256:0d727725c737b58c7bdf51f640cffb928385ec46517e0917c7f1a02f1bada8b4","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.27.44@sha256:0d727725c737b58c7bdf51f640cffb928385ec46517e0917c7f1a02f1bada8b4"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.44","digest":"sha256:b50fbadba138f6e9aba94aca09711335c489bb3b15861220cb66f6092e042dc7","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.44@sha256:b50fbadba138f6e9aba94aca09711335c489bb3b15861220cb66f6092e042dc7"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.27.44","digest":"sha256:83e48bbe12c634be8c228a576832fe45f66c529ac3659db92bddbcf2eeb6d627","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.27.44@sha256:83e48bbe12c634be8c228a576832fe45f66c529ac3659db92bddbcf2eeb6d627"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.9","digest":"sha256:e5a1569aeaf41820fa7bdee3e94468cae448133cdbf00119ad24f5b74db1ab9f","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.9@sha256:e5a1569aeaf41820fa7bdee3e94468cae448133cdbf00119ad24f5b74db1ab9f"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:0d9f1fb5fd6610c0ac1f5194a38e45a8a1e81f8a390d5142d8e4e6f26a4b3196","pinned_image":"ghcr.io/github/gh-aw-node@sha256:0d9f1fb5fd6610c0ac1f5194a38e45a8a1e81f8a390d5142d8e4e6f26a4b3196"},{"image":"ghcr.io/github/github-mcp-server:v1.9.0","digest":"sha256:881b53d6f75f69bdbc1b5b10fc2f1361717c19054143b3a8529fb5c32061a50e","pinned_image":"ghcr.io/github/github-mcp-server:v1.9.0@sha256:881b53d6f75f69bdbc1b5b10fc2f1361717c19054143b3a8529fb5c32061a50e"}],"has_pull_request_target":true} # This file was automatically generated by gh-aw (v0.86.2). DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # @@ -595,9 +595,9 @@ jobs: mkdir -p "${RUNNER_TEMP}/gh-aw/safeoutputs" mkdir -p /tmp/gh-aw/safeoutputs mkdir -p /tmp/gh-aw/mcp-logs/safeoutputs - cat > "${RUNNER_TEMP}/gh-aw/safeoutputs/config.json" << 'GH_AW_SAFE_OUTPUTS_CONFIG_ea012e85f88bb73f_EOF' - {"add_comment":{"max":4,"target":"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}"},"add_labels":{"max":3,"target":"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}"},"create_pull_request_review_comment":{"max":50,"side":"RIGHT","target":"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}"},"create_report_incomplete_issue":{},"missing_data":{},"missing_tool":{},"noop":{"max":1,"report-as-issue":"true"},"record-arm-semantic-review":{"description":"Validate and record the ARM semantic review result for auto-signoff","inputs":{"blocking_count":{"default":null,"description":"Number of verified, currently applicable Blocking findings after reconciliation","required":true,"type":"string"},"completeness":{"default":null,"description":"Whether the intended review completed reliably","options":["complete","incomplete","degraded"],"required":true,"type":"choice"},"scope":{"default":null,"description":"Whether the full PR or only a scoped subset was reviewed","options":["full","scoped"],"required":true,"type":"choice"}},"output":"ARM semantic review result recorded"},"remove_labels":{"max":3,"target":"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}"},"reply_to_pull_request_review_comment":{"max":50,"target":"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}"},"report_incomplete":{},"resolve_pull_request_review_thread":{"max":50},"submit_pull_request_review":{"footer":"if-body","max":1,"target":"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}"}} - GH_AW_SAFE_OUTPUTS_CONFIG_ea012e85f88bb73f_EOF + cat > "${RUNNER_TEMP}/gh-aw/safeoutputs/config.json" << 'GH_AW_SAFE_OUTPUTS_CONFIG_9b0d1859d02eea35_EOF' + {"add_comment":{"max":4,"target":"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}"},"add_labels":{"max":3,"target":"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}"},"create_pull_request_review_comment":{"max":50,"side":"RIGHT","target":"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}"},"create_report_incomplete_issue":{"create-issue":"${{ github.repository == 'Azure/azure-rest-api-specs' }}","title-prefix":"[incomplete]"},"missing_data":{},"missing_tool":{},"noop":{"max":1,"report-as-issue":"true"},"record-arm-semantic-review":{"description":"Validate and record the ARM semantic review result for auto-signoff","inputs":{"blocking_count":{"default":null,"description":"Number of verified, currently applicable Blocking findings after reconciliation","required":true,"type":"string"},"completeness":{"default":null,"description":"Whether the intended review completed reliably","options":["complete","incomplete","degraded"],"required":true,"type":"choice"},"scope":{"default":null,"description":"Whether the full PR or only a scoped subset was reviewed","options":["full","scoped"],"required":true,"type":"choice"}},"output":"ARM semantic review result recorded"},"remove_labels":{"max":3,"target":"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}"},"reply_to_pull_request_review_comment":{"max":50,"target":"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}"},"report_incomplete":{},"resolve_pull_request_review_thread":{"max":50},"submit_pull_request_review":{"footer":"if-body","max":1,"target":"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}"}} + GH_AW_SAFE_OUTPUTS_CONFIG_9b0d1859d02eea35_EOF - name: Generate Safe Outputs Tools env: GH_AW_TOOLS_META_JSON: | @@ -1425,7 +1425,8 @@ jobs: uses: actions/github-script@d746ffe35508b1917358783b479e04febd2b8f71 # v9.0.0 env: GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }} - GH_AW_REPORT_INCOMPLETE_CREATE_ISSUE: "true" + GH_AW_REPORT_INCOMPLETE_CREATE_ISSUE: ${{ github.repository == 'Azure/azure-rest-api-specs' }} + GH_AW_REPORT_INCOMPLETE_TITLE_PREFIX: "[incomplete]" GH_AW_WORKFLOW_NAME: "ARM API Review: Automated Workflow" GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/${{ github.repository }}/blob/${{ github.ref_name }}/.github/workflows/arm-api-review.md" with: @@ -2063,7 +2064,7 @@ jobs: GITHUB_SERVER_URL: ${{ github.server_url }} GITHUB_API_URL: ${{ github.api_url }} GH_AW_SAFE_OUTPUT_JOBS: "{\"record_arm_semantic_review\":\"\"}" - GH_AW_SAFE_OUTPUTS_HANDLER_CONFIG: "{\"add_comment\":{\"max\":4,\"target\":\"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}\"},\"add_labels\":{\"max\":3,\"target\":\"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}\"},\"create_pull_request_review_comment\":{\"max\":50,\"side\":\"RIGHT\",\"target\":\"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}\"},\"create_report_incomplete_issue\":{},\"missing_data\":{},\"missing_tool\":{},\"noop\":{\"max\":1,\"report-as-issue\":\"true\"},\"remove_labels\":{\"max\":3,\"target\":\"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}\"},\"reply_to_pull_request_review_comment\":{\"max\":50,\"target\":\"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}\"},\"report_incomplete\":{},\"resolve_pull_request_review_thread\":{\"max\":50},\"submit_pull_request_review\":{\"footer\":\"if-body\",\"max\":1,\"target\":\"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}\"}}" + GH_AW_SAFE_OUTPUTS_HANDLER_CONFIG: "{\"add_comment\":{\"max\":4,\"target\":\"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}\"},\"add_labels\":{\"max\":3,\"target\":\"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}\"},\"create_pull_request_review_comment\":{\"max\":50,\"side\":\"RIGHT\",\"target\":\"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}\"},\"create_report_incomplete_issue\":{\"create-issue\":\"${{ github.repository == 'Azure/azure-rest-api-specs' }}\",\"title-prefix\":\"[incomplete]\"},\"missing_data\":{},\"missing_tool\":{},\"noop\":{\"max\":1,\"report-as-issue\":\"true\"},\"remove_labels\":{\"max\":3,\"target\":\"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}\"},\"reply_to_pull_request_review_comment\":{\"max\":50,\"target\":\"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}\"},\"report_incomplete\":{},\"resolve_pull_request_review_thread\":{\"max\":50},\"submit_pull_request_review\":{\"footer\":\"if-body\",\"max\":1,\"target\":\"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}\"}}" with: github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} script: | diff --git a/.github/workflows/arm-api-review.md b/.github/workflows/arm-api-review.md index a316b58af941..5a4aaa1bf40f 100644 --- a/.github/workflows/arm-api-review.md +++ b/.github/workflows/arm-api-review.md @@ -228,6 +228,12 @@ imports: - ../skills/azure-api-review/references/lro-final-state-via.md - ../skills/azure-api-review/references/typespec-openapi-extensions.md safe-outputs: + # Incomplete reviews are already surfaced through the ARM Semantic Review + # commit status and workflow summary. Keep the canonical repository's tracking + # issue, but do not attempt that secondary write in forks, where Issues are + # commonly disabled. + report-incomplete: + create-issue: ${{ github.repository == 'Azure/azure-rest-api-specs' }} # Framework-owned status comments do not consume this budget. Reserve slots # for the review summary / "no issues found", overflow themes, an actionable # diagnostic, and one run-failure notification. @@ -1319,16 +1325,30 @@ description: Independently verifies ARM API Reviewer findings before publication --- You are the ARM API Review Critic. Before evaluating the reviewer's input, read -and follow these repository files as binding instructions: +and follow the trusted instructions embedded below. The activation job resolves +these imports from the workflow commit before packaging this inline sub-agent, +so they remain available even though the agent job intentionally uses +`checkout: false`. Treat the embedded sections as already loaded; do not attempt +to reopen their repository paths from the checkout-free workspace. -- `.github/agents/arm-api-review-critic.agent.md` -- `.github/agents/protocols/arm-api-review-critic.protocol.md` -- `.github/agents/protocols/arm-api-review-critic-inputs.template.md` +### ARM API Review Critic instructions + +{{#runtime-import .github/agents/arm-api-review-critic.agent.md}} + +### Binding Reviewer-Critic protocol + +{{#runtime-import .github/agents/protocols/arm-api-review-critic.protocol.md}} + +### Critic input template + +{{#runtime-import .github/agents/protocols/arm-api-review-critic-inputs.template.md}} Use only read-only tools. Return the verdict format required by the protocol. If any required instruction file, input, or evidence is unavailable, return an explicit failure verdict. Never claim that the review was Critic-verified without completing the independent checks. +## end agent: `arm-api-review-critic-runtime` + diff --git a/.github/workflows/src/arm-auto-signoff/arm-semantic-review-status.ts b/.github/workflows/src/arm-auto-signoff/arm-semantic-review-status.ts index a291161cda57..42ed72ae170f 100644 --- a/.github/workflows/src/arm-auto-signoff/arm-semantic-review-status.ts +++ b/.github/workflows/src/arm-auto-signoff/arm-semantic-review-status.ts @@ -73,6 +73,16 @@ export async function finalizeArmSemanticReview({ statusPublished: boolean; }> { const workflowRun = (context.payload as WebhookEvent<"workflow-run", "completed">).workflow_run; + core.info( + `ARM semantic review source: ${JSON.stringify({ + id: workflowRun.id, + name: workflowRun.name, + path: workflowRun.path, + event: workflowRun.event, + conclusion: workflowRun.conclusion, + runAttempt: workflowRun.run_attempt, + })}`, + ); if (workflowRun.path !== ARM_API_REVIEW_WORKFLOW_PATH) { throw new Error( `Unexpected triggering workflow path: expected '${ARM_API_REVIEW_WORKFLOW_PATH}', ` + @@ -93,7 +103,10 @@ export async function finalizeArmSemanticReview({ runId: workflowRun.id, }); if (!isFullGitSha(headSha) || !Number.isSafeInteger(issueNumber) || issueNumber <= 0) { - core.info("The reviewer run has no trusted PR/SHA correlation; status is unchanged"); + core.info( + `The reviewer run has no trusted PR/SHA correlation; status is unchanged. ` + + `This is expected when the reviewer run was skipped before pre-activation.`, + ); return EMPTY_RESULT; } diff --git a/.github/workflows/src/arm-auto-signoff/arm-semantic-review.ts b/.github/workflows/src/arm-auto-signoff/arm-semantic-review.ts index 4458e03e2897..92a12f9cb3a8 100644 --- a/.github/workflows/src/arm-auto-signoff/arm-semantic-review.ts +++ b/.github/workflows/src/arm-auto-signoff/arm-semantic-review.ts @@ -53,6 +53,7 @@ export type SemanticReviewCorrelation = Pick< export type CommitStatus = { context: string; state: string; + description?: string | null; target_url?: string | null; updated_at: string; }; diff --git a/.github/workflows/src/arm-auto-signoff/arm-universal-auto-signoff.ts b/.github/workflows/src/arm-auto-signoff/arm-universal-auto-signoff.ts index 6d830b59198a..cd07ea600e09 100644 --- a/.github/workflows/src/arm-auto-signoff/arm-universal-auto-signoff.ts +++ b/.github/workflows/src/arm-auto-signoff/arm-universal-auto-signoff.ts @@ -2,7 +2,7 @@ import { inspect } from "node:util"; import { CommitStatusState, PER_PAGE_MAX } from "../../../shared/src/github.ts"; import { byDate, invert } from "../../../shared/src/sort.ts"; import { extractInputs } from "../context.ts"; -import type { Core, GitHubScriptArgs } from "../github.ts"; +import type { Core, GitHubScriptArgs, WebhookEvent } from "../github.ts"; import { LabelAction } from "../label.ts"; import { ArmAutoSignoffLabel } from "./arm-auto-signoff-labels.ts"; import { @@ -32,9 +32,36 @@ export default async function getLabelAction({ github, context, core }: GitHubSc issueNumber: number; labelActions: ManagedLabelActions; }> { + const workflowRun = + context.eventName === "workflow_run" + ? (context.payload as WebhookEvent<"workflow-run">).workflow_run + : undefined; + core.info( + `Universal auto-signoff trigger: ${JSON.stringify({ + eventName: context.eventName, + action: context.payload.action, + workflowRun: workflowRun + ? { + id: workflowRun.id, + name: workflowRun.name, + path: workflowRun.path, + event: workflowRun.event, + conclusion: workflowRun.conclusion, + } + : undefined, + })}`, + ); const { owner, repo, head_sha, issue_number } = await extractInputs(github, context, core); + core.info( + `Universal auto-signoff correlation: ${JSON.stringify({ + owner, + repo, + issueNumber: Number.isSafeInteger(issue_number) ? issue_number : "missing", + headSha: head_sha || "missing", + })}`, + ); - return await getLabelActionImpl({ + const result = await getLabelActionImpl({ owner, repo, head_sha, @@ -42,6 +69,8 @@ export default async function getLabelAction({ github, context, core }: GitHubSc github, core, }); + core.info(`Universal auto-signoff output: ${JSON.stringify(result)}`); + return result; } /* v8 ignore stop */ @@ -70,7 +99,11 @@ export async function getLabelActionImpl({ }; if (!Number.isInteger(issue_number) || issue_number <= 0 || !head_sha) { - core.info("Missing pull request number or head SHA"); + core.info( + `Universal auto-signoff no-op: missing correlation ` + + `(issueNumber=${Number.isSafeInteger(issue_number) ? issue_number : "missing"}, ` + + `headSha=${head_sha || "missing"})`, + ); return noneResult; } @@ -81,7 +114,10 @@ export async function getLabelActionImpl({ pull_number: issue_number, }); if (pullRequest.state !== "open" || pullRequest.head.sha !== head_sha) { - core.info("Pull request is closed or its head SHA has changed"); + core.info( + `Universal auto-signoff no-op: pull request state/head mismatch ` + + `(state=${pullRequest.state}, expectedHead=${head_sha}, currentHead=${pullRequest.head.sha})`, + ); return noneResult; } @@ -94,6 +130,7 @@ export async function getLabelActionImpl({ }) ).map((label) => label.name); const hasAutoSignoff = labelNames.includes(ArmAutoSignoffLabel.ArmAutoSignedOffTest); + core.info(`Current pull request labels: ${JSON.stringify([...labelNames].sort())}`); const labelActions = await getDesiredLabelActions({ owner, @@ -104,6 +141,7 @@ export async function getLabelActionImpl({ github, core, }); + core.info(`Universal auto-signoff label actions: ${JSON.stringify(labelActions)}`); return { ...noneResult, @@ -132,8 +170,16 @@ async function getDesiredLabelActions({ core: Core; }): Promise { const labelActions = createNoneLabelActions(); - const isReadyForArmReview = - labelNames.includes("ARMReview") && !labelNames.includes("NotReadyForARMReview"); + const hasArmReview = labelNames.includes("ARMReview"); + const hasNotReadyForArmReview = labelNames.includes("NotReadyForARMReview"); + const isReadyForArmReview = hasArmReview && !hasNotReadyForArmReview; + core.info( + `ARM review readiness: ${JSON.stringify({ + hasArmReview, + hasNotReadyForArmReview, + isReadyForArmReview, + })}`, + ); if (!isReadyForArmReview) { core.info("Pull request is not ready for ARM review"); @@ -151,13 +197,29 @@ async function getDesiredLabelActions({ per_page: PER_PAGE_MAX, }); - const semanticReviewOutcome = getSemanticReviewOutcome(getLatestSemanticReviewStatus(statuses)); - core.info(`ARM Semantic Review: ${semanticReviewOutcome ?? "missing"}`); + const latestSemanticReviewStatus = getLatestSemanticReviewStatus(statuses); + const semanticReviewOutcome = getSemanticReviewOutcome(latestSemanticReviewStatus); + core.info( + `Latest ARM Semantic Review status: ${JSON.stringify({ + state: latestSemanticReviewStatus?.state ?? "missing", + description: latestSemanticReviewStatus?.description ?? "missing", + targetUrl: latestSemanticReviewStatus?.target_url ?? "missing", + updatedAt: latestSemanticReviewStatus?.updated_at ?? "missing", + outcome: semanticReviewOutcome ?? "missing", + })}`, + ); if (semanticReviewOutcome !== SemanticReviewOutcome.Passed) { if (semanticReviewOutcome === SemanticReviewOutcome.ReviewIncomplete) { - core.info("ARM semantic review requires manual signoff"); - if (!labelNames.includes(ArmAutoSignoffLabel.ArmManualSignoffRequired)) { + const hasManualSignoffRequired = labelNames.includes( + ArmAutoSignoffLabel.ArmManualSignoffRequired, + ); + core.info( + `ARM semantic review requires manual signoff; ` + + `${ArmAutoSignoffLabel.ArmManualSignoffRequired} ` + + `${hasManualSignoffRequired ? "is already present" : "will be added"}`, + ); + if (!hasManualSignoffRequired) { labelActions[ArmAutoSignoffLabel.ArmManualSignoffRequired] = LabelAction.Add; } } else { @@ -169,11 +231,25 @@ async function getDesiredLabelActions({ return labelActions; } + const hasArmChangesRequested = labelNames.includes("ARMChangesRequested"); + const hasManualSignoffRequired = labelNames.includes( + ArmAutoSignoffLabel.ArmManualSignoffRequired, + ); + const hasSuppressionReviewRequired = labelNames.includes("SuppressionReviewRequired"); + const hasApprovedSuppression = labelNames.includes("Approved-Suppression"); const labelsAllowSignoff = - !labelNames.includes("ARMChangesRequested") && - !labelNames.includes(ArmAutoSignoffLabel.ArmManualSignoffRequired) && - (!labelNames.includes("SuppressionReviewRequired") || - labelNames.includes("Approved-Suppression")); + !hasArmChangesRequested && + !hasManualSignoffRequired && + (!hasSuppressionReviewRequired || hasApprovedSuppression); + core.info( + `Auto-signoff label eligibility: ${JSON.stringify({ + hasArmChangesRequested, + hasManualSignoffRequired, + hasSuppressionReviewRequired, + hasApprovedSuppression, + labelsAllowSignoff, + })}`, + ); if (!labelsAllowSignoff) { core.info("Labels do not meet requirements for universal auto-signoff"); labelActions[ArmAutoSignoffLabel.ArmAutoSignedOffTest] = hasAutoSignoff diff --git a/.github/workflows/src/context.ts b/.github/workflows/src/context.ts index 31b29496c025..ffea074a610a 100644 --- a/.github/workflows/src/context.ts +++ b/.github/workflows/src/context.ts @@ -38,6 +38,12 @@ export async function extractInputs( if (context.eventName === "workflow_run") { const payload = context.payload as WebhookEvent<"workflow-run">; workflowRunEvent = payload.workflow_run?.event; + core.info(` payload.workflow_run.id: ${payload.workflow_run?.id ?? "undefined"}`); + core.info(` payload.workflow_run.name: ${payload.workflow_run?.name ?? "undefined"}`); + core.info(` payload.workflow_run.path: ${payload.workflow_run?.path ?? "undefined"}`); + core.info( + ` payload.workflow_run.conclusion: ${payload.workflow_run?.conclusion ?? "undefined"}`, + ); } core.info(` payload.workflow_run.event: ${workflowRunEvent}`); diff --git a/.github/workflows/test/arm-api-review-workflow.test.ts b/.github/workflows/test/arm-api-review-workflow.test.ts index 82e871fe0813..ae5b1690b59b 100644 --- a/.github/workflows/test/arm-api-review-workflow.test.ts +++ b/.github/workflows/test/arm-api-review-workflow.test.ts @@ -321,18 +321,44 @@ describe("ARM API review workflow", () => { it("wires the mandatory ARM Critic as an inline runtime subagent", async () => { const [source, compiled] = await readWorkflowFiles(); + const criticRuntime = source.slice( + source.indexOf("## agent: `arm-api-review-critic-runtime`"), + source.indexOf("## end agent: `arm-api-review-critic-runtime`") + + "## end agent: `arm-api-review-critic-runtime`".length, + ); + const criticRuntimeImports = [ + ".github/agents/arm-api-review-critic.agent.md", + ".github/agents/protocols/arm-api-review-critic.protocol.md", + ".github/agents/protocols/arm-api-review-critic-inputs.template.md", + ]; expect(source).toContain( "## agent: `arm-api-review-critic-runtime`\n---\ndescription: Independently verifies ARM API Reviewer findings before publication\n---", ); expect(source).toContain("dispatch the inline\n`arm-api-review-critic-runtime` subagent"); - expect(source).toContain("`.github/agents/arm-api-review-critic.agent.md`"); + expect(criticRuntime).toContain("Treat the embedded sections as already loaded"); + for (const importPath of criticRuntimeImports) { + expect(criticRuntime).toContain(`{{#runtime-import ${importPath}}}`); + await expect(readFile(join(ROOT, importPath), "utf8")).resolves.not.toHaveLength(0); + } + expect(criticRuntime).toContain("## end agent: `arm-api-review-critic-runtime`"); expect(source).toContain("Never claim that the review was Critic-verified"); expect(compiled).toContain("- name: Restore inline sub-agents from activation artifact"); expect(compiled).toContain('GH_AW_SUB_AGENT_DIR: ".github/agents"'); expect(compiled).toContain('GH_AW_SUB_AGENT_EXT: ".agent.md"'); }); + it("creates incomplete-review tracking issues only in the canonical repository", async () => { + const [source, compiled] = await readWorkflowFiles(); + + expect(source).toContain( + "create-issue: ${{ github.repository == 'Azure/azure-rest-api-specs' }}", + ); + expect(compiled).toContain( + "GH_AW_REPORT_INCOMPLETE_CREATE_ISSUE: ${{ github.repository == 'Azure/azure-rest-api-specs' }}", + ); + }); + it("reconciles duplicates and contradictions across every review entry point", async () => { const [ [source, compiled], diff --git a/.github/workflows/test/arm-auto-signoff/arm-universal-auto-signoff.test.ts b/.github/workflows/test/arm-auto-signoff/arm-universal-auto-signoff.test.ts index b610f21e865d..54417e3ff5cd 100644 --- a/.github/workflows/test/arm-auto-signoff/arm-universal-auto-signoff.test.ts +++ b/.github/workflows/test/arm-auto-signoff/arm-universal-auto-signoff.test.ts @@ -63,6 +63,34 @@ function run(github: ReturnType) { } describe("getLabelActionImpl", () => { + it("logs a no-op when an indirect trigger has no PR correlation", async () => { + const loggingCore = createMockCore(); + const github = createMockGithub(); + + const result = await getLabelActionImpl({ + owner, + repo, + head_sha: "", + issue_number: Number.NaN, + github, + core: loggingCore, + }); + + expect(result).toEqual({ + headSha: "", + issueNumber: Number.NaN, + labelActions: { + [ArmAutoSignoffLabel.ArmAutoSignedOffTest]: LabelAction.None, + [ArmAutoSignoffLabel.ArmManualSignoffRequired]: LabelAction.None, + }, + }); + expect(github.rest.pulls.get).not.toHaveBeenCalled(); + expect(loggingCore.info).toHaveBeenCalledWith( + "Universal auto-signoff no-op: missing correlation " + + "(issueNumber=missing, headSha=missing)", + ); + }); + it("adds the pilot label when all universal requirements pass", async () => { const github = createMockGithub({ labelNames: ["ARMReview"] }); @@ -193,6 +221,48 @@ describe("getLabelActionImpl", () => { }); }); + it("logs why an existing manual signoff label produces no new action", async () => { + const loggingCore = createMockCore(); + const github = createMockGithub({ + labelNames: ["ARMReview", ArmAutoSignoffLabel.ArmManualSignoffRequired], + statuses: [ + { + context: "ARM Semantic Review", + state: CommitStatusState.ERROR, + description: "Review incomplete: reviewer did not complete", + updated_at: "2026-01-01", + }, + ...deterministicStatuses, + ], + }); + + const result = await getLabelActionImpl({ + owner, + repo, + head_sha: headSha, + issue_number: issueNumber, + github, + core: loggingCore, + }); + + expect(result.labelActions).toEqual({ + [ArmAutoSignoffLabel.ArmAutoSignedOffTest]: LabelAction.None, + [ArmAutoSignoffLabel.ArmManualSignoffRequired]: LabelAction.None, + }); + expect(loggingCore.info).toHaveBeenCalledWith( + expect.stringContaining('"isReadyForArmReview":true'), + ); + expect(loggingCore.info).toHaveBeenCalledWith( + expect.stringContaining('"description":"Review incomplete: reviewer did not complete"'), + ); + expect(loggingCore.info).toHaveBeenCalledWith( + "ARM semantic review requires manual signoff; ARMManualSignoffRequired is already present", + ); + expect(loggingCore.info).toHaveBeenCalledWith( + 'Universal auto-signoff label actions: {"ARMAutoSignedOff-Test":"none","ARMManualSignoffRequired":"none"}', + ); + }); + it("adds manual signoff when automated coverage was scoped", async () => { const github = createMockGithub({ labelNames: ["ARMReview"], From 4d04e99bab5524c9a99c150632559364738d8ed3 Mon Sep 17 00:00:00 2001 From: Akhila Illa Date: Wed, 7 Oct 2026 13:03:04 -0500 Subject: [PATCH 08/16] Harden ARM semantic review status lifecycle - Resolve a Pending ARM Semantic Review status from the conclusion job when the record job never publishes (noop, agent or detection failure, cancel). - Split error outcomes: only scoped or oversized reviews are published as "Manual review required" and add ARMManualSignoffRequired; incomplete or failed reviews are "Review incomplete" and add no label. - Give PR events the trigger gate skips a run-scoped concurrency group so they cannot cancel an in-flight review. - Check for a newer run immediately before publishing the status. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/arm-api-review.lock.yml | 59 ++- .github/workflows/arm-api-review.md | 109 +++-- .../workflows/arm-semantic-review-status.yaml | 80 ---- .../workflows/arm-universal-auto-signoff.yaml | 2 +- .../arm-semantic-review-status.ts | 199 --------- .../arm-semantic-review-workflow.ts | 240 +++++++++-- .../arm-auto-signoff/arm-semantic-review.ts | 70 +++- .../arm-universal-auto-signoff.ts | 2 +- .../test/arm-api-review-workflow.test.ts | 48 ++- ...rm-semantic-review-status-workflow.test.ts | 148 ------- .../arm-semantic-review.test.ts | 396 +++++++++++------- .../arm-universal-auto-signoff.test.ts | 20 +- ...ntic-review-auto-signoff-implementation.md | 199 +++++---- 13 files changed, 776 insertions(+), 796 deletions(-) delete mode 100644 .github/workflows/arm-semantic-review-status.yaml delete mode 100644 .github/workflows/src/arm-auto-signoff/arm-semantic-review-status.ts delete mode 100644 .github/workflows/test/arm-auto-signoff/arm-semantic-review-status-workflow.test.ts diff --git a/.github/workflows/arm-api-review.lock.yml b/.github/workflows/arm-api-review.lock.yml index ecfacf7eec4d..26265afb418d 100644 --- a/.github/workflows/arm-api-review.lock.yml +++ b/.github/workflows/arm-api-review.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"997ed2e4774694fcb4708e93d1eb79667b6af6caf2ea4e18dd99e7e992892634","body_hash":"44117918e9efc77b0ebc76874b572ac48492669d88e865e9f45e6ddfd53691b3","compiler_version":"v0.86.2","strict":true,"agent_id":"copilot","agent_model":"gpt-5.6-sol?effort=high","detection_agent_id":"copilot","detection_agent_model":"gpt-5.6-sol?effort=high","engine_versions":{"copilot":"1.0.79"}} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"353f8005e553e4d6711b67004bb008b732d8cb1d0b209d0c341887fe6da4a38d","body_hash":"6f23a35e609eac889194018b493de3fea8132fc3bec75a836390aed113d4f083","compiler_version":"v0.86.2","strict":true,"agent_id":"copilot","agent_model":"gpt-5.6-sol?effort=high","detection_agent_id":"copilot","detection_agent_model":"gpt-5.6-sol?effort=high","engine_versions":{"copilot":"1.0.79"}} # gh-aw-manifest: {"version":1,"secrets":["GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/cache/restore","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/cache/save","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"373c709c69115d41ff229c7e5df9f8788daa9553","version":"v9"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/github-script","sha":"d746ffe35508b1917358783b479e04febd2b8f71","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"820762786026740c76f36085b0efc47a31fe5020","version":"v7.0.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"},{"repo":"github/gh-aw-actions/setup","sha":"6aab9e5b5c91c615506061f09bedd81a23babe3c","version":"v0.86.2"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.27.44","digest":"sha256:0d727725c737b58c7bdf51f640cffb928385ec46517e0917c7f1a02f1bada8b4","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.27.44@sha256:0d727725c737b58c7bdf51f640cffb928385ec46517e0917c7f1a02f1bada8b4"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.44","digest":"sha256:b50fbadba138f6e9aba94aca09711335c489bb3b15861220cb66f6092e042dc7","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.44@sha256:b50fbadba138f6e9aba94aca09711335c489bb3b15861220cb66f6092e042dc7"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.27.44","digest":"sha256:83e48bbe12c634be8c228a576832fe45f66c529ac3659db92bddbcf2eeb6d627","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.27.44@sha256:83e48bbe12c634be8c228a576832fe45f66c529ac3659db92bddbcf2eeb6d627"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.9","digest":"sha256:e5a1569aeaf41820fa7bdee3e94468cae448133cdbf00119ad24f5b74db1ab9f","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.9@sha256:e5a1569aeaf41820fa7bdee3e94468cae448133cdbf00119ad24f5b74db1ab9f"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:0d9f1fb5fd6610c0ac1f5194a38e45a8a1e81f8a390d5142d8e4e6f26a4b3196","pinned_image":"ghcr.io/github/gh-aw-node@sha256:0d9f1fb5fd6610c0ac1f5194a38e45a8a1e81f8a390d5142d8e4e6f26a4b3196"},{"image":"ghcr.io/github/github-mcp-server:v1.9.0","digest":"sha256:881b53d6f75f69bdbc1b5b10fc2f1361717c19054143b3a8529fb5c32061a50e","pinned_image":"ghcr.io/github/github-mcp-server:v1.9.0@sha256:881b53d6f75f69bdbc1b5b10fc2f1361717c19054143b3a8529fb5c32061a50e"}],"has_pull_request_target":true} # This file was automatically generated by gh-aw (v0.86.2). DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # @@ -136,7 +136,7 @@ permissions: {} concurrency: cancel-in-progress: true - group: gh-aw-${{ github.workflow }}-${{ ((github.event_name == 'issue_comment' && (github.event.comment.body != '/arm-review' || github.event.issue.pull_request == null)) || (github.event_name == 'pull_request_target' && github.event.action == 'labeled' && github.event.label.name != 'WaitForARMFeedback')) && github.run_id || github.event.issue.number || github.event.pull_request.number || github.event.inputs.pr_number || github.run_id }} + group: gh-aw-${{ github.workflow }}-${{ ((github.event_name == 'issue_comment' && (github.event.comment.body != '/arm-review' || github.event.issue.pull_request == null)) || (github.event_name == 'pull_request_target' && (github.event.pull_request.draft == true || (github.event.action == 'labeled' && github.event.label.name != 'WaitForARMFeedback') || (github.event.action != 'labeled' && !contains(github.event.pull_request.labels.*.name, 'WaitForARMFeedback'))))) && github.run_id || github.event.issue.number || github.event.pull_request.number || github.event.inputs.pr_number || github.run_id }} run-name: "ARM API Review #${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }} (${{ github.event_name }})" @@ -595,9 +595,9 @@ jobs: mkdir -p "${RUNNER_TEMP}/gh-aw/safeoutputs" mkdir -p /tmp/gh-aw/safeoutputs mkdir -p /tmp/gh-aw/mcp-logs/safeoutputs - cat > "${RUNNER_TEMP}/gh-aw/safeoutputs/config.json" << 'GH_AW_SAFE_OUTPUTS_CONFIG_9b0d1859d02eea35_EOF' - {"add_comment":{"max":4,"target":"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}"},"add_labels":{"max":3,"target":"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}"},"create_pull_request_review_comment":{"max":50,"side":"RIGHT","target":"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}"},"create_report_incomplete_issue":{"create-issue":"${{ github.repository == 'Azure/azure-rest-api-specs' }}","title-prefix":"[incomplete]"},"missing_data":{},"missing_tool":{},"noop":{"max":1,"report-as-issue":"true"},"record-arm-semantic-review":{"description":"Validate and record the ARM semantic review result for auto-signoff","inputs":{"blocking_count":{"default":null,"description":"Number of verified, currently applicable Blocking findings after reconciliation","required":true,"type":"string"},"completeness":{"default":null,"description":"Whether the intended review completed reliably","options":["complete","incomplete","degraded"],"required":true,"type":"choice"},"scope":{"default":null,"description":"Whether the full PR or only a scoped subset was reviewed","options":["full","scoped"],"required":true,"type":"choice"}},"output":"ARM semantic review result recorded"},"remove_labels":{"max":3,"target":"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}"},"reply_to_pull_request_review_comment":{"max":50,"target":"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}"},"report_incomplete":{},"resolve_pull_request_review_thread":{"max":50},"submit_pull_request_review":{"footer":"if-body","max":1,"target":"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}"}} - GH_AW_SAFE_OUTPUTS_CONFIG_9b0d1859d02eea35_EOF + cat > "${RUNNER_TEMP}/gh-aw/safeoutputs/config.json" << 'GH_AW_SAFE_OUTPUTS_CONFIG_19784e75b93a6ea2_EOF' + {"add_comment":{"max":4,"target":"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}"},"add_labels":{"max":3,"target":"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}"},"create_pull_request_review_comment":{"max":50,"side":"RIGHT","target":"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}"},"create_report_incomplete_issue":{"create-issue":"${{ github.repository == 'Azure/azure-rest-api-specs' }}","title-prefix":"[incomplete]"},"missing_data":{},"missing_tool":{},"noop":{"max":1,"report-as-issue":"true"},"record-arm-semantic-review":{"description":"Validate and publish the ARM semantic review result for auto-signoff","inputs":{"blocking_count":{"default":null,"description":"Number of verified, currently applicable Blocking findings after reconciliation","required":true,"type":"string"},"completeness":{"default":null,"description":"Whether the intended review completed reliably","options":["complete","incomplete"],"required":true,"type":"choice"},"incomplete_reason":{"default":null,"description":"Why the review is incomplete, or none when complete","options":["none","critic-unavailable","required-evidence-unavailable","tool-failure","stale-sha","discussion-data-unavailable"],"required":true,"type":"choice"},"scope":{"default":null,"description":"Whether the full PR or only a scoped subset was reviewed","options":["full","scoped"],"required":true,"type":"choice"}},"output":"ARM semantic review status published"},"remove_labels":{"max":3,"target":"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}"},"reply_to_pull_request_review_comment":{"max":50,"target":"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}"},"report_incomplete":{},"resolve_pull_request_review_thread":{"max":50},"submit_pull_request_review":{"footer":"if-body","max":1,"target":"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}"}} + GH_AW_SAFE_OUTPUTS_CONFIG_19784e75b93a6ea2_EOF - name: Generate Safe Outputs Tools env: GH_AW_TOOLS_META_JSON: | @@ -614,7 +614,7 @@ jobs: "repo_params": {}, "dynamic_tools": [ { - "description": "Validate and record the ARM semantic review result for auto-signoff", + "description": "Validate and publish the ARM semantic review result for auto-signoff", "inputSchema": { "additionalProperties": false, "properties": { @@ -626,8 +626,19 @@ jobs: "description": "Whether the intended review completed reliably", "enum": [ "complete", - "incomplete", - "degraded" + "incomplete" + ], + "type": "string" + }, + "incomplete_reason": { + "description": "Why the review is incomplete, or none when complete", + "enum": [ + "none", + "critic-unavailable", + "required-evidence-unavailable", + "tool-failure", + "stale-sha", + "discussion-data-unavailable" ], "type": "string" }, @@ -643,6 +654,7 @@ jobs: "required": [ "blocking_count", "completeness", + "incomplete_reason", "scope" ], "type": "object" @@ -1255,8 +1267,10 @@ jobs: runs-on: ubuntu-slim permissions: actions: read + contents: read issues: write pull-requests: write + statuses: write concurrency: group: "gh-aw-conclusion-arm-api-review" cancel-in-progress: false @@ -1283,6 +1297,20 @@ jobs: GH_AW_INFO_VERSION: "1.0.79" GH_AW_INFO_AWF_VERSION: "v0.27.44" GH_AW_INFO_ENGINE_ID: "copilot" + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 (source v7) + with: + persist-credentials: false + sparse-checkout: | + .github + continue-on-error: true + - name: Resolve unpublished ARM semantic review status + uses: actions/github-script@d746ffe35508b1917358783b479e04febd2b8f71 # v9.0.0 + with: + script: | + const { finalizeUnpublishedArmSemanticReview } = + await import('${{ github.workspace }}/.github/workflows/src/arm-auto-signoff/arm-semantic-review-workflow.ts'); + return await finalizeUnpublishedArmSemanticReview({ github, context, core }); + continue-on-error: true - name: Download agent output artifact id: download-agent-output continue-on-error: true @@ -1940,6 +1968,7 @@ jobs: actions: read contents: read pull-requests: read + statuses: write steps: - name: Download agent output artifact continue-on-error: true @@ -1953,23 +1982,15 @@ jobs: with: sparse-checkout: | .github - - name: Validate ARM semantic review - id: validate + - name: Publish ARM semantic review status uses: actions/github-script@d746ffe35508b1917358783b479e04febd2b8f71 # v9.0.0 env: GH_AW_AGENT_OUTPUT: ${{ runner.temp }}/gh-aw/safe-jobs/agent_output.json with: script: | - const { default: validateArmSemanticReview } = + const { default: publishArmSemanticReviewStatus } = await import('${{ github.workspace }}/.github/workflows/src/arm-auto-signoff/arm-semantic-review-workflow.ts'); - return await validateArmSemanticReview({ github, context, core }); - - name: Upload ARM semantic review receipt - uses: ./.github/actions/add-empty-artifact - env: - GH_AW_AGENT_OUTPUT: ${{ runner.temp }}/gh-aw/safe-jobs/agent_output.json - with: - name: arm-semantic-review - value: ${{ fromJson(steps.validate.outputs.result).artifactValue }} + return await publishArmSemanticReviewStatus({ github, context, core }); safe_outputs: needs: diff --git a/.github/workflows/arm-api-review.md b/.github/workflows/arm-api-review.md index 5a4aaa1bf40f..170d2d235893 100644 --- a/.github/workflows/arm-api-review.md +++ b/.github/workflows/arm-api-review.md @@ -6,11 +6,12 @@ description: > on demand via the /arm-review comment command. run-name: "ARM API Review #${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }} (${{ github.event_name }})" timeout-minutes: 30 -# Concurrency is evaluated before the job-level `if` gate. Give unrelated -# comments and non-queue label events run-scoped groups so they cannot cancel -# an active review for the same PR before being skipped. +# Concurrency is evaluated before the job-level `if` gate. Give every event the +# gate below will skip a run-scoped group so it cannot cancel an active review for +# the same PR: unrelated comments, non-queue label events, and PR events on a +# draft PR or one without `WaitForARMFeedback`. Keep this in step with the gate. concurrency: - group: "gh-aw-${{ github.workflow }}-${{ ((github.event_name == 'issue_comment' && (github.event.comment.body != '/arm-review' || github.event.issue.pull_request == null)) || (github.event_name == 'pull_request_target' && github.event.action == 'labeled' && github.event.label.name != 'WaitForARMFeedback')) && github.run_id || github.event.issue.number || github.event.pull_request.number || github.event.inputs.pr_number || github.run_id }}" + group: "gh-aw-${{ github.workflow }}-${{ ((github.event_name == 'issue_comment' && (github.event.comment.body != '/arm-review' || github.event.issue.pull_request == null)) || (github.event_name == 'pull_request_target' && (github.event.pull_request.draft == true || (github.event.action == 'labeled' && github.event.label.name != 'WaitForARMFeedback') || (github.event.action != 'labeled' && !contains(github.event.pull_request.labels.*.name, 'WaitForARMFeedback'))))) && github.run_id || github.event.issue.number || github.event.pull_request.number || github.event.inputs.pr_number || github.run_id }}" cancel-in-progress: true on: # Fork PRs ARE supported (`forks: ["*"]`), matching every other PR workflow in @@ -136,6 +137,32 @@ jobs: run_attempt: ${{ steps.resolve_target_pr.outputs.run_attempt }} target_head_sha: ${{ steps.resolve_target_pr.outputs.target_head_sha }} target_pr_number: ${{ steps.resolve_target_pr.outputs.target_pr_number }} + # Pre-activation posts a Pending `ARM Semantic Review` status, but the record job + # only runs when the agent emitted a semantic item and threat detection passed. + # The conclusion job runs after every other job regardless of their results, so it + # resolves a Pending status that this exact run still owns. That keeps a failed, + # cancelled, or noop run from leaving the PR waiting forever. The step reads no + # agent output; it takes the head SHA from the trusted `head-sha` artifact. + # Pre-steps run before the built-in conclusion steps, so `continue-on-error` + # keeps a failure here from blocking gh-aw's own failure reporting. + conclusion: + permissions: + contents: read + statuses: write + pre-steps: + - uses: actions/checkout@v7 + continue-on-error: true + with: + sparse-checkout: | + .github + - name: Resolve unpublished ARM semantic review status + continue-on-error: true + uses: actions/github-script@v9.0.0 + with: + script: | + const { finalizeUnpublishedArmSemanticReview } = + await import('${{ github.workspace }}/.github/workflows/src/arm-auto-signoff/arm-semantic-review-workflow.ts'); + return await finalizeUnpublishedArmSemanticReview({ github, context, core }); # Gate at the trigger level so the expensive agent job never starts for # ineligible events. Label / draft / comment gating that used to live in a # custom github-script step is expressed here declaratively; the remaining @@ -273,13 +300,14 @@ safe-outputs: target: "${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}" jobs: record-arm-semantic-review: - description: "Validate and record the ARM semantic review result for auto-signoff" + description: "Validate and publish the ARM semantic review result for auto-signoff" runs-on: ubuntu-slim if: needs.detection.outputs.detection_success == 'true' permissions: actions: read contents: read pull-requests: read + statuses: write inputs: scope: description: "Whether the full PR or only a scoped subset was reviewed" @@ -290,7 +318,20 @@ safe-outputs: description: "Whether the intended review completed reliably" required: true type: choice - options: ["complete", "incomplete", "degraded"] + options: ["complete", "incomplete"] + incomplete_reason: + description: "Why the review is incomplete, or none when complete" + required: true + type: choice + options: + [ + "none", + "critic-unavailable", + "required-evidence-unavailable", + "tool-failure", + "stale-sha", + "discussion-data-unavailable", + ] blocking_count: description: "Number of verified, currently applicable Blocking findings after reconciliation" required: true @@ -300,20 +341,14 @@ safe-outputs: with: sparse-checkout: | .github - - id: validate - name: Validate ARM semantic review + - name: Publish ARM semantic review status uses: actions/github-script@v9.0.0 with: script: | - const { default: validateArmSemanticReview } = + const { default: publishArmSemanticReviewStatus } = await import('${{ github.workspace }}/.github/workflows/src/arm-auto-signoff/arm-semantic-review-workflow.ts'); - return await validateArmSemanticReview({ github, context, core }); - - name: Upload ARM semantic review receipt - uses: ./.github/actions/add-empty-artifact - with: - name: "arm-semantic-review" - value: "${{ fromJson(steps.validate.outputs.result).artifactValue }}" - output: "ARM semantic review result recorded" + return await publishArmSemanticReviewStatus({ github, context, core }); + output: "ARM semantic review status published" noop: # Threat detection uses the same pinned model as the primary review so all ARM # API Reviewer experiences move together. Keep both literals identical; see @@ -576,15 +611,19 @@ Track two values for the final semantic result: - `scope` is `full` unless the size cap or file-list truncation limits coverage; otherwise it is `scoped`. - `completeness` is `complete` only when the intended review and Critic - verification finish normally. Use `degraded` when the Critic is unavailable - and `incomplete` when required files, discussions, or other evidence cannot be - fetched. + verification finish normally. Otherwise it is `incomplete`. +- `incomplete_reason` is `none` when completeness is `complete`. When incomplete, + use the most specific reason: `critic-unavailable` after all Critic retries fail; + `discussion-data-unavailable` when comments, reviews, or threads cannot be + fetched completely; `stale-sha` when the pinned review target moved or became + unreachable; `tool-failure` when a required tool operation failed; otherwise + `required-evidence-unavailable`. Before any `report_incomplete` stop after the target PR and head SHA are known, also call `record_arm_semantic_review` with the current scope, -`completeness: incomplete`, and the number of verified Blocking findings known -to remain applicable at that point. Use `"0"` if the failure occurs before -finding reconciliation. +`completeness: incomplete`, the most specific `incomplete_reason` above, and the +number of verified Blocking findings known to remain applicable at that point. +Use `"0"` if the failure occurs before finding reconciliation. ## Review Workflow @@ -833,7 +872,8 @@ considered-and-declined candidates, graph status, and iteration number. context the PR happened to go through; see [Review context parity](#review-context-parity). Because nothing verified these findings, the Step 7 label rules withhold `ARMChangesRequested` for such - a run. + a run. Record `completeness: incomplete` with + `incomplete_reason: critic-unavailable`. - If the Critic reports that the session SHA moved or is unreachable, do not post findings or mutate threads. Restart once against the new head SHA; if it moves again, call `noop` and stop. @@ -1127,7 +1167,7 @@ and never emit a marker that names neither the finding nor a degradation reason. After reconciling the current findings with existing discussion, apply label changes based on the verified findings that still apply to the current head: -- **The review is scoped, incomplete, or degraded** β†’ leave +- **The review is scoped or incomplete** β†’ leave `WaitForARMFeedback`, `ARMChangesRequested`, and `ARMSignedOff` unchanged, even when Blocking findings were identified. The PR remains in its existing queue state for manual review or a retry. @@ -1263,19 +1303,23 @@ compliant. Call `record_arm_semantic_review` exactly once after queuing the summary: - `scope`: `full` or `scoped`, as recorded during Trigger Validation; -- `completeness`: `complete`, `incomplete`, or `degraded`; +- `completeness`: `complete` or `incomplete`; +- `incomplete_reason`: `none` when complete; otherwise one of + `critic-unavailable`, `required-evidence-unavailable`, `tool-failure`, + `stale-sha`, or `discussion-data-unavailable`; - `blocking_count`: the number of verified Blocking findings that remain applicable after reconciliation. Include an unresolved finding classified as `SKIP-COVERED`; do not count fixed, resolved, Critic-dropped, or overflow-only candidates. Use `blocking_count: "0"` after a clean re-review. Do not claim `full` or -`complete` when any required evidence was unavailable. Trusted workflow -artifacts attach the pull request, head SHA, and run attempt; never reproduce -those correlation values in this model-generated result. `ARM Semantic Review - -Set Status` consumes the exact completed run's agent output and trusted -correlation artifacts, validates their shape, and publishes the final status -only after the entire reviewer workflow completes. +`complete` when any required evidence was unavailable. `complete` requires +`incomplete_reason: none`; `incomplete` requires a reason other than `none`. +Trusted workflow artifacts attach the pull request, head SHA, and run attempt; +never reproduce those correlation values in this model-generated result. The +trusted `record_arm_semantic_review` job validates the item and publishes the +head-bound status inside this reviewer run. Universal Auto-Signoff consumes the +status only after the entire reviewer workflow completes. The final status is: @@ -1284,7 +1328,8 @@ The final status is: findings; - **Manual review required** when scope is `scoped`, including oversized or truncated PRs; or -- **Review incomplete** when a full review is `incomplete` or `degraded`. +- **Review incomplete** when completeness is `incomplete`, with the reason + included in the status description. ## What to Review vs. Skip diff --git a/.github/workflows/arm-semantic-review-status.yaml b/.github/workflows/arm-semantic-review-status.yaml deleted file mode 100644 index 16abb9583519..000000000000 --- a/.github/workflows/arm-semantic-review-status.yaml +++ /dev/null @@ -1,80 +0,0 @@ -# Converts one completed ARM API Reviewer run into the durable, head-bound -# `ARM Semantic Review` commit status consumed by Universal Auto-Signoff. This -# workflow validates existing reviewer evidence; it does not run another AI review. -name: ARM Semantic Review - Set Status - -on: - workflow_run: - # workflow_run supplies the exact reviewer run ID, so finalization can read - # that run's artifacts directly instead of searching reviewer-run history. - workflows: ["ARM API Review: Automated Workflow"] - types: [completed] - -# Keep status publication isolated here. Universal Auto-Signoff needs only -# `statuses: read` when it evaluates the finalized result. -permissions: - # actions.listWorkflowRunArtifacts - actions: read - # actions/checkout - contents: read - # pulls.get, pulls.listFiles - pull-requests: read - # repos.listCommitStatusesForRef, repos.createCommitStatus - statuses: write - -jobs: - arm-semantic-review-status: - name: ARM Semantic Review - Set Status - runs-on: ubuntu-slim - - steps: - # workflow_run executes trusted default-branch automation. Check out only - # .github because finalization never needs files from the untrusted PR head. - - uses: actions/checkout@v7 - with: - sparse-checkout: | - .github - - # Download the model's structured safe outputs from the exact completed - # reviewer run; never search for a "latest" reviewer run. - - name: Download ARM API Reviewer output - continue-on-error: true - uses: actions/download-artifact@v8.0.1 - with: - name: "agent" - path: "${{ runner.temp }}/arm-semantic-review" - github-token: "${{ github.token }}" - repository: "${{ github.repository }}" - run-id: "${{ github.event.workflow_run.id }}" - - # Read the same reviewer run's trusted correlation artifacts, validate its - # structured semantic item, reject stale or superseded results, independently - # verify coverage limits, and publish the final status for the reviewed SHA. - - id: finalize - name: Finalize ARM Semantic Review - uses: actions/github-script@v9.0.0 - env: - GH_AW_AGENT_OUTPUT: "${{ runner.temp }}/arm-semantic-review/agent_output.json" - with: - script: | - const { finalizeArmSemanticReview } = - await import('${{ github.workspace }}/.github/workflows/src/arm-auto-signoff/arm-semantic-review-status.ts'); - return await finalizeArmSemanticReview({ github, context, core }); - - # Universal Auto-Signoff is triggered by this workflow's completion. These - # artifacts let it identify the exact PR and SHA without searching prior runs. - - if: fromJson(steps.finalize.outputs.result).headSha - name: Upload artifact with head SHA - uses: ./.github/actions/add-empty-artifact - with: - name: "head-sha" - value: "${{ fromJson(steps.finalize.outputs.result).headSha }}" - - # Keep PR correlation separate from the SHA artifact to match the existing - # Set Status workflow contract used by LintDiff and Avocado. - - if: fromJson(steps.finalize.outputs.result).issueNumber > 0 - name: Upload artifact with issue number - uses: ./.github/actions/add-empty-artifact - with: - name: "issue-number" - value: "${{ fromJson(steps.finalize.outputs.result).issueNumber }}" diff --git a/.github/workflows/arm-universal-auto-signoff.yaml b/.github/workflows/arm-universal-auto-signoff.yaml index e31cda444e3c..bc28cab343a8 100644 --- a/.github/workflows/arm-universal-auto-signoff.yaml +++ b/.github/workflows/arm-universal-auto-signoff.yaml @@ -20,7 +20,7 @@ on: # Re-evaluate as each mandatory status producer completes. workflows: [ - "ARM Semantic Review - Set Status", + "ARM API Review: Automated Workflow", "Swagger Avocado - Set Status", "Swagger LintDiff - Set Status", ] diff --git a/.github/workflows/src/arm-auto-signoff/arm-semantic-review-status.ts b/.github/workflows/src/arm-auto-signoff/arm-semantic-review-status.ts deleted file mode 100644 index 42ed72ae170f..000000000000 --- a/.github/workflows/src/arm-auto-signoff/arm-semantic-review-status.ts +++ /dev/null @@ -1,199 +0,0 @@ -import { readFile } from "node:fs/promises"; -import { isFullGitSha } from "../../../shared/src/git.ts"; -import { CommitStatusState, PER_PAGE_MAX } from "../../../shared/src/github.ts"; -import { ARM_API_REVIEW_WORKFLOW_PATH, getWorkflowRunArtifactInputs } from "../context.ts"; -import type { GitHubScriptArgs, WebhookEvent } from "../github.ts"; -import { - ARM_SEMANTIC_REVIEW_STATUS, - evaluateAutomatedReviewCoverage, - evaluateSemanticReview, - getLatestSemanticReviewStatus, - parseSemanticReviewResult, - type ChangedFile, - type CommitStatus, - type EvaluatedSemanticReview, -} from "./arm-semantic-review.ts"; - -const MAX_STATUS_DESCRIPTION = 140; - -const EMPTY_RESULT = { - headSha: "", - issueNumber: 0, - statusPublished: false, -}; - -async function publishStatus( - { github, core }: Pick, - owner: string, - repo: string, - headSha: string, - targetUrl: string, - status: EvaluatedSemanticReview, -): Promise { - core.info(`${ARM_SEMANTIC_REVIEW_STATUS}: ${status.state} - ${status.description}`); - await github.rest.repos.createCommitStatus({ - owner, - repo, - sha: headSha, - state: status.state, - context: ARM_SEMANTIC_REVIEW_STATUS, - description: status.description, - target_url: targetUrl, - }); -} - -function isSuperseded( - latestStatus: CommitStatus | undefined, - runId: number, - runAttempt: number, -): boolean { - const match = /\/actions\/runs\/([1-9]\d*)\/attempts\/([1-9]\d*)(?:\/|$)/.exec( - latestStatus?.target_url ?? "", - ); - if (!match) { - return false; - } - const latestRunId = Number(match[1]); - const latestAttempt = Number(match[2]); - return latestRunId > runId || (latestRunId === runId && latestAttempt > runAttempt); -} - -/** - * Converts the exact completed ARM API Reviewer run into one head-bound commit status. - * Missing or malformed evidence publishes `error`; stale SHAs are harmless because - * Universal Auto-Signoff reads statuses only for the PR's current SHA. - */ -export async function finalizeArmSemanticReview({ - github, - context, - core, -}: GitHubScriptArgs): Promise<{ - headSha: string; - issueNumber: number; - statusPublished: boolean; -}> { - const workflowRun = (context.payload as WebhookEvent<"workflow-run", "completed">).workflow_run; - core.info( - `ARM semantic review source: ${JSON.stringify({ - id: workflowRun.id, - name: workflowRun.name, - path: workflowRun.path, - event: workflowRun.event, - conclusion: workflowRun.conclusion, - runAttempt: workflowRun.run_attempt, - })}`, - ); - if (workflowRun.path !== ARM_API_REVIEW_WORKFLOW_PATH) { - throw new Error( - `Unexpected triggering workflow path: expected '${ARM_API_REVIEW_WORKFLOW_PATH}', ` + - `received '${workflowRun.path}'`, - ); - } - - const owner = workflowRun.repository.owner?.login; - if (!owner) { - throw new Error("ARM API review workflow run is missing its repository owner"); - } - const repo = workflowRun.repository.name; - const { headSha, issueNumber } = await getWorkflowRunArtifactInputs({ - github, - core, - owner, - repo, - runId: workflowRun.id, - }); - if (!isFullGitSha(headSha) || !Number.isSafeInteger(issueNumber) || issueNumber <= 0) { - core.info( - `The reviewer run has no trusted PR/SHA correlation; status is unchanged. ` + - `This is expected when the reviewer run was skipped before pre-activation.`, - ); - return EMPTY_RESULT; - } - - if (workflowRun.conclusion === "cancelled") { - core.info("Canceled reviewer run leaves its Pending status unchanged"); - return { headSha, issueNumber, statusPublished: false }; - } - - const statuses: CommitStatus[] = await github.paginate( - github.rest.repos.listCommitStatusesForRef, - { - owner, - repo, - ref: headSha, - per_page: PER_PAGE_MAX, - }, - ); - if ( - isSuperseded(getLatestSemanticReviewStatus(statuses), workflowRun.id, workflowRun.run_attempt) - ) { - core.info("A newer ARM API review run superseded this completion"); - return { headSha, issueNumber, statusPublished: false }; - } - - let status: EvaluatedSemanticReview; - try { - if (workflowRun.conclusion !== "success") { - throw new Error(`workflow concluded with ${workflowRun.conclusion ?? "an unknown result"}`); - } - const outputPath = process.env.GH_AW_AGENT_OUTPUT; - if (!outputPath) { - throw new Error("agent output is unavailable"); - } - const agentOutput = JSON.parse(await readFile(outputPath, "utf8")) as unknown; - const result = parseSemanticReviewResult(agentOutput, { - headSha, - issueNumber, - runAttempt: workflowRun.run_attempt, - }); - - // Independently verify that the PR is within automated-review coverage limits. - // The trusted file list from pulls.listFiles takes precedence over the model-reported - // scope/completeness so that an oversized or truncated PR cannot be auto-signed-off. - const { data: pullRequest } = await github.rest.pulls.get({ - owner, - repo, - pull_number: issueNumber, - }); - if (pullRequest.state !== "open" || pullRequest.head.sha !== headSha) { - core.info("Ignoring ARM API review completion for a closed PR or stale head SHA"); - return { headSha, issueNumber, statusPublished: false }; - } - const changedFiles = (await github.paginate(github.rest.pulls.listFiles, { - owner, - repo, - pull_number: issueNumber, - per_page: PER_PAGE_MAX, - })) as ChangedFile[]; - const coverage = evaluateAutomatedReviewCoverage(pullRequest.changed_files, changedFiles); - if (coverage.manualReviewRequired) { - status = { - state: CommitStatusState.ERROR, - description: `Review incomplete: ${coverage.reason ?? "automated review coverage was limited"}`, - }; - } else { - status = evaluateSemanticReview(result); - } - } catch (error) { - const fullReason = error instanceof Error ? error.message : "invalid semantic result"; - core.warning(fullReason); - const prefix = "Review incomplete: "; - const available = MAX_STATUS_DESCRIPTION - prefix.length; - const truncatedReason = - fullReason.length > available ? `${fullReason.slice(0, available - 1)}\u2026` : fullReason; - status = { - state: CommitStatusState.ERROR, - description: `${prefix}${truncatedReason}`, - }; - } - - await publishStatus( - { github, core }, - owner, - repo, - headSha, - `${workflowRun.html_url}/attempts/${workflowRun.run_attempt}`, - status, - ); - return { headSha, issueNumber, statusPublished: true }; -} diff --git a/.github/workflows/src/arm-auto-signoff/arm-semantic-review-workflow.ts b/.github/workflows/src/arm-auto-signoff/arm-semantic-review-workflow.ts index c5b42bb22912..5481bf57588f 100644 --- a/.github/workflows/src/arm-auto-signoff/arm-semantic-review-workflow.ts +++ b/.github/workflows/src/arm-auto-signoff/arm-semantic-review-workflow.ts @@ -1,47 +1,104 @@ import { readFile } from "node:fs/promises"; +import { isFullGitSha } from "../../../shared/src/git.ts"; +import { CommitStatusState, PER_PAGE_MAX } from "../../../shared/src/github.ts"; import { getWorkflowRunArtifactInputs } from "../context.ts"; -import type { GitHub, GitHubScriptArgs } from "../github.ts"; -import { parseSemanticReviewResult, type SemanticReviewResult } from "./arm-semantic-review.ts"; +import type { GitHubScriptArgs } from "../github.ts"; +import { + ARM_SEMANTIC_REVIEW_STATUS, + evaluateAutomatedReviewCoverage, + evaluateSemanticReview, + getLatestSemanticReviewStatus, + MANUAL_REVIEW_DESCRIPTION_PREFIX, + parseSemanticReviewResult, + REVIEW_INCOMPLETE_DESCRIPTION_PREFIX, + type ChangedFile, + type CommitStatus, + type EvaluatedSemanticReview, +} from "./arm-semantic-review.ts"; -export async function validateSemanticReviewResult({ - owner, - repo, - result, - github, -}: { - owner: string; - repo: string; - result: SemanticReviewResult; - github: GitHub; -}): Promise { - const { data: pullRequest } = await github.rest.pulls.get({ +const MAX_STATUS_DESCRIPTION = 140; + +const EMPTY_RESULT = { + headSha: "", + issueNumber: 0, + statusPublished: false, +}; + +function getRunStatusUrl( + { serverUrl, runId }: Pick, + owner: string, + repo: string, + runAttempt: number | string, +): string { + return `${serverUrl}/${owner}/${repo}/actions/runs/${runId}/attempts/${runAttempt}`; +} + +async function getLatestSemanticStatus( + github: GitHubScriptArgs["github"], + owner: string, + repo: string, + headSha: string, +): Promise { + const statuses: CommitStatus[] = await github.paginate( + github.rest.repos.listCommitStatusesForRef, + { + owner, + repo, + ref: headSha, + per_page: PER_PAGE_MAX, + }, + ); + return getLatestSemanticReviewStatus(statuses); +} + +async function publishStatus( + { github, core }: Pick, + owner: string, + repo: string, + headSha: string, + targetUrl: string, + status: EvaluatedSemanticReview, +): Promise { + core.info(`${ARM_SEMANTIC_REVIEW_STATUS}: ${status.state} - ${status.description}`); + await github.rest.repos.createCommitStatus({ owner, repo, - pull_number: result.issueNumber, + sha: headSha, + state: status.state, + context: ARM_SEMANTIC_REVIEW_STATUS, + description: status.description, + target_url: targetUrl, }); - if (pullRequest.state !== "open" || pullRequest.head.sha !== result.headSha) { - throw new Error("The reviewed pull request head is no longer current"); - } } -function encodeSemanticReviewReceipt(result: SemanticReviewResult): string { - return [ - result.runAttempt, - result.issueNumber, - result.headSha, - result.reviewScope, - result.completion, - result.blockingCount, - ].join("."); +function isSuperseded( + latestStatus: CommitStatus | undefined, + runId: number, + runAttempt: number, +): boolean { + const match = /\/actions\/runs\/([1-9]\d*)\/attempts\/([1-9]\d*)(?:\/|$)/.exec( + latestStatus?.target_url ?? "", + ); + if (!match) { + return false; + } + const latestRunId = Number(match[1]); + const latestAttempt = Number(match[2]); + return latestRunId > runId || (latestRunId === runId && latestAttempt > runAttempt); } -/* v8 ignore start */ -export default async function validateArmSemanticReview({ +/** + * Validates one trusted ARM reviewer safe output and publishes its head-bound status. + * This runs inside the reviewer workflow, before workflow_run consumers are triggered. + */ +export default async function publishArmSemanticReviewStatus({ github, context, core, }: GitHubScriptArgs): Promise<{ - artifactValue: string; + headSha: string; + issueNumber: number; + statusPublished: boolean; }> { const outputPath = process.env.GH_AW_AGENT_OUTPUT; if (!outputPath) { @@ -54,25 +111,122 @@ export default async function validateArmSemanticReview({ } const runAttempt = Number(runAttemptText); + const { owner, repo } = context.repo; const { headSha, issueNumber } = await getWorkflowRunArtifactInputs({ github, core, - ...context.repo, + owner, + repo, runId: context.runId, }); - const agentOutput = JSON.parse(await readFile(outputPath, "utf8")) as unknown; - const result = parseSemanticReviewResult(agentOutput, { + if (!isFullGitSha(headSha) || !Number.isSafeInteger(issueNumber) || issueNumber <= 0) { + core.info("The reviewer run has no trusted PR/SHA correlation; status is unchanged"); + return EMPTY_RESULT; + } + + let status: EvaluatedSemanticReview; + try { + const agentOutput = JSON.parse(await readFile(outputPath, "utf8")) as unknown; + const result = parseSemanticReviewResult(agentOutput, { + headSha, + issueNumber, + runAttempt, + }); + + const { data: pullRequest } = await github.rest.pulls.get({ + owner, + repo, + pull_number: issueNumber, + }); + if (pullRequest.state !== "open" || pullRequest.head.sha !== headSha) { + core.info("Ignoring ARM API review result for a closed PR or stale head SHA"); + return { headSha, issueNumber, statusPublished: false }; + } + + const changedFiles = (await github.paginate(github.rest.pulls.listFiles, { + owner, + repo, + pull_number: issueNumber, + per_page: PER_PAGE_MAX, + })) as ChangedFile[]; + const coverage = evaluateAutomatedReviewCoverage(pullRequest.changed_files, changedFiles); + status = coverage.manualReviewRequired + ? { + state: CommitStatusState.ERROR, + description: `${MANUAL_REVIEW_DESCRIPTION_PREFIX}${coverage.reason ?? "automated review coverage was limited"}`, + } + : evaluateSemanticReview(result); + } catch (error) { + // Any failure here, including a transient GitHub API error, is reported as incomplete rather + // than as a manual-review hold, so a re-run can still produce a passing result. + const fullReason = error instanceof Error ? error.message : "invalid semantic result"; + core.warning(fullReason); + const prefix = REVIEW_INCOMPLETE_DESCRIPTION_PREFIX; + const available = MAX_STATUS_DESCRIPTION - prefix.length; + const truncatedReason = + fullReason.length > available ? `${fullReason.slice(0, available - 1)}\u2026` : fullReason; + status = { + state: CommitStatusState.ERROR, + description: `${prefix}${truncatedReason}`, + }; + } + + // Checked last so the window between this read and the write is as small as possible. An + // earlier check would leave the slow PR and file-list calls inside the window, where a newer + // run's Pending could be overwritten by this older result. + const latestStatus = await getLatestSemanticStatus(github, owner, repo, headSha); + if (isSuperseded(latestStatus, context.runId, runAttempt)) { + core.info("A newer ARM API review run superseded this result"); + return { headSha, issueNumber, statusPublished: false }; + } + + await publishStatus( + { github, core }, + owner, + repo, headSha, - issueNumber, - runAttempt, - }); - await validateSemanticReviewResult({ - ...context.repo, - result, + getRunStatusUrl(context, owner, repo, runAttempt), + status, + ); + return { headSha, issueNumber, statusPublished: true }; +} + +/** + * Resolves this run's Pending status when the record job never published a result: the agent + * failed, took a noop path, was cancelled, or threat detection did not pass. It reads no agent + * output and only acts when this exact run and attempt still owns the newest status, so a newer + * run's status is never overwritten. The resulting status is Review incomplete, never a + * manual-review hold, because these outcomes are not a property of the PR. + */ +export async function finalizeUnpublishedArmSemanticReview({ + github, + context, + core, +}: GitHubScriptArgs): Promise<{ statusPublished: boolean }> { + const runAttemptText = process.env.GITHUB_RUN_ATTEMPT ?? ""; + const { owner, repo } = context.repo; + const { headSha } = await getWorkflowRunArtifactInputs({ github, + core, + owner, + repo, + runId: context.runId, + }); + if (!isFullGitSha(headSha) || !/^[1-9]\d*$/.test(runAttemptText)) { + core.info("The reviewer run has no trusted head SHA or run attempt; status is unchanged"); + return { statusPublished: false }; + } + + const targetUrl = getRunStatusUrl(context, owner, repo, runAttemptText); + const latestStatus = await getLatestSemanticStatus(github, owner, repo, headSha); + if (latestStatus?.state !== CommitStatusState.PENDING || latestStatus.target_url !== targetUrl) { + core.info("This run does not own a Pending ARM Semantic Review status; status is unchanged"); + return { statusPublished: false }; + } + + await publishStatus({ github, core }, owner, repo, headSha, targetUrl, { + state: CommitStatusState.ERROR, + description: `${REVIEW_INCOMPLETE_DESCRIPTION_PREFIX}reviewer did not publish a result`, }); - return { - artifactValue: encodeSemanticReviewReceipt(result), - }; + return { statusPublished: true }; } -/* v8 ignore stop */ diff --git a/.github/workflows/src/arm-auto-signoff/arm-semantic-review.ts b/.github/workflows/src/arm-auto-signoff/arm-semantic-review.ts index 92a12f9cb3a8..01f824df6d6c 100644 --- a/.github/workflows/src/arm-auto-signoff/arm-semantic-review.ts +++ b/.github/workflows/src/arm-auto-signoff/arm-semantic-review.ts @@ -13,9 +13,17 @@ export type ChangedFile = { export const ARM_SEMANTIC_REVIEW_STATUS = "ARM Semantic Review"; +// Both outcomes are published as a commit status in the "error" state. The description prefix is +// the only thing that tells them apart: a manual-review hold is deterministic (scope or size +// limits) and adds a sticky human label, while an incomplete review is a transient failure that +// a re-run can fix, so it never adds that label. +export const MANUAL_REVIEW_DESCRIPTION_PREFIX = "Manual review required: "; +export const REVIEW_INCOMPLETE_DESCRIPTION_PREFIX = "Review incomplete: "; + export const SemanticReviewOutcome = Object.freeze({ Passed: "passed", ChangesRequested: "changes-requested", + ManualReviewRequired: "manual-review-required", ReviewIncomplete: "review-incomplete", Pending: "pending", }); @@ -31,11 +39,21 @@ export type SemanticReviewScope = (typeof SemanticReviewScope)[keyof typeof Sema export const SemanticReviewCompletion = Object.freeze({ Complete: "complete", Incomplete: "incomplete", - Degraded: "degraded", }); export type SemanticReviewCompletion = (typeof SemanticReviewCompletion)[keyof typeof SemanticReviewCompletion]; +export const SemanticReviewIncompleteReason = Object.freeze({ + None: "none", + CriticUnavailable: "critic-unavailable", + RequiredEvidenceUnavailable: "required-evidence-unavailable", + ToolFailure: "tool-failure", + StaleSha: "stale-sha", + DiscussionDataUnavailable: "discussion-data-unavailable", +}); +export type SemanticReviewIncompleteReason = + (typeof SemanticReviewIncompleteReason)[keyof typeof SemanticReviewIncompleteReason]; + export type SemanticReviewResult = { runAttempt: number; issueNumber: number; @@ -43,6 +61,7 @@ export type SemanticReviewResult = { blockingCount: number; reviewScope: SemanticReviewScope; completion: SemanticReviewCompletion; + incompleteReason: SemanticReviewIncompleteReason; }; export type SemanticReviewCorrelation = Pick< @@ -112,40 +131,57 @@ export function parseSemanticReviewResult( } if ( result.completeness !== SemanticReviewCompletion.Complete && - result.completeness !== SemanticReviewCompletion.Incomplete && - result.completeness !== SemanticReviewCompletion.Degraded + result.completeness !== SemanticReviewCompletion.Incomplete ) { throw new Error(`Invalid ARM semantic review completion: '${String(result.completeness)}'`); } + if ( + result.incomplete_reason !== SemanticReviewIncompleteReason.None && + result.incomplete_reason !== SemanticReviewIncompleteReason.CriticUnavailable && + result.incomplete_reason !== SemanticReviewIncompleteReason.RequiredEvidenceUnavailable && + result.incomplete_reason !== SemanticReviewIncompleteReason.ToolFailure && + result.incomplete_reason !== SemanticReviewIncompleteReason.StaleSha && + result.incomplete_reason !== SemanticReviewIncompleteReason.DiscussionDataUnavailable + ) { + throw new Error( + `Invalid ARM semantic review incomplete reason: '${String(result.incomplete_reason)}'`, + ); + } + if ( + (result.completeness === SemanticReviewCompletion.Complete && + result.incomplete_reason !== SemanticReviewIncompleteReason.None) || + (result.completeness === SemanticReviewCompletion.Incomplete && + result.incomplete_reason === SemanticReviewIncompleteReason.None) + ) { + throw new Error( + `ARM semantic review completion '${result.completeness}' is inconsistent with ` + + `incomplete reason '${result.incomplete_reason}'`, + ); + } return { ...correlation, blockingCount, reviewScope: result.scope, completion: result.completeness, + incompleteReason: result.incomplete_reason, }; } export function evaluateSemanticReview(result: SemanticReviewResult): EvaluatedSemanticReview { - // Scope and completeness are checked first: Changes requested applies only to - // full, complete reviews. A scoped or incomplete review requires manual signoff + // Completeness and scope are checked first: Changes requested applies only to + // full, complete reviews. An incomplete or scoped review never authorizes signoff // regardless of the blocking count. - if (result.reviewScope === SemanticReviewScope.Scoped) { - return { - state: CommitStatusState.ERROR, - description: "Review incomplete: scoped review requires manual signoff", - }; - } if (result.completion === SemanticReviewCompletion.Incomplete) { return { state: CommitStatusState.ERROR, - description: "Review incomplete: reviewer did not complete", + description: `${REVIEW_INCOMPLETE_DESCRIPTION_PREFIX}${result.incompleteReason}`, }; } - if (result.completion === SemanticReviewCompletion.Degraded) { + if (result.reviewScope === SemanticReviewScope.Scoped) { return { state: CommitStatusState.ERROR, - description: "Review incomplete: reviewer completed in degraded mode", + description: `${MANUAL_REVIEW_DESCRIPTION_PREFIX}scoped review requires manual signoff`, }; } if (result.blockingCount > 0) { @@ -200,7 +236,7 @@ export function evaluateAutomatedReviewCoverage( } export function getSemanticReviewOutcome( - status: Pick | undefined, + status: Pick | undefined, ): SemanticReviewOutcome | undefined { if (!status) { return undefined; @@ -215,7 +251,9 @@ export function getSemanticReviewOutcome( return SemanticReviewOutcome.ChangesRequested; } if (status.state === CommitStatusState.ERROR) { - return SemanticReviewOutcome.ReviewIncomplete; + return status.description?.startsWith(MANUAL_REVIEW_DESCRIPTION_PREFIX) + ? SemanticReviewOutcome.ManualReviewRequired + : SemanticReviewOutcome.ReviewIncomplete; } return undefined; } diff --git a/.github/workflows/src/arm-auto-signoff/arm-universal-auto-signoff.ts b/.github/workflows/src/arm-auto-signoff/arm-universal-auto-signoff.ts index cd07ea600e09..98147078d059 100644 --- a/.github/workflows/src/arm-auto-signoff/arm-universal-auto-signoff.ts +++ b/.github/workflows/src/arm-auto-signoff/arm-universal-auto-signoff.ts @@ -210,7 +210,7 @@ async function getDesiredLabelActions({ ); if (semanticReviewOutcome !== SemanticReviewOutcome.Passed) { - if (semanticReviewOutcome === SemanticReviewOutcome.ReviewIncomplete) { + if (semanticReviewOutcome === SemanticReviewOutcome.ManualReviewRequired) { const hasManualSignoffRequired = labelNames.includes( ArmAutoSignoffLabel.ArmManualSignoffRequired, ); diff --git a/.github/workflows/test/arm-api-review-workflow.test.ts b/.github/workflows/test/arm-api-review-workflow.test.ts index ae5b1690b59b..91359887a094 100644 --- a/.github/workflows/test/arm-api-review-workflow.test.ts +++ b/.github/workflows/test/arm-api-review-workflow.test.ts @@ -315,6 +315,12 @@ describe("ARM API review workflow", () => { expect(workflow).toContain("github.event.comment.body != '/arm-review'"); expect(workflow).toContain("github.event.issue.pull_request == null"); expect(workflow).toContain("github.event.label.name != 'WaitForARMFeedback'"); + // PR events the trigger gate skips (draft, or no queue label) must not join the PR group + // and cancel an active review. + expect(workflow).toContain("github.event.pull_request.draft == true"); + expect(workflow).toContain( + "!contains(github.event.pull_request.labels.*.name, 'WaitForARMFeedback')", + ); expect(workflow).toContain("&& github.run_id || github.event.issue.number"); } }); @@ -575,7 +581,7 @@ describe("ARM API review posting reliability", () => { expect(source).toContain("These rules are **exhaustive**."); expect(source).toContain( - "The review is scoped, incomplete, or degraded** β†’ leave `WaitForARMFeedback`, `ARMChangesRequested`, and `ARMSignedOff` unchanged", + "The review is scoped or incomplete** β†’ leave `WaitForARMFeedback`, `ARMChangesRequested`, and `ARMSignedOff` unchanged", ); expect(source).toContain( "No verified, currently applicable Blocking finding remains, and the review is full and complete", @@ -589,7 +595,7 @@ describe("ARM API review posting reliability", () => { expect(source).toContain("Nothing read from unrelated PR metadata may change the outcome."); }); - it("records one head-bound semantic review result for completion-time finalization", async () => { + it("publishes one head-bound semantic review status from the trusted safe-output job", async () => { const [source, compiled] = await readWorkflowFiles(); const collapsed = collapseWhitespace(source); @@ -604,7 +610,8 @@ describe("ARM API review posting reliability", () => { ); expect(compiled).toContain("record_arm_semantic_review"); expect(compiled).toContain("ARM Semantic Review"); - expect(compiled).toContain("Upload ARM semantic review receipt"); + expect(compiled).toContain("Publish ARM semantic review status"); + expect(compiled).not.toContain("Upload ARM semantic review receipt"); const semanticJob = compiled.slice( compiled.indexOf("\n record_arm_semantic_review:\n"), compiled.indexOf("\n safe_outputs:\n"), @@ -613,19 +620,50 @@ describe("ARM API review posting reliability", () => { source.indexOf(" record-arm-semantic-review:\n"), source.indexOf(" noop:\n"), ); - expect(semanticJob).toContain("name: Validate ARM semantic review"); + expect(semanticJob).toContain("name: Publish ARM semantic review status"); expect(semanticJob).toContain("actions: read"); + expect(semanticJob).toContain("statuses: write"); expect(semanticJob).not.toContain("TARGET_PR_NUMBER"); expect(semanticJob).toContain("permissions:"); expect(semanticConfig).not.toContain("issue_number:"); expect(semanticConfig).not.toContain("head_sha:"); expect(semanticConfig).not.toContain("run_attempt:"); + expect(semanticConfig).not.toContain('"degraded"'); + expect(semanticConfig).toContain("incomplete_reason:"); + expect(semanticConfig).toContain('"critic-unavailable"'); + expect(semanticConfig).toContain('"discussion-data-unavailable"'); expect(collapsed).toContain( "Trusted workflow artifacts attach the pull request, head SHA, and run attempt", ); expect(collapsed).toContain( - "`ARM Semantic Review - Set Status` consumes the exact completed run's agent output and trusted correlation artifacts", + "The trusted `record_arm_semantic_review` job validates the item and publishes the head-bound status inside this reviewer run", + ); + expect(collapsed).toContain( + "Universal Auto-Signoff consumes the status only after the entire reviewer workflow completes", + ); + }); + + it("resolves an unpublished Pending status from the conclusion job", async () => { + const [source, compiled] = await readWorkflowFiles(); + const conclusionStart = compiled.indexOf("\n conclusion:\n"); + const conclusionJob = compiled.slice( + conclusionStart, + compiled.indexOf("\n detection:\n", conclusionStart), + ); + const finalizerStep = conclusionJob.indexOf( + "name: Resolve unpublished ARM semantic review status", + ); + + expect(source).toContain("finalizeUnpublishedArmSemanticReview"); + expect(conclusionStart).toBeGreaterThan(-1); + expect(conclusionJob).toContain("statuses: write"); + expect(finalizerStep).toBeGreaterThan(-1); + // The finalizer is a pre-step, so it must not be able to block gh-aw's own + // failure reporting that runs after it. + expect(conclusionJob.slice(finalizerStep, finalizerStep + 700)).toContain( + "continue-on-error: true", ); + expect(conclusionJob.indexOf("name: Process no-op messages")).toBeGreaterThan(finalizerStep); }); it("withholds ARMChangesRequested when the Critic could not verify the findings", async () => { diff --git a/.github/workflows/test/arm-auto-signoff/arm-semantic-review-status-workflow.test.ts b/.github/workflows/test/arm-auto-signoff/arm-semantic-review-status-workflow.test.ts deleted file mode 100644 index 8c684436bee5..000000000000 --- a/.github/workflows/test/arm-auto-signoff/arm-semantic-review-status-workflow.test.ts +++ /dev/null @@ -1,148 +0,0 @@ -import { readFile } from "node:fs/promises"; -import { load } from "js-yaml"; -import { join } from "node:path"; -import { describe, expect, it } from "vitest"; - -const GITHUB_ROOT = join(import.meta.dirname, "..", "..", ".."); - -type Workflow = { - name?: string; - on?: { - workflow_run?: { - workflows?: string[]; - types?: string[]; - }; - }; - permissions?: Record; - jobs?: Record< - string, - { - steps?: Array<{ - id?: string; - name?: string; - if?: string; - uses?: string; - "continue-on-error"?: boolean; - env?: Record; - with?: { - script?: string; - name?: string; - value?: string; - path?: string; - "github-token"?: string; - repository?: string; - "run-id"?: string; - }; - }>; - } - >; -}; - -async function readWorkflow(name: string): Promise<{ - source: string; - workflow: Workflow; -}> { - const source = await readFile(join(GITHUB_ROOT, "workflows", name), "utf8"); - return { - source, - workflow: load(source) as Workflow, - }; -} - -describe("ARM Semantic Review - Set Status workflow", () => { - it("finalizes only completed ARM API Reviewer runs", async () => { - const { workflow } = await readWorkflow("arm-semantic-review-status.yaml"); - - expect(workflow.name).toBe("ARM Semantic Review - Set Status"); - expect(workflow.on?.workflow_run).toEqual({ - workflows: ["ARM API Review: Automated Workflow"], - types: ["completed"], - }); - }); - - it("isolates status-write permission from Universal Auto-Signoff", async () => { - const [{ workflow: semanticStatus }, { workflow: universal }] = await Promise.all([ - readWorkflow("arm-semantic-review-status.yaml"), - readWorkflow("arm-universal-auto-signoff.yaml"), - ]); - - expect(semanticStatus.permissions).toEqual({ - actions: "read", - contents: "read", - "pull-requests": "read", - statuses: "write", - }); - expect(universal.permissions?.statuses).toBe("read"); - }); - - it("publishes PR and SHA correlation for Universal Auto-Signoff", async () => { - const { workflow } = await readWorkflow("arm-semantic-review-status.yaml"); - const steps = workflow.jobs?.["arm-semantic-review-status"]?.steps ?? []; - const finalize = steps.find((step) => step.id === "finalize"); - - expect(finalize?.with?.script).toContain("finalizeArmSemanticReview"); - expect(steps).toEqual( - expect.arrayContaining([ - expect.objectContaining({ - name: "Upload artifact with head SHA", - with: { - name: "head-sha", - value: "${{ fromJson(steps.finalize.outputs.result).headSha }}", - }, - }), - expect.objectContaining({ - name: "Upload artifact with issue number", - with: { - name: "issue-number", - value: "${{ fromJson(steps.finalize.outputs.result).issueNumber }}", - }, - }), - ]), - ); - }); - - it("downloads agent output from the exact completed reviewer run", async () => { - const { workflow } = await readWorkflow("arm-semantic-review-status.yaml"); - const steps = workflow.jobs?.["arm-semantic-review-status"]?.steps ?? []; - const download = steps.find((step) => step.name === "Download ARM API Reviewer output"); - const finalize = steps.find((step) => step.id === "finalize"); - - expect(download).toEqual( - expect.objectContaining({ - "continue-on-error": true, - uses: "actions/download-artifact@v8.0.1", - with: { - name: "agent", - path: "${{ runner.temp }}/arm-semantic-review", - "github-token": "${{ github.token }}", - repository: "${{ github.repository }}", - "run-id": "${{ github.event.workflow_run.id }}", - }, - }), - ); - expect(finalize?.env?.GH_AW_AGENT_OUTPUT).toBe( - "${{ runner.temp }}/arm-semantic-review/agent_output.json", - ); - }); - - it("makes Universal depend on semantic status completion, not reviewer completion", async () => { - const { source, workflow } = await readWorkflow("arm-universal-auto-signoff.yaml"); - const workflows = workflow.on?.workflow_run?.workflows ?? []; - - expect(workflows).toContain("ARM Semantic Review - Set Status"); - expect(workflows).not.toContain("ARM API Review: Automated Workflow"); - expect(source).not.toContain("Finalize ARM Semantic Review"); - expect(source).not.toContain("finalizeArmSemanticReview"); - }); - - it("publishes only pilot signoff and manual-review label artifacts", async () => { - const { source } = await readWorkflow("arm-universal-auto-signoff.yaml"); - - for (const label of ["ARMAutoSignedOff-Test", "ARMManualSignoffRequired"]) { - expect(source).toContain(`Upload artifact for ${label} label`); - } - for (const label of ["ARMSignedOff", "ARMAutoSignedOff", "ARMChangesRequested"]) { - expect(source).not.toContain(`Upload artifact for ${label} label`); - } - }); -}); diff --git a/.github/workflows/test/arm-auto-signoff/arm-semantic-review.test.ts b/.github/workflows/test/arm-auto-signoff/arm-semantic-review.test.ts index 14603f2ce879..d27fb20846c0 100644 --- a/.github/workflows/test/arm-auto-signoff/arm-semantic-review.test.ts +++ b/.github/workflows/test/arm-auto-signoff/arm-semantic-review.test.ts @@ -10,13 +10,15 @@ import { getSemanticReviewOutcome, parseSemanticReviewResult, SemanticReviewCompletion, + SemanticReviewIncompleteReason, SemanticReviewOutcome, SemanticReviewScope, type ChangedFile, type SemanticReviewResult, } from "../../src/arm-auto-signoff/arm-semantic-review.ts"; -import { finalizeArmSemanticReview } from "../../src/arm-auto-signoff/arm-semantic-review-status.ts"; -import validateArmSemanticReview from "../../src/arm-auto-signoff/arm-semantic-review-workflow.ts"; +import publishArmSemanticReviewStatus, { + finalizeUnpublishedArmSemanticReview, +} from "../../src/arm-auto-signoff/arm-semantic-review-workflow.ts"; import type { GitHubScriptArgs } from "../../src/github.ts"; import { createMockCore, createMockGithub } from "../mocks.ts"; @@ -27,7 +29,6 @@ const headSha = "0123456789abcdef0123456789abcdef01234567"; const runId = 456; const runAttempt = 1; const runUrl = "https://github.com/Azure/azure-rest-api-specs/actions/runs/456"; -const workflowPath = ".github/workflows/arm-api-review.lock.yml"; const passedResult: SemanticReviewResult = { runAttempt, @@ -36,6 +37,7 @@ const passedResult: SemanticReviewResult = { blockingCount: 0, reviewScope: SemanticReviewScope.Full, completion: SemanticReviewCompletion.Complete, + incompleteReason: SemanticReviewIncompleteReason.None, }; function agentOutput(result: SemanticReviewResult = passedResult) { @@ -46,6 +48,7 @@ function agentOutput(result: SemanticReviewResult = passedResult) { blocking_count: String(result.blockingCount), scope: result.reviewScope, completeness: result.completion, + incomplete_reason: result.incompleteReason, }, ], }; @@ -55,7 +58,7 @@ const defaultChangedFiles: ChangedFile[] = [ { filename: "specification/foo/foo.json", additions: 10, deletions: 5 }, ]; -function createFinalizeGithub() { +function createPublisherGithub() { const github = createMockGithub(); github.rest.actions.listWorkflowRunArtifacts.mockResolvedValue({ data: { @@ -70,49 +73,32 @@ function createFinalizeGithub() { return github; } -function createContext(conclusion: string | null = "success", path = workflowPath) { - return { - payload: { - workflow_run: { - name: `ARM API Review #${issueNumber} (issue_comment)`, - path, - conclusion, - id: runId, - run_attempt: runAttempt, - html_url: runUrl, - repository: { - name: repo, - owner: { login: owner }, - }, - }, - }, - }; -} - -async function runFinalizer({ - conclusion = "success", +async function runPublisher({ output = agentOutput(), includeOutput = true, - github = createFinalizeGithub(), - path = workflowPath, + github = createPublisherGithub(), }: { - conclusion?: string | null; output?: unknown; includeOutput?: boolean; - github?: ReturnType; - path?: string; + github?: ReturnType; } = {}) { const directory = await mkdtemp(join(tmpdir(), "arm-semantic-review-")); const outputPath = join(directory, "agent_output.json"); const previousOutputPath = process.env.GH_AW_AGENT_OUTPUT; + const previousRunAttempt = process.env.GITHUB_RUN_ATTEMPT; try { if (includeOutput) { await writeFile(outputPath, JSON.stringify(output), "utf8"); } process.env.GH_AW_AGENT_OUTPUT = outputPath; - const result = await finalizeArmSemanticReview({ + process.env.GITHUB_RUN_ATTEMPT = String(runAttempt); + const result = await publishArmSemanticReviewStatus({ github, - context: createContext(conclusion, path), + context: { + repo: { owner, repo }, + runId, + serverUrl: "https://github.com", + }, core: createMockCore(), } as unknown as GitHubScriptArgs); return { github, result }; @@ -122,6 +108,11 @@ async function runFinalizer({ } else { process.env.GH_AW_AGENT_OUTPUT = previousOutputPath; } + if (previousRunAttempt === undefined) { + delete process.env.GITHUB_RUN_ATTEMPT; + } else { + process.env.GITHUB_RUN_ATTEMPT = previousRunAttempt; + } await rm(directory, { recursive: true, force: true }); } } @@ -177,6 +168,43 @@ describe("parseSemanticReviewResult", () => { { headSha, issueNumber, runAttempt }, ), ).toThrow("Invalid ARM semantic review completion"); + expect(() => + parseSemanticReviewResult( + { + items: [{ ...agentOutput().items[0], incomplete_reason: "unknown" }], + }, + { headSha, issueNumber, runAttempt }, + ), + ).toThrow("Invalid ARM semantic review incomplete reason"); + }); + + it("rejects inconsistent completeness and reason combinations", () => { + expect(() => + parseSemanticReviewResult( + { + items: [ + { + ...agentOutput().items[0], + incomplete_reason: SemanticReviewIncompleteReason.CriticUnavailable, + }, + ], + }, + { headSha, issueNumber, runAttempt }, + ), + ).toThrow("completion 'complete' is inconsistent"); + expect(() => + parseSemanticReviewResult( + { + items: [ + { + ...agentOutput().items[0], + completeness: SemanticReviewCompletion.Incomplete, + }, + ], + }, + { headSha, issueNumber, runAttempt }, + ), + ).toThrow("completion 'incomplete' is inconsistent"); }); it("rejects invalid trusted correlation", () => { @@ -190,55 +218,6 @@ describe("parseSemanticReviewResult", () => { }); }); -describe("validateArmSemanticReview", () => { - it("records trusted correlation when model-supplied identifiers are malformed", async () => { - const directory = await mkdtemp(join(tmpdir(), "arm-semantic-review-receipt-")); - const outputPath = join(directory, "agent_output.json"); - const output = agentOutput(); - Object.assign(output.items[0], { - head_sha: "675718552ce00736684ed7a184bab", - issue_number: "999", - run_attempt: "99", - }); - const previousOutputPath = process.env.GH_AW_AGENT_OUTPUT; - const previousRunAttempt = process.env.GITHUB_RUN_ATTEMPT; - try { - await writeFile(outputPath, JSON.stringify(output), "utf8"); - process.env.GH_AW_AGENT_OUTPUT = outputPath; - process.env.GITHUB_RUN_ATTEMPT = String(runAttempt); - const github = createFinalizeGithub(); - - const result = await validateArmSemanticReview({ - github, - context: { repo: { owner, repo }, runId }, - core: createMockCore(), - } as unknown as GitHubScriptArgs); - - expect(result).toEqual({ - artifactValue: `${runAttempt}.${issueNumber}.${headSha}.full.complete.0`, - }); - expect(github.rest.actions.listWorkflowRunArtifacts).toHaveBeenCalledWith({ - owner, - repo, - run_id: runId, - per_page: PER_PAGE_MAX, - }); - } finally { - if (previousOutputPath === undefined) { - delete process.env.GH_AW_AGENT_OUTPUT; - } else { - process.env.GH_AW_AGENT_OUTPUT = previousOutputPath; - } - if (previousRunAttempt === undefined) { - delete process.env.GITHUB_RUN_ATTEMPT; - } else { - process.env.GITHUB_RUN_ATTEMPT = previousRunAttempt; - } - await rm(directory, { recursive: true, force: true }); - } - }); -}); - describe("evaluateSemanticReview", () => { it("passes when a full, complete review has no Blocking findings", () => { expect(evaluateSemanticReview(passedResult)).toEqual({ @@ -254,52 +233,50 @@ describe("evaluateSemanticReview", () => { }); }); - it.each([ - { - name: "scoped", - result: { ...passedResult, reviewScope: SemanticReviewScope.Scoped }, - description: "Review incomplete: scoped review requires manual signoff", - }, - { - name: "incomplete", - result: { ...passedResult, completion: SemanticReviewCompletion.Incomplete }, - description: "Review incomplete: reviewer did not complete", - }, - { - name: "degraded", - result: { ...passedResult, completion: SemanticReviewCompletion.Degraded }, - description: "Review incomplete: reviewer completed in degraded mode", - }, - ])("requires manual review when the semantic review is $name", ({ result, description }) => { + it("requires manual review when the semantic review is scoped", () => { + const result = { ...passedResult, reviewScope: SemanticReviewScope.Scoped }; + const description = "Manual review required: scoped review requires manual signoff"; + expect(evaluateSemanticReview(result)).toEqual({ state: CommitStatusState.ERROR, description, }); }); + it.each([ + SemanticReviewIncompleteReason.CriticUnavailable, + SemanticReviewIncompleteReason.RequiredEvidenceUnavailable, + SemanticReviewIncompleteReason.ToolFailure, + SemanticReviewIncompleteReason.StaleSha, + SemanticReviewIncompleteReason.DiscussionDataUnavailable, + ])("includes incomplete reason %s in the status", (incompleteReason) => { + const result = { + ...passedResult, + completion: SemanticReviewCompletion.Incomplete, + incompleteReason, + }; + + expect(evaluateSemanticReview(result)).toEqual({ + state: CommitStatusState.ERROR, + description: `Review incomplete: ${incompleteReason}`, + }); + }); + it.each([ { name: "scoped with blockers", result: { ...passedResult, reviewScope: SemanticReviewScope.Scoped, blockingCount: 2 }, - description: "Review incomplete: scoped review requires manual signoff", + description: "Manual review required: scoped review requires manual signoff", }, { name: "incomplete with blockers", result: { ...passedResult, completion: SemanticReviewCompletion.Incomplete, + incompleteReason: SemanticReviewIncompleteReason.CriticUnavailable, blockingCount: 1, }, - description: "Review incomplete: reviewer did not complete", - }, - { - name: "degraded with blockers", - result: { - ...passedResult, - completion: SemanticReviewCompletion.Degraded, - blockingCount: 3, - }, - description: "Review incomplete: reviewer completed in degraded mode", + description: "Review incomplete: critic-unavailable", }, ])( "requires manual review rather than changes requested for $name", @@ -320,6 +297,18 @@ describe("evaluateSemanticReview", () => { expect(getSemanticReviewOutcome({ state: "error" })).toBe( SemanticReviewOutcome.ReviewIncomplete, ); + expect( + getSemanticReviewOutcome({ + state: "error", + description: "Review incomplete: tool-failure", + }), + ).toBe(SemanticReviewOutcome.ReviewIncomplete); + expect( + getSemanticReviewOutcome({ + state: "error", + description: "Manual review required: scoped review requires manual signoff", + }), + ).toBe(SemanticReviewOutcome.ManualReviewRequired); }); }); @@ -368,9 +357,9 @@ describe("evaluateAutomatedReviewCoverage", () => { }); }); -describe("finalizeArmSemanticReview", () => { +describe("publishArmSemanticReviewStatus", () => { it("publishes Passed for a clean reviewer result", async () => { - const { github, result } = await runFinalizer(); + const { github, result } = await runPublisher(); expect(result).toEqual({ headSha, issueNumber, statusPublished: true }); expect(github.rest.repos.createCommitStatus).toHaveBeenCalledWith({ @@ -391,24 +380,8 @@ describe("finalizeArmSemanticReview", () => { }); }); - it("rejects a completion from another workflow", async () => { - const github = createFinalizeGithub(); - await expect( - runFinalizer({ - github, - path: ".github/workflows/other-workflow.yaml", - }), - ).rejects.toThrow( - "Unexpected triggering workflow path: expected " + - "'.github/workflows/arm-api-review.lock.yml', " + - "received '.github/workflows/other-workflow.yaml'", - ); - expect(github.rest.actions.listWorkflowRunArtifacts).not.toHaveBeenCalled(); - expect(github.rest.repos.createCommitStatus).not.toHaveBeenCalled(); - }); - it("publishes Changes requested for Blocking findings", async () => { - const { github } = await runFinalizer({ + const { github } = await runPublisher({ output: agentOutput({ ...passedResult, blockingCount: 3 }), }); @@ -420,25 +393,20 @@ describe("finalizeArmSemanticReview", () => { ); }); - it("publishes Review incomplete for a scoped clean review", async () => { - const { github } = await runFinalizer({ + it("publishes Manual review required for a scoped clean review", async () => { + const { github } = await runPublisher({ output: agentOutput({ ...passedResult, reviewScope: SemanticReviewScope.Scoped }), }); expect(github.rest.repos.createCommitStatus).toHaveBeenCalledWith( expect.objectContaining({ state: CommitStatusState.ERROR, - description: "Review incomplete: scoped review requires manual signoff", + description: "Manual review required: scoped review requires manual signoff", }), ); }); it.each([ - { - name: "failed workflow", - options: { conclusion: "failure" }, - description: "Review incomplete: workflow concluded with failure", - }, { name: "missing agent output", options: { includeOutput: false }, @@ -459,7 +427,7 @@ describe("finalizeArmSemanticReview", () => { description: "Review incomplete: Invalid ARM semantic review Blocking count", }, ])("publishes Review incomplete for $name", async ({ options, description }) => { - const { github } = await runFinalizer(options); + const { github } = await runPublisher(options); expect(github.rest.repos.createCommitStatus).toHaveBeenCalledWith( expect.objectContaining({ @@ -472,20 +440,13 @@ describe("finalizeArmSemanticReview", () => { expect(status.description).toContain(description); }); - it("leaves Pending unchanged when the reviewer run is canceled", async () => { - const { github, result } = await runFinalizer({ conclusion: "cancelled" }); - - expect(result).toEqual({ headSha, issueNumber, statusPublished: false }); - expect(github.rest.repos.createCommitStatus).not.toHaveBeenCalled(); - }); - it("does nothing when trusted PR/SHA correlation is missing", async () => { - const github = createFinalizeGithub(); + const github = createPublisherGithub(); github.rest.actions.listWorkflowRunArtifacts.mockResolvedValue({ data: { artifacts: [] }, }); - const execution = await runFinalizer({ github }); + const execution = await runPublisher({ github }); expect(execution.result).toEqual({ headSha: "", issueNumber: 0, @@ -504,10 +465,10 @@ describe("finalizeArmSemanticReview", () => { pullRequest: { changed_files: 1, state: "open", head: { sha: "f".repeat(40) } }, }, ])("does not publish a status when the pull request is $name", async ({ pullRequest }) => { - const github = createFinalizeGithub(); + const github = createPublisherGithub(); github.rest.pulls.get.mockResolvedValue({ data: pullRequest }); - const execution = await runFinalizer({ github }); + const execution = await runPublisher({ github }); expect(execution.result).toEqual({ headSha, @@ -518,8 +479,8 @@ describe("finalizeArmSemanticReview", () => { expect(github.rest.repos.createCommitStatus).not.toHaveBeenCalled(); }); - it("publishes Review incomplete when the PR exceeds automated review size limits", async () => { - const github = createFinalizeGithub(); + it("publishes Manual review required when the PR exceeds automated review size limits", async () => { + const github = createPublisherGithub(); const oversizedFiles = Array.from({ length: 51 }, (_, i): ChangedFile => ({ filename: `specification/foo/foo${i}.json`, additions: 1, @@ -530,18 +491,18 @@ describe("finalizeArmSemanticReview", () => { }); github.rest.pulls.listFiles.mockResolvedValue({ data: oversizedFiles }); - const { github: g } = await runFinalizer({ github }); + const { github: g } = await runPublisher({ github }); expect(g.rest.repos.createCommitStatus).toHaveBeenCalledWith( expect.objectContaining({ state: CommitStatusState.ERROR, - description: "Review incomplete: PR exceeds automated review size limits", + description: "Manual review required: PR exceeds automated review size limits", }), ); }); it("truncates an overly long error description to 140 characters", async () => { const longValue = "x".repeat(200); - const { github } = await runFinalizer({ + const { github } = await runPublisher({ output: { items: [{ ...agentOutput().items[0], scope: longValue }] }, }); @@ -553,7 +514,7 @@ describe("finalizeArmSemanticReview", () => { }); it("does not overwrite a newer review of the same SHA", async () => { - const github = createFinalizeGithub(); + const github = createPublisherGithub(); github.rest.repos.listCommitStatusesForRef.mockResolvedValue({ data: [ { @@ -565,7 +526,7 @@ describe("finalizeArmSemanticReview", () => { ], }); - const execution = await runFinalizer({ github }); + const execution = await runPublisher({ github }); expect(execution.result).toEqual({ headSha, issueNumber, @@ -574,8 +535,21 @@ describe("finalizeArmSemanticReview", () => { expect(github.rest.repos.createCommitStatus).not.toHaveBeenCalled(); }); + it("checks for a newer run only after reading the PR and its files", async () => { + const github = createPublisherGithub(); + + await runPublisher({ github }); + + const [listStatuses] = github.rest.repos.listCommitStatusesForRef.mock.invocationCallOrder; + const [getPullRequest] = github.rest.pulls.get.mock.invocationCallOrder; + const [listFiles] = github.rest.pulls.listFiles.mock.invocationCallOrder; + const [createStatus] = github.rest.repos.createCommitStatus.mock.invocationCallOrder; + expect(listStatuses).toBeGreaterThan(getPullRequest); + expect(listStatuses).toBeGreaterThan(listFiles); + expect(listStatuses).toBeLessThan(createStatus); + }); it("does not overwrite a newer attempt of the same run", async () => { - const github = createFinalizeGithub(); + const github = createPublisherGithub(); github.rest.repos.listCommitStatusesForRef.mockResolvedValue({ data: [ { @@ -587,7 +561,7 @@ describe("finalizeArmSemanticReview", () => { ], }); - const execution = await runFinalizer({ github }); + const execution = await runPublisher({ github }); expect(execution.result).toEqual({ headSha, issueNumber, @@ -596,3 +570,111 @@ describe("finalizeArmSemanticReview", () => { expect(github.rest.repos.createCommitStatus).not.toHaveBeenCalled(); }); }); + +describe("finalizeUnpublishedArmSemanticReview", () => { + const ownedPending = { + context: ARM_SEMANTIC_REVIEW_STATUS, + state: CommitStatusState.PENDING, + target_url: `${runUrl}/attempts/${runAttempt}`, + updated_at: "2026-01-01T00:00:00Z", + }; + + async function runFinalizer(statuses: unknown[], github = createPublisherGithub()) { + github.rest.repos.listCommitStatusesForRef.mockResolvedValue({ data: statuses }); + const previousRunAttempt = process.env.GITHUB_RUN_ATTEMPT; + process.env.GITHUB_RUN_ATTEMPT = String(runAttempt); + try { + const result = await finalizeUnpublishedArmSemanticReview({ + github, + context: { repo: { owner, repo }, runId, serverUrl: "https://github.com" }, + core: createMockCore(), + } as unknown as GitHubScriptArgs); + return { github, result }; + } finally { + if (previousRunAttempt === undefined) { + delete process.env.GITHUB_RUN_ATTEMPT; + } else { + process.env.GITHUB_RUN_ATTEMPT = previousRunAttempt; + } + } + } + + it("resolves a Pending status owned by this run as Review incomplete", async () => { + const { github, result } = await runFinalizer([ownedPending]); + + expect(result).toEqual({ statusPublished: true }); + expect(github.rest.repos.createCommitStatus).toHaveBeenCalledWith({ + owner, + repo, + sha: headSha, + state: CommitStatusState.ERROR, + context: ARM_SEMANTIC_REVIEW_STATUS, + description: "Review incomplete: reviewer did not publish a result", + target_url: `${runUrl}/attempts/${runAttempt}`, + }); + }); + + it("never publishes a manual-review hold for an unpublished result", async () => { + const { github } = await runFinalizer([ownedPending]); + + const status = github.rest.repos.createCommitStatus.mock.calls[0]?.[0] as { + description: string; + }; + expect(getSemanticReviewOutcome({ state: "error", description: status.description })).toBe( + SemanticReviewOutcome.ReviewIncomplete, + ); + }); + + it.each([ + { name: "there is no status", statuses: [] }, + { + name: "the record job already published a result", + statuses: [ + { + ...ownedPending, + state: CommitStatusState.SUCCESS, + updated_at: "2026-01-01T00:00:01Z", + }, + ownedPending, + ], + }, + { + name: "a newer run owns the newest status", + statuses: [ + { + ...ownedPending, + target_url: "https://github.com/Azure/azure-rest-api-specs/actions/runs/789/attempts/1", + updated_at: "2026-01-01T00:00:01Z", + }, + ownedPending, + ], + }, + { + name: "a newer attempt of this run owns the newest status", + statuses: [ + { + ...ownedPending, + target_url: `${runUrl}/attempts/2`, + updated_at: "2026-01-01T00:00:01Z", + }, + ownedPending, + ], + }, + ])("leaves the status unchanged when $name", async ({ statuses }) => { + const { github, result } = await runFinalizer(statuses); + + expect(result).toEqual({ statusPublished: false }); + expect(github.rest.repos.createCommitStatus).not.toHaveBeenCalled(); + }); + + it("leaves the status unchanged when the run has no trusted head SHA", async () => { + const github = createPublisherGithub(); + github.rest.actions.listWorkflowRunArtifacts.mockResolvedValue({ data: { artifacts: [] } }); + + const { result } = await runFinalizer([ownedPending], github); + + expect(result).toEqual({ statusPublished: false }); + expect(github.rest.repos.listCommitStatusesForRef).not.toHaveBeenCalled(); + expect(github.rest.repos.createCommitStatus).not.toHaveBeenCalled(); + }); +}); diff --git a/.github/workflows/test/arm-auto-signoff/arm-universal-auto-signoff.test.ts b/.github/workflows/test/arm-auto-signoff/arm-universal-auto-signoff.test.ts index 54417e3ff5cd..7ec0becd85ef 100644 --- a/.github/workflows/test/arm-auto-signoff/arm-universal-auto-signoff.test.ts +++ b/.github/workflows/test/arm-auto-signoff/arm-universal-auto-signoff.test.ts @@ -130,6 +130,16 @@ describe("getLabelActionImpl", () => { }); describe("auto-signoff workflow correlation", () => { + it("re-evaluates after the ARM API Reviewer publishes its status", async () => { + const workflow = await readFile( + join(GITHUB_ROOT, "workflows", "arm-universal-auto-signoff.yaml"), + "utf8", + ); + + expect(workflow).toContain('"ARM API Review: Automated Workflow"'); + expect(workflow).not.toContain("ARM Semantic Review - Set Status"); + }); + it("always publishes live-head correlation artifacts", async () => { const workflow = await readFile( join(GITHUB_ROOT, "workflows", "arm-universal-auto-signoff.yaml"), @@ -200,7 +210,7 @@ describe("getLabelActionImpl", () => { }, ); - it("adds manual signoff when semantic review is incomplete", async () => { + it("does not add manual signoff when semantic review is incomplete", async () => { const github = createMockGithub({ labelNames: ["ARMReview"], statuses: [ @@ -217,7 +227,7 @@ describe("getLabelActionImpl", () => { const result = await run(github); expect(result.labelActions).toEqual({ [ArmAutoSignoffLabel.ArmAutoSignedOffTest]: LabelAction.None, - [ArmAutoSignoffLabel.ArmManualSignoffRequired]: LabelAction.Add, + [ArmAutoSignoffLabel.ArmManualSignoffRequired]: LabelAction.None, }); }); @@ -229,7 +239,7 @@ describe("getLabelActionImpl", () => { { context: "ARM Semantic Review", state: CommitStatusState.ERROR, - description: "Review incomplete: reviewer did not complete", + description: "Manual review required: PR exceeds automated review size limits", updated_at: "2026-01-01", }, ...deterministicStatuses, @@ -253,7 +263,9 @@ describe("getLabelActionImpl", () => { expect.stringContaining('"isReadyForArmReview":true'), ); expect(loggingCore.info).toHaveBeenCalledWith( - expect.stringContaining('"description":"Review incomplete: reviewer did not complete"'), + expect.stringContaining( + '"description":"Manual review required: PR exceeds automated review size limits"', + ), ); expect(loggingCore.info).toHaveBeenCalledWith( "ARM semantic review requires manual signoff; ARMManualSignoffRequired is already present", diff --git a/documentation/arm-semantic-review-auto-signoff-implementation.md b/documentation/arm-semantic-review-auto-signoff-implementation.md index e560af983ecd..f921faeb2d47 100644 --- a/documentation/arm-semantic-review-auto-signoff-implementation.md +++ b/documentation/arm-semantic-review-auto-signoff-implementation.md @@ -24,9 +24,8 @@ and Review incomplete all block it. Universal never changes `ARMSignedOff`. ```text ARM API Reviewer produces one structured semantic item in agent_output.json - -ARM Semantic Review - Set Status - validates the item and publishes a commit status on its SHA + validates it in a trusted safe-output job + publishes a commit status on the reviewed SHA Universal Auto-Signoff reads statuses for the current PR head and decides @@ -35,8 +34,9 @@ Update Labels applies the decision if the PR head is still current ``` -The status workflow may publish a result for an older SHA. That is safe: -Universal Auto-Signoff reads statuses only for the PR's current SHA. +The reviewer may finish after the PR head changes. That is safe: the publisher +rejects stale heads, and Universal Auto-Signoff reads statuses only for the +current PR head. ## Components @@ -70,24 +70,21 @@ Responsibilities: This file is pure policy and parsing. It makes no GitHub API calls. -### Semantic Set Status orchestration - -Workflow: -[arm-semantic-review-status.yaml](../.github/workflows/arm-semantic-review-status.yaml) +### Semantic status publication -Implementation: -[arm-semantic-review-status.ts](../.github/workflows/src/arm-auto-signoff/arm-semantic-review-status.ts) +[arm-semantic-review-workflow.ts](../.github/workflows/src/arm-auto-signoff/arm-semantic-review-workflow.ts) Responsibilities: -- trigger from the exact completed reviewer run; -- download that run's `agent_output.json`; -- read trusted PR/SHA correlation artifacts; +- run as the reviewer's trusted `record_arm_semantic_review` safe-output job; +- read that run's `agent_output.json` and trusted PR/SHA correlation artifacts; - validate the semantic item once; +- independently enforce automated-review coverage limits; - publish `ARM Semantic Review` on the reviewed SHA; -- upload PR/SHA correlation for Universal Auto-Signoff. +- avoid overwriting a newer review or rerun. -Only this workflow has `statuses: write`. +Only the reviewer `pre_activation`, `record_arm_semantic_review`, and `conclusion` jobs have +`statuses: write`. The agent itself still has read-only GitHub tools. ### Universal Auto-Signoff @@ -103,7 +100,7 @@ Responsibilities: - read Semantic Review, LintDiff, and Avocado statuses for that SHA; - evaluate current labels and approvals; - manage the pilot `ARMAutoSignedOff-Test` label and add the manual-review label - when semantic evidence is incomplete. + when the review was scoped or the PR exceeded automated review limits. Universal has only `statuses: read`. @@ -121,14 +118,11 @@ flowchart TD B --> C[Agent and Critic review PR] C --> D[Publish comments and queue-label changes] C --> E[Emit record_arm_semantic_review item] - D --> F[Reviewer workflow completes] - E --> F - - F --> G[Semantic Set Status workflow] - G --> H[Download exact run agent_output.json] - H --> I[Validate attempt, PR, SHA, Blocking count, scope, and completion] - I --> J[Publish semantic status on reviewed SHA] - J --> K[Universal Auto-Signoff] + E --> F[Trusted safe-output job validates the item] + F --> G[Publish semantic status on reviewed SHA] + D --> H[Reviewer workflow completes] + G --> H + H --> K[Universal Auto-Signoff] L[LintDiff, Avocado, PR, or label event] --> K K --> M[Read all statuses for current PR head] @@ -146,43 +140,43 @@ The agent emits: ```json { "type": "record_arm_semantic_review", - "run_attempt": "1", - "issue_number": "123", - "head_sha": "abc123abc123abc123abc123abc123abc123abcd", "blocking_count": "0", "scope": "full", - "completeness": "complete" + "completeness": "complete", + "incomplete_reason": "none" } ``` Fields: -| Field | Meaning | -| ---------------- | ---------------------------------------------------------------- | -| `run_attempt` | Reviewer workflow attempt that produced the item | -| `issue_number` | Reviewed PR | -| `head_sha` | Reviewed commit | -| `blocking_count` | Verified Blocking findings still applicable after reconciliation | -| `scope` | Whether the complete PR or a size-limited subset was reviewed | -| `completeness` | Whether the review completed normally, incompletely, or degraded | +| Field | Meaning | +| ------------------- | ---------------------------------------------------------------- | +| `blocking_count` | Verified Blocking findings still applicable after reconciliation | +| `scope` | Whether the complete PR or a size-limited subset was reviewed | +| `completeness` | Whether the intended review completed | +| `incomplete_reason` | `none` when complete; otherwise the specific failure category | -The reviewer custom safe-output job exists because gh-aw requires a job for a -typed custom output. It has no token permissions and performs no authoritative -validation. +The model does not supply the run attempt, PR number, or head SHA. The custom +safe-output job reads those values from the current run and trusted artifacts, +so model output cannot redirect status publication. ## Semantic status publication -`ARM Semantic Review - Set Status` receives the exact reviewer run ID through -`workflow_run`. +The `record_arm_semantic_review` safe-output job runs inside the exact reviewer +run that produced the item. It: 1. lists that run's artifact names once; 2. reads trusted `head-sha` and `issue-number` artifacts; -3. downloads that run's `agent` artifact; -4. finds exactly one semantic item; -5. validates the attempt, PR, SHA, Blocking count, scope, and completion; -6. publishes a status on the semantic item's SHA. +3. reads the run attempt from the workflow environment; +4. finds exactly one semantic item in the safe-output payload; +5. validates the attempt, PR, SHA, Blocking count, scope, completion, and + incomplete reason; +6. confirms that the PR is open and still points to the reviewed SHA; +7. independently verifies review coverage against the trusted changed-file + list; +8. publishes a status on the reviewed SHA. Outcome mapping: @@ -190,11 +184,28 @@ Outcome mapping: | --------------------------------------------------------------------------------------------- | ---------------------------- | | Full, complete review with `blocking_count = 0` | `success`: Passed | | Full, complete review with `blocking_count > 0` | `failure`: Changes requested | -| Scoped, incomplete, or degraded review, regardless of `blocking_count` | `error`: Review incomplete | -| Workflow failure, missing output, malformed output, duplicate item, or mismatched correlation | `error`: Review incomplete | -| Canceled reviewer run | Leave Pending unchanged | +| Incomplete review, regardless of `blocking_count` | `error`: Review incomplete | +| Scoped review, or coverage limits exceeded by the trusted file list | `error`: Manual review required | +| Malformed output, duplicate item, or a failure while validating or publishing | `error`: Review incomplete | +| Agent failure, omitted semantic item, noop, or canceled reviewer run | `error`: Review incomplete, published by the `conclusion` job | | Missing trusted PR/SHA artifacts | Leave status unchanged | +Both `error` outcomes share one commit-status state and are told apart by the description +prefix (`Manual review required: ` or `Review incomplete: `). Only Manual review required is a +property of the PR itself, so only it adds the sticky `ARMManualSignoffRequired` label. +Review incomplete is a transient failure that a re-run can fix. + +### Resolving a Pending status that was never published + +`pre_activation` posts Pending before the agent starts, but `record_arm_semantic_review` only +runs when the agent emitted a semantic item and threat detection passed. The `conclusion` job +runs after every other job regardless of their results. Its pre-step reads the trusted +`head-sha` artifact and, only when this exact run and attempt still owns the newest +`ARM Semantic Review` status and it is Pending, publishes +`Review incomplete: reviewer did not publish a result`. A newer run or attempt, or a result the +record job already published, is never overwritten. The step has `continue-on-error: true` +because pre-steps run before gh-aw's own failure reporting. + The status is head-bound: ```text @@ -206,9 +217,9 @@ Target URL: exact reviewer run and attempt ## Stale-result protection -Set Status publishes only while the PR is open and its current head matches the -reviewed SHA. A completion for a closed PR or stale head leaves the existing -status unchanged. +The safe-output publisher runs only while the PR is open and its current head +matches the reviewed SHA. A completion for a closed PR or stale head leaves the +existing status unchanged. Universal independently reads statuses only for the current head SHA: @@ -220,6 +231,11 @@ No Semantic Passed exists on B Universal does not sign off ``` +A newer run for the same PR cancels an in-flight one (`cancel-in-progress`), and the publisher +checks for a newer run only immediately before it writes. Events the trigger gate would skip, +such as a push to a draft PR or to a PR without `WaitForARMFeedback`, get a run-scoped +concurrency group, so they cannot cancel a valid in-flight review. + ## Universal decision Universal fetches all commit statuses for the current head SHA and selects the @@ -239,6 +255,9 @@ Semantic Review requested changes -> remove pilot signoff Semantic Review is incomplete + -> no pilot signoff, no label change; a re-run can fix it + +Semantic Review requires manual review (scoped or oversized PR) -> add ARMManualSignoffRequired -> retain WaitForARMFeedback -> remove pilot signoff @@ -272,7 +291,7 @@ Semantic is not Passed No signoff Reviewer completes -Set Status publishes Semantic = Passed +Reviewer publishes Semantic = Passed Universal runs again All statuses pass Signoff can proceed @@ -281,7 +300,7 @@ Signoff can proceed ### Reviewer completes first ```text -Set Status publishes Semantic = Passed +Reviewer publishes Semantic = Passed Universal runs Avocado is pending No signoff @@ -309,15 +328,15 @@ Old status on A = Passed New review starts on A New Pending entry becomes newest Universal waits -Set Status publishes the new final result +Reviewer publishes the new final result ``` ### Missing or malformed semantic output ```text -Set Status publishes Review incomplete on reviewed SHA -Universal adds ARMManualSignoffRequired -No automatic signoff +Malformed emitted item -> reviewer publishes Review incomplete +No emitted item, noop, or failed/canceled reviewer -> conclusion job publishes Review incomplete +In either case, Universal cannot sign off and adds no label ``` ## Why this design @@ -331,36 +350,29 @@ Commit statuses are SHA-bound. PR labels are not. Any later event can read the current SHA's status without searching reviewer-run history or downloading old artifacts. -### Dedicated Set Status workflow - -It matches LintDiff and Avocado: - -```text -producer workflow - -> Set Status workflow - -> Universal evaluator -``` +### Direct reviewer publication -It also keeps `statuses: write` out of Universal. +The trusted safe-output job already has the exact agent output, run attempt, PR, +and SHA. Publishing there removes one workflow and one artifact handoff while +keeping `statuses: write` out of Universal. ### One authoritative validation -Validation occurs only in Set Status. The reviewer produces the typed item; Set -Status validates and publishes it. +Validation occurs only in the trusted safe-output job. The model produces the +typed item; the job validates and publishes it. ## Security and reliability properties - no untrusted PR code is checked out; - only write-access users or the trusted bot can trigger reviewer writes; -- Set Status consumes the exact completed reviewer run ID; - the semantic item is bound to the workflow attempt, PR number, and SHA; -- malformed, missing, duplicate, mismatched, scoped, incomplete, or degraded - output fails closed; +- malformed, missing, duplicate, mismatched, scoped, or incomplete output + fails closed; - only `success` permits signoff; - Universal always reads the current head SHA; - delayed label artifacts are rechecked against the live PR head; - Universal has `statuses: read`; -- only Set Status has `statuses: write`. +- only trusted reviewer jobs have `statuses: write`. ## Accepted tradeoffs @@ -372,12 +384,12 @@ The same verified findings drive: - `ARMChangesRequested`; - semantic Blocking count. -Set Status validates structure and correlation; it does not independently -re-review semantic correctness. +The safe-output job validates structure and correlation; it does not +independently re-review semantic correctness. ### Coupling to gh-aw agent output -Set Status expects: +The custom safe-output job expects: ```text artifact name: agent @@ -385,8 +397,9 @@ file: agent_output.json items[] with type record_arm_semantic_review ``` -A gh-aw format change can break extraction. That failure produces Review -incomplete and cannot authorize signoff. +A gh-aw format change can break extraction. A malformed emitted item produces +Review incomplete; a job that cannot start is resolved to Review incomplete by the +`conclusion` job. Neither can authorize signoff. ### PR-level comment and label races @@ -398,32 +411,32 @@ and the current-SHA semantic gate prevents stale auto-signoff. ```text ARM API Reviewer - -> ARM Semantic Review - Set Status -> ARM Universal Auto-Signoff -> Update Labels ``` -This reaches GitHub's supported `workflow_run` chain depth. Do not add another -downstream `workflow_run` workflow after Update Labels. +Removing the Set Status workflow shortens the `workflow_run` chain and leaves +more headroom below GitHub's supported chain-depth limit. ## Operational prerequisites - ensure `ARMAutoSignedOff-Test` and `ARMManualSignoffRequired` exist; - mirror the ARM reviewer workflow to `azure-rest-api-specs-pr`; - document who may remove manual hold labels; -- monitor incomplete/degraded rates and label transitions after rollout. +- monitor incomplete rates, stuck Pending statuses, and label transitions after + rollout. ## How to understand the implementation Read in this order: -1. [arm-semantic-review-status.yaml](../.github/workflows/arm-semantic-review-status.yaml) -2. [finalizeArmSemanticReview](../.github/workflows/src/arm-auto-signoff/arm-semantic-review-status.ts) +1. [arm-api-review.md](../.github/workflows/arm-api-review.md) +2. [publishArmSemanticReviewStatus](../.github/workflows/src/arm-auto-signoff/arm-semantic-review-workflow.ts) 3. [parseSemanticReviewResult](../.github/workflows/src/arm-auto-signoff/arm-semantic-review.ts) 4. [arm-universal-auto-signoff.ts](../.github/workflows/src/arm-auto-signoff/arm-universal-auto-signoff.ts) 5. tests: - [arm-semantic-review.test.ts](../.github/workflows/test/arm-auto-signoff/arm-semantic-review.test.ts) - - [arm-semantic-review-status-workflow.test.ts](../.github/workflows/test/arm-auto-signoff/arm-semantic-review-status-workflow.test.ts) + - [arm-api-review-workflow.test.ts](../.github/workflows/test/arm-api-review-workflow.test.ts) - [arm-universal-auto-signoff.test.ts](../.github/workflows/test/arm-auto-signoff/arm-universal-auto-signoff.test.ts) For each scenario, ask: @@ -440,15 +453,19 @@ Can signoff proceed? - [ ] Clean review publishes Passed. - [ ] Blocking review publishes Changes requested. -- [ ] Scoped, incomplete, and degraded reviews publish Review incomplete. -- [ ] Missing or malformed item publishes Review incomplete. -- [ ] Wrong attempt, PR, or SHA publishes Review incomplete. -- [ ] Canceled review leaves Pending. +- [ ] Incomplete reviews publish Review incomplete. +- [ ] Scoped or oversized reviews publish Manual review required. +- [ ] Malformed emitted item publishes Review incomplete. +- [ ] Missing item, noop, or failed/canceled reviewer is resolved to Review incomplete by the + `conclusion` job and blocks signoff, without overwriting a newer run's status. +- [ ] Model-supplied correlation is ignored; missing or invalid trusted + correlation leaves status unchanged. - [ ] New SHA cannot use old SHA's Passed result. - [ ] Same-SHA rerun publishes Pending before reevaluation. - [ ] Universal remains status-read-only. - [ ] Delayed label application rechecks the live head. - [ ] Passing evidence adds `ARMAutoSignedOff-Test` without changing `ARMSignedOff`. - [ ] Later failures remove the pilot label. -- [ ] Review incomplete adds the manual-review label without changing ARM queue labels. +- [ ] Manual review required adds the manual-review label without changing ARM queue labels; + Review incomplete adds no label. - [ ] Private repository copy is updated in coordination. From e6b44ef93d645b458ae682c23a52c8bf0a144e04 Mon Sep 17 00:00:00 2001 From: Akhila Illa Date: Wed, 7 Oct 2026 19:41:20 -0500 Subject: [PATCH 09/16] Revert concurrency rule to the simple version Skipped PR events no longer get a run-scoped concurrency group. The extra condition duplicated the trigger gate for a narrow, low-impact case. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/arm-api-review.lock.yml | 4 ++-- .github/workflows/arm-api-review.md | 9 ++++----- .github/workflows/test/arm-api-review-workflow.test.ts | 6 ------ .../arm-semantic-review-auto-signoff-implementation.md | 4 +--- 4 files changed, 7 insertions(+), 16 deletions(-) diff --git a/.github/workflows/arm-api-review.lock.yml b/.github/workflows/arm-api-review.lock.yml index 26265afb418d..e808d95c530d 100644 --- a/.github/workflows/arm-api-review.lock.yml +++ b/.github/workflows/arm-api-review.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"353f8005e553e4d6711b67004bb008b732d8cb1d0b209d0c341887fe6da4a38d","body_hash":"6f23a35e609eac889194018b493de3fea8132fc3bec75a836390aed113d4f083","compiler_version":"v0.86.2","strict":true,"agent_id":"copilot","agent_model":"gpt-5.6-sol?effort=high","detection_agent_id":"copilot","detection_agent_model":"gpt-5.6-sol?effort=high","engine_versions":{"copilot":"1.0.79"}} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"0a6ac6e055fd25945ea969b4e036b2269ab552cd1bbbe0d2caff1e8db3da9520","body_hash":"6f23a35e609eac889194018b493de3fea8132fc3bec75a836390aed113d4f083","compiler_version":"v0.86.2","strict":true,"agent_id":"copilot","agent_model":"gpt-5.6-sol?effort=high","detection_agent_id":"copilot","detection_agent_model":"gpt-5.6-sol?effort=high","engine_versions":{"copilot":"1.0.79"}} # gh-aw-manifest: {"version":1,"secrets":["GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/cache/restore","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/cache/save","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"373c709c69115d41ff229c7e5df9f8788daa9553","version":"v9"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/github-script","sha":"d746ffe35508b1917358783b479e04febd2b8f71","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"820762786026740c76f36085b0efc47a31fe5020","version":"v7.0.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"},{"repo":"github/gh-aw-actions/setup","sha":"6aab9e5b5c91c615506061f09bedd81a23babe3c","version":"v0.86.2"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.27.44","digest":"sha256:0d727725c737b58c7bdf51f640cffb928385ec46517e0917c7f1a02f1bada8b4","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.27.44@sha256:0d727725c737b58c7bdf51f640cffb928385ec46517e0917c7f1a02f1bada8b4"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.44","digest":"sha256:b50fbadba138f6e9aba94aca09711335c489bb3b15861220cb66f6092e042dc7","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.44@sha256:b50fbadba138f6e9aba94aca09711335c489bb3b15861220cb66f6092e042dc7"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.27.44","digest":"sha256:83e48bbe12c634be8c228a576832fe45f66c529ac3659db92bddbcf2eeb6d627","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.27.44@sha256:83e48bbe12c634be8c228a576832fe45f66c529ac3659db92bddbcf2eeb6d627"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.9","digest":"sha256:e5a1569aeaf41820fa7bdee3e94468cae448133cdbf00119ad24f5b74db1ab9f","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.9@sha256:e5a1569aeaf41820fa7bdee3e94468cae448133cdbf00119ad24f5b74db1ab9f"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:0d9f1fb5fd6610c0ac1f5194a38e45a8a1e81f8a390d5142d8e4e6f26a4b3196","pinned_image":"ghcr.io/github/gh-aw-node@sha256:0d9f1fb5fd6610c0ac1f5194a38e45a8a1e81f8a390d5142d8e4e6f26a4b3196"},{"image":"ghcr.io/github/github-mcp-server:v1.9.0","digest":"sha256:881b53d6f75f69bdbc1b5b10fc2f1361717c19054143b3a8529fb5c32061a50e","pinned_image":"ghcr.io/github/github-mcp-server:v1.9.0@sha256:881b53d6f75f69bdbc1b5b10fc2f1361717c19054143b3a8529fb5c32061a50e"}],"has_pull_request_target":true} # This file was automatically generated by gh-aw (v0.86.2). DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # @@ -136,7 +136,7 @@ permissions: {} concurrency: cancel-in-progress: true - group: gh-aw-${{ github.workflow }}-${{ ((github.event_name == 'issue_comment' && (github.event.comment.body != '/arm-review' || github.event.issue.pull_request == null)) || (github.event_name == 'pull_request_target' && (github.event.pull_request.draft == true || (github.event.action == 'labeled' && github.event.label.name != 'WaitForARMFeedback') || (github.event.action != 'labeled' && !contains(github.event.pull_request.labels.*.name, 'WaitForARMFeedback'))))) && github.run_id || github.event.issue.number || github.event.pull_request.number || github.event.inputs.pr_number || github.run_id }} + group: gh-aw-${{ github.workflow }}-${{ ((github.event_name == 'issue_comment' && (github.event.comment.body != '/arm-review' || github.event.issue.pull_request == null)) || (github.event_name == 'pull_request_target' && github.event.action == 'labeled' && github.event.label.name != 'WaitForARMFeedback')) && github.run_id || github.event.issue.number || github.event.pull_request.number || github.event.inputs.pr_number || github.run_id }} run-name: "ARM API Review #${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }} (${{ github.event_name }})" diff --git a/.github/workflows/arm-api-review.md b/.github/workflows/arm-api-review.md index 170d2d235893..149d0272fb9f 100644 --- a/.github/workflows/arm-api-review.md +++ b/.github/workflows/arm-api-review.md @@ -6,12 +6,11 @@ description: > on demand via the /arm-review comment command. run-name: "ARM API Review #${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }} (${{ github.event_name }})" timeout-minutes: 30 -# Concurrency is evaluated before the job-level `if` gate. Give every event the -# gate below will skip a run-scoped group so it cannot cancel an active review for -# the same PR: unrelated comments, non-queue label events, and PR events on a -# draft PR or one without `WaitForARMFeedback`. Keep this in step with the gate. +# Concurrency is evaluated before the job-level `if` gate. Give unrelated +# comments and non-queue label events run-scoped groups so they cannot cancel +# an active review for the same PR before being skipped. concurrency: - group: "gh-aw-${{ github.workflow }}-${{ ((github.event_name == 'issue_comment' && (github.event.comment.body != '/arm-review' || github.event.issue.pull_request == null)) || (github.event_name == 'pull_request_target' && (github.event.pull_request.draft == true || (github.event.action == 'labeled' && github.event.label.name != 'WaitForARMFeedback') || (github.event.action != 'labeled' && !contains(github.event.pull_request.labels.*.name, 'WaitForARMFeedback'))))) && github.run_id || github.event.issue.number || github.event.pull_request.number || github.event.inputs.pr_number || github.run_id }}" + group: "gh-aw-${{ github.workflow }}-${{ ((github.event_name == 'issue_comment' && (github.event.comment.body != '/arm-review' || github.event.issue.pull_request == null)) || (github.event_name == 'pull_request_target' && github.event.action == 'labeled' && github.event.label.name != 'WaitForARMFeedback')) && github.run_id || github.event.issue.number || github.event.pull_request.number || github.event.inputs.pr_number || github.run_id }}" cancel-in-progress: true on: # Fork PRs ARE supported (`forks: ["*"]`), matching every other PR workflow in diff --git a/.github/workflows/test/arm-api-review-workflow.test.ts b/.github/workflows/test/arm-api-review-workflow.test.ts index 5b17d88bf1bd..863ea707e3fe 100644 --- a/.github/workflows/test/arm-api-review-workflow.test.ts +++ b/.github/workflows/test/arm-api-review-workflow.test.ts @@ -315,12 +315,6 @@ describe("ARM API review workflow", () => { expect(workflow).toContain("github.event.comment.body != '/arm-review'"); expect(workflow).toContain("github.event.issue.pull_request == null"); expect(workflow).toContain("github.event.label.name != 'WaitForARMFeedback'"); - // PR events the trigger gate skips (draft, or no queue label) must not join the PR group - // and cancel an active review. - expect(workflow).toContain("github.event.pull_request.draft == true"); - expect(workflow).toContain( - "!contains(github.event.pull_request.labels.*.name, 'WaitForARMFeedback')", - ); expect(workflow).toContain("&& github.run_id || github.event.issue.number"); } }); diff --git a/documentation/arm-semantic-review-auto-signoff-implementation.md b/documentation/arm-semantic-review-auto-signoff-implementation.md index f921faeb2d47..63ef291d7371 100644 --- a/documentation/arm-semantic-review-auto-signoff-implementation.md +++ b/documentation/arm-semantic-review-auto-signoff-implementation.md @@ -232,9 +232,7 @@ Universal does not sign off ``` A newer run for the same PR cancels an in-flight one (`cancel-in-progress`), and the publisher -checks for a newer run only immediately before it writes. Events the trigger gate would skip, -such as a push to a draft PR or to a PR without `WaitForARMFeedback`, get a run-scoped -concurrency group, so they cannot cancel a valid in-flight review. +checks for a newer run only immediately before it writes. ## Universal decision From 672c8bb1d15c13c832adb63cf437af29468073f2 Mon Sep 17 00:00:00 2001 From: Akhila Illa Date: Wed, 7 Oct 2026 19:44:44 -0500 Subject: [PATCH 10/16] Recompile ARM API review lock after merging main Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/arm-api-review.lock.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/arm-api-review.lock.yml b/.github/workflows/arm-api-review.lock.yml index e808d95c530d..a1cf4977a4d6 100644 --- a/.github/workflows/arm-api-review.lock.yml +++ b/.github/workflows/arm-api-review.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"0a6ac6e055fd25945ea969b4e036b2269ab552cd1bbbe0d2caff1e8db3da9520","body_hash":"6f23a35e609eac889194018b493de3fea8132fc3bec75a836390aed113d4f083","compiler_version":"v0.86.2","strict":true,"agent_id":"copilot","agent_model":"gpt-5.6-sol?effort=high","detection_agent_id":"copilot","detection_agent_model":"gpt-5.6-sol?effort=high","engine_versions":{"copilot":"1.0.79"}} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"0a6ac6e055fd25945ea969b4e036b2269ab552cd1bbbe0d2caff1e8db3da9520","body_hash":"cb0f2e92399e5be203fc6aef3f6b833929d8c6f1b9ca4779965cdc6cd2fc770e","compiler_version":"v0.86.2","strict":true,"agent_id":"copilot","agent_model":"gpt-5.6-sol?effort=high","detection_agent_id":"copilot","detection_agent_model":"gpt-5.6-sol?effort=high","engine_versions":{"copilot":"1.0.79"}} # gh-aw-manifest: {"version":1,"secrets":["GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/cache/restore","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/cache/save","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"373c709c69115d41ff229c7e5df9f8788daa9553","version":"v9"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/github-script","sha":"d746ffe35508b1917358783b479e04febd2b8f71","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"820762786026740c76f36085b0efc47a31fe5020","version":"v7.0.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"},{"repo":"github/gh-aw-actions/setup","sha":"6aab9e5b5c91c615506061f09bedd81a23babe3c","version":"v0.86.2"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.27.44","digest":"sha256:0d727725c737b58c7bdf51f640cffb928385ec46517e0917c7f1a02f1bada8b4","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.27.44@sha256:0d727725c737b58c7bdf51f640cffb928385ec46517e0917c7f1a02f1bada8b4"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.44","digest":"sha256:b50fbadba138f6e9aba94aca09711335c489bb3b15861220cb66f6092e042dc7","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.44@sha256:b50fbadba138f6e9aba94aca09711335c489bb3b15861220cb66f6092e042dc7"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.27.44","digest":"sha256:83e48bbe12c634be8c228a576832fe45f66c529ac3659db92bddbcf2eeb6d627","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.27.44@sha256:83e48bbe12c634be8c228a576832fe45f66c529ac3659db92bddbcf2eeb6d627"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.9","digest":"sha256:e5a1569aeaf41820fa7bdee3e94468cae448133cdbf00119ad24f5b74db1ab9f","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.9@sha256:e5a1569aeaf41820fa7bdee3e94468cae448133cdbf00119ad24f5b74db1ab9f"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:0d9f1fb5fd6610c0ac1f5194a38e45a8a1e81f8a390d5142d8e4e6f26a4b3196","pinned_image":"ghcr.io/github/gh-aw-node@sha256:0d9f1fb5fd6610c0ac1f5194a38e45a8a1e81f8a390d5142d8e4e6f26a4b3196"},{"image":"ghcr.io/github/github-mcp-server:v1.9.0","digest":"sha256:881b53d6f75f69bdbc1b5b10fc2f1361717c19054143b3a8529fb5c32061a50e","pinned_image":"ghcr.io/github/github-mcp-server:v1.9.0@sha256:881b53d6f75f69bdbc1b5b10fc2f1361717c19054143b3a8529fb5c32061a50e"}],"has_pull_request_target":true} # This file was automatically generated by gh-aw (v0.86.2). DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # From ebb4c708f563f701f2486d970f9207f9697dd353 Mon Sep 17 00:00:00 2001 From: Akhila Illa Date: Thu, 8 Oct 2026 00:59:49 -0500 Subject: [PATCH 11/16] Document ARM semantic review helpers, trim validations, add skill - Document every helper, type, and constant in the ARM semantic review policy, publisher, and universal auto-signoff code. - Simplify validation: drop duplicate correlation, PR-head, and newer-run checks; publish a fixed Review incomplete description and log error details instead of echoing them into the status. - Two-tier PR size check: aggregate counters first, then the specification/ subset only when the totals exceed the cap. - Skip the conclusion finalizer when the record job already published. - Document the trusted-artifact helpers and drop the ARM-specific path guard from the generic context code. - Add tests for the not-ready, non-Error, unset output path, and unknown status branches. - Add the arm-semantic-review skill describing the design, contract, invariants, and debugging steps. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/skills/arm-semantic-review/SKILL.md | 176 +++++++ .github/workflows/arm-api-review.lock.yml | 17 +- .github/workflows/arm-api-review.md | 9 +- .../arm-semantic-review-workflow.ts | 183 ++++--- .../arm-auto-signoff/arm-semantic-review.ts | 183 ++++--- .../arm-universal-auto-signoff.ts | 37 ++ .github/workflows/src/context.ts | 30 +- .github/workflows/src/update-labels.ts | 6 + .../test/arm-api-review-workflow.test.ts | 15 +- .../arm-semantic-review.test.ts | 466 +++++++++--------- .../arm-universal-auto-signoff.test.ts | 25 + .github/workflows/test/context.test.ts | 14 +- ...ntic-review-auto-signoff-implementation.md | 56 ++- 13 files changed, 757 insertions(+), 460 deletions(-) create mode 100644 .github/skills/arm-semantic-review/SKILL.md diff --git a/.github/skills/arm-semantic-review/SKILL.md b/.github/skills/arm-semantic-review/SKILL.md new file mode 100644 index 000000000000..f493c8bac1fb --- /dev/null +++ b/.github/skills/arm-semantic-review/SKILL.md @@ -0,0 +1,176 @@ +--- +name: arm-semantic-review +description: "Maintain, review, and debug the ARM Semantic Review commit status that connects the ARM API Reviewer (gh-aw) to Universal Auto-Signoff. USE FOR: changes to .github/workflows/arm-api-review.md or its lock file's status plumbing, arm-semantic-review.ts, arm-semantic-review-workflow.ts, arm-universal-auto-signoff.ts, the record_arm_semantic_review safe output, an ARM Semantic Review status that is stuck Pending or wrong, ARMAutoSignedOff-Test or ARMManualSignoffRequired label questions. DO NOT USE FOR: reviewing ARM API specifications themselves (use azure-api-review), TypeSpec authoring, or general GitHub Actions review (use engineering-code-review)." +--- + +# ARM Semantic Review + +The ARM API Reviewer is an AI agent. Auto-signoff must not trust it directly, so its verdict is +turned into one **commit status**, `ARM Semantic Review`, on the reviewed SHA. Universal +Auto-Signoff reads that status alongside LintDiff and Avocado. + +Design record and rationale: +[arm-semantic-review-auto-signoff-implementation.md](../../../documentation/arm-semantic-review-auto-signoff-implementation.md). + +## Mental model + +```text +START pre_activation post status PENDING, upload head-sha= / issue-number= artifacts +MIDDLE agent (read-only AI) review the PR, then request record_arm_semantic_review +END record job (trusted) validate the 4 values, post SUCCESS / FAILURE / ERROR +SAFETY conclusion job if no result was posted, turn this run's PENDING into ERROR +READ Universal Auto-Signoff newest status on the current head SHA, plus labels -> label action +APPLY Update Labels re-check the live head, then add or remove the label +``` + +The model proposes. Trusted code validates and publishes. Any doubt ends as "not Passed". + +## Files + +| File | Role | +| ------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------- | +| [arm-api-review.md](../../workflows/arm-api-review.md) | Source of the reviewer workflow. Edit this, never the lock file. | +| [arm-api-review.lock.yml](../../workflows/arm-api-review.lock.yml) | Generated by `gh aw compile`. Must match the `.md`. | +| [arm-semantic-review.ts](../../workflows/src/arm-auto-signoff/arm-semantic-review.ts) | Pure policy: validate the result, map it to a status, read statuses. No I/O. | +| [arm-semantic-review-workflow.ts](../../workflows/src/arm-auto-signoff/arm-semantic-review-workflow.ts) | I/O: publish the status, finalize an unpublished one. | +| [arm-universal-auto-signoff.ts](../../workflows/src/arm-auto-signoff/arm-universal-auto-signoff.ts) | Reads statuses and labels, decides label actions. | +| [context.ts](../../workflows/src/context.ts) | `extractInputs` and the trusted-artifact parser shared by all consumers. | +| [update-labels.ts](../../workflows/src/update-labels.ts) | Applies label artifacts after re-checking the live head. | + +## The status contract + +Context `ARM Semantic Review`, bound to the reviewed SHA. `target_url` is +`.../actions/runs//attempts/`, which identifies the run that wrote it. + +| State | Description starts with | Meaning | Adds `ARMManualSignoffRequired`? | +| --------- | -------------------------- | ----------------------------------------------- | -------------------------------- | +| `pending` | `ARM API semantic review` | Review in progress | No | +| `success` | `Passed:` | Full, complete review, 0 Blocking findings | No | +| `failure` | `Changes requested:` | Full, complete review with Blocking findings | No | +| `error` | `Manual review required: ` | Scoped review, or `specification/` over the cap | **Yes** | +| `error` | `Review incomplete: ` | Failed, cancelled, unreadable, or unvalidated | No | + +Only `success` can lead to signoff. The two `error` outcomes share a state and are told apart by +the description prefix, so keep the prefixes in `arm-semantic-review.ts` as the single source. +`Manual review required` is a property of the PR. `Review incomplete` is transient, so a re-run +can still pass and it must never add the sticky label. + +## Validation, and why each check exists + +Trusted inputs (never from the model): the PR number and head SHA come from the run's +`head-sha=` and `issue-number=` artifacts, and the attempt from `GITHUB_RUN_ATTEMPT`. + +| Check | Where | If it fails | +| ----------------------------------------------------------------------------- | --------------------------- | ------------------------- | +| Trusted artifacts present, full 40-hex SHA, no conflicting duplicates | `context.ts` | Nothing published / throw | +| Output file exists and is JSON | publisher `try` | `Review incomplete` | +| Exactly one `record_arm_semantic_review` item | `parseSemanticReviewResult` | `Review incomplete` | +| `blocking_count` is a digit string; the other three fields are in closed sets | `parseSemanticReviewResult` | `Review incomplete` | +| A `complete` review must have reason `none` | `parseSemanticReviewResult` | `Review incomplete` | +| PR size within the cap, checked independently of the model's `scope` | `requiresManualReview` | `Manual review required` | +| Incomplete, then scoped, then Blocking, then Passed (first match wins) | `evaluateSemanticReview` | n/a | + +The size cap is 50 `specification/` files and 5,000 changed lines, matching the reviewer prompt. +It is two-tier: if the PR's totals are within the cap no file list is fetched, otherwise only the +`specification/` files are counted. A PR reporting zero files, or a truncated file list, requires +manual review. + +Deliberately **not** validated, because it adds code without protecting signoff: + +- the PR being open or still at the reviewed SHA (the status is SHA-bound and Universal re-checks + the live head); +- a "newer run exists" check before publishing (a late same-SHA write is stale but valid for that + SHA); +- re-validating the attempt, PR, or SHA inside the parser (already validated upstream). + +Error text from a failed validation goes to the run log only. The published description is the +fixed string `Review incomplete: result could not be validated`, so runner paths and API errors +never reach a public status. + +## Invariants: do not break these + +1. The model never supplies the SHA, PR number, or attempt. +2. Only `success` authorizes signoff; every unknown, missing, or errored case is "not Passed". +3. `statuses: write` stays in trusted jobs only (`pre_activation`, the record job, `conclusion`). + Never give it to the `agent` job, and Universal stays `statuses: read`. +4. The finalizer publishes only when this exact run and attempt owns the newest `pending` status. +5. `Review incomplete` never adds `ARMManualSignoffRequired`. +6. Do not check out or run PR code in any of these jobs (`pull_request_target`). + +## Behavior in the edge cases + +| Situation | Result | +| ------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------- | +| New commit pushed during a review | An eligible push cancels the run (`cancel-in-progress`). The old SHA's status is never read for the new head. | +| `/arm-review` while a run is in flight | Cancels the old run and starts a new one on the same SHA. | +| Agent `noop`, failure, timeout, detection failure, or cancel | The `conclusion` finalizer turns this run's `pending` into `Review incomplete`. | +| Transient GitHub API error while publishing | `Review incomplete`, no sticky label. A re-run can pass. | +| Critic unavailable | `Review incomplete: critic-unavailable`, labels unchanged. | +| Scoped or oversized PR | `Manual review required`, adds `ARMManualSignoffRequired`. | +| Non-team-member actor | No `pending`, no artifacts, no status. Signoff stays blocked. | +| Event the gate skips (other label, other comment) | The run is skipped. Those events get a private concurrency lane so they cannot cancel a review. | + +## Known limitations + +- A same-SHA re-run posts `pending` but does not revoke an existing `ARMAutoSignedOff-Test` + label until Universal next runs. This matters only once enforcement is on. +- Universal's label filter does not include `ARMChangesRequested` or `WaitForARMFeedback`, so + adding them does not trigger an immediate re-evaluation. +- The newest status named `ARM Semantic Review` wins; its creator is not verified. +- If a re-run resolves a different head SHA than the first attempt, two `head-sha=` artifacts can + conflict and the parser throws, which can leave `Pending`. +- `blocking_count` is the model's verdict. The Critic and threat detection are the only checks on + it, which is an accepted trade-off. +- If the `conclusion` job itself does not run, a `Pending` status can remain. +- A push to a draft PR, or to one without `WaitForARMFeedback`, joins the PR concurrency lane + before the gate skips it, so it can cancel a review that is already running. + +## Changing the reviewer workflow + +1. Edit `arm-api-review.md`, then regenerate the lock file with the same gh-aw version recorded in + the lock file header (`gh aw compile arm-api-review`). Confirm `git diff` shows only intended + changes. The compiler may also prune unrelated entries in `.github/aw/actions-lock.json`; + revert those. +2. A custom job cannot run after the agent: gh-aw rejects it as a dependency cycle. Post-agent + work goes in the built-in `conclusion` job via `jobs.conclusion.pre-steps`. +3. `pre_activation` is the only place that can post `pending` before the agent starts. +4. Keep the concurrency expression and the `if:` trigger gate consistent. +5. The size cap exists in the prompt (Trigger Validation) and in `arm-semantic-review.ts`. Change + both together. + +## Verify a change + +```bash +cd .github +npx vitest run workflows/test/arm-auto-signoff workflows/test/arm-api-review-workflow.test.ts workflows/test/context.test.ts +npx tsc --noEmit +npx oxlint workflows +npx oxfmt --check workflows +``` + +The tests assert the compiled lock file (job order, permissions, the finalizer condition), so a +stale lock file fails them. + +Live check on a test PR (ideally in `azure-rest-api-specs-pr`), then compare to the tables above: + +```bash +gh api repos/Azure/azure-rest-api-specs/commits//statuses \ + --jq '.[] | select(.context=="ARM Semantic Review") | [.created_at,.state,.description,.target_url]' +``` + +The `pending` status and the `head-sha=` / `issue-number=` artifacts appear on the run created +by `pre_activation`. Try: a clean review, `skip-arm-review`, a push mid-run, two `/arm-review` +comments in a row, and cancelling a run. + +## Debug a status that is stuck or wrong + +1. List the statuses for the head SHA (command above). Which `target_url` is newest, and is it + the run you expect? +2. Open that run. Did `record_arm_semantic_review` run? It needs the agent and detection to + succeed and an emitted `record_arm_semantic_review` item. +3. If it did not run, check whether the `conclusion` finalizer step ran. It is skipped when the + record job succeeded, and it acts only on a `pending` this exact run owns. +4. If the description is `Review incomplete: result could not be validated`, read the warning in + the record job's log for the real reason. +5. If Universal did not act, check its run log: it prints the event, correlation, labels, and the + status it read. A "state/head mismatch" means the PR head moved on. diff --git a/.github/workflows/arm-api-review.lock.yml b/.github/workflows/arm-api-review.lock.yml index a1cf4977a4d6..db1ac323c6cb 100644 --- a/.github/workflows/arm-api-review.lock.yml +++ b/.github/workflows/arm-api-review.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"0a6ac6e055fd25945ea969b4e036b2269ab552cd1bbbe0d2caff1e8db3da9520","body_hash":"cb0f2e92399e5be203fc6aef3f6b833929d8c6f1b9ca4779965cdc6cd2fc770e","compiler_version":"v0.86.2","strict":true,"agent_id":"copilot","agent_model":"gpt-5.6-sol?effort=high","detection_agent_id":"copilot","detection_agent_model":"gpt-5.6-sol?effort=high","engine_versions":{"copilot":"1.0.79"}} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"add9d2d9a555b9c420520ed210e6647e23212c8b879f27937d4f71a0d678a4e0","body_hash":"cb0f2e92399e5be203fc6aef3f6b833929d8c6f1b9ca4779965cdc6cd2fc770e","compiler_version":"v0.86.2","strict":true,"agent_id":"copilot","agent_model":"gpt-5.6-sol?effort=high","detection_agent_id":"copilot","detection_agent_model":"gpt-5.6-sol?effort=high","engine_versions":{"copilot":"1.0.79"}} # gh-aw-manifest: {"version":1,"secrets":["GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/cache/restore","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/cache/save","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"373c709c69115d41ff229c7e5df9f8788daa9553","version":"v9"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/github-script","sha":"d746ffe35508b1917358783b479e04febd2b8f71","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"820762786026740c76f36085b0efc47a31fe5020","version":"v7.0.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"},{"repo":"github/gh-aw-actions/setup","sha":"6aab9e5b5c91c615506061f09bedd81a23babe3c","version":"v0.86.2"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.27.44","digest":"sha256:0d727725c737b58c7bdf51f640cffb928385ec46517e0917c7f1a02f1bada8b4","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.27.44@sha256:0d727725c737b58c7bdf51f640cffb928385ec46517e0917c7f1a02f1bada8b4"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.44","digest":"sha256:b50fbadba138f6e9aba94aca09711335c489bb3b15861220cb66f6092e042dc7","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.44@sha256:b50fbadba138f6e9aba94aca09711335c489bb3b15861220cb66f6092e042dc7"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.27.44","digest":"sha256:83e48bbe12c634be8c228a576832fe45f66c529ac3659db92bddbcf2eeb6d627","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.27.44@sha256:83e48bbe12c634be8c228a576832fe45f66c529ac3659db92bddbcf2eeb6d627"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.9","digest":"sha256:e5a1569aeaf41820fa7bdee3e94468cae448133cdbf00119ad24f5b74db1ab9f","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.9@sha256:e5a1569aeaf41820fa7bdee3e94468cae448133cdbf00119ad24f5b74db1ab9f"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:0d9f1fb5fd6610c0ac1f5194a38e45a8a1e81f8a390d5142d8e4e6f26a4b3196","pinned_image":"ghcr.io/github/gh-aw-node@sha256:0d9f1fb5fd6610c0ac1f5194a38e45a8a1e81f8a390d5142d8e4e6f26a4b3196"},{"image":"ghcr.io/github/github-mcp-server:v1.9.0","digest":"sha256:881b53d6f75f69bdbc1b5b10fc2f1361717c19054143b3a8529fb5c32061a50e","pinned_image":"ghcr.io/github/github-mcp-server:v1.9.0@sha256:881b53d6f75f69bdbc1b5b10fc2f1361717c19054143b3a8529fb5c32061a50e"}],"has_pull_request_target":true} # This file was automatically generated by gh-aw (v0.86.2). DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # @@ -595,9 +595,9 @@ jobs: mkdir -p "${RUNNER_TEMP}/gh-aw/safeoutputs" mkdir -p /tmp/gh-aw/safeoutputs mkdir -p /tmp/gh-aw/mcp-logs/safeoutputs - cat > "${RUNNER_TEMP}/gh-aw/safeoutputs/config.json" << 'GH_AW_SAFE_OUTPUTS_CONFIG_19784e75b93a6ea2_EOF' - {"add_comment":{"max":4,"target":"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}"},"add_labels":{"max":3,"target":"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}"},"create_pull_request_review_comment":{"max":50,"side":"RIGHT","target":"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}"},"create_report_incomplete_issue":{"create-issue":"${{ github.repository == 'Azure/azure-rest-api-specs' }}","title-prefix":"[incomplete]"},"missing_data":{},"missing_tool":{},"noop":{"max":1,"report-as-issue":"true"},"record-arm-semantic-review":{"description":"Validate and publish the ARM semantic review result for auto-signoff","inputs":{"blocking_count":{"default":null,"description":"Number of verified, currently applicable Blocking findings after reconciliation","required":true,"type":"string"},"completeness":{"default":null,"description":"Whether the intended review completed reliably","options":["complete","incomplete"],"required":true,"type":"choice"},"incomplete_reason":{"default":null,"description":"Why the review is incomplete, or none when complete","options":["none","critic-unavailable","required-evidence-unavailable","tool-failure","stale-sha","discussion-data-unavailable"],"required":true,"type":"choice"},"scope":{"default":null,"description":"Whether the full PR or only a scoped subset was reviewed","options":["full","scoped"],"required":true,"type":"choice"}},"output":"ARM semantic review status published"},"remove_labels":{"max":3,"target":"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}"},"reply_to_pull_request_review_comment":{"max":50,"target":"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}"},"report_incomplete":{},"resolve_pull_request_review_thread":{"max":50},"submit_pull_request_review":{"footer":"if-body","max":1,"target":"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}"}} - GH_AW_SAFE_OUTPUTS_CONFIG_19784e75b93a6ea2_EOF + cat > "${RUNNER_TEMP}/gh-aw/safeoutputs/config.json" << 'GH_AW_SAFE_OUTPUTS_CONFIG_ade4c9f8f4c77740_EOF' + {"add_comment":{"max":4,"target":"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}"},"add_labels":{"max":3,"target":"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}"},"create_pull_request_review_comment":{"max":50,"side":"RIGHT","target":"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}"},"create_report_incomplete_issue":{},"missing_data":{},"missing_tool":{},"noop":{"max":1,"report-as-issue":"true"},"record-arm-semantic-review":{"description":"Validate and publish the ARM semantic review result for auto-signoff","inputs":{"blocking_count":{"default":null,"description":"Number of verified, currently applicable Blocking findings after reconciliation","required":true,"type":"string"},"completeness":{"default":null,"description":"Whether the intended review completed reliably","options":["complete","incomplete"],"required":true,"type":"choice"},"incomplete_reason":{"default":null,"description":"Why the review is incomplete, or none when complete","options":["none","critic-unavailable","required-evidence-unavailable","tool-failure","stale-sha","discussion-data-unavailable"],"required":true,"type":"choice"},"scope":{"default":null,"description":"Whether the full PR or only a scoped subset was reviewed","options":["full","scoped"],"required":true,"type":"choice"}},"output":"ARM semantic review status published"},"remove_labels":{"max":3,"target":"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}"},"reply_to_pull_request_review_comment":{"max":50,"target":"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}"},"report_incomplete":{},"resolve_pull_request_review_thread":{"max":50},"submit_pull_request_review":{"footer":"if-body","max":1,"target":"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}"}} + GH_AW_SAFE_OUTPUTS_CONFIG_ade4c9f8f4c77740_EOF - name: Generate Safe Outputs Tools env: GH_AW_TOOLS_META_JSON: | @@ -1297,13 +1297,15 @@ jobs: GH_AW_INFO_VERSION: "1.0.79" GH_AW_INFO_AWF_VERSION: "v0.27.44" GH_AW_INFO_ENGINE_ID: "copilot" - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 (source v7) + - if: needs.record_arm_semantic_review.result != 'success' + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 (source v7) with: persist-credentials: false sparse-checkout: | .github continue-on-error: true - name: Resolve unpublished ARM semantic review status + if: needs.record_arm_semantic_review.result != 'success' uses: actions/github-script@d746ffe35508b1917358783b479e04febd2b8f71 # v9.0.0 with: script: | @@ -1453,8 +1455,7 @@ jobs: uses: actions/github-script@d746ffe35508b1917358783b479e04febd2b8f71 # v9.0.0 env: GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }} - GH_AW_REPORT_INCOMPLETE_CREATE_ISSUE: ${{ github.repository == 'Azure/azure-rest-api-specs' }} - GH_AW_REPORT_INCOMPLETE_TITLE_PREFIX: "[incomplete]" + GH_AW_REPORT_INCOMPLETE_CREATE_ISSUE: "true" GH_AW_WORKFLOW_NAME: "ARM API Review: Automated Workflow" GH_AW_WORKFLOW_SOURCE_URL: "${{ github.server_url }}/${{ github.repository }}/blob/${{ github.ref_name }}/.github/workflows/arm-api-review.md" with: @@ -2085,7 +2086,7 @@ jobs: GITHUB_SERVER_URL: ${{ github.server_url }} GITHUB_API_URL: ${{ github.api_url }} GH_AW_SAFE_OUTPUT_JOBS: "{\"record_arm_semantic_review\":\"\"}" - GH_AW_SAFE_OUTPUTS_HANDLER_CONFIG: "{\"add_comment\":{\"max\":4,\"target\":\"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}\"},\"add_labels\":{\"max\":3,\"target\":\"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}\"},\"create_pull_request_review_comment\":{\"max\":50,\"side\":\"RIGHT\",\"target\":\"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}\"},\"create_report_incomplete_issue\":{\"create-issue\":\"${{ github.repository == 'Azure/azure-rest-api-specs' }}\",\"title-prefix\":\"[incomplete]\"},\"missing_data\":{},\"missing_tool\":{},\"noop\":{\"max\":1,\"report-as-issue\":\"true\"},\"remove_labels\":{\"max\":3,\"target\":\"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}\"},\"reply_to_pull_request_review_comment\":{\"max\":50,\"target\":\"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}\"},\"report_incomplete\":{},\"resolve_pull_request_review_thread\":{\"max\":50},\"submit_pull_request_review\":{\"footer\":\"if-body\",\"max\":1,\"target\":\"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}\"}}" + GH_AW_SAFE_OUTPUTS_HANDLER_CONFIG: "{\"add_comment\":{\"max\":4,\"target\":\"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}\"},\"add_labels\":{\"max\":3,\"target\":\"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}\"},\"create_pull_request_review_comment\":{\"max\":50,\"side\":\"RIGHT\",\"target\":\"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}\"},\"create_report_incomplete_issue\":{},\"missing_data\":{},\"missing_tool\":{},\"noop\":{\"max\":1,\"report-as-issue\":\"true\"},\"remove_labels\":{\"max\":3,\"target\":\"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}\"},\"reply_to_pull_request_review_comment\":{\"max\":50,\"target\":\"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}\"},\"report_incomplete\":{},\"resolve_pull_request_review_thread\":{\"max\":50},\"submit_pull_request_review\":{\"footer\":\"if-body\",\"max\":1,\"target\":\"${{ github.event.pull_request.number || github.event.issue.number || github.event.inputs.pr_number }}\"}}" with: github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} script: | diff --git a/.github/workflows/arm-api-review.md b/.github/workflows/arm-api-review.md index 149d0272fb9f..b76e1ccb32aa 100644 --- a/.github/workflows/arm-api-review.md +++ b/.github/workflows/arm-api-review.md @@ -142,6 +142,7 @@ jobs: # resolves a Pending status that this exact run still owns. That keeps a failed, # cancelled, or noop run from leaving the PR waiting forever. The step reads no # agent output; it takes the head SHA from the trusted `head-sha` artifact. + # It is skipped when the record job succeeded, since that job already published. # Pre-steps run before the built-in conclusion steps, so `continue-on-error` # keeps a failure here from blocking gh-aw's own failure reporting. conclusion: @@ -150,11 +151,13 @@ jobs: statuses: write pre-steps: - uses: actions/checkout@v7 + if: needs.record_arm_semantic_review.result != 'success' continue-on-error: true with: sparse-checkout: | .github - name: Resolve unpublished ARM semantic review status + if: needs.record_arm_semantic_review.result != 'success' continue-on-error: true uses: actions/github-script@v9.0.0 with: @@ -254,12 +257,6 @@ imports: - ../skills/azure-api-review/references/lro-final-state-via.md - ../skills/azure-api-review/references/typespec-openapi-extensions.md safe-outputs: - # Incomplete reviews are already surfaced through the ARM Semantic Review - # commit status and workflow summary. Keep the canonical repository's tracking - # issue, but do not attempt that secondary write in forks, where Issues are - # commonly disabled. - report-incomplete: - create-issue: ${{ github.repository == 'Azure/azure-rest-api-specs' }} # Framework-owned status comments do not consume this budget. Reserve slots # for the review summary / "no issues found", overflow themes, an actionable # diagnostic, and one run-failure notification. diff --git a/.github/workflows/src/arm-auto-signoff/arm-semantic-review-workflow.ts b/.github/workflows/src/arm-auto-signoff/arm-semantic-review-workflow.ts index 5481bf57588f..d0ac0a9cf3b3 100644 --- a/.github/workflows/src/arm-auto-signoff/arm-semantic-review-workflow.ts +++ b/.github/workflows/src/arm-auto-signoff/arm-semantic-review-workflow.ts @@ -1,56 +1,89 @@ import { readFile } from "node:fs/promises"; -import { isFullGitSha } from "../../../shared/src/git.ts"; import { CommitStatusState, PER_PAGE_MAX } from "../../../shared/src/github.ts"; import { getWorkflowRunArtifactInputs } from "../context.ts"; import type { GitHubScriptArgs } from "../github.ts"; import { ARM_SEMANTIC_REVIEW_STATUS, - evaluateAutomatedReviewCoverage, evaluateSemanticReview, + exceedsSpecificationLimits, getLatestSemanticReviewStatus, + isWithinAutomatedReviewLimits, MANUAL_REVIEW_DESCRIPTION_PREFIX, parseSemanticReviewResult, REVIEW_INCOMPLETE_DESCRIPTION_PREFIX, - type ChangedFile, type CommitStatus, + type PullRequestCounts, type EvaluatedSemanticReview, } from "./arm-semantic-review.ts"; -const MAX_STATUS_DESCRIPTION = 140; +/* + * I/O for the `ARM Semantic Review` commit status. These functions run in trusted reviewer jobs + * (`record_arm_semantic_review` and the `conclusion` job) and treat the model's output as + * untrusted data. The PR number, head SHA, and run attempt always come from the run's own + * artifacts and environment, never from the model. + * + * GitHub caps commit status descriptions at 140 characters. Every description published here is a + * short fixed string, so none can exceed it. + */ +/** Returned when the run has no trusted PR/SHA correlation and no status was published. */ const EMPTY_RESULT = { headSha: "", issueNumber: 0, statusPublished: false, }; +/** + * Builds the `target_url` of this run and attempt. It is published on every status this run + * writes and is how a later reader tells whether a status belongs to this exact run, which is what + * lets the finalizer avoid overwriting a newer run. + * + * Reads the attempt from `GITHUB_RUN_ATTEMPT`, which GitHub Actions always sets. + */ function getRunStatusUrl( - { serverUrl, runId }: Pick, + { serverUrl, runId }: GitHubScriptArgs["context"], owner: string, repo: string, - runAttempt: number | string, -): string { - return `${serverUrl}/${owner}/${repo}/actions/runs/${runId}/attempts/${runAttempt}`; +) { + return `${serverUrl}/${owner}/${repo}/actions/runs/${runId}/attempts/${process.env.GITHUB_RUN_ATTEMPT}`; } -async function getLatestSemanticStatus( +/** + * Independently checks the PR against the automated-review limits, since the model's `scope` is + * untrusted. The file list is fetched only when the PR's aggregate counters are over the limits. + * + * @param issueNumber The PR number, from trusted artifacts. + * @param pullRequest The aggregate counters from `pulls.get`. + * @returns `true` when a human must review because the `specification/` changes exceed the limits. + */ +async function requiresManualReview( github: GitHubScriptArgs["github"], owner: string, repo: string, - headSha: string, -): Promise { - const statuses: CommitStatus[] = await github.paginate( - github.rest.repos.listCommitStatusesForRef, - { - owner, - repo, - ref: headSha, - per_page: PER_PAGE_MAX, - }, - ); - return getLatestSemanticReviewStatus(statuses); + issueNumber: number, + pullRequest: PullRequestCounts, +): Promise { + if (isWithinAutomatedReviewLimits(pullRequest)) { + return false; + } + if (pullRequest.changed_files < 1) { + return true; + } + const changedFiles = await github.paginate(github.rest.pulls.listFiles, { + owner, + repo, + pull_number: issueNumber, + per_page: PER_PAGE_MAX, + }); + return exceedsSpecificationLimits(pullRequest.changed_files, changedFiles); } +/** + * Writes one `ARM Semantic Review` commit status on the reviewed SHA. + * + * @param targetUrl The `target_url` identifying this run and attempt. + * @param status The state and description to publish. + */ async function publishStatus( { github, core }: Pick, owner: string, @@ -71,25 +104,23 @@ async function publishStatus( }); } -function isSuperseded( - latestStatus: CommitStatus | undefined, - runId: number, - runAttempt: number, -): boolean { - const match = /\/actions\/runs\/([1-9]\d*)\/attempts\/([1-9]\d*)(?:\/|$)/.exec( - latestStatus?.target_url ?? "", - ); - if (!match) { - return false; - } - const latestRunId = Number(match[1]); - const latestAttempt = Number(match[2]); - return latestRunId > runId || (latestRunId === runId && latestAttempt > runAttempt); -} - /** * Validates one trusted ARM reviewer safe output and publishes its head-bound status. * This runs inside the reviewer workflow, before workflow_run consumers are triggered. + * + * Steps: read the trusted `head-sha` and `issue-number` artifacts; parse and validate the model's + * result; independently check the PR's size; map the evidence to a status; publish it. Any failure + * while validating or reading the PR is published as `Review incomplete` with a fixed description, + * and the details go to the run log. + * + * The status is bound to the reviewed SHA, so a result for a commit that is no longer the PR head + * is harmless: Universal Auto-Signoff reads only the current head's statuses. + * + * Requires `GH_AW_AGENT_OUTPUT` (the path to `agent_output.json`) in the environment. + * + * @returns The correlation used and whether a status was published. Nothing is published when the + * run has no trusted PR/SHA correlation. + * @throws If the trusted artifacts conflict with each other. */ export default async function publishArmSemanticReviewStatus({ github, @@ -100,17 +131,6 @@ export default async function publishArmSemanticReviewStatus({ issueNumber: number; statusPublished: boolean; }> { - const outputPath = process.env.GH_AW_AGENT_OUTPUT; - if (!outputPath) { - throw new Error("GH_AW_AGENT_OUTPUT is unavailable"); - } - - const runAttemptText = process.env.GITHUB_RUN_ATTEMPT; - if (!runAttemptText || !/^[1-9]\d*$/.test(runAttemptText)) { - throw new Error(`Invalid workflow run attempt: '${runAttemptText ?? ""}'`); - } - - const runAttempt = Number(runAttemptText); const { owner, repo } = context.repo; const { headSha, issueNumber } = await getWorkflowRunArtifactInputs({ github, @@ -119,73 +139,46 @@ export default async function publishArmSemanticReviewStatus({ repo, runId: context.runId, }); - if (!isFullGitSha(headSha) || !Number.isSafeInteger(issueNumber) || issueNumber <= 0) { + if (!headSha || !(issueNumber > 0)) { core.info("The reviewer run has no trusted PR/SHA correlation; status is unchanged"); return EMPTY_RESULT; } let status: EvaluatedSemanticReview; try { - const agentOutput = JSON.parse(await readFile(outputPath, "utf8")) as unknown; - const result = parseSemanticReviewResult(agentOutput, { - headSha, - issueNumber, - runAttempt, - }); + const agentOutput = JSON.parse( + await readFile(process.env.GH_AW_AGENT_OUTPUT ?? "", "utf8"), + ) as unknown; + const result = parseSemanticReviewResult(agentOutput); const { data: pullRequest } = await github.rest.pulls.get({ owner, repo, pull_number: issueNumber, }); - if (pullRequest.state !== "open" || pullRequest.head.sha !== headSha) { - core.info("Ignoring ARM API review result for a closed PR or stale head SHA"); - return { headSha, issueNumber, statusPublished: false }; - } - - const changedFiles = (await github.paginate(github.rest.pulls.listFiles, { - owner, - repo, - pull_number: issueNumber, - per_page: PER_PAGE_MAX, - })) as ChangedFile[]; - const coverage = evaluateAutomatedReviewCoverage(pullRequest.changed_files, changedFiles); - status = coverage.manualReviewRequired + status = (await requiresManualReview(github, owner, repo, issueNumber, pullRequest)) ? { state: CommitStatusState.ERROR, - description: `${MANUAL_REVIEW_DESCRIPTION_PREFIX}${coverage.reason ?? "automated review coverage was limited"}`, + description: `${MANUAL_REVIEW_DESCRIPTION_PREFIX}PR exceeds automated review size limits`, } : evaluateSemanticReview(result); } catch (error) { // Any failure here, including a transient GitHub API error, is reported as incomplete rather - // than as a manual-review hold, so a re-run can still produce a passing result. - const fullReason = error instanceof Error ? error.message : "invalid semantic result"; - core.warning(fullReason); - const prefix = REVIEW_INCOMPLETE_DESCRIPTION_PREFIX; - const available = MAX_STATUS_DESCRIPTION - prefix.length; - const truncatedReason = - fullReason.length > available ? `${fullReason.slice(0, available - 1)}\u2026` : fullReason; + // than as a manual-review hold, so a re-run can still produce a passing result. The details go + // to the run log: raw error text can include runner paths and must not reach a public status. + core.warning(error instanceof Error ? error.message : String(error)); status = { state: CommitStatusState.ERROR, - description: `${prefix}${truncatedReason}`, + description: `${REVIEW_INCOMPLETE_DESCRIPTION_PREFIX}result could not be validated`, }; } - // Checked last so the window between this read and the write is as small as possible. An - // earlier check would leave the slow PR and file-list calls inside the window, where a newer - // run's Pending could be overwritten by this older result. - const latestStatus = await getLatestSemanticStatus(github, owner, repo, headSha); - if (isSuperseded(latestStatus, context.runId, runAttempt)) { - core.info("A newer ARM API review run superseded this result"); - return { headSha, issueNumber, statusPublished: false }; - } - await publishStatus( { github, core }, owner, repo, headSha, - getRunStatusUrl(context, owner, repo, runAttempt), + getRunStatusUrl(context, owner, repo), status, ); return { headSha, issueNumber, statusPublished: true }; @@ -197,13 +190,15 @@ export default async function publishArmSemanticReviewStatus({ * output and only acts when this exact run and attempt still owns the newest status, so a newer * run's status is never overwritten. The resulting status is Review incomplete, never a * manual-review hold, because these outcomes are not a property of the PR. + * + * @returns Whether a status was published. + * @throws If the trusted artifacts conflict with each other. */ export async function finalizeUnpublishedArmSemanticReview({ github, context, core, }: GitHubScriptArgs): Promise<{ statusPublished: boolean }> { - const runAttemptText = process.env.GITHUB_RUN_ATTEMPT ?? ""; const { owner, repo } = context.repo; const { headSha } = await getWorkflowRunArtifactInputs({ github, @@ -212,13 +207,17 @@ export async function finalizeUnpublishedArmSemanticReview({ repo, runId: context.runId, }); - if (!isFullGitSha(headSha) || !/^[1-9]\d*$/.test(runAttemptText)) { - core.info("The reviewer run has no trusted head SHA or run attempt; status is unchanged"); + if (!headSha) { + core.info("The reviewer run has no trusted head SHA; status is unchanged"); return { statusPublished: false }; } - const targetUrl = getRunStatusUrl(context, owner, repo, runAttemptText); - const latestStatus = await getLatestSemanticStatus(github, owner, repo, headSha); + const targetUrl = getRunStatusUrl(context, owner, repo); + const statuses: CommitStatus[] = await github.paginate( + github.rest.repos.listCommitStatusesForRef, + { owner, repo, ref: headSha, per_page: PER_PAGE_MAX }, + ); + const latestStatus = getLatestSemanticReviewStatus(statuses); if (latestStatus?.state !== CommitStatusState.PENDING || latestStatus.target_url !== targetUrl) { core.info("This run does not own a Pending ARM Semantic Review status; status is unchanged"); return { statusPublished: false }; diff --git a/.github/workflows/src/arm-auto-signoff/arm-semantic-review.ts b/.github/workflows/src/arm-auto-signoff/arm-semantic-review.ts index 01f824df6d6c..45d2af22360c 100644 --- a/.github/workflows/src/arm-auto-signoff/arm-semantic-review.ts +++ b/.github/workflows/src/arm-auto-signoff/arm-semantic-review.ts @@ -1,25 +1,57 @@ import { CommitStatusState } from "../../../shared/src/github.ts"; import { byDate, invert } from "../../../shared/src/sort.ts"; -import { isFullGitSha } from "../../../shared/src/git.ts"; +/* + * Pure policy for the `ARM Semantic Review` commit status: how the ARM API Reviewer's structured + * result is validated, how it maps to a status, and how the status is read back by consumers. + * This file makes no GitHub API calls; `arm-semantic-review-workflow.ts` does the I/O. + */ + +/** + * Size limits for automated review. They mirror the cap in the reviewer prompt + * (`arm-api-review.md`, Trigger Validation): a PR above either limit is reviewed only in part, so + * it needs a human. Keep both copies in step. + */ const MAX_SPECIFICATION_FILES = 50; const MAX_SPECIFICATION_LINES = 5_000; +/** GitHub caps the pull request file list at this many entries without reporting truncation. */ +const MAX_LISTED_FILES = 3_000; + +/** One entry of the pull request file list (`pulls.listFiles`). */ export type ChangedFile = { filename: string; additions: number; deletions: number; }; +/** The aggregate counters on a pull request object (`pulls.get`). */ +export type PullRequestCounts = { + changed_files: number; + additions: number; + deletions: number; +}; + +/** The commit status context that carries the semantic review result for a SHA. */ export const ARM_SEMANTIC_REVIEW_STATUS = "ARM Semantic Review"; -// Both outcomes are published as a commit status in the "error" state. The description prefix is -// the only thing that tells them apart: a manual-review hold is deterministic (scope or size -// limits) and adds a sticky human label, while an incomplete review is a transient failure that -// a re-run can fix, so it never adds that label. +/* + * Both outcomes below are published as a commit status in the "error" state. The description + * prefix is the only thing that tells them apart: a manual-review hold is deterministic (scope or + * size limits) and adds a sticky human label, while an incomplete review is a transient failure + * that a re-run can fix, so it never adds that label. + */ + +/** Description prefix of an `error` status that requires a human reviewer (scoped or oversized). */ export const MANUAL_REVIEW_DESCRIPTION_PREFIX = "Manual review required: "; + +/** Description prefix of an `error` status for a review that did not finish or could not be read. */ export const REVIEW_INCOMPLETE_DESCRIPTION_PREFIX = "Review incomplete: "; +/** + * What a consumer concludes from the latest `ARM Semantic Review` status. Only `Passed` can + * contribute to auto-signoff. + */ export const SemanticReviewOutcome = Object.freeze({ Passed: "passed", ChangesRequested: "changes-requested", @@ -30,12 +62,14 @@ export const SemanticReviewOutcome = Object.freeze({ export type SemanticReviewOutcome = (typeof SemanticReviewOutcome)[keyof typeof SemanticReviewOutcome]; +/** How much of the PR the reviewer covered: the whole PR, or a size-limited subset. */ export const SemanticReviewScope = Object.freeze({ Full: "full", Scoped: "scoped", }); export type SemanticReviewScope = (typeof SemanticReviewScope)[keyof typeof SemanticReviewScope]; +/** Whether the reviewer and the Critic finished normally. */ export const SemanticReviewCompletion = Object.freeze({ Complete: "complete", Incomplete: "incomplete", @@ -43,6 +77,10 @@ export const SemanticReviewCompletion = Object.freeze({ export type SemanticReviewCompletion = (typeof SemanticReviewCompletion)[keyof typeof SemanticReviewCompletion]; +/** + * Why a review is incomplete, or `none` when it is complete. The set is closed so that model + * output can never place free text in a public status description. + */ export const SemanticReviewIncompleteReason = Object.freeze({ None: "none", CriticUnavailable: "critic-unavailable", @@ -54,21 +92,16 @@ export const SemanticReviewIncompleteReason = Object.freeze({ export type SemanticReviewIncompleteReason = (typeof SemanticReviewIncompleteReason)[keyof typeof SemanticReviewIncompleteReason]; +/** The validated form of the reviewer's `record_arm_semantic_review` item. */ export type SemanticReviewResult = { - runAttempt: number; - issueNumber: number; - headSha: string; + /** Verified Blocking findings still applicable after reconciliation. */ blockingCount: number; reviewScope: SemanticReviewScope; completion: SemanticReviewCompletion; incompleteReason: SemanticReviewIncompleteReason; }; -export type SemanticReviewCorrelation = Pick< - SemanticReviewResult, - "runAttempt" | "issueNumber" | "headSha" ->; - +/** The fields of a commit status (`repos.listCommitStatusesForRef`) that this feature reads. */ export type CommitStatus = { context: string; state: string; @@ -77,32 +110,32 @@ export type CommitStatus = { updated_at: string; }; +/** The commit status state and description to publish for a validated result. */ export type EvaluatedSemanticReview = { state: CommitStatusState; description: string; }; +/** Narrows to a plain object, excluding `null` and arrays. */ function isRecord(value: unknown): value is Record { return typeof value === "object" && value !== null && !Array.isArray(value); } /** - * Combines exactly one semantic result from gh-aw's agent output with trusted - * workflow correlation. + * Extracts and validates exactly one semantic result from gh-aw's agent output + * (`agent_output.json`). The PR, head SHA, and run attempt are never read from the model's + * output; the caller takes them from trusted artifacts. + * + * Every field the model supplies is checked against a closed set, so an unknown value can never + * fall through toward a passing status. + * + * @param agentOutput The parsed contents of `agent_output.json`. + * @returns The validated result. + * @throws If the output has no `items` array, does not contain exactly one + * `record_arm_semantic_review` item, has a malformed or unknown field, or reports a complete + * review that carries an incomplete reason. */ -export function parseSemanticReviewResult( - agentOutput: unknown, - correlation: SemanticReviewCorrelation, -): SemanticReviewResult { - if ( - !Number.isSafeInteger(correlation.runAttempt) || - correlation.runAttempt <= 0 || - !Number.isSafeInteger(correlation.issueNumber) || - correlation.issueNumber <= 0 || - !isFullGitSha(correlation.headSha) - ) { - throw new Error("ARM semantic review correlation is missing or invalid"); - } +export function parseSemanticReviewResult(agentOutput: unknown): SemanticReviewResult { if (!isRecord(agentOutput) || !Array.isArray(agentOutput.items)) { throw new Error("ARM API Reviewer output is missing its items array"); } @@ -116,12 +149,7 @@ export function parseSemanticReviewResult( } const result = results[0]; - const blockingCount = Number(result.blocking_count); - if ( - typeof result.blocking_count !== "string" || - !/^(0|[1-9]\d*)$/.test(result.blocking_count) || - !Number.isSafeInteger(blockingCount) - ) { + if (typeof result.blocking_count !== "string" || !/^(0|[1-9]\d*)$/.test(result.blocking_count)) { throw new Error( `Invalid ARM semantic review Blocking count: '${String(result.blocking_count)}'`, ); @@ -147,27 +175,33 @@ export function parseSemanticReviewResult( `Invalid ARM semantic review incomplete reason: '${String(result.incomplete_reason)}'`, ); } + // A complete review must not carry a failure reason, or it could be read as a pass. if ( - (result.completeness === SemanticReviewCompletion.Complete && - result.incomplete_reason !== SemanticReviewIncompleteReason.None) || - (result.completeness === SemanticReviewCompletion.Incomplete && - result.incomplete_reason === SemanticReviewIncompleteReason.None) + result.completeness === SemanticReviewCompletion.Complete && + result.incomplete_reason !== SemanticReviewIncompleteReason.None ) { throw new Error( - `ARM semantic review completion '${result.completeness}' is inconsistent with ` + + `ARM semantic review completion 'complete' is inconsistent with ` + `incomplete reason '${result.incomplete_reason}'`, ); } return { - ...correlation, - blockingCount, + blockingCount: Number(result.blocking_count), reviewScope: result.scope, completion: result.completeness, incompleteReason: result.incomplete_reason, }; } +/** + * Maps a validated result to the commit status to publish. The first matching rule wins: + * incomplete, then scoped, then Blocking findings, then Passed. `success` is only reachable for + * a full, complete review with no Blocking findings. + * + * @param result A result returned by `parseSemanticReviewResult`. + * @returns The state and description to publish. + */ export function evaluateSemanticReview(result: SemanticReviewResult): EvaluatedSemanticReview { // Completeness and scope are checked first: Changes requested applies only to // full, complete reviews. An incomplete or scoped review never authorizes signoff @@ -197,25 +231,37 @@ export function evaluateSemanticReview(result: SemanticReviewResult): EvaluatedS } /** - * Independently verifies that the PR is within automated-review coverage limits. - * A full-and-complete agent result must still be rejected if the trusted file list - * shows the PR was too large or the list was truncated. + * Cheap pre-check on the PR's aggregate counters. When the whole PR is within the limits, its + * `specification/` subset necessarily is too, so no file list is needed. A PR reporting zero + * files is not "within": GitHub reports zero counters when a diff is too large to compute. + * + * @param pullRequest The aggregate counters from `pulls.get`. + * @returns `true` when no file list is needed and the PR is within the automated-review limits. */ -export function evaluateAutomatedReviewCoverage( +export function isWithinAutomatedReviewLimits(pullRequest: PullRequestCounts): boolean { + return ( + pullRequest.changed_files >= 1 && + pullRequest.changed_files <= MAX_SPECIFICATION_FILES && + pullRequest.additions + pullRequest.deletions <= MAX_SPECIFICATION_LINES + ); +} + +/** + * Independent check of the `specification/` subset, because the model's own `scope` is untrusted. + * Only needed when the aggregate counters exceed the limits. A truncated file list cannot prove + * coverage, so it also requires manual review. + * + * @param changedFileCount `changed_files` from `pulls.get`, used to detect truncation. + * @param changedFiles The paginated `pulls.listFiles` result. + * @returns `true` when the `specification/` subset exceeds the limits or coverage is unprovable. + */ +export function exceedsSpecificationLimits( changedFileCount: number, changedFiles: ChangedFile[], -): { manualReviewRequired: boolean; reason?: string } { - const fileListTruncated = - changedFiles.length >= 3_000 || - (changedFileCount === 0 && changedFiles.length > 0) || - changedFileCount > changedFiles.length; - if (fileListTruncated) { - return { - manualReviewRequired: true, - reason: "changed-file list was truncated", - }; +): boolean { + if (changedFiles.length >= MAX_LISTED_FILES || changedFileCount > changedFiles.length) { + return true; } - const specificationFiles = changedFiles.filter((file) => file.filename.startsWith("specification/"), ); @@ -223,18 +269,20 @@ export function evaluateAutomatedReviewCoverage( (total, file) => total + file.additions + file.deletions, 0, ); - if ( + return ( specificationFiles.length > MAX_SPECIFICATION_FILES || specificationLines > MAX_SPECIFICATION_LINES - ) { - return { - manualReviewRequired: true, - reason: "PR exceeds automated review size limits", - }; - } - return { manualReviewRequired: false }; + ); } +/** + * Interprets a commit status as a semantic review outcome. An `error` status is a manual-review + * hold only when its description starts with `MANUAL_REVIEW_DESCRIPTION_PREFIX`; any other + * `error` is an incomplete review. + * + * @param status The latest `ARM Semantic Review` status, if any. + * @returns The outcome, or `undefined` when there is no status or its state is unrecognized. + */ export function getSemanticReviewOutcome( status: Pick | undefined, ): SemanticReviewOutcome | undefined { @@ -258,6 +306,13 @@ export function getSemanticReviewOutcome( return undefined; } +/** + * Selects the newest `ARM Semantic Review` status. A SHA can hold many statuses for the same + * context (pending, then a result, then a re-run), and only the newest applies. + * + * @param statuses All commit statuses for a SHA. + * @returns The newest matching status, or `undefined` when there is none. + */ export function getLatestSemanticReviewStatus(statuses: CommitStatus[]): CommitStatus | undefined { return statuses .filter((status) => status.context.toLowerCase() === ARM_SEMANTIC_REVIEW_STATUS.toLowerCase()) diff --git a/.github/workflows/src/arm-auto-signoff/arm-universal-auto-signoff.ts b/.github/workflows/src/arm-auto-signoff/arm-universal-auto-signoff.ts index afb143173b4d..3326076f92f7 100644 --- a/.github/workflows/src/arm-auto-signoff/arm-universal-auto-signoff.ts +++ b/.github/workflows/src/arm-auto-signoff/arm-universal-auto-signoff.ts @@ -11,13 +11,20 @@ import { SemanticReviewOutcome, } from "./arm-semantic-review.ts"; +/** Deterministic commit statuses that must be `success`, in addition to the semantic review. */ const requiredStatusNames = ["Swagger LintDiff", "Swagger Avocado"]; +/** + * The labels this workflow manages, each with the action to take. `ARMAutoSignedOff-Test` records + * the pilot decision. `ARMManualSignoffRequired` is added (and never removed here) when a human + * must review. `ARMSignedOff` is deliberately not managed: the pilot never changes it. + */ export type ManagedLabelActions = { "ARMAutoSignedOff-Test": LabelAction; ARMManualSignoffRequired: LabelAction; }; +/** Returns a label-action set that changes nothing. */ function createNoneLabelActions(): ManagedLabelActions { return { [ArmAutoSignoffLabel.ArmAutoSignedOffTest]: LabelAction.None, @@ -27,6 +34,13 @@ function createNoneLabelActions(): ManagedLabelActions { /* v8 ignore start */ +/** + * Workflow entry point. Resolves the PR and head SHA from the triggering event (see + * `extractInputs`), logs the correlation for diagnosis, and returns the label actions to apply. + * + * @returns The correlated head SHA and PR number, which the workflow uploads as artifacts so + * `Update Labels` can re-check the live head, and the label actions to apply. + */ export default async function getLabelAction({ github, context, core }: GitHubScriptArgs): Promise<{ headSha: string; issueNumber: number; @@ -74,6 +88,15 @@ export default async function getLabelAction({ github, context, core }: GitHubSc } /* v8 ignore stop */ +/** + * Decides label actions for a PR head. It first re-reads the PR and takes no action when it is + * closed or its head is no longer `head_sha`, so a stale event cannot change current labels. + * Otherwise it reads the PR's labels and the head SHA's commit statuses and defers to + * `getDesiredLabelActions`. + * + * @returns The correlation passed in and the label actions; no action when correlation is missing + * or stale. + */ export async function getLabelActionImpl({ owner, repo, @@ -146,6 +169,20 @@ export async function getLabelActionImpl({ }; } +/** + * Applies the auto-signoff policy for one head SHA. In order: + * + * 1. Not ready for ARM review (`ARMReview` missing or `NotReadyForARMReview` present): no signoff. + * 2. `ARM Semantic Review` is not `Passed`: no signoff. Only a manual-review hold + * (`Manual review required`) also adds `ARMManualSignoffRequired`; a pending, failed, or + * incomplete review adds nothing, because a re-run can still pass. + * 3. Labels block signoff (`ARMChangesRequested`, `ARMManualSignoffRequired`, or an unapproved + * `SuppressionReviewRequired`): no signoff. + * 4. Any required status (`Swagger LintDiff`, `Swagger Avocado`) is not `success`: no signoff. + * 5. Otherwise add `ARMAutoSignedOff-Test`. + * + * Whenever signoff is withheld, an existing `ARMAutoSignedOff-Test` is removed. + */ async function getDesiredLabelActions({ owner, repo, diff --git a/.github/workflows/src/context.ts b/.github/workflows/src/context.ts index ffea074a610a..c07d968a3fdf 100644 --- a/.github/workflows/src/context.ts +++ b/.github/workflows/src/context.ts @@ -12,11 +12,18 @@ export type PullRequest = export type RestEndpointMethodTypes = import("@octokit/plugin-rest-endpoint-methods").RestEndpointMethodTypes; -export const ARM_API_REVIEW_WORKFLOW_PATH = ".github/workflows/arm-api-review.lock.yml"; - /** * Extracts inputs from context based on event name and properties. * run_id is only defined for "workflow_run:completed" events. + * + * For "workflow_run:completed", `workflow_run.event` is the event that triggered the completed + * (upstream) run, and decides where the head SHA and PR number come from: + * - "pull_request": the payload only (fork code could forge artifacts). + * - "pull_request_target": the payload, overridden by the upstream run's trusted artifacts. + * - "issue_comment", "workflow_run", "check_run", "workflow_dispatch": artifacts only. + * + * `head_sha` may be "" and `issue_number` may be NaN when they can't be determined. + * @throws If artifacts conflict with each other or with the event, or the event is unsupported. */ export async function extractInputs( github: GitHub, @@ -139,8 +146,8 @@ export async function extractInputs( // For pull_request, do NOT attempt to extract the issue number from an artifact, since this could be modified // in a fork PR. // - // For pull_request_target, trusted artifacts carry the PR head SHA because workflow_run.head_sha identifies - // the base branch commit. The issue number is also read from the artifacts and checked against the event. + // For pull_request_target, the workflow runs from the base branch, so its artifacts are trusted and + // override the SHA and are checked against the PR number below. const pullRequest = payload.workflow_run.pull_requests?.find((pr) => pr !== null); if (pullRequest) { @@ -223,9 +230,8 @@ export async function extractInputs( runId: payload.workflow_run.id, }); if (artifactInputs.headSha) { + // The artifact can be newer than the SHA that triggered the run if a push landed in between. head_sha = artifactInputs.headSha; - } else if (payload.workflow_run.path === ARM_API_REVIEW_WORKFLOW_PATH) { - head_sha = ""; } if (artifactInputs.issueNumber) { if (issue_number && issue_number !== artifactInputs.issueNumber) { @@ -309,6 +315,12 @@ export async function extractInputs( return inputs; } +/** + * Reads the `head-sha=` and `issue-number=` artifact names of a workflow run. + * Only call this for runs whose artifacts come from trusted code (see `extractInputs`). + * @returns `headSha` ("" if absent) and `issueNumber` (NaN if absent or invalid). + * @throws If the artifacts hold an invalid or conflicting head SHA, or conflicting PR numbers. + */ export async function getWorkflowRunArtifactInputs({ github, core, @@ -333,6 +345,11 @@ export async function getWorkflowRunArtifactInputs({ return parseWorkflowRunArtifactInputs(artifactNames, core); } +/** + * Parses `head-sha=` and `issue-number=` from artifact names; other names are ignored. + * @returns `headSha` ("" if absent) and `issueNumber` (NaN if absent or invalid). + * @throws If a head SHA is invalid, or two head SHAs or two PR numbers conflict. + */ export function parseWorkflowRunArtifactInputs( artifactNames: string[], core: Core, @@ -356,6 +373,7 @@ export function parseWorkflowRunArtifactInputs( } headSha = value; } else if (key === "issue-number") { + // Stricter than parseInt, which would accept "12abc". const parsedValue = /^[1-9]\d*$/.test(value) ? Number(value) : NaN; if (Number.isSafeInteger(parsedValue)) { if (Number.isSafeInteger(issueNumber) && issueNumber !== parsedValue) { diff --git a/.github/workflows/src/update-labels.ts b/.github/workflows/src/update-labels.ts index a9ac022f394e..4604c1c9a50b 100644 --- a/.github/workflows/src/update-labels.ts +++ b/.github/workflows/src/update-labels.ts @@ -12,6 +12,11 @@ export default async function updateLabels({ github, context, core }: GitHubScri await updateLabelsImpl({ owner, repo, head_sha, issue_number, run_id, github, core }); } +/** + * Adds or removes PR labels from the `label-=true|false` artifacts of one workflow run. + * If the run also published a `head-sha=` artifact, changes are skipped when the PR is closed or + * its head has moved on, so a delayed run can't label a newer commit. + */ export async function updateLabelsImpl({ owner, repo, @@ -56,6 +61,7 @@ export async function updateLabelsImpl({ }); const artifactNames: string[] = artifacts.map((a) => a.name); + // Producers opt in to the stale-head check below by publishing a head-sha artifact. const hasHeadShaArtifact = artifactNames.some((name) => name.startsWith("head-sha=")); core.info(`artifactNames: ${JSON.stringify(artifactNames)}`); diff --git a/.github/workflows/test/arm-api-review-workflow.test.ts b/.github/workflows/test/arm-api-review-workflow.test.ts index 863ea707e3fe..2c93eb4c5e47 100644 --- a/.github/workflows/test/arm-api-review-workflow.test.ts +++ b/.github/workflows/test/arm-api-review-workflow.test.ts @@ -348,17 +348,6 @@ describe("ARM API review workflow", () => { expect(compiled).toContain('GH_AW_SUB_AGENT_EXT: ".agent.md"'); }); - it("creates incomplete-review tracking issues only in the canonical repository", async () => { - const [source, compiled] = await readWorkflowFiles(); - - expect(source).toContain( - "create-issue: ${{ github.repository == 'Azure/azure-rest-api-specs' }}", - ); - expect(compiled).toContain( - "GH_AW_REPORT_INCOMPLETE_CREATE_ISSUE: ${{ github.repository == 'Azure/azure-rest-api-specs' }}", - ); - }); - it("reconciles duplicates and contradictions across every review entry point", async () => { const [ [source, compiled], @@ -657,6 +646,10 @@ describe("ARM API review posting reliability", () => { expect(conclusionJob.slice(finalizerStep, finalizerStep + 700)).toContain( "continue-on-error: true", ); + // The record job already published when it succeeded, so the finalizer's reads are skipped. + expect(conclusionJob.slice(finalizerStep, finalizerStep + 200)).toContain( + "if: needs.record_arm_semantic_review.result != 'success'", + ); expect(conclusionJob.indexOf("name: Process no-op messages")).toBeGreaterThan(finalizerStep); }); diff --git a/.github/workflows/test/arm-auto-signoff/arm-semantic-review.test.ts b/.github/workflows/test/arm-auto-signoff/arm-semantic-review.test.ts index d27fb20846c0..93c8b934f1ec 100644 --- a/.github/workflows/test/arm-auto-signoff/arm-semantic-review.test.ts +++ b/.github/workflows/test/arm-auto-signoff/arm-semantic-review.test.ts @@ -5,8 +5,9 @@ import { describe, expect, it } from "vitest"; import { CommitStatusState, PER_PAGE_MAX } from "../../../shared/src/github.ts"; import { ARM_SEMANTIC_REVIEW_STATUS, - evaluateAutomatedReviewCoverage, evaluateSemanticReview, + exceedsSpecificationLimits, + isWithinAutomatedReviewLimits, getSemanticReviewOutcome, parseSemanticReviewResult, SemanticReviewCompletion, @@ -31,9 +32,6 @@ const runAttempt = 1; const runUrl = "https://github.com/Azure/azure-rest-api-specs/actions/runs/456"; const passedResult: SemanticReviewResult = { - runAttempt, - issueNumber, - headSha, blockingCount: 0, reviewScope: SemanticReviewScope.Full, completion: SemanticReviewCompletion.Complete, @@ -54,9 +52,7 @@ function agentOutput(result: SemanticReviewResult = passedResult) { }; } -const defaultChangedFiles: ChangedFile[] = [ - { filename: "specification/foo/foo.json", additions: 10, deletions: 5 }, -]; +const smallPullRequest = { changed_files: 1, additions: 10, deletions: 5 }; function createPublisherGithub() { const github = createMockGithub(); @@ -65,21 +61,20 @@ function createPublisherGithub() { artifacts: [{ name: `head-sha=${headSha}` }, { name: `issue-number=${issueNumber}` }], }, }); - // Default: small PR well within coverage limits - github.rest.pulls.get.mockResolvedValue({ - data: { changed_files: 1, state: "open", head: { sha: headSha } }, - }); - github.rest.pulls.listFiles.mockResolvedValue({ data: defaultChangedFiles }); + // Default: small PR well within automated review limits + github.rest.pulls.get.mockResolvedValue({ data: smallPullRequest }); return github; } async function runPublisher({ output = agentOutput(), includeOutput = true, + setOutputPath = true, github = createPublisherGithub(), }: { output?: unknown; includeOutput?: boolean; + setOutputPath?: boolean; github?: ReturnType; } = {}) { const directory = await mkdtemp(join(tmpdir(), "arm-semantic-review-")); @@ -90,8 +85,13 @@ async function runPublisher({ if (includeOutput) { await writeFile(outputPath, JSON.stringify(output), "utf8"); } - process.env.GH_AW_AGENT_OUTPUT = outputPath; + if (setOutputPath) { + process.env.GH_AW_AGENT_OUTPUT = outputPath; + } else { + delete process.env.GH_AW_AGENT_OUTPUT; + } process.env.GITHUB_RUN_ATTEMPT = String(runAttempt); + const core = createMockCore(); const result = await publishArmSemanticReviewStatus({ github, context: { @@ -99,9 +99,9 @@ async function runPublisher({ runId, serverUrl: "https://github.com", }, - core: createMockCore(), + core, } as unknown as GitHubScriptArgs); - return { github, result }; + return { github, result, core }; } finally { if (previousOutputPath === undefined) { delete process.env.GH_AW_AGENT_OUTPUT; @@ -119,12 +119,10 @@ async function runPublisher({ describe("parseSemanticReviewResult", () => { it("parses one valid result", () => { - expect(parseSemanticReviewResult(agentOutput(), { headSha, issueNumber, runAttempt })).toEqual( - passedResult, - ); + expect(parseSemanticReviewResult(agentOutput())).toEqual(passedResult); }); - it("uses trusted correlation instead of model-supplied identifiers", () => { + it("ignores model-supplied identifiers", () => { const output = agentOutput(); Object.assign(output.items[0], { head_sha: "675718552ce00736684ed7a184bab", @@ -132,92 +130,53 @@ describe("parseSemanticReviewResult", () => { run_attempt: "99", }); - expect(parseSemanticReviewResult(output, { headSha, issueNumber, runAttempt })).toEqual( - passedResult, - ); + expect(parseSemanticReviewResult(output)).toEqual(passedResult); }); - it("rejects missing, duplicate, or invalid semantic results", () => { - expect(() => - parseSemanticReviewResult({ items: [] }, { headSha, issueNumber, runAttempt }), - ).toThrow("Expected one ARM semantic review result, found 0"); + it("rejects missing or duplicate semantic results", () => { + expect(() => parseSemanticReviewResult({ items: [] })).toThrow( + "Expected one ARM semantic review result, found 0", + ); expect(() => - parseSemanticReviewResult( - { - items: [ - ...agentOutput().items, - ...agentOutput({ ...passedResult, blockingCount: 1 }).items, - ], - }, - { headSha, issueNumber, runAttempt }, - ), + parseSemanticReviewResult({ + items: [ + ...agentOutput().items, + ...agentOutput({ ...passedResult, blockingCount: 1 }).items, + ], + }), ).toThrow("Expected one ARM semantic review result, found 2"); - expect(() => - parseSemanticReviewResult( - { - items: [{ ...agentOutput().items[0], scope: "partial" }], - }, - { headSha, issueNumber, runAttempt }, - ), - ).toThrow("Invalid ARM semantic review scope"); - expect(() => - parseSemanticReviewResult( - { - items: [{ ...agentOutput().items[0], completeness: "unknown" }], - }, - { headSha, issueNumber, runAttempt }, - ), - ).toThrow("Invalid ARM semantic review completion"); - expect(() => - parseSemanticReviewResult( - { - items: [{ ...agentOutput().items[0], incomplete_reason: "unknown" }], - }, - { headSha, issueNumber, runAttempt }, - ), - ).toThrow("Invalid ARM semantic review incomplete reason"); + expect(() => parseSemanticReviewResult({})).toThrow("missing its items array"); }); - it("rejects inconsistent completeness and reason combinations", () => { - expect(() => - parseSemanticReviewResult( - { - items: [ - { - ...agentOutput().items[0], - incomplete_reason: SemanticReviewIncompleteReason.CriticUnavailable, - }, - ], - }, - { headSha, issueNumber, runAttempt }, - ), - ).toThrow("completion 'complete' is inconsistent"); + it.each([ + { field: "blocking_count", value: "-1", message: "Invalid ARM semantic review Blocking count" }, + { field: "blocking_count", value: 0, message: "Invalid ARM semantic review Blocking count" }, + { field: "scope", value: "partial", message: "Invalid ARM semantic review scope" }, + { field: "completeness", value: "unknown", message: "Invalid ARM semantic review completion" }, + { + field: "incomplete_reason", + value: "unknown", + message: "Invalid ARM semantic review incomplete reason", + }, + ])("rejects an invalid $field", ({ field, value, message }) => { expect(() => - parseSemanticReviewResult( - { - items: [ - { - ...agentOutput().items[0], - completeness: SemanticReviewCompletion.Incomplete, - }, - ], - }, - { headSha, issueNumber, runAttempt }, - ), - ).toThrow("completion 'incomplete' is inconsistent"); + parseSemanticReviewResult({ items: [{ ...agentOutput().items[0], [field]: value }] }), + ).toThrow(message); }); - it("rejects invalid trusted correlation", () => { + it("rejects a complete review that carries an incomplete reason", () => { expect(() => - parseSemanticReviewResult(agentOutput(), { - headSha: "675718552ce00736684ed7a184bab", - issueNumber, - runAttempt, + parseSemanticReviewResult({ + items: [ + { + ...agentOutput().items[0], + incomplete_reason: SemanticReviewIncompleteReason.CriticUnavailable, + }, + ], }), - ).toThrow("ARM semantic review correlation is missing or invalid"); + ).toThrow("completion 'complete' is inconsistent"); }); }); - describe("evaluateSemanticReview", () => { it("passes when a full, complete review has no Blocking findings", () => { expect(evaluateSemanticReview(passedResult)).toEqual({ @@ -309,54 +268,86 @@ describe("evaluateSemanticReview", () => { description: "Manual review required: scoped review requires manual signoff", }), ).toBe(SemanticReviewOutcome.ManualReviewRequired); + expect(getSemanticReviewOutcome(undefined)).toBeUndefined(); + expect(getSemanticReviewOutcome({ state: "unexpected" })).toBeUndefined(); }); }); -describe("evaluateAutomatedReviewCoverage", () => { - const smallFile: ChangedFile = { - filename: "specification/foo/foo.json", - additions: 10, - deletions: 5, - }; +describe("isWithinAutomatedReviewLimits", () => { + it.each([ + { name: "a small PR", pr: smallPullRequest, expected: true }, + { + name: "exactly the file limit", + pr: { changed_files: 50, additions: 1, deletions: 0 }, + expected: true, + }, + { + name: "one file over the limit", + pr: { changed_files: 51, additions: 1, deletions: 0 }, + expected: false, + }, + { + name: "exactly the line limit", + pr: { changed_files: 1, additions: 3_000, deletions: 2_000 }, + expected: true, + }, + { + name: "over the line limit", + pr: { changed_files: 1, additions: 4_000, deletions: 2_000 }, + expected: false, + }, + // GitHub reports zero counters when the diff is too large to compute. + { + name: "zero reported files", + pr: { changed_files: 0, additions: 0, deletions: 0 }, + expected: false, + }, + ])("$name", ({ pr, expected }) => { + expect(isWithinAutomatedReviewLimits(pr)).toBe(expected); + }); +}); - it("passes a small PR within limits", () => { - expect(evaluateAutomatedReviewCoverage(1, [smallFile])).toEqual({ - manualReviewRequired: false, - }); +describe("exceedsSpecificationLimits", () => { + const specFile = (name: string, additions = 1, deletions = 0): ChangedFile => ({ + filename: `specification/foo/${name}.json`, + additions, + deletions, + }); + const otherFile = (name: string, additions = 1): ChangedFile => ({ + filename: `documentation/${name}.md`, + additions, + deletions: 0, }); - it("requires manual review when the file list is truncated", () => { - expect(evaluateAutomatedReviewCoverage(10, [smallFile])).toEqual({ - manualReviewRequired: true, - reason: "changed-file list was truncated", - }); + it("ignores files and lines outside specification/", () => { + const files = [ + specFile("a"), + ...Array.from({ length: 60 }, (_, i) => otherFile(`doc${i}`, 200)), + ]; + expect(exceedsSpecificationLimits(files.length, files)).toBe(false); }); - it("requires manual review when specification file count exceeds limit", () => { - const manyFiles = Array.from({ length: 51 }, (_, i): ChangedFile => ({ - filename: `specification/foo/foo${i}.json`, - additions: 1, - deletions: 0, - })); - expect(evaluateAutomatedReviewCoverage(51, manyFiles)).toEqual({ - manualReviewRequired: true, - reason: "PR exceeds automated review size limits", - }); + it("allows exactly the specification file limit", () => { + const files = Array.from({ length: 50 }, (_, i) => specFile(`f${i}`)); + expect(exceedsSpecificationLimits(files.length, files)).toBe(false); }); - it("requires manual review when specification line count exceeds limit", () => { - const bigFile: ChangedFile = { - filename: "specification/foo/foo.json", - additions: 4_000, - deletions: 2_000, - }; - expect(evaluateAutomatedReviewCoverage(1, [bigFile])).toEqual({ - manualReviewRequired: true, - reason: "PR exceeds automated review size limits", - }); + it("flags more specification files than the limit", () => { + const files = Array.from({ length: 51 }, (_, i) => specFile(`f${i}`)); + expect(exceedsSpecificationLimits(files.length, files)).toBe(true); + }); + + it("flags more specification lines than the limit", () => { + const files = [specFile("big", 4_000, 2_000)]; + expect(exceedsSpecificationLimits(files.length, files)).toBe(true); }); -}); + it("flags a truncated file list", () => { + expect(exceedsSpecificationLimits(10, [specFile("a")])).toBe(true); + const capped = Array.from({ length: 3_000 }, (_, i) => otherFile(`d${i}`)); + expect(exceedsSpecificationLimits(3_000, capped)).toBe(true); + }); +}); describe("publishArmSemanticReviewStatus", () => { it("publishes Passed for a clean reviewer result", async () => { const { github, result } = await runPublisher(); @@ -406,16 +397,89 @@ describe("publishArmSemanticReviewStatus", () => { ); }); + it("does not list files when the PR is within the aggregate limits", async () => { + const { github } = await runPublisher(); + + expect(github.rest.pulls.listFiles).not.toHaveBeenCalled(); + }); + + it("publishes Passed when only non-specification files push the PR over the limits", async () => { + const github = createPublisherGithub(); + const files = [ + { filename: "specification/foo/foo.json", additions: 10, deletions: 5 }, + ...Array.from({ length: 60 }, (_, i) => ({ + filename: `documentation/doc${i}.md`, + additions: 1, + deletions: 0, + })), + ]; + github.rest.pulls.get.mockResolvedValue({ + data: { changed_files: files.length, additions: 70, deletions: 5 }, + }); + github.rest.pulls.listFiles.mockResolvedValue({ data: files }); + + const { github: g } = await runPublisher({ github }); + expect(g.rest.pulls.listFiles).toHaveBeenCalled(); + expect(g.rest.repos.createCommitStatus).toHaveBeenCalledWith( + expect.objectContaining({ state: CommitStatusState.SUCCESS }), + ); + }); + + it.each([ + { + name: "too many specification files", + pr: { changed_files: 51, additions: 51, deletions: 0 }, + files: Array.from({ length: 51 }, (_, i) => ({ + filename: `specification/foo/f${i}.json`, + additions: 1, + deletions: 0, + })), + }, + { + name: "a truncated file list", + pr: { changed_files: 51, additions: 51, deletions: 0 }, + files: [{ filename: "specification/foo/foo.json", additions: 1, deletions: 0 }], + }, + ])("publishes Manual review required for $name", async ({ pr, files }) => { + const github = createPublisherGithub(); + github.rest.pulls.get.mockResolvedValue({ data: pr }); + github.rest.pulls.listFiles.mockResolvedValue({ data: files }); + + const { github: g } = await runPublisher({ github }); + expect(g.rest.repos.createCommitStatus).toHaveBeenCalledWith( + expect.objectContaining({ + state: CommitStatusState.ERROR, + description: "Manual review required: PR exceeds automated review size limits", + }), + ); + }); + + it("publishes Manual review required without listing files when the PR reports zero files", async () => { + const github = createPublisherGithub(); + github.rest.pulls.get.mockResolvedValue({ + data: { changed_files: 0, additions: 0, deletions: 0 }, + }); + + const { github: g } = await runPublisher({ github }); + expect(g.rest.pulls.listFiles).not.toHaveBeenCalled(); + expect(g.rest.repos.createCommitStatus).toHaveBeenCalledWith( + expect.objectContaining({ + state: CommitStatusState.ERROR, + description: "Manual review required: PR exceeds automated review size limits", + }), + ); + }); + it.each([ { name: "missing agent output", options: { includeOutput: false }, - description: "Review incomplete: ENOENT", + logged: "ENOENT", }, { name: "missing semantic item", options: { output: { items: [] } }, - description: "Review incomplete: Expected one ARM semantic review result, found 0", + logged: "Expected one ARM semantic review result, found 0", }, { name: "invalid Blocking count", @@ -424,147 +488,67 @@ describe("publishArmSemanticReviewStatus", () => { items: [{ ...agentOutput().items[0], blocking_count: "not-a-number" }], }, }, - description: "Review incomplete: Invalid ARM semantic review Blocking count", + logged: "Invalid ARM semantic review Blocking count", }, - ])("publishes Review incomplete for $name", async ({ options, description }) => { - const { github } = await runPublisher(options); + ])("publishes Review incomplete for $name", async ({ options, logged }) => { + const { github, core } = await runPublisher(options); expect(github.rest.repos.createCommitStatus).toHaveBeenCalledWith( expect.objectContaining({ state: CommitStatusState.ERROR, + description: "Review incomplete: result could not be validated", }), ); - const status = github.rest.repos.createCommitStatus.mock.calls[0]?.[0] as { - description: string; - }; - expect(status.description).toContain(description); + // The details go to the run log, not the public status description. + expect(core.warning).toHaveBeenCalledWith(expect.stringContaining(logged)); }); - it("does nothing when trusted PR/SHA correlation is missing", async () => { - const github = createPublisherGithub(); - github.rest.actions.listWorkflowRunArtifacts.mockResolvedValue({ - data: { artifacts: [] }, - }); + it("publishes Review incomplete when the output path is not set", async () => { + const { github } = await runPublisher({ setOutputPath: false }); - const execution = await runPublisher({ github }); - expect(execution.result).toEqual({ - headSha: "", - issueNumber: 0, - statusPublished: false, - }); - expect(github.rest.repos.createCommitStatus).not.toHaveBeenCalled(); + expect(github.rest.repos.createCommitStatus).toHaveBeenCalledWith( + expect.objectContaining({ + state: CommitStatusState.ERROR, + description: "Review incomplete: result could not be validated", + }), + ); }); - it.each([ - { - name: "closed", - pullRequest: { changed_files: 1, state: "closed", head: { sha: headSha } }, - }, - { - name: "stale", - pullRequest: { changed_files: 1, state: "open", head: { sha: "f".repeat(40) } }, - }, - ])("does not publish a status when the pull request is $name", async ({ pullRequest }) => { + it("logs a thrown value that is not an Error", async () => { const github = createPublisherGithub(); - github.rest.pulls.get.mockResolvedValue({ data: pullRequest }); - - const execution = await runPublisher({ github }); + github.rest.pulls.get.mockRejectedValue("plain string failure"); - expect(execution.result).toEqual({ - headSha, - issueNumber, - statusPublished: false, - }); - expect(github.rest.pulls.listFiles).not.toHaveBeenCalled(); - expect(github.rest.repos.createCommitStatus).not.toHaveBeenCalled(); + const { github: g, core } = await runPublisher({ github }); + expect(g.rest.repos.createCommitStatus).toHaveBeenCalledWith( + expect.objectContaining({ state: CommitStatusState.ERROR }), + ); + expect(core.warning).toHaveBeenCalledWith("plain string failure"); }); - it("publishes Manual review required when the PR exceeds automated review size limits", async () => { + it("publishes Review incomplete without exposing the error when the PR cannot be read", async () => { const github = createPublisherGithub(); - const oversizedFiles = Array.from({ length: 51 }, (_, i): ChangedFile => ({ - filename: `specification/foo/foo${i}.json`, - additions: 1, - deletions: 0, - })); - github.rest.pulls.get.mockResolvedValue({ - data: { changed_files: 51, state: "open", head: { sha: headSha } }, - }); - github.rest.pulls.listFiles.mockResolvedValue({ data: oversizedFiles }); + github.rest.pulls.get.mockRejectedValue(new Error("boom: /home/runner/work/_temp/secret")); - const { github: g } = await runPublisher({ github }); + const { github: g, core } = await runPublisher({ github }); expect(g.rest.repos.createCommitStatus).toHaveBeenCalledWith( expect.objectContaining({ state: CommitStatusState.ERROR, - description: "Manual review required: PR exceeds automated review size limits", + description: "Review incomplete: result could not be validated", }), ); + expect(core.warning).toHaveBeenCalledWith(expect.stringContaining("boom")); }); - it("truncates an overly long error description to 140 characters", async () => { - const longValue = "x".repeat(200); - const { github } = await runPublisher({ - output: { items: [{ ...agentOutput().items[0], scope: longValue }] }, - }); - - const call = github.rest.repos.createCommitStatus.mock.calls[0]?.[0] as { - description: string; - }; - expect(call.description.length).toBeLessThanOrEqual(140); - expect(call.description).toMatch(/^Review incomplete:/); - }); - - it("does not overwrite a newer review of the same SHA", async () => { - const github = createPublisherGithub(); - github.rest.repos.listCommitStatusesForRef.mockResolvedValue({ - data: [ - { - context: ARM_SEMANTIC_REVIEW_STATUS, - state: CommitStatusState.PENDING, - target_url: "https://github.com/Azure/azure-rest-api-specs/actions/runs/789/attempts/1", - updated_at: "2026-01-02T00:00:00Z", - }, - ], - }); - - const execution = await runPublisher({ github }); - expect(execution.result).toEqual({ - headSha, - issueNumber, - statusPublished: false, - }); - expect(github.rest.repos.createCommitStatus).not.toHaveBeenCalled(); - }); - - it("checks for a newer run only after reading the PR and its files", async () => { - const github = createPublisherGithub(); - - await runPublisher({ github }); - - const [listStatuses] = github.rest.repos.listCommitStatusesForRef.mock.invocationCallOrder; - const [getPullRequest] = github.rest.pulls.get.mock.invocationCallOrder; - const [listFiles] = github.rest.pulls.listFiles.mock.invocationCallOrder; - const [createStatus] = github.rest.repos.createCommitStatus.mock.invocationCallOrder; - expect(listStatuses).toBeGreaterThan(getPullRequest); - expect(listStatuses).toBeGreaterThan(listFiles); - expect(listStatuses).toBeLessThan(createStatus); - }); - it("does not overwrite a newer attempt of the same run", async () => { + it("does nothing when trusted PR/SHA correlation is missing", async () => { const github = createPublisherGithub(); - github.rest.repos.listCommitStatusesForRef.mockResolvedValue({ - data: [ - { - context: ARM_SEMANTIC_REVIEW_STATUS, - state: CommitStatusState.PENDING, - target_url: `https://github.com/Azure/azure-rest-api-specs/actions/runs/${runId}/attempts/2`, - updated_at: "2026-01-02T00:00:00Z", - }, - ], + github.rest.actions.listWorkflowRunArtifacts.mockResolvedValue({ + data: { artifacts: [] }, }); const execution = await runPublisher({ github }); expect(execution.result).toEqual({ - headSha, - issueNumber, + headSha: "", + issueNumber: 0, statusPublished: false, }); expect(github.rest.repos.createCommitStatus).not.toHaveBeenCalled(); diff --git a/.github/workflows/test/arm-auto-signoff/arm-universal-auto-signoff.test.ts b/.github/workflows/test/arm-auto-signoff/arm-universal-auto-signoff.test.ts index 7ec0becd85ef..9c18b20d2b9a 100644 --- a/.github/workflows/test/arm-auto-signoff/arm-universal-auto-signoff.test.ts +++ b/.github/workflows/test/arm-auto-signoff/arm-universal-auto-signoff.test.ts @@ -113,6 +113,31 @@ describe("getLabelActionImpl", () => { expect(result.labelActions[ArmAutoSignoffLabel.ArmAutoSignedOffTest]).toBe(LabelAction.None); }); + it.each([ + { name: "ARMReview is missing", labelNames: [] as string[] }, + { name: "NotReadyForARMReview is present", labelNames: ["ARMReview", "NotReadyForARMReview"] }, + ])("does not read statuses and removes the pilot label when $name", async ({ labelNames }) => { + const github = createMockGithub({ + labelNames: [...labelNames, ArmAutoSignoffLabel.ArmAutoSignedOffTest], + }); + + const result = await run(github); + expect(result.labelActions).toEqual({ + [ArmAutoSignoffLabel.ArmAutoSignedOffTest]: LabelAction.Remove, + [ArmAutoSignoffLabel.ArmManualSignoffRequired]: LabelAction.None, + }); + expect(github.rest.repos.listCommitStatusesForRef).not.toHaveBeenCalled(); + }); + + it("takes no action when the PR is not ready and has no pilot label", async () => { + const github = createMockGithub({ labelNames: [] }); + + const result = await run(github); + expect(result.labelActions).toEqual({ + [ArmAutoSignoffLabel.ArmAutoSignedOffTest]: LabelAction.None, + [ArmAutoSignoffLabel.ArmManualSignoffRequired]: LabelAction.None, + }); + }); it("treats a manual signoff label as a hard stop without removing it", async () => { const github = createMockGithub({ labelNames: [ diff --git a/.github/workflows/test/context.test.ts b/.github/workflows/test/context.test.ts index 233a3a94c6f9..ea1eaf0deee2 100644 --- a/.github/workflows/test/context.test.ts +++ b/.github/workflows/test/context.test.ts @@ -234,7 +234,7 @@ describe("extractInputs", () => { action: "completed", workflow_run: { event: "pull_request_target", - head_sha: "base-branch-sha", + head_sha: "payload-sha", id: 456, repository: { name: "TestRepoName", @@ -262,7 +262,7 @@ describe("extractInputs", () => { }); }); - it("does not substitute the base SHA when ARM review correlation is missing", async () => { + it("workflow_run:completed:pull_request_target keeps the payload SHA when artifacts are missing", async () => { const github = createMockGithub(); github.rest.actions.listWorkflowRunArtifacts.mockResolvedValue({ data: { artifacts: [] }, @@ -272,10 +272,8 @@ describe("extractInputs", () => { payload: { action: "completed", workflow_run: { - name: "ARM API Review #123 (pull_request_target)", - path: ".github/workflows/arm-api-review.lock.yml", event: "pull_request_target", - head_sha: "base-branch-sha", + head_sha: "payload-sha", id: 456, repository: { name: "TestRepoName", @@ -291,7 +289,7 @@ describe("extractInputs", () => { await expect(extractInputs(github, context, createMockCore())).resolves.toEqual({ owner: "TestRepoOwnerLogin", repo: "TestRepoName", - head_sha: "", + head_sha: "payload-sha", issue_number: 123, run_id: 456, }); @@ -332,7 +330,7 @@ describe("extractInputs", () => { }); }); - it("does not substitute the current head when ARM API review artifacts are missing", async () => { + it("workflow_run:completed:workflow_dispatch does not fall back to the payload SHA when artifacts are missing", async () => { const github = createMockGithub(); github.rest.actions.listWorkflowRunArtifacts.mockResolvedValue({ data: { artifacts: [] }, @@ -342,8 +340,6 @@ describe("extractInputs", () => { payload: { action: "completed", workflow_run: { - name: "ARM API Review: Automated Workflow", - display_title: "ARM API Review #123 (workflow_dispatch)", event: "workflow_dispatch", head_sha: "default-branch-sha", id: 456, diff --git a/documentation/arm-semantic-review-auto-signoff-implementation.md b/documentation/arm-semantic-review-auto-signoff-implementation.md index 63ef291d7371..0bae4667142d 100644 --- a/documentation/arm-semantic-review-auto-signoff-implementation.md +++ b/documentation/arm-semantic-review-auto-signoff-implementation.md @@ -63,7 +63,7 @@ Responsibilities: Responsibilities: - parse exactly one structured semantic item; -- validate run attempt, PR number, SHA, Blocking count, scope, and completion; +- validate the Blocking count, scope, completion, and incomplete reason; - map the evidence to Passed, Changes requested, or Review incomplete; - interpret GitHub commit-status states; - select the newest semantic status for a SHA. @@ -79,9 +79,9 @@ Responsibilities: - run as the reviewer's trusted `record_arm_semantic_review` safe-output job; - read that run's `agent_output.json` and trusted PR/SHA correlation artifacts; - validate the semantic item once; -- independently enforce automated-review coverage limits; +- independently enforce automated-review size limits; - publish `ARM Semantic Review` on the reviewed SHA; -- avoid overwriting a newer review or rerun. +- resolve a Pending status that was never published (from the `conclusion` job). Only the reviewer `pre_activation`, `record_arm_semantic_review`, and `conclusion` jobs have `statuses: write`. The agent itself still has read-only GitHub tools. @@ -167,16 +167,24 @@ run that produced the item. It: -1. lists that run's artifact names once; -2. reads trusted `head-sha` and `issue-number` artifacts; -3. reads the run attempt from the workflow environment; -4. finds exactly one semantic item in the safe-output payload; -5. validates the attempt, PR, SHA, Blocking count, scope, completion, and - incomplete reason; -6. confirms that the PR is open and still points to the reviewed SHA; -7. independently verifies review coverage against the trusted changed-file - list; -8. publishes a status on the reviewed SHA. +1. reads the trusted `head-sha` and `issue-number` artifacts from that run; +2. finds exactly one semantic item in the safe-output payload; +3. validates the Blocking count, scope, completion, and incomplete reason, and + rejects a complete review that carries an incomplete reason; +4. independently checks the PR's size against the automated-review limits; +5. publishes a status on the reviewed SHA. + +The size check mirrors the reviewer prompt's cap of 50 `specification/` files and 5,000 +`specification/` changed lines. It is two-tier: + +- **Fast path:** if the PR's total `changed_files`, `additions`, and `deletions` are within the + cap, its `specification/` subset necessarily is too, and no file list is requested. +- **Slow path:** only when the totals exceed the cap, the changed files are listed and only the + `specification/` files and lines are counted. A list that is truncated (3,000 entries, or + fewer entries than `changed_files`) cannot prove coverage and requires manual review. + +A PR reporting zero files requires manual review without listing files, because GitHub reports +zero counters when a diff is too large to compute. Outcome mapping: @@ -185,7 +193,7 @@ Outcome mapping: | Full, complete review with `blocking_count = 0` | `success`: Passed | | Full, complete review with `blocking_count > 0` | `failure`: Changes requested | | Incomplete review, regardless of `blocking_count` | `error`: Review incomplete | -| Scoped review, or coverage limits exceeded by the trusted file list | `error`: Manual review required | +| Scoped review, or a PR whose `specification/` changes exceed the automated-review limits | `error`: Manual review required | | Malformed output, duplicate item, or a failure while validating or publishing | `error`: Review incomplete | | Agent failure, omitted semantic item, noop, or canceled reviewer run | `error`: Review incomplete, published by the `conclusion` job | | Missing trusted PR/SHA artifacts | Leave status unchanged | @@ -195,6 +203,9 @@ prefix (`Manual review required: ` or `Review incomplete: `). Only Manual review property of the PR itself, so only it adds the sticky `ARMManualSignoffRequired` label. Review incomplete is a transient failure that a re-run can fix. +A failure while validating or publishing is reported with a fixed description, +`Review incomplete: result could not be validated`. The details go to the run log, because raw error text can include runner paths and must not appear in a public status. + ### Resolving a Pending status that was never published `pre_activation` posts Pending before the agent starts, but `record_arm_semantic_review` only @@ -203,8 +214,9 @@ runs after every other job regardless of their results. Its pre-step reads the t `head-sha` artifact and, only when this exact run and attempt still owns the newest `ARM Semantic Review` status and it is Pending, publishes `Review incomplete: reviewer did not publish a result`. A newer run or attempt, or a result the -record job already published, is never overwritten. The step has `continue-on-error: true` -because pre-steps run before gh-aw's own failure reporting. +record job already published, is never overwritten. The step is skipped when the record job +succeeded, since that job already published, and has `continue-on-error: true` because pre-steps +run before gh-aw's own failure reporting. The status is head-bound: @@ -217,11 +229,8 @@ Target URL: exact reviewer run and attempt ## Stale-result protection -The safe-output publisher runs only while the PR is open and its current head -matches the reviewed SHA. A completion for a closed PR or stale head leaves the -existing status unchanged. - -Universal independently reads statuses only for the current head SHA: +The status is bound to the reviewed SHA, so a result for a commit that is no longer the PR head +is harmless. Universal reads only the current head SHA's statuses: ```text Reviewer publishes Passed on SHA A @@ -231,8 +240,9 @@ No Semantic Passed exists on B Universal does not sign off ``` -A newer run for the same PR cancels an in-flight one (`cancel-in-progress`), and the publisher -checks for a newer run only immediately before it writes. +A newer run for the same PR cancels an in-flight one (`cancel-in-progress`). Two runs on the +same SHA review the same code, so if an older run still publishes late, the result is stale +but valid for that SHA. ## Universal decision From 93a19bd40c364f719b8fa66be2c4f49f271a6a73 Mon Sep 17 00:00:00 2001 From: Akhila Illa Date: Thu, 8 Oct 2026 12:59:40 -0500 Subject: [PATCH 12/16] Address review feedback: shorter README, YAML block list - Replace the long documentation/ implementation guide with a short README next to the code, and point the skill at it. - Use a YAML block sequence for the Universal workflow's upstream list. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/skills/arm-semantic-review/SKILL.md | 4 +- .../workflows/arm-universal-auto-signoff.yaml | 8 +- .../workflows/src/arm-auto-signoff/README.md | 59 +++ ...ntic-review-auto-signoff-implementation.md | 479 ------------------ 4 files changed, 64 insertions(+), 486 deletions(-) create mode 100644 .github/workflows/src/arm-auto-signoff/README.md delete mode 100644 documentation/arm-semantic-review-auto-signoff-implementation.md diff --git a/.github/skills/arm-semantic-review/SKILL.md b/.github/skills/arm-semantic-review/SKILL.md index f493c8bac1fb..8c0648e69199 100644 --- a/.github/skills/arm-semantic-review/SKILL.md +++ b/.github/skills/arm-semantic-review/SKILL.md @@ -9,8 +9,8 @@ The ARM API Reviewer is an AI agent. Auto-signoff must not trust it directly, so turned into one **commit status**, `ARM Semantic Review`, on the reviewed SHA. Universal Auto-Signoff reads that status alongside LintDiff and Avocado. -Design record and rationale: -[arm-semantic-review-auto-signoff-implementation.md](../../../documentation/arm-semantic-review-auto-signoff-implementation.md). +Short overview next to the code: +[README.md](../../workflows/src/arm-auto-signoff/README.md). ## Mental model diff --git a/.github/workflows/arm-universal-auto-signoff.yaml b/.github/workflows/arm-universal-auto-signoff.yaml index 613b2be63694..b63e65c623f4 100644 --- a/.github/workflows/arm-universal-auto-signoff.yaml +++ b/.github/workflows/arm-universal-auto-signoff.yaml @@ -19,11 +19,9 @@ on: # zizmor: ignore[dangerous-triggers] Fork PR updates require this; only base workflow_run: # Re-evaluate as each mandatory status producer completes. workflows: - [ - "ARM API Review: Automated Workflow", - "Swagger Avocado - Set Status", - "Swagger LintDiff - Set Status", - ] + - "ARM API Review: Automated Workflow" + - "Swagger Avocado - Set Status" + - "Swagger LintDiff - Set Status" types: [completed] permissions: diff --git a/.github/workflows/src/arm-auto-signoff/README.md b/.github/workflows/src/arm-auto-signoff/README.md new file mode 100644 index 000000000000..d8eef7760e7f --- /dev/null +++ b/.github/workflows/src/arm-auto-signoff/README.md @@ -0,0 +1,59 @@ +# ARM Semantic Review and Universal Auto-Signoff + +The ARM API Reviewer is an AI agent, so auto-signoff does not trust it directly. Its verdict is +turned into one commit status, `ARM Semantic Review`, on the reviewed SHA. Universal Auto-Signoff +reads that status alongside `Swagger LintDiff` and `Swagger Avocado`. + +Universal runs in parallel with the legacy ARM Auto SignOff workflows, which remain the +production path. It only manages the pilot label `ARMAutoSignedOff-Test` and never changes +`ARMSignedOff`. + +## Flow + +```text +arm-api-review (reviewer) + pre_activation post status PENDING, upload head-sha= / issue-number= artifacts + agent review the PR, request record_arm_semantic_review + record_arm_semantic_review validate the result, post SUCCESS / FAILURE / ERROR + conclusion if no result was posted, turn this run's PENDING into ERROR + +arm-universal-auto-signoff newest status per context on the current head SHA + labels +update-labels re-check the live head, then add or remove the label +``` + +The model proposes; trusted code validates and publishes. Any doubt ends as "not Passed". + +## Signoff policy + +```text +ARM Semantic Review passed for the current PR head +AND Swagger LintDiff and Swagger Avocado passed for the current PR head +AND labels and approvals allow signoff += add ARMAutoSignedOff-Test +``` + +## Status contract + +| State | Description starts with | Adds `ARMManualSignoffRequired` | +| --------- | -------------------------- | -------------------------------- | +| `pending` | `ARM API semantic review` | No | +| `success` | `Passed:` | No | +| `failure` | `Changes requested:` | No | +| `error` | `Manual review required: ` | Yes (scoped or oversized PR) | +| `error` | `Review incomplete: ` | No (transient, a re-run can fix) | + +Only `success` can lead to signoff. A status is bound to its SHA, so a result for a commit that is +no longer the PR head is never read for the new head. + +## Files + +| File | Role | +| ------------------------------------------------------------------------ | ------------------------------------------------------------------- | +| [arm-semantic-review.ts](./arm-semantic-review.ts) | Pure policy: validate the result, map it to a status, read statuses | +| [arm-semantic-review-workflow.ts](./arm-semantic-review-workflow.ts) | Publish the status; resolve an unpublished one | +| [arm-universal-auto-signoff.ts](./arm-universal-auto-signoff.ts) | Read statuses and labels, decide label actions | +| [arm-api-review.md](../../arm-api-review.md) | Reviewer workflow source (the `.lock.yml` is generated from it) | +| [arm-universal-auto-signoff.yaml](../../arm-universal-auto-signoff.yaml) | Universal workflow | + +The design rationale, validation rules, invariants, edge cases, and debugging steps are in the +[arm-semantic-review skill](../../../skills/arm-semantic-review/SKILL.md). diff --git a/documentation/arm-semantic-review-auto-signoff-implementation.md b/documentation/arm-semantic-review-auto-signoff-implementation.md deleted file mode 100644 index 0bae4667142d..000000000000 --- a/documentation/arm-semantic-review-auto-signoff-implementation.md +++ /dev/null @@ -1,479 +0,0 @@ -# ARM Semantic Review and Universal Auto-Signoff - -## Purpose - -This guide explains the pilot integration between the ARM API Reviewer and -Universal Auto-Signoff. The legacy ARM Auto SignOff workflows remain the -production signoff path while Universal runs in parallel. - -The policy is intentionally small: - -```text -ARM Semantic Review passed for the current PR head -AND Swagger LintDiff passed for the current PR head -AND Swagger Avocado passed for the current PR head -AND current labels and approvals allow signoff -= add ARMAutoSignedOff-Test -``` - -Only `Passed` authorizes the pilot label. Missing, Pending, Changes requested, -and Review incomplete all block it. Universal never changes `ARMSignedOff`. - -## Mental model - -```text -ARM API Reviewer - produces one structured semantic item in agent_output.json - validates it in a trusted safe-output job - publishes a commit status on the reviewed SHA - -Universal Auto-Signoff - reads statuses for the current PR head and decides - -Update Labels - applies the decision if the PR head is still current -``` - -The reviewer may finish after the PR head changes. That is safe: the publisher -rejects stale heads, and Universal Auto-Signoff reads statuses only for the -current PR head. - -## Components - -### ARM API Reviewer - -Source: -[arm-api-review.md](../.github/workflows/arm-api-review.md) - -Generated workflow: -[arm-api-review.lock.yml](../.github/workflows/arm-api-review.lock.yml) - -Responsibilities: - -- review the cumulative PR diff at a pinned head SHA; -- use the Critic to verify findings; -- reconcile current findings with existing PR discussion; -- publish comments and ARM queue-label changes; -- emit one `record_arm_semantic_review` item. - -### Semantic policy - -[arm-semantic-review.ts](../.github/workflows/src/arm-auto-signoff/arm-semantic-review.ts) - -Responsibilities: - -- parse exactly one structured semantic item; -- validate the Blocking count, scope, completion, and incomplete reason; -- map the evidence to Passed, Changes requested, or Review incomplete; -- interpret GitHub commit-status states; -- select the newest semantic status for a SHA. - -This file is pure policy and parsing. It makes no GitHub API calls. - -### Semantic status publication - -[arm-semantic-review-workflow.ts](../.github/workflows/src/arm-auto-signoff/arm-semantic-review-workflow.ts) - -Responsibilities: - -- run as the reviewer's trusted `record_arm_semantic_review` safe-output job; -- read that run's `agent_output.json` and trusted PR/SHA correlation artifacts; -- validate the semantic item once; -- independently enforce automated-review size limits; -- publish `ARM Semantic Review` on the reviewed SHA; -- resolve a Pending status that was never published (from the `conclusion` job). - -Only the reviewer `pre_activation`, `record_arm_semantic_review`, and `conclusion` jobs have -`statuses: write`. The agent itself still has read-only GitHub tools. - -### Universal Auto-Signoff - -Workflow: -[arm-universal-auto-signoff.yaml](../.github/workflows/arm-universal-auto-signoff.yaml) - -Decision logic: -[arm-universal-auto-signoff.ts](../.github/workflows/src/arm-auto-signoff/arm-universal-auto-signoff.ts) - -Responsibilities: - -- resolve the current PR and current head SHA; -- read Semantic Review, LintDiff, and Avocado statuses for that SHA; -- evaluate current labels and approvals; -- manage the pilot `ARMAutoSignedOff-Test` label and add the manual-review label - when the review was scoped or the PR exceeded automated review limits. - -Universal has only `statuses: read`. - -### Label application - -[update-labels.ts](../.github/workflows/src/update-labels.ts) applies label -artifacts. When a producer supplies a head-SHA artifact, it confirms that the PR -is still open and still has that head before mutating labels. - -## End-to-end flow - -```mermaid -flowchart TD - A[Eligible reviewer trigger] --> B[Set Semantic Review Pending on reviewed SHA] - B --> C[Agent and Critic review PR] - C --> D[Publish comments and queue-label changes] - C --> E[Emit record_arm_semantic_review item] - E --> F[Trusted safe-output job validates the item] - F --> G[Publish semantic status on reviewed SHA] - D --> H[Reviewer workflow completes] - G --> H - H --> K[Universal Auto-Signoff] - - L[LintDiff, Avocado, PR, or label event] --> K - K --> M[Read all statuses for current PR head] - M --> N{All requirements pass?} - N -- No --> O[Remove stale pilot signoff] - N -- Yes --> P[Add ARMAutoSignedOff-Test artifact] - P --> Q[Update Labels rechecks current head] - Q --> R[Apply label] -``` - -## Structured semantic item - -The agent emits: - -```json -{ - "type": "record_arm_semantic_review", - "blocking_count": "0", - "scope": "full", - "completeness": "complete", - "incomplete_reason": "none" -} -``` - -Fields: - -| Field | Meaning | -| ------------------- | ---------------------------------------------------------------- | -| `blocking_count` | Verified Blocking findings still applicable after reconciliation | -| `scope` | Whether the complete PR or a size-limited subset was reviewed | -| `completeness` | Whether the intended review completed | -| `incomplete_reason` | `none` when complete; otherwise the specific failure category | - -The model does not supply the run attempt, PR number, or head SHA. The custom -safe-output job reads those values from the current run and trusted artifacts, -so model output cannot redirect status publication. - -## Semantic status publication - -The `record_arm_semantic_review` safe-output job runs inside the exact reviewer -run that produced the item. - -It: - -1. reads the trusted `head-sha` and `issue-number` artifacts from that run; -2. finds exactly one semantic item in the safe-output payload; -3. validates the Blocking count, scope, completion, and incomplete reason, and - rejects a complete review that carries an incomplete reason; -4. independently checks the PR's size against the automated-review limits; -5. publishes a status on the reviewed SHA. - -The size check mirrors the reviewer prompt's cap of 50 `specification/` files and 5,000 -`specification/` changed lines. It is two-tier: - -- **Fast path:** if the PR's total `changed_files`, `additions`, and `deletions` are within the - cap, its `specification/` subset necessarily is too, and no file list is requested. -- **Slow path:** only when the totals exceed the cap, the changed files are listed and only the - `specification/` files and lines are counted. A list that is truncated (3,000 entries, or - fewer entries than `changed_files`) cannot prove coverage and requires manual review. - -A PR reporting zero files requires manual review without listing files, because GitHub reports -zero counters when a diff is too large to compute. - -Outcome mapping: - -| Evidence | Status | -| --------------------------------------------------------------------------------------------- | ---------------------------- | -| Full, complete review with `blocking_count = 0` | `success`: Passed | -| Full, complete review with `blocking_count > 0` | `failure`: Changes requested | -| Incomplete review, regardless of `blocking_count` | `error`: Review incomplete | -| Scoped review, or a PR whose `specification/` changes exceed the automated-review limits | `error`: Manual review required | -| Malformed output, duplicate item, or a failure while validating or publishing | `error`: Review incomplete | -| Agent failure, omitted semantic item, noop, or canceled reviewer run | `error`: Review incomplete, published by the `conclusion` job | -| Missing trusted PR/SHA artifacts | Leave status unchanged | - -Both `error` outcomes share one commit-status state and are told apart by the description -prefix (`Manual review required: ` or `Review incomplete: `). Only Manual review required is a -property of the PR itself, so only it adds the sticky `ARMManualSignoffRequired` label. -Review incomplete is a transient failure that a re-run can fix. - -A failure while validating or publishing is reported with a fixed description, -`Review incomplete: result could not be validated`. The details go to the run log, because raw error text can include runner paths and must not appear in a public status. - -### Resolving a Pending status that was never published - -`pre_activation` posts Pending before the agent starts, but `record_arm_semantic_review` only -runs when the agent emitted a semantic item and threat detection passed. The `conclusion` job -runs after every other job regardless of their results. Its pre-step reads the trusted -`head-sha` artifact and, only when this exact run and attempt still owns the newest -`ARM Semantic Review` status and it is Pending, publishes -`Review incomplete: reviewer did not publish a result`. A newer run or attempt, or a result the -record job already published, is never overwritten. The step is skipped when the record job -succeeded, since that job already published, and has `continue-on-error: true` because pre-steps -run before gh-aw's own failure reporting. - -The status is head-bound: - -```text -SHA: reviewed SHA -Context: ARM Semantic Review -State: success | failure | error | pending -Target URL: exact reviewer run and attempt -``` - -## Stale-result protection - -The status is bound to the reviewed SHA, so a result for a commit that is no longer the PR head -is harmless. Universal reads only the current head SHA's statuses: - -```text -Reviewer publishes Passed on SHA A -PR head is now SHA B -Universal reads statuses only for SHA B -No Semantic Passed exists on B -Universal does not sign off -``` - -A newer run for the same PR cancels an in-flight one (`cancel-in-progress`). Two runs on the -same SHA review the same code, so if an older run still publishes late, the result is stale -but valid for that SHA. - -## Universal decision - -Universal fetches all commit statuses for the current head SHA and selects the -newest entry for: - -- `ARM Semantic Review`; -- `Swagger LintDiff`; -- `Swagger Avocado`. - -Decision: - -```text -Semantic Review is missing or Pending - -> wait and remove stale pilot signoff - -Semantic Review requested changes - -> remove pilot signoff - -Semantic Review is incomplete - -> no pilot signoff, no label change; a re-run can fix it - -Semantic Review requires manual review (scoped or oversized PR) - -> add ARMManualSignoffRequired - -> retain WaitForARMFeedback - -> remove pilot signoff - -ARMManualSignoffRequired is present - -> no pilot signoff - -LintDiff or Avocado is not successful - -> no pilot signoff - -required approval is missing - -> no pilot signoff - -otherwise - -> add ARMAutoSignedOff-Test -``` - -`ARMAutoSignedOff-Test` records the pilot decision without changing production -signoff. `ARMManualSignoffRequired` remains an explicit human veto and is never -removed automatically. - -## Event-order examples - -### Avocado completes first - -```text -Semantic = Pending -Avocado = Passed -Universal runs -Semantic is not Passed -No signoff - -Reviewer completes -Reviewer publishes Semantic = Passed -Universal runs again -All statuses pass -Signoff can proceed -``` - -### Reviewer completes first - -```text -Reviewer publishes Semantic = Passed -Universal runs -Avocado is pending -No signoff - -Avocado completes -Universal runs again -All statuses pass -Signoff can proceed -``` - -### New commit during review - -```text -Reviewer publishes result on SHA A -Current PR head is SHA B -Universal reads B -No semantic status for B -No signoff -``` - -### Same-SHA rerun - -```text -Old status on A = Passed -New review starts on A -New Pending entry becomes newest -Universal waits -Reviewer publishes the new final result -``` - -### Missing or malformed semantic output - -```text -Malformed emitted item -> reviewer publishes Review incomplete -No emitted item, noop, or failed/canceled reviewer -> conclusion job publishes Review incomplete -In either case, Universal cannot sign off and adds no label -``` - -## Why this design - -### Commit status instead of labels - -Commit statuses are SHA-bound. PR labels are not. - -### Commit status instead of artifact-only lookup - -Any later event can read the current SHA's status without searching reviewer-run -history or downloading old artifacts. - -### Direct reviewer publication - -The trusted safe-output job already has the exact agent output, run attempt, PR, -and SHA. Publishing there removes one workflow and one artifact handoff while -keeping `statuses: write` out of Universal. - -### One authoritative validation - -Validation occurs only in the trusted safe-output job. The model produces the -typed item; the job validates and publishes it. - -## Security and reliability properties - -- no untrusted PR code is checked out; -- only write-access users or the trusted bot can trigger reviewer writes; -- the semantic item is bound to the workflow attempt, PR number, and SHA; -- malformed, missing, duplicate, mismatched, scoped, or incomplete output - fails closed; -- only `success` permits signoff; -- Universal always reads the current head SHA; -- delayed label artifacts are rechecked against the live PR head; -- Universal has `statuses: read`; -- only trusted reviewer jobs have `statuses: write`. - -## Accepted tradeoffs - -### The model is the semantic source of truth - -The same verified findings drive: - -- comments authors act on; -- `ARMChangesRequested`; -- semantic Blocking count. - -The safe-output job validates structure and correlation; it does not -independently re-review semantic correctness. - -### Coupling to gh-aw agent output - -The custom safe-output job expects: - -```text -artifact name: agent -file: agent_output.json -items[] with type record_arm_semantic_review -``` - -A gh-aw format change can break extraction. A malformed emitted item produces -Review incomplete; a job that cannot start is resolved to Review incomplete by the -`conclusion` job. Neither can authorize signoff. - -### PR-level comment and label races - -Comments and queue labels are not transactional. A canceled run can publish -some outputs before cancellation. Reconciliation reduces duplicate comments, -and the current-SHA semantic gate prevents stale auto-signoff. - -### Workflow chain depth - -```text -ARM API Reviewer - -> ARM Universal Auto-Signoff - -> Update Labels -``` - -Removing the Set Status workflow shortens the `workflow_run` chain and leaves -more headroom below GitHub's supported chain-depth limit. - -## Operational prerequisites - -- ensure `ARMAutoSignedOff-Test` and `ARMManualSignoffRequired` exist; -- mirror the ARM reviewer workflow to `azure-rest-api-specs-pr`; -- document who may remove manual hold labels; -- monitor incomplete rates, stuck Pending statuses, and label transitions after - rollout. - -## How to understand the implementation - -Read in this order: - -1. [arm-api-review.md](../.github/workflows/arm-api-review.md) -2. [publishArmSemanticReviewStatus](../.github/workflows/src/arm-auto-signoff/arm-semantic-review-workflow.ts) -3. [parseSemanticReviewResult](../.github/workflows/src/arm-auto-signoff/arm-semantic-review.ts) -4. [arm-universal-auto-signoff.ts](../.github/workflows/src/arm-auto-signoff/arm-universal-auto-signoff.ts) -5. tests: - - [arm-semantic-review.test.ts](../.github/workflows/test/arm-auto-signoff/arm-semantic-review.test.ts) - - [arm-api-review-workflow.test.ts](../.github/workflows/test/arm-api-review-workflow.test.ts) - - [arm-universal-auto-signoff.test.ts](../.github/workflows/test/arm-auto-signoff/arm-universal-auto-signoff.test.ts) - -For each scenario, ask: - -```text -What SHA did the reviewer inspect? -What semantic status exists on that SHA? -What is the PR's current SHA? -What statuses does Universal see on the current SHA? -Can signoff proceed? -``` - -## Review checklist - -- [ ] Clean review publishes Passed. -- [ ] Blocking review publishes Changes requested. -- [ ] Incomplete reviews publish Review incomplete. -- [ ] Scoped or oversized reviews publish Manual review required. -- [ ] Malformed emitted item publishes Review incomplete. -- [ ] Missing item, noop, or failed/canceled reviewer is resolved to Review incomplete by the - `conclusion` job and blocks signoff, without overwriting a newer run's status. -- [ ] Model-supplied correlation is ignored; missing or invalid trusted - correlation leaves status unchanged. -- [ ] New SHA cannot use old SHA's Passed result. -- [ ] Same-SHA rerun publishes Pending before reevaluation. -- [ ] Universal remains status-read-only. -- [ ] Delayed label application rechecks the live head. -- [ ] Passing evidence adds `ARMAutoSignedOff-Test` without changing `ARMSignedOff`. -- [ ] Later failures remove the pilot label. -- [ ] Manual review required adds the manual-review label without changing ARM queue labels; - Review incomplete adds no label. -- [ ] Private repository copy is updated in coordination. From dd85a0f75ce16b2b543d4c164f823164ed84c7cb Mon Sep 17 00:00:00 2001 From: Akhila Illa Date: Fri, 9 Oct 2026 15:00:33 -0500 Subject: [PATCH 13/16] Address review feedback: scope check, one artifact, retries - Verify the reviewer's scope claim with one pure helper pair in arm-semantic-review.ts (checkPullRequestSize, addSpecificationChanges). Only a "full, complete" claim is verified; a PR within the totals needs no file list, otherwise only specification/ changes are counted, stopping at the first page already over the limits. A rename counts by its old or new path. The status states the reason, and each step logs what it saw. - Upload one head-sha=;issue-number= artifact instead of two, and split names on ";" in the shared artifact parser. - Retry transient API errors in the record and finalizer steps, and set persist-credentials: false on their checkouts. - Drop the Update Labels stale-head guard from this PR; it changes every producer that publishes a head-sha artifact and overlaps #46996. - Update the skill and README to match. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/skills/arm-semantic-review/SKILL.md | 72 ++- .github/workflows/arm-api-review.lock.yml | 29 +- .github/workflows/arm-api-review.md | 23 +- .../workflows/src/arm-auto-signoff/README.md | 7 +- .../arm-semantic-review-workflow.ts | 123 +++-- .../arm-auto-signoff/arm-semantic-review.ts | 144 +++--- .github/workflows/src/context.ts | 9 +- .github/workflows/src/update-labels.ts | 25 -- .../test/arm-api-review-workflow.test.ts | 7 + .../arm-semantic-review.test.ts | 421 +++++++++++++----- .github/workflows/test/context.test.ts | 51 +++ .github/workflows/test/mocks.ts | 7 +- .github/workflows/test/update-labels.test.ts | 44 +- 13 files changed, 632 insertions(+), 330 deletions(-) diff --git a/.github/skills/arm-semantic-review/SKILL.md b/.github/skills/arm-semantic-review/SKILL.md index 8c0648e69199..497dd5524d1d 100644 --- a/.github/skills/arm-semantic-review/SKILL.md +++ b/.github/skills/arm-semantic-review/SKILL.md @@ -15,12 +15,13 @@ Short overview next to the code: ## Mental model ```text -START pre_activation post status PENDING, upload head-sha= / issue-number= artifacts +START pre_activation post status PENDING, upload one head-sha=;issue-number= artifact MIDDLE agent (read-only AI) review the PR, then request record_arm_semantic_review END record job (trusted) validate the 4 values, post SUCCESS / FAILURE / ERROR SAFETY conclusion job if no result was posted, turn this run's PENDING into ERROR -READ Universal Auto-Signoff newest status on the current head SHA, plus labels -> label action -APPLY Update Labels re-check the live head, then add or remove the label +READ Universal Auto-Signoff re-check the PR is open at the evaluated head, read the newest status + on that SHA plus labels -> label action +APPLY Update Labels add or remove the label as emitted ``` The model proposes. Trusted code validates and publishes. Any doubt ends as "not Passed". @@ -35,7 +36,6 @@ The model proposes. Trusted code validates and publishes. Any doubt ends as "not | [arm-semantic-review-workflow.ts](../../workflows/src/arm-auto-signoff/arm-semantic-review-workflow.ts) | I/O: publish the status, finalize an unpublished one. | | [arm-universal-auto-signoff.ts](../../workflows/src/arm-auto-signoff/arm-universal-auto-signoff.ts) | Reads statuses and labels, decides label actions. | | [context.ts](../../workflows/src/context.ts) | `extractInputs` and the trusted-artifact parser shared by all consumers. | -| [update-labels.ts](../../workflows/src/update-labels.ts) | Applies label artifacts after re-checking the live head. | ## The status contract @@ -57,24 +57,40 @@ can still pass and it must never add the sticky label. ## Validation, and why each check exists -Trusted inputs (never from the model): the PR number and head SHA come from the run's -`head-sha=` and `issue-number=` artifacts, and the attempt from `GITHUB_RUN_ATTEMPT`. - -| Check | Where | If it fails | -| ----------------------------------------------------------------------------- | --------------------------- | ------------------------- | -| Trusted artifacts present, full 40-hex SHA, no conflicting duplicates | `context.ts` | Nothing published / throw | -| Output file exists and is JSON | publisher `try` | `Review incomplete` | -| Exactly one `record_arm_semantic_review` item | `parseSemanticReviewResult` | `Review incomplete` | -| `blocking_count` is a digit string; the other three fields are in closed sets | `parseSemanticReviewResult` | `Review incomplete` | -| A `complete` review must have reason `none` | `parseSemanticReviewResult` | `Review incomplete` | -| PR size within the cap, checked independently of the model's `scope` | `requiresManualReview` | `Manual review required` | -| Incomplete, then scoped, then Blocking, then Passed (first match wins) | `evaluateSemanticReview` | n/a | - -The size cap is 50 `specification/` files and 5,000 changed lines, matching the reviewer prompt. -It is two-tier: if the PR's totals are within the cap no file list is fetched, otherwise only the -`specification/` files are counted. A PR reporting zero files, or a truncated file list, requires -manual review. - +Trusted inputs (never from the model): the PR number and head SHA come from the run's correlation +artifact, named `head-sha=;issue-number=` (`;` joins the pairs so one upload carries both), +and the attempt from `GITHUB_RUN_ATTEMPT`. + +| Check | Where | If it fails | +| ----------------------------------------------------------------------------- | -------------------------------------- | ------------------------- | +| Trusted artifacts present, full 40-hex SHA, no conflicting duplicates | `context.ts` | Nothing published / throw | +| Output file exists and is JSON | publisher `try` | `Review incomplete` | +| Exactly one `record_arm_semantic_review` item | `parseSemanticReviewResult` | `Review incomplete` | +| `blocking_count` is a digit string; the other three fields are in closed sets | `parseSemanticReviewResult` | `Review incomplete` | +| A `complete` review must have reason `none` | `parseSemanticReviewResult` | `Review incomplete` | +| PR size within the cap, checked independently of the model's `scope` | `requiresManualReview` (workflow file) | `Manual review required` | +| Incomplete, then scoped, then Blocking, then Passed (first match wins) | `evaluateSemanticReview` | n/a | + +### Verifying the model's scope + +The model reports `scope: full|scoped`, but it is untrusted. For a **full, complete** claim, the +trusted code re-derives scope with the reviewer prompt's rule: more than 50 `specification/` files +or 5,000 `specification/` lines needs a human. The rules are pure helpers in +`arm-semantic-review.ts` (`checkPullRequestSize` and `addSpecificationChanges`); the workflow file's +`getOversizeReason` does the API calls and logging. Any other claim (scoped, incomplete) already +withholds signoff, so the check is skipped. + +1. Zero changed files (GitHub could not compute the diff) or 3,000 or more requires manual review + without listing files. 3,000 is the most files GitHub's file-listing API will return, so a PR + that large cannot be proven fully covered. +2. If the PR's totals are within the cap, its `specification/` subset is too: no file list, so + most PRs cost one `pulls.get` call. +3. Otherwise count only the `specification/` files, and stop paginating as soon as the count is + over the cap. A renamed or moved file counts when either its old or its new path is under + `specification/`. + +The status says why, for example `Manual review required: specification/ changes exceed 50 files +or 5000 lines`. Reasons are fixed strings, never model output. Each step logs what it saw. Deliberately **not** validated, because it adds code without protecting signoff: - the PR being open or still at the reviewed SHA (the status is SHA-bound and Universal re-checks @@ -112,6 +128,10 @@ never reach a public status. ## Known limitations +- Update Labels applies the emitted label action without re-checking the PR head. Universal checks + just before emitting, so a push in the short window between the two can leave the pilot label + stale until the next evaluation. A stale-head guard in Update Labels is a separate, cross-cutting + change (it affects every producer that publishes a `head-sha=` artifact). - A same-SHA re-run posts `pending` but does not revoke an existing `ARMAutoSignedOff-Test` label until Universal next runs. This matters only once enforcement is on. - Universal's label filter does not include `ARMChangesRequested` or `WaitForARMFeedback`, so @@ -121,7 +141,11 @@ never reach a public status. conflict and the parser throws, which can leave `Pending`. - `blocking_count` is the model's verdict. The Critic and threat detection are the only checks on it, which is an accepted trade-off. -- If the `conclusion` job itself does not run, a `Pending` status can remain. +- A `Pending` status is resolved in almost every failure, but can still remain when the + `conclusion` job does not run, the finalizer's own API calls still fail after their retries, + the runner or GitHub is lost mid-run, or the finalizer cannot read the head SHA (the + conflicting-artifacts case above). Nothing signs off in any of these cases, because only + `success` does. - A push to a draft PR, or to one without `WaitForARMFeedback`, joins the PR concurrency lane before the gate skips it, so it can cancel a review that is already running. @@ -158,7 +182,7 @@ gh api repos/Azure/azure-rest-api-specs/commits//statuses \ --jq '.[] | select(.context=="ARM Semantic Review") | [.created_at,.state,.description,.target_url]' ``` -The `pending` status and the `head-sha=` / `issue-number=` artifacts appear on the run created +The `pending` status and the `head-sha=;issue-number=` artifact appear on the run created by `pre_activation`. Try: a clean review, `skip-arm-review`, a push mid-run, two `/arm-review` comments in a row, and cancelling a run. diff --git a/.github/workflows/arm-api-review.lock.yml b/.github/workflows/arm-api-review.lock.yml index db1ac323c6cb..0d55de94937c 100644 --- a/.github/workflows/arm-api-review.lock.yml +++ b/.github/workflows/arm-api-review.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"add9d2d9a555b9c420520ed210e6647e23212c8b879f27937d4f71a0d678a4e0","body_hash":"cb0f2e92399e5be203fc6aef3f6b833929d8c6f1b9ca4779965cdc6cd2fc770e","compiler_version":"v0.86.2","strict":true,"agent_id":"copilot","agent_model":"gpt-5.6-sol?effort=high","detection_agent_id":"copilot","detection_agent_model":"gpt-5.6-sol?effort=high","engine_versions":{"copilot":"1.0.79"}} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"abec1bffb3995c8bc5ea29c0b14653bbbabb26a77b4c975bb4f69f23b01280fc","body_hash":"cb0f2e92399e5be203fc6aef3f6b833929d8c6f1b9ca4779965cdc6cd2fc770e","compiler_version":"v0.86.2","strict":true,"agent_id":"copilot","agent_model":"gpt-5.6-sol?effort=high","detection_agent_id":"copilot","detection_agent_model":"gpt-5.6-sol?effort=high","engine_versions":{"copilot":"1.0.79"}} # gh-aw-manifest: {"version":1,"secrets":["GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/cache/restore","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/cache/save","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"373c709c69115d41ff229c7e5df9f8788daa9553","version":"v9"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/github-script","sha":"d746ffe35508b1917358783b479e04febd2b8f71","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"820762786026740c76f36085b0efc47a31fe5020","version":"v7.0.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"},{"repo":"github/gh-aw-actions/setup","sha":"6aab9e5b5c91c615506061f09bedd81a23babe3c","version":"v0.86.2"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.27.44","digest":"sha256:0d727725c737b58c7bdf51f640cffb928385ec46517e0917c7f1a02f1bada8b4","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.27.44@sha256:0d727725c737b58c7bdf51f640cffb928385ec46517e0917c7f1a02f1bada8b4"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.44","digest":"sha256:b50fbadba138f6e9aba94aca09711335c489bb3b15861220cb66f6092e042dc7","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.44@sha256:b50fbadba138f6e9aba94aca09711335c489bb3b15861220cb66f6092e042dc7"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.27.44","digest":"sha256:83e48bbe12c634be8c228a576832fe45f66c529ac3659db92bddbcf2eeb6d627","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.27.44@sha256:83e48bbe12c634be8c228a576832fe45f66c529ac3659db92bddbcf2eeb6d627"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.9","digest":"sha256:e5a1569aeaf41820fa7bdee3e94468cae448133cdbf00119ad24f5b74db1ab9f","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.9@sha256:e5a1569aeaf41820fa7bdee3e94468cae448133cdbf00119ad24f5b74db1ab9f"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:0d9f1fb5fd6610c0ac1f5194a38e45a8a1e81f8a390d5142d8e4e6f26a4b3196","pinned_image":"ghcr.io/github/gh-aw-node@sha256:0d9f1fb5fd6610c0ac1f5194a38e45a8a1e81f8a390d5142d8e4e6f26a4b3196"},{"image":"ghcr.io/github/github-mcp-server:v1.9.0","digest":"sha256:881b53d6f75f69bdbc1b5b10fc2f1361717c19054143b3a8529fb5c32061a50e","pinned_image":"ghcr.io/github/github-mcp-server:v1.9.0@sha256:881b53d6f75f69bdbc1b5b10fc2f1361717c19054143b3a8529fb5c32061a50e"}],"has_pull_request_target":true} # This file was automatically generated by gh-aw (v0.86.2). DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # @@ -98,19 +98,11 @@ on: # run: "mkdir -p \"$RUNNER_TEMP/arm-semantic-review\"\n: > \"$RUNNER_TEMP/arm-semantic-review/empty.txt\"\n" # shell: bash # - if: steps.check_membership.outputs.is_team_member == 'true' - # name: Upload ARM semantic review head SHA + # name: Upload ARM semantic review correlation # uses: actions/upload-artifact@v7 # with: # if-no-files-found: error - # name: head-sha=${{ steps.resolve_target_pr.outputs.target_head_sha }} - # overwrite: true - # path: ${{ runner.temp }}/arm-semantic-review/empty.txt - # - if: steps.check_membership.outputs.is_team_member == 'true' - # name: Upload ARM semantic review issue number - # uses: actions/upload-artifact@v7 - # with: - # if-no-files-found: error - # name: issue-number=${{ steps.resolve_target_pr.outputs.target_pr_number }} + # name: head-sha=${{ steps.resolve_target_pr.outputs.target_head_sha }};issue-number=${{ steps.resolve_target_pr.outputs.target_pr_number }} # overwrite: true # path: ${{ runner.temp }}/arm-semantic-review/empty.txt # - env: @@ -1308,6 +1300,7 @@ jobs: if: needs.record_arm_semantic_review.result != 'success' uses: actions/github-script@d746ffe35508b1917358783b479e04febd2b8f71 # v9.0.0 with: + retries: 3 script: | const { finalizeUnpublishedArmSemanticReview } = await import('${{ github.workspace }}/.github/workflows/src/arm-auto-signoff/arm-semantic-review-workflow.ts'); @@ -1919,20 +1912,12 @@ jobs: mkdir -p "$RUNNER_TEMP/arm-semantic-review" : > "$RUNNER_TEMP/arm-semantic-review/empty.txt" shell: bash - - name: Upload ARM semantic review head SHA + - name: Upload ARM semantic review correlation if: steps.check_membership.outputs.is_team_member == 'true' uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 (source v7) with: if-no-files-found: error - name: head-sha=${{ steps.resolve_target_pr.outputs.target_head_sha }} - overwrite: true - path: ${{ runner.temp }}/arm-semantic-review/empty.txt - - name: Upload ARM semantic review issue number - if: steps.check_membership.outputs.is_team_member == 'true' - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 (source v7) - with: - if-no-files-found: error - name: issue-number=${{ steps.resolve_target_pr.outputs.target_pr_number }} + name: head-sha=${{ steps.resolve_target_pr.outputs.target_head_sha }};issue-number=${{ steps.resolve_target_pr.outputs.target_pr_number }} overwrite: true path: ${{ runner.temp }}/arm-semantic-review/empty.txt - name: Set ARM semantic review pending @@ -1981,6 +1966,7 @@ jobs: env: GH_AW_AGENT_OUTPUT: ${{ runner.temp }}/gh-aw/safe-jobs/agent_output.json with: + persist-credentials: false sparse-checkout: | .github - name: Publish ARM semantic review status @@ -1988,6 +1974,7 @@ jobs: env: GH_AW_AGENT_OUTPUT: ${{ runner.temp }}/gh-aw/safe-jobs/agent_output.json with: + retries: 3 script: | const { default: publishArmSemanticReviewStatus } = await import('${{ github.workspace }}/.github/workflows/src/arm-auto-signoff/arm-semantic-review-workflow.ts'); diff --git a/.github/workflows/arm-api-review.md b/.github/workflows/arm-api-review.md index b76e1ccb32aa..fac27e7ef25f 100644 --- a/.github/workflows/arm-api-review.md +++ b/.github/workflows/arm-api-review.md @@ -77,19 +77,13 @@ on: run: | mkdir -p "$RUNNER_TEMP/arm-semantic-review" : > "$RUNNER_TEMP/arm-semantic-review/empty.txt" - - name: Upload ARM semantic review head SHA + - name: Upload ARM semantic review correlation if: steps.check_membership.outputs.is_team_member == 'true' uses: actions/upload-artifact@v7 with: - name: "head-sha=${{ steps.resolve_target_pr.outputs.target_head_sha }}" - path: "${{ runner.temp }}/arm-semantic-review/empty.txt" - if-no-files-found: error - overwrite: true - - name: Upload ARM semantic review issue number - if: steps.check_membership.outputs.is_team_member == 'true' - uses: actions/upload-artifact@v7 - with: - name: "issue-number=${{ steps.resolve_target_pr.outputs.target_pr_number }}" + # One artifact carries both values to save an upload step. `;` separates the + # `key=value` pairs; `parseWorkflowRunArtifactInputs` in context.ts reads each pair. + name: "head-sha=${{ steps.resolve_target_pr.outputs.target_head_sha }};issue-number=${{ steps.resolve_target_pr.outputs.target_pr_number }}" path: "${{ runner.temp }}/arm-semantic-review/empty.txt" if-no-files-found: error overwrite: true @@ -136,12 +130,13 @@ jobs: run_attempt: ${{ steps.resolve_target_pr.outputs.run_attempt }} target_head_sha: ${{ steps.resolve_target_pr.outputs.target_head_sha }} target_pr_number: ${{ steps.resolve_target_pr.outputs.target_pr_number }} + # Pre-activation posts a Pending `ARM Semantic Review` status, but the record job # only runs when the agent emitted a semantic item and threat detection passed. # The conclusion job runs after every other job regardless of their results, so it # resolves a Pending status that this exact run still owns. That keeps a failed, # cancelled, or noop run from leaving the PR waiting forever. The step reads no - # agent output; it takes the head SHA from the trusted `head-sha` artifact. + # agent output; it takes the head SHA from the trusted correlation artifact. # It is skipped when the record job succeeded, since that job already published. # Pre-steps run before the built-in conclusion steps, so `continue-on-error` # keeps a failure here from blocking gh-aw's own failure reporting. @@ -154,6 +149,7 @@ jobs: if: needs.record_arm_semantic_review.result != 'success' continue-on-error: true with: + persist-credentials: false sparse-checkout: | .github - name: Resolve unpublished ARM semantic review status @@ -161,6 +157,8 @@ jobs: continue-on-error: true uses: actions/github-script@v9.0.0 with: + # Retry transient API errors so a one-off failure does not leave the status Pending. + retries: 3 script: | const { finalizeUnpublishedArmSemanticReview } = await import('${{ github.workspace }}/.github/workflows/src/arm-auto-signoff/arm-semantic-review-workflow.ts'); @@ -335,11 +333,14 @@ safe-outputs: steps: - uses: actions/checkout@v7 with: + persist-credentials: false sparse-checkout: | .github - name: Publish ARM semantic review status uses: actions/github-script@v9.0.0 with: + # Retry transient API errors so a one-off failure does not leave the status Pending. + retries: 3 script: | const { default: publishArmSemanticReviewStatus } = await import('${{ github.workspace }}/.github/workflows/src/arm-auto-signoff/arm-semantic-review-workflow.ts'); diff --git a/.github/workflows/src/arm-auto-signoff/README.md b/.github/workflows/src/arm-auto-signoff/README.md index d8eef7760e7f..024c337e2a61 100644 --- a/.github/workflows/src/arm-auto-signoff/README.md +++ b/.github/workflows/src/arm-auto-signoff/README.md @@ -12,13 +12,14 @@ production path. It only manages the pilot label `ARMAutoSignedOff-Test` and nev ```text arm-api-review (reviewer) - pre_activation post status PENDING, upload head-sha= / issue-number= artifacts + pre_activation post status PENDING, upload one head-sha=;issue-number= artifact agent review the PR, request record_arm_semantic_review record_arm_semantic_review validate the result, post SUCCESS / FAILURE / ERROR conclusion if no result was posted, turn this run's PENDING into ERROR -arm-universal-auto-signoff newest status per context on the current head SHA + labels -update-labels re-check the live head, then add or remove the label +arm-universal-auto-signoff re-check the PR is open at the evaluated head, read the newest status + per context on that SHA + labels, decide the label action +update-labels add or remove the label ``` The model proposes; trusted code validates and publishes. Any doubt ends as "not Passed". diff --git a/.github/workflows/src/arm-auto-signoff/arm-semantic-review-workflow.ts b/.github/workflows/src/arm-auto-signoff/arm-semantic-review-workflow.ts index d0ac0a9cf3b3..0175d1ef7276 100644 --- a/.github/workflows/src/arm-auto-signoff/arm-semantic-review-workflow.ts +++ b/.github/workflows/src/arm-auto-signoff/arm-semantic-review-workflow.ts @@ -3,16 +3,19 @@ import { CommitStatusState, PER_PAGE_MAX } from "../../../shared/src/github.ts"; import { getWorkflowRunArtifactInputs } from "../context.ts"; import type { GitHubScriptArgs } from "../github.ts"; import { + addSpecificationChanges, ARM_SEMANTIC_REVIEW_STATUS, + checkPullRequestSize, evaluateSemanticReview, - exceedsSpecificationLimits, getLatestSemanticReviewStatus, - isWithinAutomatedReviewLimits, MANUAL_REVIEW_DESCRIPTION_PREFIX, parseSemanticReviewResult, REVIEW_INCOMPLETE_DESCRIPTION_PREFIX, + SemanticReviewCompletion, + SemanticReviewScope, + SPECIFICATION_OVER_LIMITS_REASON, type CommitStatus, - type PullRequestCounts, + type SpecificationTotals, type EvaluatedSemanticReview, } from "./arm-semantic-review.ts"; @@ -49,35 +52,73 @@ function getRunStatusUrl( } /** - * Independently checks the PR against the automated-review limits, since the model's `scope` is - * untrusted. The file list is fetched only when the PR's aggregate counters are over the limits. + * Independently verifies that the PR is small enough for the reviewer to have covered all of it, + * because the model's own `scope` is untrusted. The rules live in `arm-semantic-review.ts`; this + * does the API calls. It costs one `pulls.get` call, which is all most PRs need. Only when the + * PR's totals are over the limits does it list the files, page by page, and it stops as soon as + * the `specification/` changes are over the limits. * - * @param issueNumber The PR number, from trusted artifacts. - * @param pullRequest The aggregate counters from `pulls.get`. - * @returns `true` when a human must review because the `specification/` changes exceed the limits. + * @returns A short fixed reason when a human must review, or `undefined` when the PR is within + * the limits. The reason is published in the status, so it must never include model output. */ -async function requiresManualReview( +async function getOversizeReason( github: GitHubScriptArgs["github"], + core: GitHubScriptArgs["core"], owner: string, repo: string, issueNumber: number, - pullRequest: PullRequestCounts, -): Promise { - if (isWithinAutomatedReviewLimits(pullRequest)) { - return false; - } - if (pullRequest.changed_files < 1) { - return true; - } - const changedFiles = await github.paginate(github.rest.pulls.listFiles, { +): Promise { + const { data: pullRequest } = await github.rest.pulls.get({ owner, repo, pull_number: issueNumber, - per_page: PER_PAGE_MAX, }); - return exceedsSpecificationLimits(pullRequest.changed_files, changedFiles); -} + core.info( + `PR #${issueNumber} size: ${pullRequest.changed_files} files, ` + + `+${pullRequest.additions} -${pullRequest.deletions} lines`, + ); + + const size = checkPullRequestSize(pullRequest); + if (size.outcome === "manual-review") { + core.info(`Cannot be proven within the limits: ${size.reason}`); + return size.reason; + } + if (size.outcome === "within") { + core.info( + "Within the limits overall, so the specification/ subset is too: no file list needed", + ); + return undefined; + } + core.info("Over the limits overall: counting only the specification/ changes"); + const totals: SpecificationTotals = { files: 0, lines: 0 }; + let pages = 0; + let overLimit = false; + await github.paginate( + github.rest.pulls.listFiles, + { owner, repo, pull_number: issueNumber, per_page: PER_PAGE_MAX }, + (response, done) => { + pages++; + overLimit = addSpecificationChanges(totals, response.data); + core.info( + ` page ${pages}: ${totals.files} specification/ files, ${totals.lines} lines so far`, + ); + if (overLimit) { + core.info(" already over the limits, so the remaining pages are not needed"); + done(); + } + return []; + }, + ); + + if (overLimit) { + return SPECIFICATION_OVER_LIMITS_REASON; + } + core.info( + `specification/ changes are within the limits: ${totals.files} files, ${totals.lines} lines`, + ); + return undefined; +} /** * Writes one `ARM Semantic Review` commit status on the reviewed SHA. * @@ -143,6 +184,7 @@ export default async function publishArmSemanticReviewStatus({ core.info("The reviewer run has no trusted PR/SHA correlation; status is unchanged"); return EMPTY_RESULT; } + core.info(`Publishing for PR #${issueNumber} at ${headSha}`); let status: EvaluatedSemanticReview; try { @@ -150,18 +192,30 @@ export default async function publishArmSemanticReviewStatus({ await readFile(process.env.GH_AW_AGENT_OUTPUT ?? "", "utf8"), ) as unknown; const result = parseSemanticReviewResult(agentOutput); + core.info( + `Reviewer reported: scope=${result.reviewScope}, completeness=${result.completion}, ` + + `incomplete reason=${result.incompleteReason}, Blocking findings=${result.blockingCount}`, + ); - const { data: pullRequest } = await github.rest.pulls.get({ - owner, - repo, - pull_number: issueNumber, - }); - status = (await requiresManualReview(github, owner, repo, issueNumber, pullRequest)) - ? { + status = evaluateSemanticReview(result); + if ( + result.completion === SemanticReviewCompletion.Complete && + result.reviewScope === SemanticReviewScope.Full + ) { + // Only a claim of a full, complete review can authorize signoff, so only that claim needs + // verifying. A scoped or incomplete review (the oversized PRs, where the check is the most + // expensive) is already withheld. + const reason = await getOversizeReason(github, core, owner, repo, issueNumber); + if (reason) { + core.info(`The reviewer claimed a full review, but ${reason}: manual review required`); + status = { state: CommitStatusState.ERROR, - description: `${MANUAL_REVIEW_DESCRIPTION_PREFIX}PR exceeds automated review size limits`, - } - : evaluateSemanticReview(result); + description: `${MANUAL_REVIEW_DESCRIPTION_PREFIX}${reason}`, + }; + } + } else { + core.info("The reviewer did not claim a full, complete review: skipping the size check"); + } } catch (error) { // Any failure here, including a transient GitHub API error, is reported as incomplete rather // than as a manual-review hold, so a re-run can still produce a passing result. The details go @@ -213,13 +267,18 @@ export async function finalizeUnpublishedArmSemanticReview({ } const targetUrl = getRunStatusUrl(context, owner, repo); + core.info(`Checking whether ${targetUrl} left a Pending status on ${headSha}`); const statuses: CommitStatus[] = await github.paginate( github.rest.repos.listCommitStatusesForRef, { owner, repo, ref: headSha, per_page: PER_PAGE_MAX }, ); const latestStatus = getLatestSemanticReviewStatus(statuses); + core.info( + `Latest ${ARM_SEMANTIC_REVIEW_STATUS}: ${latestStatus?.state ?? "none"} ` + + `(${latestStatus?.target_url ?? "no url"})`, + ); if (latestStatus?.state !== CommitStatusState.PENDING || latestStatus.target_url !== targetUrl) { - core.info("This run does not own a Pending ARM Semantic Review status; status is unchanged"); + core.info("This run does not own a Pending status; status is unchanged"); return { statusPublished: false }; } diff --git a/.github/workflows/src/arm-auto-signoff/arm-semantic-review.ts b/.github/workflows/src/arm-auto-signoff/arm-semantic-review.ts index 45d2af22360c..21ef7e7d675a 100644 --- a/.github/workflows/src/arm-auto-signoff/arm-semantic-review.ts +++ b/.github/workflows/src/arm-auto-signoff/arm-semantic-review.ts @@ -7,31 +7,6 @@ import { byDate, invert } from "../../../shared/src/sort.ts"; * This file makes no GitHub API calls; `arm-semantic-review-workflow.ts` does the I/O. */ -/** - * Size limits for automated review. They mirror the cap in the reviewer prompt - * (`arm-api-review.md`, Trigger Validation): a PR above either limit is reviewed only in part, so - * it needs a human. Keep both copies in step. - */ -const MAX_SPECIFICATION_FILES = 50; -const MAX_SPECIFICATION_LINES = 5_000; - -/** GitHub caps the pull request file list at this many entries without reporting truncation. */ -const MAX_LISTED_FILES = 3_000; - -/** One entry of the pull request file list (`pulls.listFiles`). */ -export type ChangedFile = { - filename: string; - additions: number; - deletions: number; -}; - -/** The aggregate counters on a pull request object (`pulls.get`). */ -export type PullRequestCounts = { - changed_files: number; - additions: number; - deletions: number; -}; - /** The commit status context that carries the semantic review result for a SHA. */ export const ARM_SEMANTIC_REVIEW_STATUS = "ARM Semantic Review"; @@ -230,51 +205,102 @@ export function evaluateSemanticReview(result: SemanticReviewResult): EvaluatedS }; } +/* + * Automated-review size limits. They mirror the cap in the reviewer prompt (`arm-api-review.md`, + * Trigger Validation); keep both in step. A PR above either limit is only partly reviewed, so a + * human must review it. The model's own `scope` is untrusted, so these are checked independently. + */ +const MAX_SPECIFICATION_FILES = 50; +const MAX_SPECIFICATION_LINES = 5_000; + +// GitHub's file-listing API returns at most this many files for a PR and does not say when it cut +// the list off, so a PR this large cannot be proven fully covered. +const MAX_LISTED_FILES = 3_000; + +/** The aggregate counters on a pull request object (`pulls.get`). */ +export type PullRequestCounts = { + changed_files: number; + additions: number; + deletions: number; +}; + +/** One entry of the pull request file list (`pulls.listFiles`). */ +export type ChangedFile = { + filename: string; + /** Set only for a renamed or moved file: the path it had before the PR. */ + previous_filename?: string; + additions: number; + deletions: number; +}; + +/** Running totals of the `specification/` changes seen so far. */ +export type SpecificationTotals = { files: number; lines: number }; + +/** The status reason for a PR whose `specification/` changes exceed the limits. */ +export const SPECIFICATION_OVER_LIMITS_REASON = `specification/ changes exceed ${MAX_SPECIFICATION_FILES} files or ${MAX_SPECIFICATION_LINES} lines`; + +/** What the PR's aggregate counters alone say about its size. */ +export type SizeCheck = + | { outcome: "within" } + | { outcome: "manual-review"; reason: string } + | { outcome: "list-files" }; + /** - * Cheap pre-check on the PR's aggregate counters. When the whole PR is within the limits, its - * `specification/` subset necessarily is too, so no file list is needed. A PR reporting zero - * files is not "within": GitHub reports zero counters when a diff is too large to compute. + * First, cheapest step of the size check: decide from the PR's aggregate counters alone. * - * @param pullRequest The aggregate counters from `pulls.get`. - * @returns `true` when no file list is needed and the PR is within the automated-review limits. + * - Zero changed files means GitHub could not compute the diff, and 3,000 or more means its file + * list is cut off. Neither can prove coverage, so a human must review. + * - If the whole PR is within the limits, so is its `specification/` subset: nothing more to check. + * - Otherwise the PR may still be fine (for example many docs files and few specification + * files), so the caller must list the files and count only `specification/` changes. + * + * The reasons are fixed strings because they are published in the status; never add model output. */ -export function isWithinAutomatedReviewLimits(pullRequest: PullRequestCounts): boolean { - return ( - pullRequest.changed_files >= 1 && - pullRequest.changed_files <= MAX_SPECIFICATION_FILES && - pullRequest.additions + pullRequest.deletions <= MAX_SPECIFICATION_LINES - ); +export function checkPullRequestSize(pullRequest: PullRequestCounts): SizeCheck { + const { changed_files, additions, deletions } = pullRequest; + if (changed_files < 1) { + return { outcome: "manual-review", reason: "GitHub could not compute the PR's size" }; + } + if (changed_files >= MAX_LISTED_FILES) { + return { + outcome: "manual-review", + reason: `PR has ${MAX_LISTED_FILES} or more changed files`, + }; + } + if ( + changed_files <= MAX_SPECIFICATION_FILES && + additions + deletions <= MAX_SPECIFICATION_LINES + ) { + return { outcome: "within" }; + } + return { outcome: "list-files" }; } /** - * Independent check of the `specification/` subset, because the model's own `scope` is untrusted. - * Only needed when the aggregate counters exceed the limits. A truncated file list cannot prove - * coverage, so it also requires manual review. + * Second step of the size check, run once per page of the file list: add that page's + * `specification/` changes to the running `totals`. A renamed or moved file counts when its old or + * its new path is under `specification/`, so a file moved out of it is not missed. * - * @param changedFileCount `changed_files` from `pulls.get`, used to detect truncation. - * @param changedFiles The paginated `pulls.listFiles` result. - * @returns `true` when the `specification/` subset exceeds the limits or coverage is unprovable. + * Totals only grow, so once they are over the limits the answer is final and a caller can stop + * reading pages. + * + * @returns `true` when the totals now exceed the limits. */ -export function exceedsSpecificationLimits( - changedFileCount: number, - changedFiles: ChangedFile[], +export function addSpecificationChanges( + totals: SpecificationTotals, + files: ChangedFile[], ): boolean { - if (changedFiles.length >= MAX_LISTED_FILES || changedFileCount > changedFiles.length) { - return true; + for (const file of files) { + if ( + file.filename.startsWith("specification/") || + file.previous_filename?.startsWith("specification/") + ) { + totals.files++; + totals.lines += file.additions + file.deletions; + } } - const specificationFiles = changedFiles.filter((file) => - file.filename.startsWith("specification/"), - ); - const specificationLines = specificationFiles.reduce( - (total, file) => total + file.additions + file.deletions, - 0, - ); - return ( - specificationFiles.length > MAX_SPECIFICATION_FILES || - specificationLines > MAX_SPECIFICATION_LINES - ); + return totals.files > MAX_SPECIFICATION_FILES || totals.lines > MAX_SPECIFICATION_LINES; } - /** * Interprets a commit status as a semantic review outcome. An `error` status is a manual-review * hold only when its description starts with `MANUAL_REVIEW_DESCRIPTION_PREFIX`; any other diff --git a/.github/workflows/src/context.ts b/.github/workflows/src/context.ts index c07d968a3fdf..60549c600fba 100644 --- a/.github/workflows/src/context.ts +++ b/.github/workflows/src/context.ts @@ -316,8 +316,8 @@ export async function extractInputs( } /** - * Reads the `head-sha=` and `issue-number=` artifact names of a workflow run. - * Only call this for runs whose artifacts come from trusted code (see `extractInputs`). + * Reads the `head-sha=` and `issue-number=` pairs from the artifact names of a workflow + * run. Only call this for runs whose artifacts come from trusted code (see `extractInputs`). * @returns `headSha` ("" if absent) and `issueNumber` (NaN if absent or invalid). * @throws If the artifacts hold an invalid or conflicting head SHA, or conflicting PR numbers. */ @@ -347,6 +347,9 @@ export async function getWorkflowRunArtifactInputs({ /** * Parses `head-sha=` and `issue-number=` from artifact names; other names are ignored. + * A producer may carry both in one artifact by joining the pairs with `;`, for example + * `head-sha=;issue-number=`, which saves an upload step. Names without `;` are a single + * pair, so existing single-value artifacts parse unchanged. * @returns `headSha` ("" if absent) and `issueNumber` (NaN if absent or invalid). * @throws If a head SHA is invalid, or two head SHAs or two PR numbers conflict. */ @@ -356,7 +359,7 @@ export function parseWorkflowRunArtifactInputs( ): { headSha: string; issueNumber: number } { let headSha = ""; let issueNumber = NaN; - for (const artifactName of artifactNames) { + for (const artifactName of artifactNames.flatMap((name) => name.split(";"))) { const firstEquals = artifactName.indexOf("="); if (firstEquals === -1) { continue; diff --git a/.github/workflows/src/update-labels.ts b/.github/workflows/src/update-labels.ts index 4604c1c9a50b..312c3fa2dad0 100644 --- a/.github/workflows/src/update-labels.ts +++ b/.github/workflows/src/update-labels.ts @@ -12,11 +12,6 @@ export default async function updateLabels({ github, context, core }: GitHubScri await updateLabelsImpl({ owner, repo, head_sha, issue_number, run_id, github, core }); } -/** - * Adds or removes PR labels from the `label-=true|false` artifacts of one workflow run. - * If the run also published a `head-sha=` artifact, changes are skipped when the PR is closed or - * its head has moved on, so a delayed run can't label a newer commit. - */ export async function updateLabelsImpl({ owner, repo, @@ -61,8 +56,6 @@ export async function updateLabelsImpl({ }); const artifactNames: string[] = artifacts.map((a) => a.name); - // Producers opt in to the stale-head check below by publishing a head-sha artifact. - const hasHeadShaArtifact = artifactNames.some((name) => name.startsWith("head-sha=")); core.info(`artifactNames: ${JSON.stringify(artifactNames)}`); @@ -108,24 +101,6 @@ export async function updateLabelsImpl({ ); } - if ( - (labelsToAdd.length > 0 || labelsToRemove.length > 0) && - hasHeadShaArtifact && - isFullGitSha(head_sha) && - Number.isInteger(issue_number) && - issue_number > 0 - ) { - const { data: pullRequest } = await github.rest.pulls.get({ - owner, - repo, - pull_number: issue_number, - }); - if (pullRequest.state !== "open" || pullRequest.head.sha !== head_sha) { - core.info("Skipping label update because the pull request is closed or its head changed"); - return; - } - } - const pullRequestUrl = `https://github.com/${owner}/${repo}/pull/${issue_number}`; core.info(`pull request url: ${pullRequestUrl}`); diff --git a/.github/workflows/test/arm-api-review-workflow.test.ts b/.github/workflows/test/arm-api-review-workflow.test.ts index 2c93eb4c5e47..76b7b4ba46a6 100644 --- a/.github/workflows/test/arm-api-review-workflow.test.ts +++ b/.github/workflows/test/arm-api-review-workflow.test.ts @@ -583,6 +583,13 @@ describe("ARM API review posting reliability", () => { const collapsed = collapseWhitespace(source); expect(source).toContain("name: Set ARM semantic review pending"); + // One upload carries both correlation values; the old per-value steps must not return. + expect(source).toContain("name: Upload ARM semantic review correlation"); + expect(source).not.toContain("name: Upload ARM semantic review head SHA"); + expect(source).not.toContain("name: Upload ARM semantic review issue number"); + expect(compiled).toContain( + "head-sha=${{ steps.resolve_target_pr.outputs.target_head_sha }};issue-number=${{ steps.resolve_target_pr.outputs.target_pr_number }}", + ); expect(source).toContain('context: "ARM Semantic Review"'); expect(source).toContain("record-arm-semantic-review:"); expect(source).toContain("scope:"); diff --git a/.github/workflows/test/arm-auto-signoff/arm-semantic-review.test.ts b/.github/workflows/test/arm-auto-signoff/arm-semantic-review.test.ts index 93c8b934f1ec..653a026fe236 100644 --- a/.github/workflows/test/arm-auto-signoff/arm-semantic-review.test.ts +++ b/.github/workflows/test/arm-auto-signoff/arm-semantic-review.test.ts @@ -6,9 +6,9 @@ import { CommitStatusState, PER_PAGE_MAX } from "../../../shared/src/github.ts"; import { ARM_SEMANTIC_REVIEW_STATUS, evaluateSemanticReview, - exceedsSpecificationLimits, - isWithinAutomatedReviewLimits, getSemanticReviewOutcome, + addSpecificationChanges, + checkPullRequestSize, parseSemanticReviewResult, SemanticReviewCompletion, SemanticReviewIncompleteReason, @@ -16,6 +16,7 @@ import { SemanticReviewScope, type ChangedFile, type SemanticReviewResult, + type SpecificationTotals, } from "../../src/arm-auto-signoff/arm-semantic-review.ts"; import publishArmSemanticReviewStatus, { finalizeUnpublishedArmSemanticReview, @@ -52,7 +53,7 @@ function agentOutput(result: SemanticReviewResult = passedResult) { }; } -const smallPullRequest = { changed_files: 1, additions: 10, deletions: 5 }; +const smallPullRequest = { number: issueNumber, changed_files: 1, additions: 10, deletions: 5 }; function createPublisherGithub() { const github = createMockGithub(); @@ -273,79 +274,109 @@ describe("evaluateSemanticReview", () => { }); }); -describe("isWithinAutomatedReviewLimits", () => { +describe("checkPullRequestSize", () => { it.each([ - { name: "a small PR", pr: smallPullRequest, expected: true }, + { name: "a small PR", counts: [1, 10, 5], expected: { outcome: "within" } }, + { name: "exactly 50 files", counts: [50, 50, 0], expected: { outcome: "within" } }, + { name: "exactly 5,000 lines", counts: [1, 3_000, 2_000], expected: { outcome: "within" } }, + { name: "51 files", counts: [51, 51, 0], expected: { outcome: "list-files" } }, + { name: "5,001 lines", counts: [1, 5_001, 0], expected: { outcome: "list-files" } }, { - name: "exactly the file limit", - pr: { changed_files: 50, additions: 1, deletions: 0 }, - expected: true, - }, - { - name: "one file over the limit", - pr: { changed_files: 51, additions: 1, deletions: 0 }, - expected: false, - }, - { - name: "exactly the line limit", - pr: { changed_files: 1, additions: 3_000, deletions: 2_000 }, - expected: true, + // GitHub reports zero counters when the diff is too large for it to compute. + name: "zero reported files", + counts: [0, 0, 0], + expected: { outcome: "manual-review", reason: "GitHub could not compute the PR's size" }, }, { - name: "over the line limit", - pr: { changed_files: 1, additions: 4_000, deletions: 2_000 }, - expected: false, + name: "one file under GitHub's listing cap", + counts: [2_999, 3_000, 0], + expected: { outcome: "list-files" }, }, - // GitHub reports zero counters when the diff is too large to compute. { - name: "zero reported files", - pr: { changed_files: 0, additions: 0, deletions: 0 }, - expected: false, + // GitHub lists at most 3,000 files and does not say when it cut the list off. + name: "GitHub's listing cap", + counts: [3_000, 3_000, 0], + expected: { outcome: "manual-review", reason: "PR has 3000 or more changed files" }, }, - ])("$name", ({ pr, expected }) => { - expect(isWithinAutomatedReviewLimits(pr)).toBe(expected); + ])("$name", ({ counts: [changed_files, additions, deletions], expected }) => { + expect(checkPullRequestSize({ changed_files, additions, deletions })).toEqual(expected); }); }); -describe("exceedsSpecificationLimits", () => { - const specFile = (name: string, additions = 1, deletions = 0): ChangedFile => ({ +describe("addSpecificationChanges", () => { + const spec = (name: string, lines = 1): ChangedFile => ({ filename: `specification/foo/${name}.json`, - additions, - deletions, + additions: lines, + deletions: 0, }); - const otherFile = (name: string, additions = 1): ChangedFile => ({ + const other = (name: string): ChangedFile => ({ filename: `documentation/${name}.md`, - additions, + additions: 100, + deletions: 0, + }); + const renamed = (from: string, to: string): ChangedFile => ({ + filename: to, + previous_filename: from, + additions: 0, deletions: 0, }); + const times = (count: number, build: (i: number) => T) => + Array.from({ length: count }, (_, i) => build(i)); + const add = (files: ChangedFile[]) => { + const totals: SpecificationTotals = { files: 0, lines: 0 }; + return { over: addSpecificationChanges(totals, files), totals }; + }; - it("ignores files and lines outside specification/", () => { - const files = [ - specFile("a"), - ...Array.from({ length: 60 }, (_, i) => otherFile(`doc${i}`, 200)), - ]; - expect(exceedsSpecificationLimits(files.length, files)).toBe(false); + it("is false at exactly the limits and true just over", () => { + expect(add(times(50, (i) => spec(`f${i}`))).over).toBe(false); + expect(add(times(51, (i) => spec(`f${i}`))).over).toBe(true); + expect(add([spec("big", 5_000)]).over).toBe(false); + expect(add([spec("big", 5_001)]).over).toBe(true); }); - it("allows exactly the specification file limit", () => { - const files = Array.from({ length: 50 }, (_, i) => specFile(`f${i}`)); - expect(exceedsSpecificationLimits(files.length, files)).toBe(false); + it("does not count files or lines outside specification/", () => { + const { over, totals } = add([spec("a"), ...times(500, (i) => other(`d${i}`))]); + + expect(over).toBe(false); + expect(totals).toEqual({ files: 1, lines: 1 }); }); - it("flags more specification files than the limit", () => { - const files = Array.from({ length: 51 }, (_, i) => specFile(`f${i}`)); - expect(exceedsSpecificationLimits(files.length, files)).toBe(true); + it("accumulates across pages", () => { + const totals: SpecificationTotals = { files: 0, lines: 0 }; + + expect( + addSpecificationChanges( + totals, + times(30, (i) => spec(`a${i}`)), + ), + ).toBe(false); + expect( + addSpecificationChanges( + totals, + times(21, (i) => spec(`b${i}`)), + ), + ).toBe(true); + expect(totals.files).toBe(51); }); - it("flags more specification lines than the limit", () => { - const files = [specFile("big", 4_000, 2_000)]; - expect(exceedsSpecificationLimits(files.length, files)).toBe(true); + it.each([ + { name: "moved into specification/", from: "eng", to: "specification/foo" }, + { name: "moved out of specification/", from: "specification/foo", to: "archive" }, + { name: "renamed within specification/", from: "specification/foo", to: "specification/bar" }, + ])("counts a file $name once", ({ from, to }) => { + const { over, totals } = add( + times(51, (i) => renamed(`${from}/f${i}.json`, `${to}/f${i}.json`)), + ); + + expect(over).toBe(true); + expect(totals.files).toBe(51); }); - it("flags a truncated file list", () => { - expect(exceedsSpecificationLimits(10, [specFile("a")])).toBe(true); - const capped = Array.from({ length: 3_000 }, (_, i) => otherFile(`d${i}`)); - expect(exceedsSpecificationLimits(3_000, capped)).toBe(true); + it("ignores a rename that never touches specification/", () => { + const { over, totals } = add(times(51, (i) => renamed(`eng/a${i}.json`, `docs/a${i}.json`))); + + expect(over).toBe(false); + expect(totals.files).toBe(0); }); }); describe("publishArmSemanticReviewStatus", () => { @@ -397,79 +428,241 @@ describe("publishArmSemanticReviewStatus", () => { ); }); - it("does not list files when the PR is within the aggregate limits", async () => { - const { github } = await runPublisher(); - - expect(github.rest.pulls.listFiles).not.toHaveBeenCalled(); - }); + describe("PR size limits", () => { + const OVER_LIMITS = + "Manual review required: specification/ changes exceed 50 files or 5000 lines"; + const spec = (name: string, lines = 1) => ({ + filename: `specification/foo/${name}.json`, + additions: lines, + deletions: 0, + }); + const doc = (name: string) => ({ + filename: `documentation/${name}.md`, + additions: 1, + deletions: 0, + }); + const times = (count: number, build: (i: number) => T) => + Array.from({ length: count }, (_, i) => build(i)); + + /** Publishes for a PR with the given totals; `files` is what `pulls.listFiles` returns. */ + async function publish( + totals: { changed_files: number; additions: number; deletions: number }, + files: unknown[] = [], + ) { + const github = createPublisherGithub(); + github.rest.pulls.get.mockResolvedValue({ data: { number: issueNumber, ...totals } }); + github.rest.pulls.listFiles.mockResolvedValue({ data: files }); + return (await runPublisher({ github })).github; + } + const published = (github: ReturnType) => + github.rest.repos.createCommitStatus.mock.calls[0][0] as { + state: string; + description: string; + }; + const publishedState = (github: ReturnType) => + published(github).state; + const publishedDescription = (github: ReturnType) => + published(github).description; + + it.each([ + { name: "a small PR", totals: { changed_files: 1, additions: 10, deletions: 5 } }, + { name: "exactly 50 files", totals: { changed_files: 50, additions: 50, deletions: 0 } }, + { + name: "exactly 5,000 lines", + totals: { changed_files: 1, additions: 3_000, deletions: 2_000 }, + }, + ])("does not list files for $name", async ({ totals }) => { + const github = await publish(totals); - it("publishes Passed when only non-specification files push the PR over the limits", async () => { - const github = createPublisherGithub(); - const files = [ - { filename: "specification/foo/foo.json", additions: 10, deletions: 5 }, - ...Array.from({ length: 60 }, (_, i) => ({ - filename: `documentation/doc${i}.md`, - additions: 1, - deletions: 0, - })), - ]; - github.rest.pulls.get.mockResolvedValue({ - data: { changed_files: files.length, additions: 70, deletions: 5 }, + expect(github.rest.pulls.listFiles).not.toHaveBeenCalled(); + expect(publishedState(github)).toBe(CommitStatusState.SUCCESS); }); - github.rest.pulls.listFiles.mockResolvedValue({ data: files }); - const { github: g } = await runPublisher({ github }); - expect(g.rest.pulls.listFiles).toHaveBeenCalled(); - expect(g.rest.repos.createCommitStatus).toHaveBeenCalledWith( - expect.objectContaining({ state: CommitStatusState.SUCCESS }), + it.each([ + { + name: "zero reported files", + changed_files: 0, + reason: "GitHub could not compute the PR's size", + }, + // GitHub lists at most 3,000 files and does not say when it truncated the list. + { + name: "3,000 or more files", + changed_files: 3_000, + reason: "PR has 3000 or more changed files", + }, + ])( + "requires manual review without listing files for $name", + async ({ changed_files, reason }) => { + const github = await publish({ changed_files, additions: 0, deletions: 0 }); + + expect(github.rest.pulls.listFiles).not.toHaveBeenCalled(); + expect(publishedDescription(github)).toBe(`Manual review required: ${reason}`); + }, ); - }); - it.each([ - { - name: "too many specification files", - pr: { changed_files: 51, additions: 51, deletions: 0 }, - files: Array.from({ length: 51 }, (_, i) => ({ - filename: `specification/foo/f${i}.json`, - additions: 1, - deletions: 0, - })), - }, - { - name: "a truncated file list", - pr: { changed_files: 51, additions: 51, deletions: 0 }, - files: [{ filename: "specification/foo/foo.json", additions: 1, deletions: 0 }], - }, - ])("publishes Manual review required for $name", async ({ pr, files }) => { - const github = createPublisherGithub(); - github.rest.pulls.get.mockResolvedValue({ data: pr }); - github.rest.pulls.listFiles.mockResolvedValue({ data: files }); + it("lists the files of the trusted PR when the totals are over the limits", async () => { + const github = await publish({ changed_files: 51, additions: 51, deletions: 0 }, [spec("a")]); - const { github: g } = await runPublisher({ github }); - expect(g.rest.repos.createCommitStatus).toHaveBeenCalledWith( - expect.objectContaining({ - state: CommitStatusState.ERROR, - description: "Manual review required: PR exceeds automated review size limits", - }), - ); - }); + expect(github.rest.pulls.listFiles).toHaveBeenCalledWith( + expect.objectContaining({ owner, repo, pull_number: issueNumber }), + ); + }); - it("publishes Manual review required without listing files when the PR reports zero files", async () => { - const github = createPublisherGithub(); - github.rest.pulls.get.mockResolvedValue({ - data: { changed_files: 0, additions: 0, deletions: 0 }, + it("passes when only files outside specification/ push the PR over the limits", async () => { + const files = [spec("a"), ...times(60, (i) => doc(`d${i}`))]; + const github = await publish({ changed_files: 61, additions: 61, deletions: 0 }, files); + + expect(publishedState(github)).toBe(CommitStatusState.SUCCESS); }); - const { github: g } = await runPublisher({ github }); - expect(g.rest.pulls.listFiles).not.toHaveBeenCalled(); - expect(g.rest.repos.createCommitStatus).toHaveBeenCalledWith( - expect.objectContaining({ - state: CommitStatusState.ERROR, - description: "Manual review required: PR exceeds automated review size limits", - }), + it.each([ + { name: "too many specification files", files: times(51, (i) => spec(`f${i}`)) }, + { name: "too many specification lines", files: [spec("big", 5_001)] }, + ])("requires manual review for $name", async ({ files }) => { + const additions = files.reduce((total, file) => total + file.additions, 0); + const github = await publish({ changed_files: 51, additions, deletions: 0 }, files); + + expect(publishedDescription(github)).toBe(OVER_LIMITS); + }); + + it("overrides a Changes requested result when the PR is over the limits", async () => { + // The model claimed a full review of an oversized PR, so its Blocking count is not trusted. + const github = createPublisherGithub(); + github.rest.pulls.get.mockResolvedValue({ + data: { number: issueNumber, changed_files: 51, additions: 51, deletions: 0 }, + }); + github.rest.pulls.listFiles.mockResolvedValue({ data: times(51, (i) => spec(`f${i}`)) }); + + const { github: g } = await runPublisher({ + github, + output: agentOutput({ ...passedResult, blockingCount: 2 }), + }); + expect(publishedDescription(g)).toBe(OVER_LIMITS); + }); + + it.each([ + { + name: "a scoped review", + result: { ...passedResult, reviewScope: SemanticReviewScope.Scoped }, + description: "Manual review required: scoped review requires manual signoff", + }, + { + name: "an incomplete review", + result: { + ...passedResult, + completion: SemanticReviewCompletion.Incomplete, + incompleteReason: SemanticReviewIncompleteReason.ToolFailure, + }, + description: "Review incomplete: tool-failure", + }, + ])( + "skips the size check for $name, which already withholds signoff", + async ({ result, description }) => { + const github = createPublisherGithub(); + github.rest.pulls.get.mockResolvedValue({ + data: { number: issueNumber, changed_files: 500, additions: 9_000, deletions: 0 }, + }); + + const { github: g } = await runPublisher({ github, output: agentOutput(result) }); + expect(g.rest.pulls.get).not.toHaveBeenCalled(); + expect(g.rest.pulls.listFiles).not.toHaveBeenCalled(); + expect(publishedDescription(g)).toBe(description); + }, ); - }); + it("logs each step of the size decision", async () => { + const github = createPublisherGithub(); + github.rest.pulls.get.mockResolvedValue({ + data: { number: issueNumber, changed_files: 61, additions: 61, deletions: 0 }, + }); + github.rest.pulls.listFiles.mockResolvedValue({ + data: [spec("a"), ...times(60, (i) => doc(`d${i}`))], + }); + + const { core } = await runPublisher({ github }); + const logged = core.info.mock.calls.map(([message]) => String(message)); + expect(logged).toEqual( + expect.arrayContaining([ + expect.stringContaining("Reviewer reported: scope=full, completeness=complete"), + expect.stringContaining("size: 61 files, +61 -0 lines"), + expect.stringContaining("counting only the specification/ changes"), + expect.stringContaining("page 1: 1 specification/ files, 1 lines so far"), + expect.stringContaining("specification/ changes are within the limits: 1 files, 1 lines"), + ]), + ); + }); + + describe("paginating the file list", () => { + const PAGE_SIZE = 100; + const page = (start: number, count: number, prefix: string) => + Array.from({ length: count }, (_, i) => ({ + filename: `${prefix}/f${start + i}.json`, + additions: 1, + deletions: 0, + })); + + /** Serves `pages` through paginate's map function, honouring `done()` like Octokit does. */ + function withPages(pages: ReturnType[], changedFiles: number) { + const github = createPublisherGithub(); + github.rest.pulls.get.mockResolvedValue({ + data: { + number: issueNumber, + changed_files: changedFiles, + additions: changedFiles, + deletions: 0, + }, + }); + const requested: number[] = []; + const originalPaginate = github.paginate; + type MapPage = (response: { data: unknown[] }, done: () => void) => unknown[]; + github.paginate = (async (fn: unknown, params: unknown, mapFn?: MapPage) => { + if (fn !== github.rest.pulls.listFiles || !mapFn) { + return originalPaginate(fn as never, params as never); + } + const result: unknown[] = []; + let stopped = false; + for (const [index, data] of pages.entries()) { + requested.push(index); + result.push(...mapFn({ data }, () => (stopped = true))); + if (stopped) break; + } + return result; + }) as typeof github.paginate; + return { github, requested }; + } + + it("stops after the first page when its specification/ files already exceed the limits", async () => { + const { github, requested } = withPages( + [ + page(0, PAGE_SIZE, "specification/foo"), + page(PAGE_SIZE, PAGE_SIZE, "specification/foo"), + page(2 * PAGE_SIZE, 50, "specification/foo"), + ], + 250, + ); + + const { github: g } = await runPublisher({ github }); + expect(requested).toEqual([0]); + expect(publishedDescription(g)).toBe(OVER_LIMITS); + }); + + it("keeps paginating while the specification/ files seen are still within the limits", async () => { + // 250 files, but only 10 are under specification/, so no page can settle the answer early. + const { github, requested } = withPages( + [ + page(0, PAGE_SIZE, "documentation"), + page(PAGE_SIZE, PAGE_SIZE, "documentation"), + [...page(2 * PAGE_SIZE, 40, "documentation"), ...page(0, 10, "specification/foo")], + ], + 250, + ); + + const { github: g } = await runPublisher({ github }); + expect(requested).toEqual([0, 1, 2]); + expect(publishedState(g)).toBe(CommitStatusState.SUCCESS); + }); + }); + }); it.each([ { name: "missing agent output", diff --git a/.github/workflows/test/context.test.ts b/.github/workflows/test/context.test.ts index ea1eaf0deee2..0ff31d1628ff 100644 --- a/.github/workflows/test/context.test.ts +++ b/.github/workflows/test/context.test.ts @@ -653,6 +653,57 @@ describe("extractInputs", () => { }); }); + it("workflow_run:completed:workflow_run reads both values from one combined artifact", async () => { + const github = createMockGithub(); + github.rest.actions.listWorkflowRunArtifacts.mockResolvedValue({ + data: { + artifacts: [{ name: `head-sha=${fullGitSha};issue-number=123` }, { name: "unrelated=1" }], + }, + }); + + const context = { + eventName: "workflow_run", + payload: { + action: "completed", + workflow_run: { + event: "workflow_run", + head_sha: "def456", + id: 456, + repository: { name: "TestRepoName", owner: { login: "TestRepoOwnerLogin" } }, + }, + }, + }; + + await expect(extractInputs(github, context, createMockCore())).resolves.toEqual({ + owner: "TestRepoOwnerLogin", + repo: "TestRepoName", + head_sha: fullGitSha, + issue_number: 123, + run_id: 456, + }); + + // A combined artifact is validated exactly like separate ones. + github.rest.actions.listWorkflowRunArtifacts.mockResolvedValue({ + data: { artifacts: [{ name: "head-sha=not-full-git-sha;issue-number=123" }] }, + }); + await expect(extractInputs(github, context, createMockCore())).rejects.toThrow( + "head-sha is not a valid full git SHA", + ); + + // It also conflicts with a separate artifact that disagrees. + github.rest.actions.listWorkflowRunArtifacts.mockResolvedValue({ + data: { + artifacts: [ + { name: `head-sha=${fullGitSha};issue-number=123` }, + { name: "issue-number=456" }, + ], + }, + }); + await expect(extractInputs(github, context, createMockCore())).rejects.toThrow( + "Conflicting issue-number artifacts", + ); + }); + it("workflow_run:completed:unsupported", async () => { const context = { eventName: "workflow_run", diff --git a/.github/workflows/test/mocks.ts b/.github/workflows/test/mocks.ts index 50fd3a2a019f..1142002d9326 100644 --- a/.github/workflows/test/mocks.ts +++ b/.github/workflows/test/mocks.ts @@ -22,9 +22,14 @@ function createMockGithubImpl() { paginate: async ( func: (input: T) => Promise<{ data: Array | Record> }>, params: T, + mapFn?: (response: { data: Array | Record> }, done: () => void) => U[], ) => { // Assume all test data fits in single page - const data = (await func(params)).data; + const response = await func(params); + if (mapFn) { + return mapFn(response, () => {}); + } + const data = response.data; // Simulate normalization performed by real impl return Array.isArray(data) ? data : data[Object.keys(data)[0]]; diff --git a/.github/workflows/test/update-labels.test.ts b/.github/workflows/test/update-labels.test.ts index a9cdb8acddba..ce82eeb279d4 100644 --- a/.github/workflows/test/update-labels.test.ts +++ b/.github/workflows/test/update-labels.test.ts @@ -5,14 +5,6 @@ import { fullGitSha } from "../../shared/test/examples.ts"; import updateLabelsSrc, { updateLabelsImpl } from "../src/update-labels.ts"; import { createMockCore, createMockGithub, createMockRequestError } from "./mocks.ts"; -function createLabelMockGithub(currentHeadSha = fullGitSha) { - const github = createMockGithub(); - github.rest.pulls.get.mockResolvedValue({ - data: { state: "open", head: { sha: currentHeadSha } }, - }); - return github; -} - function updateLabels(asyncFunctionArgs: unknown) { return updateLabelsSrc(asyncFunctionArgs as GitHubScriptArgs); } @@ -21,7 +13,7 @@ describe("updateLabels", () => { it("loads inputs from context", async () => { const core = createMockCore(); - const github = createLabelMockGithub(); + const github = createMockGithub(); github.rest.actions.listWorkflowRunArtifacts.mockResolvedValue({ data: { artifacts: [{ name: "label-foo=true" }], @@ -70,7 +62,7 @@ describe("updateLabels", () => { describe("updateLabelsImpl", () => { it("throws if no run_id", async () => { - const github = createLabelMockGithub(); + const github = createMockGithub(); await expect( updateLabelsImpl({ @@ -91,7 +83,7 @@ describe("updateLabelsImpl", () => { }); it("handles missing issue_number", async () => { - const github = createLabelMockGithub(); + const github = createMockGithub(); github.rest.actions.listWorkflowRunArtifacts.mockResolvedValue({ data: { @@ -136,7 +128,7 @@ describe("updateLabelsImpl", () => { }); it("adds and removes labels for artifacts", async () => { - const github = createLabelMockGithub(); + const github = createMockGithub(); github.rest.actions.listWorkflowRunArtifacts.mockResolvedValue({ data: { artifacts: [ @@ -191,7 +183,7 @@ describe("updateLabelsImpl", () => { }); it("throws for invalid label value", async () => { - const github = createLabelMockGithub(); + const github = createMockGithub(); github.rest.actions.listWorkflowRunArtifacts.mockResolvedValue({ data: { artifacts: [ @@ -229,7 +221,7 @@ describe("updateLabelsImpl", () => { }); it("throws for invalid label name", async () => { - const github = createLabelMockGithub(); + const github = createMockGithub(); github.rest.actions.listWorkflowRunArtifacts.mockResolvedValue({ data: { artifacts: [ @@ -265,7 +257,7 @@ describe("updateLabelsImpl", () => { }); it.each([404, 500, 501])("handles error removing label (%s)", async (status) => { - const github = createLabelMockGithub(); + const github = createMockGithub(); github.rest.actions.listWorkflowRunArtifacts.mockResolvedValue({ data: { artifacts: [{ name: "label-foo=false" }], @@ -303,26 +295,4 @@ describe("updateLabelsImpl", () => { name: "foo", }); }); - - it("does not apply labels for a stale pull request head", async () => { - const github = createLabelMockGithub("fedcba9876543210fedcba9876543210fedcba98"); - github.rest.actions.listWorkflowRunArtifacts.mockResolvedValue({ - data: { - artifacts: [{ name: "label-foo=true" }, { name: `head-sha=${fullGitSha}` }], - }, - }); - - await updateLabelsImpl({ - owner: "owner", - repo: "repo", - head_sha: fullGitSha, - issue_number: 123, - run_id: 456, - github, - core: createMockCore(), - }); - - expect(github.rest.issues.addLabels).not.toHaveBeenCalled(); - expect(github.rest.issues.removeLabel).not.toHaveBeenCalled(); - }); }); From 64053a388095380092a9511a61ddba1bcb92e66c Mon Sep 17 00:00:00 2001 From: Akhila Illa Date: Mon, 12 Oct 2026 12:06:27 -0500 Subject: [PATCH 14/16] Update logging --- .../src/arm-auto-signoff/arm-universal-auto-signoff.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/src/arm-auto-signoff/arm-universal-auto-signoff.ts b/.github/workflows/src/arm-auto-signoff/arm-universal-auto-signoff.ts index 3326076f92f7..633a791ab184 100644 --- a/.github/workflows/src/arm-auto-signoff/arm-universal-auto-signoff.ts +++ b/.github/workflows/src/arm-auto-signoff/arm-universal-auto-signoff.ts @@ -254,7 +254,12 @@ async function getDesiredLabelActions({ labelActions[ArmAutoSignoffLabel.ArmManualSignoffRequired] = LabelAction.Add; } } else { - core.info("ARM semantic review has not passed"); + core.info( + `ARM semantic review has not passed: ${semanticReviewOutcome ?? "missing"}` + + (latestSemanticReviewStatus?.description + ? ` (${latestSemanticReviewStatus.description})` + : ""), + ); } labelActions[ArmAutoSignoffLabel.ArmAutoSignedOffTest] = hasAutoSignoff ? LabelAction.Remove From b081a73810e94e7993b2d7a2f00df2afa55ef689 Mon Sep 17 00:00:00 2001 From: Akhila Illa Date: Mon, 12 Oct 2026 15:59:35 -0500 Subject: [PATCH 15/16] Updates based on comments --- .../workflows/arm-universal-auto-signoff.yaml | 2 + .../arm-universal-auto-signoff.ts | 4 +- .../arm-universal-auto-signoff.test.ts | 87 +++++++++++++++++-- 3 files changed, 86 insertions(+), 7 deletions(-) diff --git a/.github/workflows/arm-universal-auto-signoff.yaml b/.github/workflows/arm-universal-auto-signoff.yaml index b63e65c623f4..6c0756b5eb34 100644 --- a/.github/workflows/arm-universal-auto-signoff.yaml +++ b/.github/workflows/arm-universal-auto-signoff.yaml @@ -104,6 +104,7 @@ jobs: # The label updater uses these trusted artifacts to associate this workflow run with its PR. - if: | always() && + github.event_name == 'workflow_run' && fromJson(steps.get-label-action.outputs.result).headSha name: Upload artifact with head SHA uses: ./.github/actions/add-empty-artifact @@ -113,6 +114,7 @@ jobs: - if: | always() && + github.event_name == 'workflow_run' && fromJson(steps.get-label-action.outputs.result).issueNumber > 0 name: Upload artifact with issue number uses: ./.github/actions/add-empty-artifact diff --git a/.github/workflows/src/arm-auto-signoff/arm-universal-auto-signoff.ts b/.github/workflows/src/arm-auto-signoff/arm-universal-auto-signoff.ts index 633a791ab184..b9ae9f14a13b 100644 --- a/.github/workflows/src/arm-auto-signoff/arm-universal-auto-signoff.ts +++ b/.github/workflows/src/arm-auto-signoff/arm-universal-auto-signoff.ts @@ -38,8 +38,8 @@ function createNoneLabelActions(): ManagedLabelActions { * Workflow entry point. Resolves the PR and head SHA from the triggering event (see * `extractInputs`), logs the correlation for diagnosis, and returns the label actions to apply. * - * @returns The correlated head SHA and PR number, which the workflow uploads as artifacts so - * `Update Labels` can re-check the live head, and the label actions to apply. + * @returns The correlated head SHA and PR number, which the workflow uploads as artifacts on + * `workflow_run` triggers so `Update Labels` can find the PR, and the label actions to apply. */ export default async function getLabelAction({ github, context, core }: GitHubScriptArgs): Promise<{ headSha: string; diff --git a/.github/workflows/test/arm-auto-signoff/arm-universal-auto-signoff.test.ts b/.github/workflows/test/arm-auto-signoff/arm-universal-auto-signoff.test.ts index 9c18b20d2b9a..7d44eae90df4 100644 --- a/.github/workflows/test/arm-auto-signoff/arm-universal-auto-signoff.test.ts +++ b/.github/workflows/test/arm-auto-signoff/arm-universal-auto-signoff.test.ts @@ -165,7 +165,7 @@ describe("getLabelActionImpl", () => { expect(workflow).not.toContain("ARM Semantic Review - Set Status"); }); - it("always publishes live-head correlation artifacts", async () => { + it("publishes correlation artifacts only for workflow_run triggers", async () => { const workflow = await readFile( join(GITHUB_ROOT, "workflows", "arm-universal-auto-signoff.yaml"), "utf8", @@ -175,11 +175,11 @@ describe("getLabelActionImpl", () => { expect(headArtifact).toBeGreaterThan(-1); expect(issueArtifact).toBeGreaterThan(headArtifact); - expect(workflow.slice(headArtifact - 180, headArtifact)).not.toContain( - "github.event_name ==", + expect(workflow.slice(headArtifact - 180, headArtifact)).toContain( + "github.event_name == 'workflow_run'", ); - expect(workflow.slice(issueArtifact - 180, issueArtifact)).not.toContain( - "github.event_name ==", + expect(workflow.slice(issueArtifact - 180, issueArtifact)).toContain( + "github.event_name == 'workflow_run'", ); }); }); @@ -256,6 +256,83 @@ describe("getLabelActionImpl", () => { }); }); + it("logs the reason when semantic review is incomplete", async () => { + const loggingCore = createMockCore(); + const github = createMockGithub({ + labelNames: ["ARMReview"], + statuses: [ + { + context: "ARM Semantic Review", + state: CommitStatusState.ERROR, + description: "Review incomplete: tool-failure", + updated_at: "2026-01-01", + }, + ...deterministicStatuses, + ], + }); + + await getLabelActionImpl({ + owner, + repo, + head_sha: headSha, + issue_number: issueNumber, + github, + core: loggingCore, + }); + + expect(loggingCore.info).toHaveBeenCalledWith( + "ARM semantic review has not passed: review-incomplete (Review incomplete: tool-failure)", + ); + }); + + it("logs the reason when semantic review requested changes", async () => { + const loggingCore = createMockCore(); + const github = createMockGithub({ + labelNames: ["ARMReview"], + statuses: [ + { + context: "ARM Semantic Review", + state: CommitStatusState.FAILURE, + description: "Changes requested: 2 Blocking finding(s)", + updated_at: "2026-01-01", + }, + ...deterministicStatuses, + ], + }); + + await getLabelActionImpl({ + owner, + repo, + head_sha: headSha, + issue_number: issueNumber, + github, + core: loggingCore, + }); + + expect(loggingCore.info).toHaveBeenCalledWith( + "ARM semantic review has not passed: changes-requested (Changes requested: 2 Blocking finding(s))", + ); + }); + + it("logs a missing semantic review status", async () => { + const loggingCore = createMockCore(); + const github = createMockGithub({ + labelNames: ["ARMReview"], + statuses: deterministicStatuses, + }); + + await getLabelActionImpl({ + owner, + repo, + head_sha: headSha, + issue_number: issueNumber, + github, + core: loggingCore, + }); + + expect(loggingCore.info).toHaveBeenCalledWith("ARM semantic review has not passed: missing"); + }); + it("logs why an existing manual signoff label produces no new action", async () => { const loggingCore = createMockCore(); const github = createMockGithub({ From daa38ae21f1afdcf922b0b5633cd860458ed10ac Mon Sep 17 00:00:00 2001 From: Akhila Illa Date: Mon, 12 Oct 2026 18:33:29 -0500 Subject: [PATCH 16/16] Delete the skill file to bypass merge blockers --- .github/skills/arm-semantic-review/SKILL.md | 200 ------------------ .../workflows/src/arm-auto-signoff/README.md | 3 - 2 files changed, 203 deletions(-) delete mode 100644 .github/skills/arm-semantic-review/SKILL.md diff --git a/.github/skills/arm-semantic-review/SKILL.md b/.github/skills/arm-semantic-review/SKILL.md deleted file mode 100644 index 497dd5524d1d..000000000000 --- a/.github/skills/arm-semantic-review/SKILL.md +++ /dev/null @@ -1,200 +0,0 @@ ---- -name: arm-semantic-review -description: "Maintain, review, and debug the ARM Semantic Review commit status that connects the ARM API Reviewer (gh-aw) to Universal Auto-Signoff. USE FOR: changes to .github/workflows/arm-api-review.md or its lock file's status plumbing, arm-semantic-review.ts, arm-semantic-review-workflow.ts, arm-universal-auto-signoff.ts, the record_arm_semantic_review safe output, an ARM Semantic Review status that is stuck Pending or wrong, ARMAutoSignedOff-Test or ARMManualSignoffRequired label questions. DO NOT USE FOR: reviewing ARM API specifications themselves (use azure-api-review), TypeSpec authoring, or general GitHub Actions review (use engineering-code-review)." ---- - -# ARM Semantic Review - -The ARM API Reviewer is an AI agent. Auto-signoff must not trust it directly, so its verdict is -turned into one **commit status**, `ARM Semantic Review`, on the reviewed SHA. Universal -Auto-Signoff reads that status alongside LintDiff and Avocado. - -Short overview next to the code: -[README.md](../../workflows/src/arm-auto-signoff/README.md). - -## Mental model - -```text -START pre_activation post status PENDING, upload one head-sha=;issue-number= artifact -MIDDLE agent (read-only AI) review the PR, then request record_arm_semantic_review -END record job (trusted) validate the 4 values, post SUCCESS / FAILURE / ERROR -SAFETY conclusion job if no result was posted, turn this run's PENDING into ERROR -READ Universal Auto-Signoff re-check the PR is open at the evaluated head, read the newest status - on that SHA plus labels -> label action -APPLY Update Labels add or remove the label as emitted -``` - -The model proposes. Trusted code validates and publishes. Any doubt ends as "not Passed". - -## Files - -| File | Role | -| ------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------- | -| [arm-api-review.md](../../workflows/arm-api-review.md) | Source of the reviewer workflow. Edit this, never the lock file. | -| [arm-api-review.lock.yml](../../workflows/arm-api-review.lock.yml) | Generated by `gh aw compile`. Must match the `.md`. | -| [arm-semantic-review.ts](../../workflows/src/arm-auto-signoff/arm-semantic-review.ts) | Pure policy: validate the result, map it to a status, read statuses. No I/O. | -| [arm-semantic-review-workflow.ts](../../workflows/src/arm-auto-signoff/arm-semantic-review-workflow.ts) | I/O: publish the status, finalize an unpublished one. | -| [arm-universal-auto-signoff.ts](../../workflows/src/arm-auto-signoff/arm-universal-auto-signoff.ts) | Reads statuses and labels, decides label actions. | -| [context.ts](../../workflows/src/context.ts) | `extractInputs` and the trusted-artifact parser shared by all consumers. | - -## The status contract - -Context `ARM Semantic Review`, bound to the reviewed SHA. `target_url` is -`.../actions/runs//attempts/`, which identifies the run that wrote it. - -| State | Description starts with | Meaning | Adds `ARMManualSignoffRequired`? | -| --------- | -------------------------- | ----------------------------------------------- | -------------------------------- | -| `pending` | `ARM API semantic review` | Review in progress | No | -| `success` | `Passed:` | Full, complete review, 0 Blocking findings | No | -| `failure` | `Changes requested:` | Full, complete review with Blocking findings | No | -| `error` | `Manual review required: ` | Scoped review, or `specification/` over the cap | **Yes** | -| `error` | `Review incomplete: ` | Failed, cancelled, unreadable, or unvalidated | No | - -Only `success` can lead to signoff. The two `error` outcomes share a state and are told apart by -the description prefix, so keep the prefixes in `arm-semantic-review.ts` as the single source. -`Manual review required` is a property of the PR. `Review incomplete` is transient, so a re-run -can still pass and it must never add the sticky label. - -## Validation, and why each check exists - -Trusted inputs (never from the model): the PR number and head SHA come from the run's correlation -artifact, named `head-sha=;issue-number=` (`;` joins the pairs so one upload carries both), -and the attempt from `GITHUB_RUN_ATTEMPT`. - -| Check | Where | If it fails | -| ----------------------------------------------------------------------------- | -------------------------------------- | ------------------------- | -| Trusted artifacts present, full 40-hex SHA, no conflicting duplicates | `context.ts` | Nothing published / throw | -| Output file exists and is JSON | publisher `try` | `Review incomplete` | -| Exactly one `record_arm_semantic_review` item | `parseSemanticReviewResult` | `Review incomplete` | -| `blocking_count` is a digit string; the other three fields are in closed sets | `parseSemanticReviewResult` | `Review incomplete` | -| A `complete` review must have reason `none` | `parseSemanticReviewResult` | `Review incomplete` | -| PR size within the cap, checked independently of the model's `scope` | `requiresManualReview` (workflow file) | `Manual review required` | -| Incomplete, then scoped, then Blocking, then Passed (first match wins) | `evaluateSemanticReview` | n/a | - -### Verifying the model's scope - -The model reports `scope: full|scoped`, but it is untrusted. For a **full, complete** claim, the -trusted code re-derives scope with the reviewer prompt's rule: more than 50 `specification/` files -or 5,000 `specification/` lines needs a human. The rules are pure helpers in -`arm-semantic-review.ts` (`checkPullRequestSize` and `addSpecificationChanges`); the workflow file's -`getOversizeReason` does the API calls and logging. Any other claim (scoped, incomplete) already -withholds signoff, so the check is skipped. - -1. Zero changed files (GitHub could not compute the diff) or 3,000 or more requires manual review - without listing files. 3,000 is the most files GitHub's file-listing API will return, so a PR - that large cannot be proven fully covered. -2. If the PR's totals are within the cap, its `specification/` subset is too: no file list, so - most PRs cost one `pulls.get` call. -3. Otherwise count only the `specification/` files, and stop paginating as soon as the count is - over the cap. A renamed or moved file counts when either its old or its new path is under - `specification/`. - -The status says why, for example `Manual review required: specification/ changes exceed 50 files -or 5000 lines`. Reasons are fixed strings, never model output. Each step logs what it saw. -Deliberately **not** validated, because it adds code without protecting signoff: - -- the PR being open or still at the reviewed SHA (the status is SHA-bound and Universal re-checks - the live head); -- a "newer run exists" check before publishing (a late same-SHA write is stale but valid for that - SHA); -- re-validating the attempt, PR, or SHA inside the parser (already validated upstream). - -Error text from a failed validation goes to the run log only. The published description is the -fixed string `Review incomplete: result could not be validated`, so runner paths and API errors -never reach a public status. - -## Invariants: do not break these - -1. The model never supplies the SHA, PR number, or attempt. -2. Only `success` authorizes signoff; every unknown, missing, or errored case is "not Passed". -3. `statuses: write` stays in trusted jobs only (`pre_activation`, the record job, `conclusion`). - Never give it to the `agent` job, and Universal stays `statuses: read`. -4. The finalizer publishes only when this exact run and attempt owns the newest `pending` status. -5. `Review incomplete` never adds `ARMManualSignoffRequired`. -6. Do not check out or run PR code in any of these jobs (`pull_request_target`). - -## Behavior in the edge cases - -| Situation | Result | -| ------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------- | -| New commit pushed during a review | An eligible push cancels the run (`cancel-in-progress`). The old SHA's status is never read for the new head. | -| `/arm-review` while a run is in flight | Cancels the old run and starts a new one on the same SHA. | -| Agent `noop`, failure, timeout, detection failure, or cancel | The `conclusion` finalizer turns this run's `pending` into `Review incomplete`. | -| Transient GitHub API error while publishing | `Review incomplete`, no sticky label. A re-run can pass. | -| Critic unavailable | `Review incomplete: critic-unavailable`, labels unchanged. | -| Scoped or oversized PR | `Manual review required`, adds `ARMManualSignoffRequired`. | -| Non-team-member actor | No `pending`, no artifacts, no status. Signoff stays blocked. | -| Event the gate skips (other label, other comment) | The run is skipped. Those events get a private concurrency lane so they cannot cancel a review. | - -## Known limitations - -- Update Labels applies the emitted label action without re-checking the PR head. Universal checks - just before emitting, so a push in the short window between the two can leave the pilot label - stale until the next evaluation. A stale-head guard in Update Labels is a separate, cross-cutting - change (it affects every producer that publishes a `head-sha=` artifact). -- A same-SHA re-run posts `pending` but does not revoke an existing `ARMAutoSignedOff-Test` - label until Universal next runs. This matters only once enforcement is on. -- Universal's label filter does not include `ARMChangesRequested` or `WaitForARMFeedback`, so - adding them does not trigger an immediate re-evaluation. -- The newest status named `ARM Semantic Review` wins; its creator is not verified. -- If a re-run resolves a different head SHA than the first attempt, two `head-sha=` artifacts can - conflict and the parser throws, which can leave `Pending`. -- `blocking_count` is the model's verdict. The Critic and threat detection are the only checks on - it, which is an accepted trade-off. -- A `Pending` status is resolved in almost every failure, but can still remain when the - `conclusion` job does not run, the finalizer's own API calls still fail after their retries, - the runner or GitHub is lost mid-run, or the finalizer cannot read the head SHA (the - conflicting-artifacts case above). Nothing signs off in any of these cases, because only - `success` does. -- A push to a draft PR, or to one without `WaitForARMFeedback`, joins the PR concurrency lane - before the gate skips it, so it can cancel a review that is already running. - -## Changing the reviewer workflow - -1. Edit `arm-api-review.md`, then regenerate the lock file with the same gh-aw version recorded in - the lock file header (`gh aw compile arm-api-review`). Confirm `git diff` shows only intended - changes. The compiler may also prune unrelated entries in `.github/aw/actions-lock.json`; - revert those. -2. A custom job cannot run after the agent: gh-aw rejects it as a dependency cycle. Post-agent - work goes in the built-in `conclusion` job via `jobs.conclusion.pre-steps`. -3. `pre_activation` is the only place that can post `pending` before the agent starts. -4. Keep the concurrency expression and the `if:` trigger gate consistent. -5. The size cap exists in the prompt (Trigger Validation) and in `arm-semantic-review.ts`. Change - both together. - -## Verify a change - -```bash -cd .github -npx vitest run workflows/test/arm-auto-signoff workflows/test/arm-api-review-workflow.test.ts workflows/test/context.test.ts -npx tsc --noEmit -npx oxlint workflows -npx oxfmt --check workflows -``` - -The tests assert the compiled lock file (job order, permissions, the finalizer condition), so a -stale lock file fails them. - -Live check on a test PR (ideally in `azure-rest-api-specs-pr`), then compare to the tables above: - -```bash -gh api repos/Azure/azure-rest-api-specs/commits//statuses \ - --jq '.[] | select(.context=="ARM Semantic Review") | [.created_at,.state,.description,.target_url]' -``` - -The `pending` status and the `head-sha=;issue-number=` artifact appear on the run created -by `pre_activation`. Try: a clean review, `skip-arm-review`, a push mid-run, two `/arm-review` -comments in a row, and cancelling a run. - -## Debug a status that is stuck or wrong - -1. List the statuses for the head SHA (command above). Which `target_url` is newest, and is it - the run you expect? -2. Open that run. Did `record_arm_semantic_review` run? It needs the agent and detection to - succeed and an emitted `record_arm_semantic_review` item. -3. If it did not run, check whether the `conclusion` finalizer step ran. It is skipped when the - record job succeeded, and it acts only on a `pending` this exact run owns. -4. If the description is `Review incomplete: result could not be validated`, read the warning in - the record job's log for the real reason. -5. If Universal did not act, check its run log: it prints the event, correlation, labels, and the - status it read. A "state/head mismatch" means the PR head moved on. diff --git a/.github/workflows/src/arm-auto-signoff/README.md b/.github/workflows/src/arm-auto-signoff/README.md index 024c337e2a61..aa1ee9b7d1b3 100644 --- a/.github/workflows/src/arm-auto-signoff/README.md +++ b/.github/workflows/src/arm-auto-signoff/README.md @@ -55,6 +55,3 @@ no longer the PR head is never read for the new head. | [arm-universal-auto-signoff.ts](./arm-universal-auto-signoff.ts) | Read statuses and labels, decide label actions | | [arm-api-review.md](../../arm-api-review.md) | Reviewer workflow source (the `.lock.yml` is generated from it) | | [arm-universal-auto-signoff.yaml](../../arm-universal-auto-signoff.yaml) | Universal workflow | - -The design rationale, validation rules, invariants, edge cases, and debugging steps are in the -[arm-semantic-review skill](../../../skills/arm-semantic-review/SKILL.md).