ci-integration.yml

# Tier 2 integration suite - runs only when a PR is added to the merge queue.
#
# Design (microsoft/apm#770):
#   - Tier 1 (ci.yml) runs unit tests + binary build on every PR push and on
#     every merge_group event. No secrets needed there.
#   - Tier 2 (this workflow) runs smoke + integration + release-validation
#     against the tentative merge commit that the merge queue creates.
#
# Required-check satisfaction at PR time:
#   - Branch protection requires only `gate` (from `merge-gate.yml`), which
#     aggregates all PR-time signals. The four check names produced by THIS
#     workflow (Build/Smoke/Integration/Release Validation - all Linux) are
#     not required at PR time; they only run on the `gh-readonly-queue/main/*`
#     SHA the merge queue creates, against the tentative merge commit.
#   - This workflow intentionally does NOT trigger on pull_request events.
#     Doing so would let a fork PR check out and run code with the secrets
#     declared below, which is a supply-chain attack vector. Keep this file
#     `merge_group`-only forever.
#
# Trust model:
#   - merge_group only fires when a user with write access enqueues a PR.
#   - The gh-readonly-queue/main/* ref contains the PR's code merged into
#     main and is created by GitHub, not by the contributor.
#   - Trust boundary = write-access grant, managed in repo settings.
#     Write access is granted only to vetted contributors.
#   - Fork PRs without write access never reach this workflow and never see
#     CI secrets.
#
# Binary is built inline rather than downloaded from ci.yml so this workflow
# tests exactly the merge-queue tentative merge SHA without cross-workflow
# artifact plumbing.
name: Integration Tests

env:
  PYTHON_VERSION: '3.12'

on:
  merge_group:
    branches: [ main ]
    types: [ checks_requested ]

permissions:
  contents: read

jobs:
  build:
    name: Build (Linux)
    runs-on: ubuntu-24.04
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

      - name: Set up Python
        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
        with:
          python-version: ${{ env.PYTHON_VERSION }}

      - name: Install uv
        uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
        with:
          enable-cache: true

      - name: Install dependencies
        run: uv sync --frozen --extra dev --extra build

      - name: Install UPX
        run: |
          sudo apt-get update
          sudo apt-get install -y upx-ucl

      - name: Build binary
        run: |
          chmod +x scripts/build-binary.sh
          uv run ./scripts/build-binary.sh

      - name: Upload binary
        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
        with:
          name: apm-mq-linux-x86_64
          path: |
            ./dist/apm-linux-x86_64
            ./dist/apm-linux-x86_64.sha256
          include-hidden-files: true
          retention-days: 7
          if-no-files-found: error

  smoke-test:
    name: Smoke Test (Linux)
    needs: [build]
    runs-on: ubuntu-24.04
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

      # Artifact root is ./dist/ (least common ancestor of the uploaded paths
      # in the build job), so download to ./dist/ to preserve the
      # dist/apm-linux-x86_64/apm layout the conftest expects.
      - name: Download binary
        uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
        with:
          name: apm-mq-linux-x86_64
          path: ./dist/

      - name: Make binary executable
        run: chmod +x ./dist/apm-linux-x86_64/apm

      - name: Set up Python
        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
        with:
          python-version: ${{ env.PYTHON_VERSION }}

      - name: Install uv
        uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
        with:
          enable-cache: true

      - name: Install dependencies
        run: uv sync --frozen --extra dev

      # Smoke target is test_core_smoke.py: a hermetic, network-free pre-flight
      # over the README's three promises (portable / secure / governed). It
      # runs against the built binary in ~10s and fails the merge queue early
      # if the bundle is broken before the 30-min integration suite even
      # starts. The legacy test_runtime_smoke.py covers the experimental
      # `apm run` execution layer and stays in the heavy integration job
      # under requires_runtime_* markers.
      - name: Run core smoke tests
        env:
          APM_E2E_TESTS: "1"
          APM_BINARY_PATH: ${{ github.workspace }}/dist/apm-linux-x86_64/apm
        run: uv run pytest tests/integration/test_core_smoke.py -v

  scaling-guards:
    name: Scaling Guards (Linux)
    runs-on: ubuntu-24.04
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

      - name: Set up Python
        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
        with:
          python-version: ${{ env.PYTHON_VERSION }}

      - name: Install uv
        uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
        with:
          enable-cache: true

      - name: Install dependencies
        run: uv sync --frozen --extra dev

      # Scaling guards are hermetic, network-free tests that catch
      # O(n^2) algorithmic regressions by comparing wall-clock ratios
      # at two input sizes. They run in ~5s and need no binary or
      # secrets -- parallel with the build job for zero added latency.
      - name: Run scaling guards
        run: uv run pytest tests/benchmarks/test_scaling_guards.py -v --tb=short

  integration-tests-shard:
    name: Integration Tests Shard ${{ matrix.shard }} (Linux)
    needs: [build, smoke-test]
    runs-on: ubuntu-24.04
    strategy:
      fail-fast: false
      matrix:
        # 4-way sharding via pytest-split keeps wall-clock low. Each shard
        # is an independent runner so HOME-mutating tests never race
        # across shards. xdist within a shard gives an additional 2x; the
        # 4 HOME-mutating files carry an xdist_group("home_env") marker
        # so xdist serializes them on a single worker (race-safe).
        shard: [1, 2, 3, 4]
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

      # Artifact root is ./dist/ (least common ancestor of the uploaded paths in
      # the build job), so download to ./dist/ to preserve dist/apm-linux-x86_64/apm.
      - name: Download binary
        uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
        with:
          name: apm-mq-linux-x86_64
          path: ./dist/

      - name: Make binary executable
        run: chmod +x ./dist/apm-linux-x86_64/apm

      - name: Set up Node.js
        uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
        with:
          node-version: '24'

      - name: Set up Python
        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
        with:
          python-version: ${{ env.PYTHON_VERSION }}

      - name: Install uv
        uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
        with:
          enable-cache: true

      - name: Install Unix shell startup test runtimes
        run: |
          sudo apt-get update
          sudo apt-get install -y fish zsh

      - name: Install test dependencies
        run: uv sync --frozen --extra dev

      # APM resolves its cache dir to ~/.cache/apm by default on Linux
      # (XDG_CACHE_HOME unset -> $HOME/.cache/apm). actions/cache restores
      # straight into that path, so the apm binary picks it up natively
      # with no symlink or env override required.
      - name: Compute cache week bucket
        id: cache-bucket
        run: echo "week=$(date -u +'%Y-%U')" >> "$GITHUB_OUTPUT"

      - name: Cache APM resolved git/HTTP fetches
        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
        with:
          path: ~/.cache/apm
          key: apm-cache-shard${{ matrix.shard }}-${{ steps.cache-bucket.outputs.week }}-${{ hashFiles('uv.lock') }}
          restore-keys: |
            apm-cache-shard${{ matrix.shard }}-${{ steps.cache-bucket.outputs.week }}-
            apm-cache-shard${{ matrix.shard }}-

      - name: Run integration tests (sharded + parallelized)
        env:
          APM_E2E_TESTS: "1"
          APM_RUN_INTEGRATION_TESTS: "1"
          GITHUB_APM_PAT: ${{ secrets.GH_CLI_PAT }}
          ADO_APM_PAT: ${{ secrets.ADO_APM_PAT }}
          # Sharded + parallel pytest invocation passed through to
          # scripts/test-integration.sh, which still owns runtime setup
          # (apm runtime setup copilot/codex/llm) and token resolution.
          # --dist loadgroup is required: it is the only xdist scheduler
          # that honors pytest.mark.xdist_group, which we use to keep
          # HOME-mutating tests serialized on a single worker.
          PYTEST_EXTRA_ARGS: "--splits 4 --group ${{ matrix.shard }} -n 2 --dist loadgroup --maxfail 10 --cov --cov-fail-under=0"
        run: |
          chmod +x scripts/test-integration.sh
          uv run ./scripts/test-integration.sh
        # Keep an outer cap above the runtime setup and sharded integration
        # suite so runner variance does not turn into an unbounded job.
        timeout-minutes: 20

      - name: Rename coverage data for shard
        if: always()
        run: |
          if [ -f .coverage ]; then
            mv .coverage .coverage.shard-${{ matrix.shard }}
          fi

      - name: Upload coverage data
        if: always()
        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
        with:
          name: integration-coverage-shard-${{ matrix.shard }}
          path: .coverage.shard-${{ matrix.shard }}
          include-hidden-files: true
          retention-days: 7
          if-no-files-found: ignore

  # Fan-in job that preserves the gate-required check name
  # "Integration Tests (Linux)". Branch protection / merge-gate.yml
  # require this exact name; shard jobs above are not required directly.
  integration-tests:
    name: Integration Tests (Linux)
    needs: [integration-tests-shard, scaling-guards]
    if: always()
    runs-on: ubuntu-24.04
    steps:
      # Coverage summary runs first (with if: always()) so we get
      # observability even when some shards fail. The shard-result
      # gate runs last so it can still fail the job.
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

      - name: Set up Python
        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
        with:
          python-version: ${{ env.PYTHON_VERSION }}

      - name: Install coverage
        run: pip install "coverage[toml]"

      - name: Download shard coverage
        if: always()
        uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
        with:
          pattern: integration-coverage-shard-*
          path: coverage-shards/
          merge-multiple: true
        continue-on-error: true

      - name: Combine and summarise coverage
        if: always()
        run: |
          if find coverage-shards -type f -name '.coverage*' 2>/dev/null | grep -q .; then
            coverage combine coverage-shards/
            coverage json -o coverage.json
            python3 scripts/coverage-summary.py coverage.json --title "Integration Test Coverage"
          else
            echo "No coverage shard files found; skipping integration coverage summary."
          fi
        continue-on-error: true

      # Integration coverage gate -- raise this floor when baseline improves.
      # Current floor: 55% (actual ~71%, 16pt margin for community PRs).
      - name: Enforce integration coverage gate
        if: always()
        run: |
          if [ -f .coverage ]; then
            coverage report --fail-under=70
          else
            echo "ERROR: No combined coverage data -- the 'Combine and summarise coverage' step must produce .coverage before the gate can run."
            exit 1
          fi

      - name: Aggregate shard results
        run: |
          if [[ "${{ needs.integration-tests-shard.result }}" != "success" ]]; then
            echo "One or more integration test shards failed: ${{ needs.integration-tests-shard.result }}"
            exit 1
          fi
          echo "All integration test shards passed."

      - name: Aggregate scaling-guards result
        run: |
          if [[ "${{ needs.scaling-guards.result }}" != "success" ]]; then
            echo "Scaling guards failed: ${{ needs.scaling-guards.result }}"
            exit 1
          fi
          echo "Scaling guards passed."

  release-validation:
    name: Release Validation (Linux)
    needs: [build, integration-tests]
    runs-on: ubuntu-24.04
    steps:
      - name: Checkout test scripts
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          sparse-checkout: scripts
          path: repo

      - name: Set up Node.js
        uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
        with:
          node-version: '24'

      - name: Set up Python
        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
        with:
          python-version: ${{ env.PYTHON_VERSION }}

      # See note in integration-tests: download to dist/ to preserve layout.
      - name: Download binary
        uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
        with:
          name: apm-mq-linux-x86_64
          path: /tmp/apm-isolated-test/dist/

      - name: Make binary executable
        run: chmod +x /tmp/apm-isolated-test/dist/apm-linux-x86_64/apm

      - name: Create APM symlink for testing
        run: |
          cd /tmp/apm-isolated-test
          ln -s "$(pwd)/dist/apm-linux-x86_64/apm" "$(pwd)/apm"
          echo "/tmp/apm-isolated-test" >> $GITHUB_PATH

      - name: Prepare test scripts
        run: |
          cp -r repo/scripts /tmp/apm-isolated-test/scripts

      - name: Run release validation tests
        env:
          APM_E2E_TESTS: "1"
          GITHUB_APM_PAT: ${{ secrets.GH_CLI_PAT }}
          ADO_APM_PAT: ${{ secrets.ADO_APM_PAT }}
        run: |
          cd /tmp/apm-isolated-test
          chmod +x scripts/test-release-validation.sh
          ./scripts/test-release-validation.sh
        timeout-minutes: 20