Periodic: Cua Sandbox Live Fleet E2E #690
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: "Periodic: Cua Sandbox Live Fleet E2E" | |
| on: | |
| schedule: | |
| - cron: "7/15 * * * *" | |
| push: | |
| branches: [main] | |
| paths: | |
| - "libs/python/cua-sandbox/**" | |
| - "libs/python/cua-fleet/**" | |
| - ".github/workflows/periodic-cua-sandbox-live.yml" | |
| - ".github/scripts/tests/test_periodic_cua_sandbox_live.py" | |
| workflow_dispatch: | |
| inputs: | |
| lane: | |
| description: "Lane to run" | |
| required: true | |
| default: both | |
| type: choice | |
| options: [both, main-source, published-package] | |
| suite: | |
| description: "Suite to run" | |
| required: true | |
| default: both | |
| type: choice | |
| options: [both, ephemeral, pool] | |
| force_failure: | |
| description: "Fail after setup to certify alerting" | |
| required: true | |
| default: false | |
| type: boolean | |
| permissions: | |
| contents: read | |
| jobs: | |
| prepare: | |
| if: github.repository == 'trycua/cua' | |
| runs-on: ubuntu-latest | |
| outputs: | |
| matrix: ${{ steps.matrix.outputs.matrix }} | |
| steps: | |
| - id: matrix | |
| shell: bash | |
| env: | |
| EVENT_NAME: ${{ github.event_name }} | |
| REQUESTED_LANE: ${{ inputs.lane }} | |
| REQUESTED_SUITE: ${{ inputs.suite }} | |
| run: | | |
| set -euo pipefail | |
| if [[ "$EVENT_NAME" == "push" ]]; then | |
| matrix='{"include":[{"lane":"main-source","suite":"ephemeral"}]}' | |
| else | |
| lanes=("main-source" "published-package") | |
| if [[ "$EVENT_NAME" == "workflow_dispatch" && "$REQUESTED_LANE" != "both" ]]; then | |
| lanes=("$REQUESTED_LANE") | |
| fi | |
| suites=("ephemeral" "pool") | |
| if [[ "$EVENT_NAME" == "workflow_dispatch" && "${REQUESTED_SUITE:-both}" != "both" ]]; then | |
| suites=("$REQUESTED_SUITE") | |
| fi | |
| include="" | |
| for lane in "${lanes[@]}"; do | |
| for suite in "${suites[@]}"; do | |
| include+="{\"lane\":\"$lane\",\"suite\":\"$suite\"}," | |
| done | |
| done | |
| matrix="{\"include\":[${include%,}]}" | |
| fi | |
| echo "matrix=$matrix" >> "$GITHUB_OUTPUT" | |
| live: | |
| if: github.repository == 'trycua/cua' | |
| needs: prepare | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 25 | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.prepare.outputs.matrix) }} | |
| concurrency: | |
| group: periodic-cua-sandbox-live-${{ github.event_name }}-${{ matrix.lane }}-${{ matrix.suite }} | |
| cancel-in-progress: ${{ github.event_name == 'schedule' }} | |
| env: | |
| CUA_FLEET_BASE_URL: https://run.cua.ai | |
| CUA_LIVE_E2E_LANE: ${{ matrix.lane }} | |
| CUA_LIVE_E2E_SUITE: ${{ matrix.suite }} | |
| CUA_LIVE_E2E_EVENT: ${{ github.event_name }} | |
| CUA_LIVE_E2E_NAMESPACE: cua-live-${{ matrix.lane }}-${{ github.event_name == 'workflow_dispatch' && github.run_id || github.event_name }} | |
| CUA_LIVE_E2E_POOL_WARM_NAMESPACE: cua-live-pool-warm-${{ matrix.lane }}-${{ github.event_name == 'workflow_dispatch' && 'manual' || github.event_name }} | |
| CUA_LIVE_E2E_POOL_COLD_NAMESPACE: cua-live-pool-cold-${{ matrix.lane }}-${{ github.event_name == 'workflow_dispatch' && 'manual' || github.event_name }} | |
| CUA_LIVE_E2E_ARTIFACT_DIR: /tmp/cua-live-e2e | |
| CUA_TELEMETRY_ENABLED: "false" | |
| CUA_LIVE_E2E_SIGNED_URLS: "true" | |
| steps: | |
| - name: Check Fleet OAuth credentials | |
| env: | |
| CUA_CLIENT_ID: ${{ secrets.CUA_CLIENT_ID }} | |
| CUA_CLIENT_SECRET: ${{ secrets.CUA_CLIENT_SECRET }} | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| if [[ -z "$CUA_CLIENT_ID" || -z "$CUA_CLIENT_SECRET" ]]; then | |
| echo "CUA_CLIENT_ID and CUA_CLIENT_SECRET must both be configured." | |
| exit 1 | |
| fi | |
| - name: Checkout main | |
| uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 | |
| with: | |
| ref: ${{ github.event_name == 'push' && github.sha || github.ref }} | |
| - name: Record checked out source SHA | |
| id: source_sha | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| source_sha="$(git rev-parse HEAD)" | |
| echo "source_sha=$source_sha" >> "$GITHUB_OUTPUT" | |
| echo "CUA_LIVE_E2E_SOURCE_SHA=$source_sha" >> "$GITHUB_ENV" | |
| - name: Set up Python | |
| uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 | |
| with: | |
| python-version: "3.12" | |
| - name: Install uv | |
| run: python -m pip install uv | |
| - name: Install main source | |
| if: matrix.lane == 'main-source' | |
| run: | | |
| uv pip install --system \ | |
| --index https://wheels.cua.ai/simple \ | |
| --default-index https://pypi.org/simple \ | |
| -e libs/python/cua-sandbox pytest pytest-asyncio | |
| - name: Install published package | |
| if: matrix.lane == 'published-package' | |
| run: | | |
| uv pip install --system \ | |
| --index https://wheels.cua.ai/simple \ | |
| --default-index https://pypi.org/simple \ | |
| --upgrade cua-sandbox pytest pytest-asyncio | |
| - name: Prepare isolated live test suite | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| suite_root="$(mktemp -d /tmp/cua-live-e2e-suite.XXXXXX)" | |
| mkdir -p "$suite_root/tests/live" | |
| cp libs/python/cua-sandbox/tests/__init__.py "$suite_root/tests/" | |
| cp libs/python/cua-sandbox/tests/live/*.py "$suite_root/tests/live/" | |
| echo "CUA_LIVE_E2E_TEST_ROOT=$suite_root" >> "$GITHUB_ENV" | |
| - name: Record installed versions | |
| id: versions | |
| shell: bash | |
| run: | | |
| set -o pipefail | |
| python - <<'PY' | tee -a "$GITHUB_OUTPUT" | |
| from importlib.metadata import version | |
| print("sandbox=" + version("cua-sandbox")) | |
| print("fleet=" + version("cua-fleet")) | |
| PY | |
| - name: Write controlled failure diagnostics | |
| if: github.event_name == 'workflow_dispatch' && inputs.force_failure | |
| run: | | |
| python - <<'PY' | |
| import json | |
| import os | |
| from pathlib import Path | |
| artifact_dir = Path(os.environ["CUA_LIVE_E2E_ARTIFACT_DIR"]) | |
| artifact_dir.mkdir(parents=True, exist_ok=True) | |
| (artifact_dir / "summary.json").write_text( | |
| json.dumps( | |
| { | |
| "lane": os.environ["CUA_LIVE_E2E_LANE"], | |
| "suite": os.environ["CUA_LIVE_E2E_SUITE"], | |
| "namespace": os.environ["CUA_LIVE_E2E_NAMESPACE"], | |
| "source_sha": os.environ["CUA_LIVE_E2E_SOURCE_SHA"], | |
| "error": {"type": "ControlledFailure"}, | |
| }, | |
| indent=2, | |
| sort_keys=True, | |
| ) | |
| + "\n" | |
| ) | |
| PY | |
| - name: Controlled alert test failure | |
| if: github.event_name == 'workflow_dispatch' && inputs.force_failure | |
| run: exit 1 | |
| - name: Run live Fleet smoke | |
| if: ${{ matrix.suite == 'ephemeral' && !(github.event_name == 'workflow_dispatch' && inputs.force_failure) }} | |
| env: | |
| CUA_CLIENT_ID: ${{ secrets.CUA_CLIENT_ID }} | |
| CUA_CLIENT_SECRET: ${{ secrets.CUA_CLIENT_SECRET }} | |
| run: | | |
| PYTHONPATH="$CUA_LIVE_E2E_TEST_ROOT" python -m pytest -q -s \ | |
| "$CUA_LIVE_E2E_TEST_ROOT/tests/live/test_fleet_ephemeral.py" | |
| - name: Run live Fleet pool smoke | |
| if: ${{ matrix.suite == 'pool' && !(github.event_name == 'workflow_dispatch' && inputs.force_failure) }} | |
| env: | |
| CUA_CLIENT_ID: ${{ secrets.CUA_CLIENT_ID }} | |
| CUA_CLIENT_SECRET: ${{ secrets.CUA_CLIENT_SECRET }} | |
| run: | | |
| PYTHONPATH="$CUA_LIVE_E2E_TEST_ROOT" python -m pytest -q -s \ | |
| "$CUA_LIVE_E2E_TEST_ROOT/tests/live/test_fleet_pool_persistent.py" | |
| - name: Upload failure diagnostics | |
| if: failure() | |
| uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4 | |
| with: | |
| name: cua-sandbox-live-${{ matrix.lane }}-${{ matrix.suite }}-${{ github.run_id }}-${{ github.run_attempt }} | |
| path: /tmp/cua-live-e2e | |
| if-no-files-found: warn | |
| retention-days: 7 | |
| - name: Alert Alertmanager | |
| if: failure() | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| curl --silent --show-error --fail \ | |
| -X POST https://am.cua.ai/api/v2/alerts \ | |
| -H 'Content-Type: application/json' \ | |
| -d '[{ | |
| "labels": { | |
| "alertname": "PeriodicCuaSandboxLiveE2EFailed", | |
| "severity": "critical", | |
| "service": "cua-sandbox", | |
| "job": "periodic-cua-sandbox-live", | |
| "lane": "${{ matrix.lane }}", | |
| "suite": "${{ matrix.suite }}" | |
| }, | |
| "annotations": { | |
| "summary": "Cua Sandbox live Fleet E2E failed (${{ matrix.lane }}/${{ matrix.suite }})", | |
| "description": "Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}", | |
| "source_sha": "${{ steps.source_sha.outputs.source_sha }}", | |
| "package_version": "${{ steps.versions.outputs.sandbox }}", | |
| "image_digest": "sha256:5b9cb82f482834f7541901b87be956e7544d0db13fabc0b372cbc5eca5a74180" | |
| }, | |
| "generatorURL": "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" | |
| }]' |