release-constraints.yml

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#
# Resolves the constraints that a release ships, rather than tagging whatever the
# `constraints-X-Y` branch happened to hold when the release ran.
#
# The stage is derived from the version, so the two cannot be mismatched by hand:
#
#   * a pre-release - beta (`3.4.0b1`) or candidate (`3.3.1rc1`) - pins the providers at the versions
#     PyPI holds (rc provider waves included) and lands on a branch of its own, so it never moves the
#     branch every other build reads. Re-running it for the same pre-release replaces that branch and
#     its tag, so the two always describe the run that produced them;
#   * a final (`3.3.1`) pins the providers at their released versions, ignoring any candidate, and
#     commits onto `constraints-X-Y` itself, which is what makes the released constraints the
#     baseline everything downstream reads.
#
# A final therefore cannot be produced by retagging a pre-release: a released version must never
# pin an rc.
---
name: Release constraints
on:  # yamllint disable-line rule:truthy
  workflow_dispatch:
    inputs:
      version:
        description: "Version the constraints belong to, e.g. 3.4.0b1, 3.3.1rc1 or 3.3.1"
        required: true
        type: string
      ref:
        description: "Ref the constraints are resolved from, e.g. v3-3-stable or the release tag"
        required: true
        type: string
permissions:
  contents: read
concurrency:
  group: release-constraints-${{ inputs.version }}
  cancel-in-progress: false
jobs:
  build-info:
    timeout-minutes: 10
    name: "Build info"
    runs-on: ["ubuntu-22.04"]
    if: contains(fromJSON('[
      "ashb",
      "eladkal",
      "ephraimbuddy",
      "jedcunningham",
      "kaxil",
      "pierrejeambrun",
      "potiuk",
      "utkarsharma2",
      "vatsrahul1001",
      "vincbeck",
      ]'), github.event.sender.login)
    outputs:
      python-versions: ${{ steps.selective-checks.outputs.python-versions }}
      python-versions-list-as-string: ${{ steps.selective-checks.outputs.python-versions-list-as-string }}
      default-branch: ${{ steps.selective-checks.outputs.default-branch }}
      default-constraints-branch: ${{ steps.selective-checks.outputs.default-constraints-branch }}
      constraints-branch: ${{ steps.stage.outputs.constraints-branch }}
      target-branch: ${{ steps.stage.outputs.target-branch }}
      allow-pre-releases: ${{ steps.stage.outputs.allow-pre-releases }}
    steps:
      - name: "Cleanup repo"
        shell: bash
        run: sudo rm -rf ${GITHUB_WORKSPACE}/*
      - name: "Checkout ${{ inputs.ref }}"
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
        with:
          ref: ${{ inputs.ref }}
          fetch-depth: 2
          persist-credentials: false
      - name: "Install Breeze"
        uses: ./.github/actions/breeze
        id: breeze
      - name: "Save github context to file"
        # See ci-amd.yml for the full rationale: avoids ARG_MAX on big PRs by writing the
        # github context to a file, and the single-quoted heredoc makes the zizmor
        # template-injection finding a false positive (no bash expansion happens inside it).
        shell: bash
        run: |  # zizmor: ignore[template-injection]
          cat > "${RUNNER_TEMP}/github_context.json" << '__GITHUB_CONTEXT_END__'
          ${{ toJson(github) }}
          __GITHUB_CONTEXT_END__
      - name: Selective checks
        id: selective-checks
        env:
          PR_LABELS: "[]"
          COMMIT_REF: "${{ inputs.ref }}"
          VERBOSE: "false"
          GITHUB_CONTEXT_INPUT: "${{ runner.temp }}/github_context.json"
        run: breeze ci selective-check 2>> ${GITHUB_OUTPUT}
      - name: "Derive the release stage from the version"
        id: stage
        shell: bash
        env:
          VERSION: ${{ inputs.version }}
        run: |
          if [[ ! "${VERSION}" =~ ^[0-9]+\.[0-9]+\.[0-9]+((b|rc)[0-9]+)?$ ]]; then
            echo "'${VERSION}' is not a release version - expected X.Y.Z, X.Y.ZbN or X.Y.ZrcN." >&2
            exit 1
          fi
          major="$(echo "${VERSION}" | cut -d. -f1)"
          minor="$(echo "${VERSION}" | cut -d. -f2)"
          constraints_branch="constraints-${major}-${minor}"
          echo "constraints-branch=${constraints_branch}" >> "${GITHUB_OUTPUT}"
          # A pre-release - beta or rc - pins the providers at the versions PyPI holds (rc provider
          # waves included) and lands on a branch and tag of its own, so it never moves the shared
          # constraints-X-Y branch. A final pins released providers and commits onto the shared branch.
          if [[ "${VERSION}" =~ (b|rc)[0-9]+$ ]]; then
            echo "allow-pre-releases=true" >> "${GITHUB_OUTPUT}"
            echo "target-branch=constraints-${VERSION}" >> "${GITHUB_OUTPUT}"
          else
            echo "allow-pre-releases=false" >> "${GITHUB_OUTPUT}"
            echo "target-branch=${constraints_branch}" >> "${GITHUB_OUTPUT}"
          fi
      - name: "Parameters summary"
        shell: bash
        env:
          VERSION: ${{ inputs.version }}
          REF: ${{ inputs.ref }}
          CONSTRAINTS_BRANCH: ${{ steps.stage.outputs.constraints-branch }}
          TARGET_BRANCH: ${{ steps.stage.outputs.target-branch }}
          ALLOW_PRE_RELEASES: ${{ steps.stage.outputs.allow-pre-releases }}
        run: |
          {
            echo "## Release constraints"
            echo ""
            echo "| Parameter | Value |"
            echo "|---|---|"
            echo "| Version | \`${VERSION}\` |"
            echo "| Resolved from ref | \`${REF}\` |"
            echo "| Branched off | \`${CONSTRAINTS_BRANCH}\` |"
            echo "| Committed to | \`${TARGET_BRANCH}\` |"
            echo "| Pre-releases allowed | \`${ALLOW_PRE_RELEASES}\` |"
            echo "| Tagged | \`constraints-${VERSION}\` |"
          } | tee -a "${GITHUB_STEP_SUMMARY}"

  build-ci-images:
    name: "Build CI images"
    needs: [build-info]
    uses: ./.github/workflows/ci-image-build.yml
    permissions:
      contents: read
      packages: write
    with:
      runners: '["ubuntu-22.04"]'
      platform: "linux/amd64"
      push-image: "false"
      upload-image-artifact: "true"
      upload-mount-cache-artifact: "false"
      python-versions: ${{ needs.build-info.outputs.python-versions }}
      branch: ${{ needs.build-info.outputs.default-branch }}
      constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }}
      checkout-ref: ${{ inputs.ref }}
      use-uv: "true"
      upgrade-to-newer-dependencies: "false"
      docker-cache: "registry"
      disable-airflow-repo-cache: "false"

  generate-constraints:
    name: "Generate constraints"
    needs: [build-info, build-ci-images]
    uses: ./.github/workflows/generate-constraints.yml
    with:
      runners: '["ubuntu-22.04"]'
      platform: "linux/amd64"
      python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }}
      python-versions: ${{ needs.build-info.outputs.python-versions }}
      generate-pypi-constraints: "true"
      # Only the PyPI constraints are what a release ships; the other modes serve CI, and
      # regenerating them here would move them for reasons unrelated to the release.
      generate-no-providers-constraints: "false"
      # A release pins the providers as published on PyPI, so nothing is built from the sources
      # here: a locally built provider wheel would answer the resolution and then be left out of
      # the constraints, which is exactly the version the release needed pinned.
      pypi-providers-only: "true"
      allow-pre-releases: ${{ needs.build-info.outputs.allow-pre-releases }}
      debug-resources: "false"
      checkout-ref: ${{ inputs.ref }}
      use-uv: "true"
    secrets:
      SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}

  publish-constraints:
    runs-on: ["ubuntu-22.04"]
    timeout-minutes: 20
    name: "Publish and tag constraints"
    needs: [build-info, generate-constraints]
    permissions:
      contents: write
      packages: read
    env:
      VERSION: ${{ inputs.version }}
      CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.constraints-branch }}
      TARGET_BRANCH: ${{ needs.build-info.outputs.target-branch }}
      ALLOW_PRE_RELEASES: ${{ needs.build-info.outputs.allow-pre-releases }}
    steps:
      - name: "Cleanup repo"
        shell: bash
        run: sudo rm -rf ${GITHUB_WORKSPACE}/*
      - name: "Checkout ${{ inputs.ref }}"
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
        with:
          ref: ${{ inputs.ref }}
          persist-credentials: false
      # Branched off the shared constraints branch in both cases; what differs is where the commit
      # ends up, which the checkout below decides.
      - name: "Checkout ${{ needs.build-info.outputs.constraints-branch }}"
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
        with:
          path: "constraints"
          ref: ${{ needs.build-info.outputs.constraints-branch }}
          persist-credentials: true
          fetch-depth: 0
      - name: "Download constraints from the generate-constraints job"
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c  # v8.0.1
        with:
          pattern: constraints-*
          path: ./files
      # A candidate's branch and tag belong to that candidate alone, so re-running for the same
      # rc replaces them rather than adding to them: the branch would otherwise already hold the
      # previous run's constraints (making the push a non-fast-forward) and the tag already exist.
      # A final never gets this - it commits onto the shared constraints-X-Y branch, whose history
      # every other build reads.
      - name: "Delete the previous ${{ needs.build-info.outputs.target-branch }} branch and tag"
        if: needs.build-info.outputs.allow-pre-releases == 'true'
        working-directory: "constraints"
        shell: bash
        run: |
          if git ls-remote --exit-code origin "refs/heads/${TARGET_BRANCH}" > /dev/null; then
            echo "Deleting the existing '${TARGET_BRANCH}' branch."
            git push origin --delete "refs/heads/${TARGET_BRANCH}"
          fi
          if git ls-remote --exit-code origin "refs/tags/constraints-${VERSION}" > /dev/null; then
            echo "Deleting the existing 'constraints-${VERSION}' tag."
            git push origin --delete "refs/tags/constraints-${VERSION}"
          fi
          git tag --delete "constraints-${VERSION}" > /dev/null 2>&1 || true
          git branch --delete --force "${TARGET_BRANCH}" > /dev/null 2>&1 || true
      - name: "Switch to ${{ needs.build-info.outputs.target-branch }}"
        working-directory: "constraints"
        shell: bash
        run: git switch -c "${TARGET_BRANCH}" 2>/dev/null || git switch "${TARGET_BRANCH}"
      - name: "Diff in constraints for Python: ${{ needs.build-info.outputs.python-versions-list-as-string }}"
        run: ./scripts/ci/constraints/ci_diff_constraints.sh
      - name: "Commit changed constraint files"
        run: ./scripts/ci/constraints/ci_commit_constraints.sh
      - name: "Push ${{ needs.build-info.outputs.target-branch }}"
        working-directory: "constraints"
        shell: bash
        # Qualify as refs/heads/ -- for a release candidate the branch and the tag share a name
        # (both constraints-<VERSION>), so a bare refspec is ambiguous ("matches more than one").
        run: git push origin "refs/heads/${TARGET_BRANCH}"
      - name: "Tag constraints-${{ inputs.version }}"
        working-directory: "constraints"
        shell: bash
        run: |
          git tag -a "constraints-${VERSION}" -m "Constraints for Apache Airflow ${VERSION}"
          git push origin "refs/tags/constraints-${VERSION}"
      - name: "Summary"
        shell: bash
        run: |
          {
            echo "Constraints for \`${VERSION}\` are on \`${TARGET_BRANCH}\`,"
            echo "tagged \`constraints-${VERSION}\`."
          } | tee -a "${GITHUB_STEP_SUMMARY}"