[v3-3-test] Keep not-ready providers out of the all extra (#71165) (#71167)

A not-ready provider has never been published, so naming it in an extra makes
that extra impossible to satisfy - there is no version on PyPI to resolve to.
It broke constraints generation for 3.3.1rc1, where apache-airflow[all] could
not resolve because it required apache-airflow-providers-ibm-mq.

Only the extras change. The source tree still holds those providers, so they
stay in the workspace and under mypy - which is how CI installs and checks them.
(cherry picked from commit fb459de)

Author: potiuk

.github/workflows/additional-ci-image-checks.yml

@@ -91,15 +91,18 @@ jobs:
   # We only push CI cache as PROD cache usually does not gain as much from fresh cache because
   # it uses prepared airflow and provider distributions that invalidate the cache anyway most of the time
   #
-  # Runs on `push` as well, not only on the scheduled canary. Release branches (`v*-*-test`) are built
-  # by `push` only, and their other refresh path -- the Regular cache push in `finalize-tests.yml` -- sits
-  # behind the whole test matrix. A branch that is red therefore never refreshes its cache at all, and the
-  # staler the cache gets the slower it is to turn green again. `main` is unaffected either way: it is not
-  # built on push, so it keeps refreshing through the scheduled canary.
+  # Only for the scheduled canary and manual dispatch -- both of which run to completion. Push events
+  # are refreshed by `refresh-image-cache.yml` instead: this workflow is part of the CI run, which
+  # cancels in progress on the next push, and a cache refresh that keeps being cancelled leaves the
+  # branch building cold forever (see that workflow's header for the loop it broke).
   #
   # Pull requests stay excluded -- `canary-run` is also true for a PR carrying the `canary` label, and
   # pushing cache from an unmerged branch is not something a label should authorise. This is the same
   # condition the Regular cache push uses.
+  #
+  # Both paths cover every Python version: `refresh-image-cache.yml` forces it, and on `schedule` /
+  # `workflow_dispatch` selective checks always resolves `python-versions` to all of them
+  # (`_should_run_all_tests_and_versions`), which `test_all_python_versions_on_scheduled_canary` pins.
   push-early-buildx-cache-to-github-registry:
     name: Push Early Image Cache
     uses: ./.github/workflows/push-image-cache.yml
@@ -121,7 +124,9 @@ jobs:
       include-success-outputs: ${{ inputs.include-success-outputs }}
       docker-cache: ${{ inputs.docker-cache }}
       disable-airflow-repo-cache: ${{ inputs.disable-airflow-repo-cache }}
-    if: inputs.canary-run == 'true' && github.event_name != 'pull_request'
+    if: >
+      inputs.canary-run == 'true' && github.event_name != 'pull_request'
+      && github.event_name != 'push'
 
   # Check that after earlier cache push, breeze command will build quickly
   # This build is a bit slow from in-the scratch builds, so we should run it only in

.github/workflows/refresh-image-cache.yml

@@ -0,0 +1,136 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+---
+name: Refresh image cache
+# Refreshes the CI image cache in the GitHub registry, in its own workflow run.
+#
+# It lives outside `ci-amd.yml` / `ci-arm.yml` because those cancel in progress runs on the next
+# push (`cancel-in-progress: true`), and a cache refresh is precisely the job that must not be
+# cancelled: a branch with no cache builds cold (~20 minutes per Python version), which widens the
+# window in which the next push cancels the run, which leaves the cache missing. `v3-3-test` sat in
+# exactly that loop -- its `linux/amd64` cache existed for the default Python version only, because
+# that was the single matrix entry that finished before the next push killed the rest.
+#
+# Cancelling here would recreate the loop, so this workflow does not cancel. A newer run queues
+# behind the one in flight instead of replacing it.
+on:  # yamllint disable-line rule:truthy
+  workflow_dispatch:
+    inputs:
+      platform:
+        description: "Platform to refresh the cache for"
+        type: choice
+        options:
+          - "both"
+          - "linux/amd64"
+          - "linux/arm64"
+        default: "both"
+  push:
+    # Release-prep and providers branches only. `main` refreshes its cache through the scheduled
+    # canary in `ci-amd.yml` / `ci-arm.yml`, which nothing cancels -- it is not built on push.
+    branches:
+      - v[0-9]+-[0-9]+-test
+      - providers-[a-z]+-?[a-z]*/v[0-9]+-[0-9]+
+permissions:
+  contents: read
+concurrency:
+  group: refresh-image-cache-${{ github.ref }}
+  cancel-in-progress: false
+jobs:
+  build-info:
+    name: "Build info"
+    # No outputs yet at this stage, so the runner has to be hard-coded to a public one.
+    runs-on: ["ubuntu-22.04"]
+    outputs:
+      default-branch: ${{ steps.selective-checks.outputs.default-branch }}
+      default-constraints-branch: ${{ steps.selective-checks.outputs.default-constraints-branch }}
+      disable-airflow-repo-cache: ${{ steps.selective-checks.outputs.disable-airflow-repo-cache }}
+      docker-cache: ${{ steps.selective-checks.outputs.docker-cache }}
+      include-success-outputs: ${{ steps.selective-checks.outputs.include-success-outputs }}
+      python-versions: ${{ steps.selective-checks.outputs.python-versions }}
+      use-uv: ${{ steps.selective-checks.outputs.force-pip == 'true' && 'false' || 'true' }}
+    steps:
+      - name: "Cleanup repo"
+        shell: bash
+        run: sudo rm -rf ${GITHUB_WORKSPACE}/*
+      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
+        with:
+          persist-credentials: false
+      - name: "Install Breeze"
+        uses: ./.github/actions/breeze
+      - name: Selective checks
+        id: selective-checks
+        env:
+          # A cache is only useful for the Python versions that are actually built, so the refresh
+          # always covers all of them. Selective checks would otherwise narrow `python-versions`
+          # down to the default one -- on a push where only text files changed it does exactly
+          # that (`_should_run_all_tests_and_versions`), which is the other way a branch ends up
+          # with cache for one Python version and nothing for the rest.
+          PR_LABELS: '["all versions"]'
+          COMMIT_REF: "${{ github.sha }}"
+          VERBOSE: "false"
+        run: breeze ci selective-check 2>> ${GITHUB_OUTPUT}
+
+  refresh-ci-cache-amd:
+    name: "Refresh cache linux/amd64"
+    needs: [build-info]
+    uses: ./.github/workflows/push-image-cache.yml
+    permissions:
+      contents: read
+      # Only granted for `push` / `workflow_dispatch` on apache/airflow itself, never for forks.
+      packages: write
+    with:
+      runners: '["ubuntu-22.04"]'
+      platform: "linux/amd64"
+      cache-type: "Refresh"
+      # PROD cache is built from distributions prepared earlier in a CI run, which a standalone
+      # refresh has no artifact for. CI cache is also where the build time actually goes.
+      include-prod-images: "false"
+      # `latest` is what CI pulls when it has no cache at all; moving it belongs to a full canary
+      # run that ran the tests, not to a cache refresh.
+      push-latest-images: "false"
+      python-versions: ${{ needs.build-info.outputs.python-versions }}
+      branch: ${{ needs.build-info.outputs.default-branch }}
+      constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }}
+      use-uv: ${{ needs.build-info.outputs.use-uv }}
+      include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }}
+      docker-cache: ${{ needs.build-info.outputs.docker-cache }}
+      disable-airflow-repo-cache: ${{ needs.build-info.outputs.disable-airflow-repo-cache }}
+    if: inputs.platform == 'both' || inputs.platform == 'linux/amd64' || github.event_name == 'push'
+
+  refresh-ci-cache-arm:
+    name: "Refresh cache linux/arm64"
+    needs: [build-info]
+    uses: ./.github/workflows/push-image-cache.yml
+    permissions:
+      contents: read
+      packages: write
+    with:
+      runners: '["ubuntu-22.04-arm"]'
+      platform: "linux/arm64"
+      cache-type: "Refresh"
+      include-prod-images: "false"
+      push-latest-images: "false"
+      python-versions: ${{ needs.build-info.outputs.python-versions }}
+      branch: ${{ needs.build-info.outputs.default-branch }}
+      constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }}
+      use-uv: ${{ needs.build-info.outputs.use-uv }}
+      include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }}
+      docker-cache: ${{ needs.build-info.outputs.docker-cache }}
+      disable-airflow-repo-cache: ${{ needs.build-info.outputs.disable-airflow-repo-cache }}
+    if: inputs.platform == 'both' || inputs.platform == 'linux/arm64' || github.event_name == 'push'

dev/breeze/tests/test_selective_checks.py

@@ -27,6 +27,7 @@
 from airflow_breeze.global_constants import (
     ALLOWED_KUBERNETES_VERSIONS,
     ALLOWED_PYTHON_MAJOR_MINOR_VERSIONS,
+    CURRENT_PYTHON_MAJOR_MINOR_VERSIONS,
     DEFAULT_KUBERNETES_VERSION,
     DEFAULT_PYTHON_MAJOR_MINOR_VERSION,
     JAVA_SDK_VERSION,
@@ -55,6 +56,9 @@
 ALL_KUBERNETES_VERSIONS_AS_LIST = "[" + ", ".join([f"'{v}'" for v in ALLOWED_KUBERNETES_VERSIONS]) + "]"
 ALL_PYTHON_VERSIONS_AS_STRING = " ".join(ALLOWED_PYTHON_MAJOR_MINOR_VERSIONS)
 ALL_PYTHON_VERSIONS_AS_LIST = "[" + ", ".join([f"'{v}'" for v in ALLOWED_PYTHON_MAJOR_MINOR_VERSIONS]) + "]"
+CURRENT_PYTHON_VERSIONS_AS_LIST = (
+    "[" + ", ".join([f"'{v}'" for v in CURRENT_PYTHON_MAJOR_MINOR_VERSIONS]) + "]"
+)
 
 DEFAULT_HELM_K8S_VERSION = ALLOWED_KUBERNETES_VERSIONS[0].lstrip("v")
 LAST_HELM_K8S_VERSION = ALLOWED_KUBERNETES_VERSIONS[-1].lstrip("v")
@@ -2692,6 +2696,40 @@ def test_no_commit_provided_trigger_full_build_for_any_event_type(mock_get, gith
     )
 
 
+# The image cache is pushed for `python-versions`, so a narrowed list leaves the remaining
+# versions with no cache at all and they build cold on every run. The two tests below pin the
+# two paths that refresh it: the canary in `additional-ci-image-checks.yml` and the standalone
+# `refresh-image-cache.yml`.
+
+
+@pytest.mark.parametrize(
+    "github_event",
+    [GithubEvents.SCHEDULE, GithubEvents.WORKFLOW_DISPATCH],
+)
+def test_all_python_versions_on_scheduled_canary(github_event):
+    """Only a `push` may narrow to the default version on a text-only change; the canary may not."""
+    stderr = SelectiveChecks(
+        files=("INTHEWILD.md",),
+        commit_ref=NEUTRAL_COMMIT,
+        github_event=github_event,
+        pr_labels=(),
+        default_branch="main",
+    )
+    assert_outputs_are_printed({"python-versions": CURRENT_PYTHON_VERSIONS_AS_LIST}, str(stderr))
+
+
+def test_all_python_versions_with_all_versions_label():
+    """`refresh-image-cache.yml` forces this label, on the event that would otherwise narrow."""
+    stderr = SelectiveChecks(
+        files=("INTHEWILD.md",),
+        commit_ref=NEUTRAL_COMMIT,
+        github_event=GithubEvents.PUSH,
+        pr_labels=("all versions",),
+        default_branch="main",
+    )
+    assert_outputs_are_printed({"python-versions": CURRENT_PYTHON_VERSIONS_AS_LIST}, str(stderr))
+
+
 @pytest.mark.parametrize(
     "github_event",
     [

providers/apache/kafka/src/airflow/providers/apache/kafka/plugins/__init__.py

@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.

providers/openai/src/airflow/providers/openai/example_dags/__init__.py

@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.

pyproject.toml

@@ -251,9 +251,6 @@ apache-airflow = "airflow.__main__:main"
 "http" = [
     "apache-airflow-providers-http>=4.13.2"
 ]
-"ibm.mq" = [
-    "apache-airflow-providers-ibm-mq>=0.1.0; platform_machine !=\"aarch64\" and platform_machine !=\"arm64\""
-]
 "imap" = [
     "apache-airflow-providers-imap>=3.8.0"
 ]
@@ -458,7 +455,6 @@ apache-airflow = "airflow.__main__:main"
     "apache-airflow-providers-grpc>=3.7.0",
     "apache-airflow-providers-hashicorp>=4.0.0",
     "apache-airflow-providers-http>=4.13.2",
-    "apache-airflow-providers-ibm-mq>=0.1.0; platform_machine !=\"aarch64\" and platform_machine !=\"arm64\"",
     "apache-airflow-providers-imap>=3.8.0",
     "apache-airflow-providers-influxdb>=2.8.0",
     "apache-airflow-providers-informatica>=0.1.1",

scripts/ci/prek/common_prek_utils.py

@@ -588,19 +588,30 @@ def get_provider_base_dir_from_path(file_path: Path) -> Path | None:
     return None
 
 
-def get_all_provider_ids(exclude_suspended_providers: bool = False) -> list[str]:
+def get_all_provider_ids(
+    exclude_suspended_providers: bool = False, exclude_not_ready_providers: bool = False
+) -> list[str]:
     """
     Get all providers from the new provider structure
+
+    :param exclude_suspended_providers: skip providers whose state is ``suspended``
+    :param exclude_not_ready_providers: skip providers whose state is ``not-ready`` - those have
+        never been published, so anything describing what is installable must leave them out
     """
     all_provider_ids = []
+    excluded_states = set()
+    if exclude_suspended_providers:
+        excluded_states.add("suspended")
+    if exclude_not_ready_providers:
+        excluded_states.add("not-ready")
     for provider_file in AIRFLOW_PROVIDERS_ROOT_PATH.rglob("provider.yaml"):
         if provider_file.is_relative_to(AIRFLOW_PROVIDERS_ROOT_PATH / "src"):
             continue
-        if exclude_suspended_providers:
+        if excluded_states:
             import yaml
 
             provider_info = yaml.safe_load(provider_file.read_text())
-            if provider_info.get("state") == "suspended":
+            if provider_info.get("state") in excluded_states:
                 continue
         provider_id = get_provider_id_from_path(provider_file)
         if provider_id:

scripts/ci/prek/update_airflow_pyproject_toml.py

@@ -269,9 +269,20 @@ def get_exclusion_marker(provider_dependencies: dict[str, Any]) -> str:
             all_optional_dependencies.append(f'"{optional}" = [\n    "apache-airflow-core[{optional}]"\n]\n')
     optional_airflow_task_sdk_dependencies = get_optional_dependencies(AIRFLOW_TASK_SDK_PYPROJECT_TOML_FILE)
     all_optional_dependencies.append('"all-task-sdk" = [\n    "apache-airflow-task-sdk[all]"\n]\n')
+    # Two lists, because the sections below describe two different things.
+    #
+    # `all_providers` describes the source tree: mypy has to type-check a not-ready provider and uv
+    # has to keep it in the workspace, which is how CI installs it. Only suspended providers drop out.
+    #
+    # `released_providers` describes what is installable from PyPI. A not-ready provider has never
+    # been published, so naming it in an extra makes that extra unsatisfiable - there is no version
+    # of it to resolve to.
     all_providers = sorted(get_all_provider_ids(exclude_suspended_providers=True))
+    released_providers = sorted(
+        get_all_provider_ids(exclude_suspended_providers=True, exclude_not_ready_providers=True)
+    )
     all_provider_lines = []
-    for provider_id in all_providers:
+    for provider_id in released_providers:
         distribution_name = provider_distribution_name(provider_id)
         min_provider_version, comment = find_min_provider_version(provider_id)
         exclusion_marker = get_exclusion_marker(all_providers_dependencies.get(provider_id, {}))
@@ -288,10 +299,15 @@ def get_exclusion_marker(provider_dependencies: dict[str, Any]) -> str:
             all_provider_lines.append(f'    "{distribution_name}",\n')
     all_optional_dependencies.append('"all" = [\n')
     optional_apache_airflow_dependencies = get_optional_dependencies(AIRFLOW_PYPROJECT_TOML_FILE)
+    # Filtered against every provider id rather than against `all_providers`: a provider left out of
+    # `all_providers` still has an extra named after it, and testing only against the included ones
+    # would sweep that extra in here instead of dropping it - which is how a not-ready provider would
+    # come back into `all` through the side door.
+    every_provider_id = set(get_all_provider_ids())
     all_local_extras = [
         extra
         for extra in sorted(optional_apache_airflow_dependencies)
-        if extra not in all_providers and not extra.startswith("all")
+        if extra not in every_provider_id and not extra.startswith("all")
     ]
     all_optional_dependencies.append(f'    "apache-airflow[{",".join(all_local_extras)}]",\n')
     all_optional_dependencies.append('    "apache-airflow-core[all]",\n')