Flag missing conn-fields when hook-only files change in static checks

Author: dabla

providers/.pre-commit-config.yaml

@@ -232,7 +232,10 @@ repos:
         name: Validate provider.yaml files
         entry: ../scripts/ci/prek/check_provider_yaml_files.py
         language: python
-        files: ^.*/provider\.yaml$
+        files: >
+          (?x)
+          ^.*/provider\.yaml$|
+          ^.*/src/airflow/providers/.*/hooks/[^/]+\.py$
         exclude: ^.*/.venv/.*$
         require_serial: true
       - id: check-cli-definition-imports

scripts/ci/prek/check_provider_conn_fields.py

@@ -46,6 +46,11 @@ def check_conn_fields_for_entry(
       at all — the entry is then skipped entirely (no ``conn-fields`` check), or
     - raises any other ``Exception`` to signal an unexpected failure (converted
       here into an error string so callers never need to catch it).
+
+    When the hook returns widgets, the ``conn-fields`` section in the provider YAML
+    must exactly match.  If ``conn-fields`` is absent, every widget key is treated
+    as missing from YAML (because any custom field missing from ``conn-fields`` will
+    be invisible in the new React connection UI).
     """
     hook_class_name: str = conn_type_entry["hook-class-name"]
     connection_type: str = conn_type_entry.get("connection-type", "?")
@@ -61,11 +66,8 @@ def check_conn_fields_for_entry(
     if widget_keys is None:
         return []
 
-    conn_fields = conn_type_entry.get("conn-fields")
-    if conn_fields is None:
-        # No conn-fields declared: the new UI simply exposes no custom fields for this
-        # connection type, which is intentional.  Nothing to validate.
-        return []
+    # Treat absent conn-fields as an empty dict so any hook widget is flagged as missing.
+    conn_fields: dict = conn_type_entry.get("conn-fields") or {}
 
     error = build_mismatch_error(
         set(conn_fields.keys()), widget_keys, connection_type, yaml_file_path, hook_class_name

scripts/ci/prek/check_provider_yaml_files.py

@@ -23,6 +23,7 @@
 # ///
 from __future__ import annotations
 
+import pathlib
 import sys
 
 from common_prek_utils import (
@@ -33,11 +34,41 @@
 
 initialize_breeze_prek(__name__, __file__)
 
-files_to_test = sys.argv[1:]
+
+def _resolve_provider_yaml_files(raw_files: list[str]) -> list[str]:
+    """
+    Accept a mix of provider.yaml paths and Python source files.
+
+    When a Python source file is passed (e.g. a hook whose
+    ``get_connection_form_widgets()`` was edited), map it to the
+    ``provider.yaml`` at the root of the same provider package so the
+    conn-fields check runs even when only the hook changes.
+
+    All paths are relative to the ``providers/`` directory, as supplied by
+    prek.  The first path segment is the provider package name
+    (e.g. ``samba/src/airflow/...`` → ``samba/provider.yaml``).
+    """
+    result: set[str] = set()
+    for f in raw_files:
+        p = pathlib.PurePosixPath(f)
+        if p.name == "provider.yaml":
+            result.add(f)
+        else:
+            # Map any Python file to the provider.yaml of its package root.
+            # Path structure: <provider-pkg>/<rest...>
+            # e.g. samba/src/airflow/providers/samba/hooks/samba.py
+            parts = p.parts
+            if parts:
+                result.add(f"{parts[0]}/provider.yaml")
+    return sorted(result)
+
+
+files_to_test = _resolve_provider_yaml_files(sys.argv[1:])
 cmd_result = run_command_via_breeze_run(
     ["python3", "/opt/airflow/scripts/in_container/run_provider_yaml_files_check.py", *files_to_test],
     backend="sqlite",
     warn_image_upgrade_needed=True,
     extra_env={"PYTHONWARNINGS": "default"},
 )
 validate_cmd_result(cmd_result, include_ci_env_check=True)
+

scripts/tests/ci/prek/test_check_conn_fields_match_form_widgets.py

@@ -107,8 +107,8 @@ class TestCheckConnFieldsForEntry:
             pytest.param([], _get_keys(), id="empty-on-both-sides"),
             pytest.param(["a"], _skip, id="skip-hook-without-get-connection-form-widgets"),
             pytest.param(None, _skip, id="skip-missing-conn-fields-when-hook-has-no-widgets"),
-            # Hook with widgets but no conn-fields is allowed: new UI intentionally omits custom fields.
-            pytest.param(None, _get_keys("field_a"), id="no-conn-fields-with-hook-widgets-is-ok"),
+            # When hook has no custom widgets, absent conn-fields is fine.
+            pytest.param(None, lambda _: set(), id="no-conn-fields-no-hook-widgets-is-ok"),
         ],
     )
     def test_no_errors(self, conn_fields, get_keys):
@@ -123,6 +123,10 @@ def test_no_errors(self, conn_fields, get_keys):
             ),
             pytest.param(["a"], _raise, "boom", id="unexpected-exception-message"),
             pytest.param(["a"], _raise, HOOK_CLASS, id="unexpected-exception-mentions-hook-class"),
+            # absent conn-fields with hook widgets must be flagged (every widget key is "missing")
+            pytest.param(
+                None, _get_keys("auth_protocol"), "auth_protocol", id="absent-conn-fields-with-hook-widgets"
+            ),
         ],
     )
     def test_one_error_containing(self, conn_fields, get_keys, expected_in_error):