Skip to content

Commit c24e6fa

Browse files
committed
Skip the Python-only parse checks for non-Python Dag files
The Airflow module pre-import and the Dag version inflation check read a Dag file as Python source, which fails for a binary file such as a JAR. Both now run only for .py files.
1 parent 7fe4da6 commit c24e6fa

2 files changed

Lines changed: 48 additions & 2 deletions

File tree

‎airflow-core/src/airflow/dag_processing/processor.py‎

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,11 @@
7676
from airflow.sdk.importers import DagSourceCode # noqa: TC001
7777
from airflow.serialization.definitions.dag import SerializedDAG
7878
from airflow.serialization.serialized_objects import DagSerialization, LazyDeserializedDAG
79-
from airflow.utils.dag_version_inflation_checker import check_dag_file_stability
79+
from airflow.utils.dag_version_inflation_checker import (
80+
DagVersionInflationCheckLevel,
81+
DagVersionInflationCheckResult,
82+
check_dag_file_stability,
83+
)
8084
from airflow.utils.file import iter_airflow_imports
8185
from airflow.utils.helpers import prune_dict
8286
from airflow.utils.log.logging_mixin import LoggingMixin
@@ -175,6 +179,11 @@ class DagFileParsingResult(BaseModel):
175179
]
176180

177181

182+
def _is_python_source(file_path: str | os.PathLike[str]) -> bool:
183+
"""Return whether a Dag file is Python source, which the AST-based pre-parse steps can read."""
184+
return Path(file_path).suffix.lower() == ".py"
185+
186+
178187
def _pre_import_airflow_modules(file_path: str, log: FilteringBoundLogger) -> None:
179188
"""
180189
Pre-import Airflow modules found in the given file.
@@ -186,6 +195,8 @@ def _pre_import_airflow_modules(file_path: str, log: FilteringBoundLogger) -> No
186195
:param file_path: Path to the file to scan for imports
187196
:param log: Logger instance to use for warnings
188197
"""
198+
if not _is_python_source(file_path):
199+
return
189200
if not conf.getboolean("dag_processor", "parsing_pre_import_modules", fallback=True):
190201
return
191202

@@ -226,7 +237,11 @@ def _parse_file_entrypoint():
226237
def _parse_file(msg: DagFileParseRequest, log: FilteringBoundLogger) -> DagFileParsingResult | None:
227238
# TODO: Set known_pool names on DagBag!
228239

229-
stability_check_result = check_dag_file_stability(os.fspath(msg.file))
240+
stability_check_result = (
241+
check_dag_file_stability(os.fspath(msg.file))
242+
if _is_python_source(msg.file)
243+
else DagVersionInflationCheckResult(check_level=DagVersionInflationCheckLevel.off)
244+
)
230245

231246
# Callback runs must not be blocked by the stability check: callbacks for
232247
# already-scheduled runs still have to execute, and they never produce a

‎airflow-core/tests/unit/dag_processing/test_processor.py‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,7 @@
9494
from airflow.utils.state import TaskInstanceState
9595

9696
from tests_common.test_utils.config import conf_vars, env_vars
97+
from unit.dag_processing.fake_importers import FAKE_IMPORTER, task_sdk_importers
9798

9899
if TYPE_CHECKING:
99100
from kgb import SpyAgency
@@ -433,6 +434,16 @@ def test__pre_import_airflow_modules_when_disabled(self):
433434
mock_iter.assert_not_called()
434435
logger.warning.assert_not_called()
435436

437+
def test__pre_import_airflow_modules_skips_non_python_files(self):
438+
logger = MagicMock(spec=FilteringBoundLogger)
439+
with (
440+
env_vars({"AIRFLOW__DAG_PROCESSOR__PARSING_PRE_IMPORT_MODULES": "true"}),
441+
patch("airflow.dag_processing.processor.iter_airflow_imports") as mock_iter,
442+
):
443+
_pre_import_airflow_modules("bundle.jar", logger)
444+
445+
mock_iter.assert_not_called()
446+
436447
def test__pre_import_airflow_modules_when_enabled(self):
437448
logger = MagicMock(spec=FilteringBoundLogger)
438449
with (
@@ -695,6 +706,26 @@ def fake_collect_dags(self, *args, **kwargs):
695706
assert called is True
696707

697708

709+
def test_parse_file_skips_python_checks_for_other_importers(tmp_path):
710+
native = tmp_path / "native.fake"
711+
native.write_text("native_dag\n")
712+
713+
with (
714+
task_sdk_importers(FAKE_IMPORTER),
715+
patch("airflow.dag_processing.processor.check_dag_file_stability", autospec=True) as stability_check,
716+
):
717+
result = _parse_file(
718+
DagFileParseRequest(file=str(native), bundle_path=tmp_path, bundle_name="testing"),
719+
log=structlog.get_logger(),
720+
)
721+
722+
assert result is not None
723+
assert result.import_errors == {}
724+
assert [dag.dag_id for dag in result.serialized_dags] == ["native_dag"]
725+
assert result.serialized_dags[0].data["dag"]["relative_fileloc"] == "native.fake"
726+
stability_check.assert_not_called()
727+
728+
698729
@conf_vars({("dag_processor", "dag_version_inflation_check_level"): "error"})
699730
def test_parse_file_static_check_with_error():
700731
result = _parse_file(

0 commit comments

Comments
 (0)