TS SDK: serialize a native Dag to Dag JSON

Author: jason810496

.gitignore

@@ -340,3 +340,8 @@ dev/registry/providers.json
 # absolute paths written by sandbox-add-project-root.sh; per
 # https://github.com/apache/magpie/issues/197).
 /.claude/settings.local.json
+
+# Generated by ts-sdk/tests/conformance/serialize_typescript.ts, to hand to
+# serialize_python.py --verify. The recorded Python side is committed; this one
+# is rebuilt on demand.
+ts-sdk/tests/conformance/serialized_typescript.json

.pre-commit-config.yaml

@@ -318,6 +318,18 @@ repos:
           (?x)
           ^airflow-core/src/airflow/serialization/schema\.json$|
           ^ts-sdk/schema/dag-schema\.json$
+      - id: check-ts-sdk-conformance-recording
+        name: Check the recorded Python Dag serialization the TS SDK is compared against
+        description: "Re-run Airflow's serializer over the conformance fixtures and fail if the recording drifted"
+        entry: ./ts-sdk/scripts/ci/prek/check_conformance_recording.py
+        language: python
+        pass_filenames: false
+        require_serial: true
+        files: >
+          (?x)
+          ^airflow-core/src/airflow/serialization/serialized_objects\.py$|
+          ^airflow-core/src/airflow/serialization/schema\.json$|
+          ^ts-sdk/tests/conformance/.*$
       - id: check-go-version-in-sync
         name: Check Go toolchain version is consistent across build files
         entry: ./scripts/ci/prek/check_go_version_in_sync.py

airflow-core/docs/authoring-and-scheduling/language-sdks/typescript.rst

@@ -346,6 +346,29 @@ an id of either.
 
 .. warning:: A cycle is not allowed. The Dag is rejected when it is read, naming the tasks on the cycle.
 
+Serialization
+~~~~~~~~~~~~~
+
+A native Dag serializes into the same Dag JSON a Python Dag produces, so the scheduler reads it
+without knowing which language declared it.
+
+``schedule`` accepts what maps to a stock timetable: unset, ``@once``, ``@continuous``, or a cron
+expression. A cron preset such as ``@daily`` is recorded as the expression it stands for. Anything
+else names a Python object a TypeScript bundle cannot point at, and is rejected.
+
+Every task of a native Dag runs on the Node coordinator, so it needs the queue the deployment routes
+there. Set it once on the Dag and each task inherits it:
+
+.. code-block:: typescript
+
+    const dag = new Dag("ts_etl", { schedule: "@daily", queue: "typescript" });
+
+    // ...and one task that needs its own.
+    dag.task("heavy", heavyHandler, { queue: "typescript_large" })();
+
+``queue`` on a task wins over the Dag's. See :ref:`typescript-sdk/coordinator-config` for the
+``queue_to_coordinator`` entry that sends that queue to the coordinator.
+
 ``new Dag`` and ``dag.task`` both take a trailing spec of Airflow options:
 ``{ schedule: "@daily", tags: ["etl"] }`` for the Dag, ``{ retries: 2, retryDelay: 30 }`` for a task.
 

ts-sdk/.prettierignore

@@ -21,3 +21,9 @@ node_modules/
 
 # Byte-exact bundle artifacts: reformatting one invalidates its embedded digests.
 tests/cli/fixtures/*.min.mjs
+
+# Written by the conformance harness, whose two sides have to agree byte for
+# byte: reformatting either would make serialize_python.py --check fail against
+# output it had just produced.
+tests/conformance/serialized_python.json
+tests/conformance/serialized_typescript.json

ts-sdk/scripts/ci/prek/check_conformance_recording.py

@@ -0,0 +1,65 @@
+#!/usr/bin/env python3
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+"""
+Keep the recorded Python serialization output the conformance suite compares against honest.
+
+``ts-sdk/tests/conformance/serialized_python.json`` is what Airflow's own serializer
+produced for the fixtures in ``test_dags.json``. The TypeScript conformance suite diffs
+its output against that recording, so the recording going stale would leave the suite
+passing against an Airflow that no longer serializes the same way.
+"""
+
+from __future__ import annotations
+
+import subprocess
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[4] / "scripts" / "ci" / "prek"))
+
+from common_prek_utils import AIRFLOW_ROOT_PATH, console
+
+if __name__ not in ("__main__", "__mp_main__"):
+    raise SystemExit(
+        "This file is intended to be executed as an executable program. You cannot use it as a module."
+        f"To run this script, run the ./{__file__} command"
+    )
+
+SCRIPT = Path("ts-sdk/tests/conformance/serialize_python.py")
+
+if __name__ == "__main__":
+    result = subprocess.run(
+        ["uv", "run", "--project", "airflow-core", "python", str(SCRIPT), "--check"],
+        cwd=AIRFLOW_ROOT_PATH,
+        check=False,
+        capture_output=True,
+        text=True,
+    )
+    if result.returncode == 0:
+        sys.exit(0)
+    message = (
+        f"{SCRIPT} --check failed, so the recorded Python output no longer matches what "
+        "Airflow's serializer produces.\n"
+        f"Re-record it with `uv run --project airflow-core python {SCRIPT}` and review the diff: "
+        "a change there is a change in the parity contract.\n\n"
+        f"{result.stdout}{result.stderr}"
+    )
+    if console:
+        console.print(f"[yellow]{message}[/]")
+    else:
+        print(message, file=sys.stderr)
+    raise SystemExit(result.returncode)