Run Dag processors without metadata database access

airflow-core/docs/howto/docker-compose/docker-compose.yaml

@@ -155,7 +155,14 @@ services:
     <<: *airflow-common
     command: dag-processor
     healthcheck:
-      test: ["CMD-SHELL", 'airflow jobs check --job-type DagProcessorJob --hostname "$${HOSTNAME}"']
+      test:
+        - CMD-SHELL
+        - >-
+          if [ -n "$${AIRFLOW__DAG_PROCESSOR__EXECUTION_API_TOKEN_FILE:-}" ]; then
+            airflow dag-processor --check-health;
+          else
+            airflow jobs check --job-type DagProcessorJob --hostname "$${HOSTNAME}";
+          fi
       interval: 30s
       timeout: 10s
       retries: 5

airflow-core/docs/migrations-ref.rst

@@ -39,7 +39,11 @@ Here's the list of all the Database Migrations that are executed via when you ru
 +-------------------------+------------------+-------------------+--------------------------------------------------------------+
 | Revision ID             | Revises ID       | Airflow Version   | Description                                                  |
 +=========================+==================+===================+==============================================================+
-| ``c71065a1b14f`` (head) | ``a5d7b9c13e40`` | ``3.5.0``         | Add session_id and registration_id to Job.                   |
+| ``687eb253de24`` (head) | ``e941ab8243b7`` | ``3.5.0``         | Add source revisions for Dag processor reconciliation.       |
++-------------------------+------------------+-------------------+--------------------------------------------------------------+
+| ``e941ab8243b7``        | ``c71065a1b14f`` | ``3.5.0``         | Add checkpoints for Dag result publication.                  |
++-------------------------+------------------+-------------------+--------------------------------------------------------------+
+| ``c71065a1b14f``        | ``a5d7b9c13e40`` | ``3.5.0``         | Add session_id and registration_id to Job.                   |
 +-------------------------+------------------+-------------------+--------------------------------------------------------------+
 | ``a5d7b9c13e40``        | ``c3e7a9182f64`` | ``3.4.0``         | Preserve exact task execution attribution in asset state.    |
 +-------------------------+------------------+-------------------+--------------------------------------------------------------+

airflow-core/docs/security/dag-processor-api/docker-compose.yaml

@@ -0,0 +1,53 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+---
+# Run independently of the quick-start Compose file so its credentials are not inherited.
+services:
+  dag-processor:
+    image: ${AIRFLOW_IMAGE_NAME:?Set an Airflow image with processor API support}
+    command: [dag-processor]
+    user: "${AIRFLOW_UID:-50000}:0"
+    environment:
+      USER: airflow
+      AIRFLOW_CONFIG: /tmp/airflow.cfg
+      AIRFLOW__CORE__LOAD_EXAMPLES: "False"
+      AIRFLOW__CORE__EXECUTION_API_SERVER_URL: http://airflow-apiserver:8080/execution/
+      AIRFLOW__DAG_PROCESSOR__EXECUTION_API_TOKEN_FILE: /run/dag-processor/processor.jwt
+      AIRFLOW__DAG_PROCESSOR__DAG_BUNDLE_STORAGE_PATH: /tmp/dag-bundles
+      AIRFLOW__DAG_PROCESSOR__HEALTH_CHECK_FILE: /tmp/dag-processor-health.json
+      CONNECTION_CHECK_MAX_COUNT: "0"
+    volumes:
+      - ./tokens:/run/dag-processor:ro
+      - ./dags:/opt/airflow/dags:ro
+      - ./logs:/opt/airflow/logs
+    read_only: true
+    tmpfs:
+      - /tmp
+    cap_drop: [ALL]
+    security_opt: [no-new-privileges:true]
+    healthcheck:
+      test: [CMD, airflow, dag-processor, --check-health]
+      interval: 30s
+      timeout: 10s
+      start_period: 30s
+      retries: 5
+    networks: [processor-api]
+    restart: unless-stopped
+networks:
+  processor-api:
+    name: ${DAG_PROCESSOR_API_NETWORK:-dag-processor-api}
+    external: true

airflow-core/docs/security/dag-processor-api/helm-values.yaml

@@ -0,0 +1,74 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+---
+# Install as a separate processor-only release, using an image with API mode.
+airflowVersion: "3.5.0"
+defaultAirflowTag: "3.5.0"
+executor: LocalExecutor
+scheduler:
+  enabled: false
+apiServer:
+  enabled: false
+triggerer:
+  enabled: false
+createUserJob:
+  enabled: false
+migrateDatabaseJob:
+  enabled: false
+postgresql:
+  enabled: false
+redis:
+  enabled: false
+statsd:
+  enabled: false
+rbac:
+  create: false
+enableBuiltInSecretEnvVars:
+  AIRFLOW__CORE__FERNET_KEY: false
+  AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: false
+  AIRFLOW_CONN_AIRFLOW_DB: false
+  AIRFLOW__API__SECRET_KEY: false
+  AIRFLOW__API_AUTH__JWT_SECRET: false
+  AIRFLOW__CELERY__RESULT_BACKEND: false
+  AIRFLOW__CELERY__BROKER_URL: false
+config:
+  core:
+    execution_api_server_url: http://airflow-api-server.airflow.svc.cluster.local:8080/execution/
+  dag_processor:
+    execution_api_token_file: /run/dag-processor/processor.jwt
+dagProcessor:
+  replicas: 1
+  waitForMigrations:
+    enabled: false
+  serviceAccount:
+    automountServiceAccountToken: false
+  env:
+    - name: CONNECTION_CHECK_MAX_COUNT
+      value: "0"
+  extraVolumes:
+    - name: processor-token
+      secret:
+        secretName: dag-processor-session
+        defaultMode: 0440
+  extraVolumeMounts:
+    - name: processor-token
+      mountPath: /run/dag-processor
+      readOnly: true
+dags:
+  persistence:
+    enabled: true
+    existingClaim: processor-dags

airflow-core/docs/security/dag-processor-api/network-policy.yaml

@@ -0,0 +1,51 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+---
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: dag-processor-api-only
+spec:
+  podSelector:
+    matchLabels:
+      component: dag-processor
+  policyTypes: [Ingress, Egress]
+  ingress: []
+  egress:
+    - to:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: airflow
+          podSelector:
+            matchLabels:
+              component: api-server
+              release: airflow
+      ports:
+        - protocol: TCP
+          port: 8080
+    - to:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: kube-system
+          podSelector:
+            matchLabels:
+              k8s-app: kube-dns
+      ports:
+        - protocol: UDP
+          port: 53
+        - protocol: TCP
+          port: 53