|
| 1 | +# Licensed to the Apache Software Foundation (ASF) under one |
| 2 | +# or more contributor license agreements. See the NOTICE file |
| 3 | +# distributed with this work for additional information |
| 4 | +# regarding copyright ownership. The ASF licenses this file |
| 5 | +# to you under the Apache License, Version 2.0 (the |
| 6 | +# "License"); you may not use this file except in compliance |
| 7 | +# with the License. You may obtain a copy of the License at |
| 8 | +# |
| 9 | +# http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | +# |
| 11 | +# Unless required by applicable law or agreed to in writing, |
| 12 | +# software distributed under the License is distributed on an |
| 13 | +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| 14 | +# KIND, either express or implied. See the License for the |
| 15 | +# specific language governing permissions and limitations |
| 16 | +# under the License. |
| 17 | + |
| 18 | +""" |
| 19 | +Add dynamic region storage. |
| 20 | +
|
| 21 | +Revision ID: 54a27b6f9d01 |
| 22 | +Revises: e7c2a91bd540 |
| 23 | +Create Date: 2026-09-27 20:00:00.000000 |
| 24 | +""" |
| 25 | + |
| 26 | +from __future__ import annotations |
| 27 | + |
| 28 | +from textwrap import dedent |
| 29 | +from uuid import UUID |
| 30 | + |
| 31 | +import sqlalchemy as sa |
| 32 | +from alembic import op |
| 33 | + |
| 34 | +from airflow.migrations.utils import raise_if_rows_exist |
| 35 | +from airflow.models.base import StringID |
| 36 | +from airflow.utils.sqlalchemy import CompactUUID, UtcDateTime, compact_uuid_default |
| 37 | + |
| 38 | +revision = "54a27b6f9d01" |
| 39 | +down_revision = "e7c2a91bd540" |
| 40 | +branch_labels = None |
| 41 | +depends_on = None |
| 42 | +airflow_version = "3.4.0" |
| 43 | + |
| 44 | +_SENTINEL = UUID(int=0) |
| 45 | +_KEYS = ( |
| 46 | + ( |
| 47 | + "task_instance", |
| 48 | + "task_instance_current_key", |
| 49 | + ("dag_id", "task_id", "run_id", "map_index", "working_set"), |
| 50 | + "working_set IS NOT NULL", |
| 51 | + ), |
| 52 | + ( |
| 53 | + "task_instance", |
| 54 | + "task_instance_try_key", |
| 55 | + ("dag_id", "task_id", "run_id", "map_index", "try_number"), |
| 56 | + None, |
| 57 | + ), |
| 58 | + ("task_state_store", "task_state_store_uq", ("dag_run_id", "task_id", "map_index", "key"), None), |
| 59 | +) |
| 60 | + |
| 61 | + |
| 62 | +def _replace_unique(table_name, constraint_name, columns): |
| 63 | + context = op.get_context() |
| 64 | + dialect = context.dialect.name |
| 65 | + if dialect in ("postgresql", "mysql"): |
| 66 | + quote = context.dialect.identifier_preparer.quote |
| 67 | + column_list = ", ".join(quote(name) for name in columns) |
| 68 | + if dialect == "postgresql": |
| 69 | + sql = f""" |
| 70 | + ALTER TABLE {table_name} |
| 71 | + DROP CONSTRAINT {constraint_name}, |
| 72 | + ADD CONSTRAINT {constraint_name} UNIQUE ({column_list}) |
| 73 | + """ |
| 74 | + else: |
| 75 | + sql = f""" |
| 76 | + ALTER TABLE {table_name} |
| 77 | + DROP INDEX {constraint_name}, |
| 78 | + ADD UNIQUE KEY {constraint_name} ({column_list}) |
| 79 | + """ |
| 80 | + op.execute(dedent(sql)) |
| 81 | + else: |
| 82 | + with op.get_context().autocommit_block(): |
| 83 | + foreign_keys = op.get_bind().exec_driver_sql("PRAGMA foreign_keys").scalar_one() |
| 84 | + op.execute("PRAGMA foreign_keys=OFF") |
| 85 | + try: |
| 86 | + with op.batch_alter_table(table_name) as batch_op: |
| 87 | + batch_op.drop_constraint(constraint_name, type_="unique") |
| 88 | + batch_op.create_unique_constraint(constraint_name, columns) |
| 89 | + finally: |
| 90 | + op.execute(f"PRAGMA foreign_keys={foreign_keys}") |
| 91 | + |
| 92 | + |
| 93 | +def _build_collision_query(table_name, columns, where): |
| 94 | + """Select old-key groups holding more than one row, scanning only DagRuns that have regions.""" |
| 95 | + table = sa.table(table_name, *(sa.column(name) for name in columns)) |
| 96 | + region = sa.table("dynamic_region", sa.column("dag_id"), sa.column("run_id")) |
| 97 | + if table_name == "task_state_store": |
| 98 | + dag_run = sa.table("dag_run", sa.column("id"), sa.column("dag_id"), sa.column("run_id")) |
| 99 | + has_region = ( |
| 100 | + sa.select(1) |
| 101 | + .select_from( |
| 102 | + dag_run.join( |
| 103 | + region, sa.and_(region.c.dag_id == dag_run.c.dag_id, region.c.run_id == dag_run.c.run_id) |
| 104 | + ) |
| 105 | + ) |
| 106 | + .where(dag_run.c.id == table.c.dag_run_id) |
| 107 | + .exists() |
| 108 | + ) |
| 109 | + else: |
| 110 | + has_region = ( |
| 111 | + sa.select(1) |
| 112 | + .select_from(region) |
| 113 | + .where(region.c.dag_id == table.c.dag_id, region.c.run_id == table.c.run_id) |
| 114 | + .exists() |
| 115 | + ) |
| 116 | + query = sa.select(1).select_from(table).where(has_region) |
| 117 | + if where: |
| 118 | + query = query.where(sa.text(where)) |
| 119 | + return query.group_by(*(table.c[name] for name in columns)).having(sa.func.count() > 1).limit(1) |
| 120 | + |
| 121 | + |
| 122 | +def _assert_downgrade_is_lossless(): |
| 123 | + """ |
| 124 | + Abort the downgrade if dropping ``region_id`` would merge rows, before any DDL runs. |
| 125 | +
|
| 126 | + MySQL DDL cannot be rolled back, so every check has to happen first. The SQL guard also works |
| 127 | + in offline mode but does not support SQLite, which is checked by running the query directly. |
| 128 | + The query scans the whole ``task_instance`` table, so expect it to be slow on very large installations. |
| 129 | + """ |
| 130 | + dialect = op.get_context().dialect |
| 131 | + for table_name, _, columns, where in _KEYS: |
| 132 | + query = _build_collision_query(table_name, columns, where) |
| 133 | + message = ( |
| 134 | + f"Cannot downgrade: {table_name} has region rows colliding on the old key. " |
| 135 | + "Delete the affected DagRuns first." |
| 136 | + ) |
| 137 | + if dialect.name == "sqlite": |
| 138 | + if op.get_bind().execute(query).scalar_one_or_none() is not None: |
| 139 | + raise RuntimeError(message) |
| 140 | + else: |
| 141 | + sql = str(query.compile(dialect=dialect, compile_kwargs={"literal_binds": True})) |
| 142 | + raise_if_rows_exist(sql, message, op) |
| 143 | + |
| 144 | + |
| 145 | +def _configure_index_builds(): |
| 146 | + if op.get_bind().dialect.name == "postgresql": |
| 147 | + op.execute("SET LOCAL statement_timeout = 0") |
| 148 | + op.execute( |
| 149 | + "SELECT set_config('maintenance_work_mem', '256MB', true) " |
| 150 | + "WHERE (SELECT setting::bigint FROM pg_settings WHERE name = 'maintenance_work_mem') < 262144" |
| 151 | + ) |
| 152 | + |
| 153 | + |
| 154 | +def upgrade(): |
| 155 | + _configure_index_builds() |
| 156 | + op.create_table( |
| 157 | + "dynamic_region", |
| 158 | + sa.Column("id", CompactUUID(), nullable=False), |
| 159 | + sa.Column("dag_id", StringID(), nullable=False), |
| 160 | + sa.Column("run_id", StringID(), nullable=False), |
| 161 | + sa.Column("node_id", StringID(), nullable=False), |
| 162 | + sa.Column("parent_region_id", CompactUUID(), nullable=True), |
| 163 | + sa.Column("parent_region_index", sa.Integer(), nullable=True), |
| 164 | + sa.Column("forked_from_region_id", CompactUUID(), nullable=True), |
| 165 | + sa.Column("resumes_from_index", sa.Integer(), nullable=False, server_default="0"), |
| 166 | + sa.Column("created_at", UtcDateTime(), nullable=False), |
| 167 | + sa.PrimaryKeyConstraint("id", name="dynamic_region_pkey"), |
| 168 | + sa.ForeignKeyConstraint( |
| 169 | + ["dag_id", "run_id"], |
| 170 | + ["dag_run.dag_id", "dag_run.run_id"], |
| 171 | + name="dynamic_region_dag_run_fkey", |
| 172 | + ondelete="CASCADE", |
| 173 | + ), |
| 174 | + sa.ForeignKeyConstraint( |
| 175 | + ["parent_region_id"], |
| 176 | + ["dynamic_region.id"], |
| 177 | + name="dynamic_region_parent_region_id_fkey", |
| 178 | + ondelete="CASCADE", |
| 179 | + ), |
| 180 | + # Fork lineage is unbounded; a cascading self-FK would exceed MySQL's cascade depth limit. |
| 181 | + sa.UniqueConstraint("forked_from_region_id", name="dynamic_region_forked_from_region_id_uq"), |
| 182 | + sa.CheckConstraint( |
| 183 | + "(parent_region_id IS NULL AND parent_region_index IS NULL) OR " |
| 184 | + "(parent_region_id IS NOT NULL AND parent_region_index IS NOT NULL)", |
| 185 | + name="parent_coordinates_paired", |
| 186 | + ), |
| 187 | + sa.CheckConstraint("resumes_from_index >= 0", name="resumes_from_index_nonnegative"), |
| 188 | + ) |
| 189 | + op.create_index( |
| 190 | + "idx_dynamic_region_slot", |
| 191 | + "dynamic_region", |
| 192 | + ["dag_id", "run_id", "node_id", "parent_region_id", "parent_region_index"], |
| 193 | + ) |
| 194 | + op.create_index("idx_dynamic_region_parent_region_id", "dynamic_region", ["parent_region_id"]) |
| 195 | + for table_name in ("task_instance", "task_state_store"): |
| 196 | + op.add_column( |
| 197 | + table_name, |
| 198 | + sa.Column( |
| 199 | + "region_id", CompactUUID(), nullable=False, server_default=compact_uuid_default(_SENTINEL) |
| 200 | + ), |
| 201 | + ) |
| 202 | + for table_name, constraint_name, columns, _ in _KEYS: |
| 203 | + new_columns = list(columns) |
| 204 | + new_columns.insert(new_columns.index("map_index"), "region_id") |
| 205 | + _replace_unique(table_name, constraint_name, new_columns) |
| 206 | + |
| 207 | + |
| 208 | +def downgrade(): |
| 209 | + _configure_index_builds() |
| 210 | + _assert_downgrade_is_lossless() |
| 211 | + for table_name, constraint_name, columns, _ in _KEYS: |
| 212 | + _replace_unique(table_name, constraint_name, list(columns)) |
| 213 | + for table_name in ("task_instance", "task_state_store"): |
| 214 | + op.drop_column(table_name, "region_id") |
| 215 | + op.drop_table("dynamic_region") |
0 commit comments