From 2dcba6cae53c4b69c6f4da3a96bf04093564e889 Mon Sep 17 00:00:00 2001
From: bornunique911 <ravirajmehta911@gmail.com>
Date: Thu, 6 Aug 2026 00:53:24 +0530
Subject: [PATCH 01/20] Fix SQLite migration compatibility and idempotency

---
 ...e_add_document_metadata_to_cre_and_node.py |  43 +++++++
 ...fa_add_embedding_vec_to_embeddings_for_.py |  33 +++++
 ...b3c4d5e_add_artifact_ingest_persistence.py | 114 +++++++++---------
 3 files changed, 130 insertions(+), 60 deletions(-)
 create mode 100644 migrations/versions/055dbd9f8bfe_add_document_metadata_to_cre_and_node.py
 create mode 100644 migrations/versions/967016ee10fa_add_embedding_vec_to_embeddings_for_.py

diff --git a/migrations/versions/055dbd9f8bfe_add_document_metadata_to_cre_and_node.py b/migrations/versions/055dbd9f8bfe_add_document_metadata_to_cre_and_node.py
new file mode 100644
index 000000000..65e67b6d6
--- /dev/null
+++ b/migrations/versions/055dbd9f8bfe_add_document_metadata_to_cre_and_node.py
@@ -0,0 +1,43 @@
+"""add document_metadata to cre and node
+
+Revision ID: 055dbd9f8bfe
+Revises: d4e5f6a7b8c9
+Create Date: 2026-08-06 00:13:49.191527
+
+"""
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.engine.reflection import Inspector
+
+revision = '055dbd9f8bfe'
+down_revision = 'd4e5f6a7b8c9'   # or whatever the current head is
+branch_labels = None
+depends_on = None
+
+def column_exists(table, column):
+    conn = op.get_bind()
+    inspector = Inspector.from_engine(conn)
+    columns = [c['name'] for c in inspector.get_columns(table)]
+    return column in columns
+
+def upgrade():
+    # Add to 'cre' table if missing
+    if not column_exists('cre', 'document_metadata'):
+        op.add_column('cre', sa.Column('document_metadata', sa.Text(), nullable=True))
+        print("Added document_metadata to cre")
+    else:
+        print("Column document_metadata already exists in cre, skipping.")
+
+    # Add to 'node' table if missing
+    if not column_exists('node', 'document_metadata'):
+        op.add_column('node', sa.Column('document_metadata', sa.Text(), nullable=True))
+        print("Added document_metadata to node")
+    else:
+        print("Column document_metadata already exists in node, skipping.")
+
+def downgrade():
+    # Remove columns (optional, but we want a clean downgrade)
+    # Note: SQLite does not support DROP COLUMN directly, but you can use batch.
+    # For simplicity, we'll just raise an error or skip.
+    pass
diff --git a/migrations/versions/967016ee10fa_add_embedding_vec_to_embeddings_for_.py b/migrations/versions/967016ee10fa_add_embedding_vec_to_embeddings_for_.py
new file mode 100644
index 000000000..725dc1b43
--- /dev/null
+++ b/migrations/versions/967016ee10fa_add_embedding_vec_to_embeddings_for_.py
@@ -0,0 +1,33 @@
+"""add embedding_vec to embeddings for SQLite
+
+Revision ID: 967016ee10fa
+Revises: 055dbd9f8bfe
+Create Date: 2026-08-06 00:13:49.191527
+
+"""
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.engine.reflection import Inspector
+
+revision = '967016ee10fa'
+down_revision = '055dbd9f8bfe'   # <-- set this to the head revision you found
+branch_labels = None
+depends_on = None
+
+def column_exists(table, column):
+    conn = op.get_bind()
+    inspector = Inspector.from_engine(conn)
+    columns = [c['name'] for c in inspector.get_columns(table)]
+    return column in columns
+
+def upgrade():
+    if not column_exists('embeddings', 'embedding_vec'):
+        op.add_column('embeddings', sa.Column('embedding_vec', sa.Text(), nullable=True))
+        print("Added embedding_vec to embeddings")
+    else:
+        print("Column embedding_vec already exists in embeddings, skipping.")
+
+def downgrade():
+    # SQLite does not support DROP COLUMN directly; skip or use batch if needed.
+    pass
\ No newline at end of file
diff --git a/migrations/versions/9f1a2b3c4d5e_add_artifact_ingest_persistence.py b/migrations/versions/9f1a2b3c4d5e_add_artifact_ingest_persistence.py
index f313016b6..e56959f63 100644
--- a/migrations/versions/9f1a2b3c4d5e_add_artifact_ingest_persistence.py
+++ b/migrations/versions/9f1a2b3c4d5e_add_artifact_ingest_persistence.py
@@ -8,75 +8,69 @@
 
 from alembic import op
 import sqlalchemy as sa
-
+from sqlalchemy.engine.reflection import Inspector
 
 revision = "9f1a2b3c4d5e"
 down_revision = "a1b2c3d4e5f6"
 branch_labels = None
 depends_on = None
 
+def table_exists(table_name):
+    conn = op.get_bind()
+    inspector = Inspector.from_engine(conn)
+    return table_name in inspector.get_table_names()
 
 def upgrade():
-    op.create_table(
-        "artifact_ingest_event",
-        sa.Column("id", sa.String(), primary_key=True),
-        sa.Column("run_id", sa.String(), nullable=False),
-        sa.Column("artifact_id", sa.String(), nullable=False),
-        sa.Column("harvest_mode", sa.String(), nullable=False),
-        sa.Column("event_type", sa.String(), nullable=False),
-        sa.Column("source_json", sa.Text(), nullable=False),
-        sa.Column("locator_json", sa.Text(), nullable=False),
-        sa.Column("artifact_json", sa.Text(), nullable=False),
-        sa.Column("harvest_json", sa.Text(), nullable=False),
-        sa.Column("observed_at", sa.DateTime(), nullable=False),
-        sa.Column("created_at", sa.DateTime(), nullable=False),
-        sa.ForeignKeyConstraint(
-            ["run_id"],
-            ["import_run.id"],
-            onupdate="CASCADE",
-            ondelete="CASCADE",
-        ),
-    )
-    op.create_unique_constraint(
-        "uq_artifact_ingest_event_run_artifact",
-        "artifact_ingest_event",
-        ["run_id", "artifact_id"],
-    )
-
-    op.create_table(
-        "ingest_chunk",
-        sa.Column("id", sa.String(), primary_key=True),
-        sa.Column("artifact_event_id", sa.String(), nullable=False),
-        sa.Column("chunk_id", sa.String(), nullable=False),
-        sa.Column("text", sa.Text(), nullable=False),
-        sa.Column("char_count", sa.Integer(), nullable=False),
-        sa.Column("span_json", sa.Text(), nullable=False),
-        sa.Column("delta_json", sa.Text(), nullable=True),
-        sa.Column("created_at", sa.DateTime(), nullable=False),
-        sa.ForeignKeyConstraint(
-            ["artifact_event_id"],
-            ["artifact_ingest_event.id"],
-            onupdate="CASCADE",
-            ondelete="CASCADE",
-        ),
-    )
-    op.create_unique_constraint(
-        "uq_ingest_chunk_artifact_chunk",
-        "ingest_chunk",
-        ["artifact_event_id", "chunk_id"],
-    )
+    # Create artifact_ingest_event only if it doesn't exist
+    if not table_exists("artifact_ingest_event"):
+        op.create_table(
+            "artifact_ingest_event",
+            sa.Column("id", sa.String(), primary_key=True),
+            sa.Column("run_id", sa.String(), nullable=False),
+            sa.Column("artifact_id", sa.String(), nullable=False),
+            sa.Column("harvest_mode", sa.String(), nullable=False),
+            sa.Column("event_type", sa.String(), nullable=False),
+            sa.Column("source_json", sa.Text(), nullable=False),
+            sa.Column("locator_json", sa.Text(), nullable=False),
+            sa.Column("artifact_json", sa.Text(), nullable=False),
+            sa.Column("harvest_json", sa.Text(), nullable=False),
+            sa.Column("observed_at", sa.DateTime(), nullable=False),
+            sa.Column("created_at", sa.DateTime(), nullable=False),
+            sa.ForeignKeyConstraint(
+                ["run_id"],
+                ["import_run.id"],
+                onupdate="CASCADE",
+                ondelete="CASCADE",
+            ),
+            sa.UniqueConstraint("run_id", "artifact_id", name="uq_artifact_ingest_event_run_artifact"),
+        )
+    else:
+        print("Table artifact_ingest_event already exists, skipping creation.")
 
+    # Create ingest_chunk only if it doesn't exist
+    if not table_exists("ingest_chunk"):
+        op.create_table(
+            "ingest_chunk",
+            sa.Column("id", sa.String(), primary_key=True),
+            sa.Column("artifact_event_id", sa.String(), nullable=False),
+            sa.Column("chunk_id", sa.String(), nullable=False),
+            sa.Column("text", sa.Text(), nullable=False),
+            sa.Column("char_count", sa.Integer(), nullable=False),
+            sa.Column("span_json", sa.Text(), nullable=False),
+            sa.Column("delta_json", sa.Text(), nullable=True),
+            sa.Column("created_at", sa.DateTime(), nullable=False),
+            sa.ForeignKeyConstraint(
+                ["artifact_event_id"],
+                ["artifact_ingest_event.id"],
+                onupdate="CASCADE",
+                ondelete="CASCADE",
+            ),
+            sa.UniqueConstraint("artifact_event_id", "chunk_id", name="uq_ingest_chunk_artifact_chunk"),
+        )
+    else:
+        print("Table ingest_chunk already exists, skipping creation.")
 
 def downgrade():
-    op.drop_constraint(
-        "uq_ingest_chunk_artifact_chunk",
-        "ingest_chunk",
-        type_="unique",
-    )
+    # Drop tables in reverse order; constraints are dropped automatically with the tables.
     op.drop_table("ingest_chunk")
-    op.drop_constraint(
-        "uq_artifact_ingest_event_run_artifact",
-        "artifact_ingest_event",
-        type_="unique",
-    )
-    op.drop_table("artifact_ingest_event")
+    op.drop_table("artifact_ingest_event")
\ No newline at end of file

From 187dff7237b879c399090d442679f01ddf3dc420 Mon Sep 17 00:00:00 2001
From: bornunique911 <ravirajmehta911@gmail.com>
Date: Thu, 6 Aug 2026 02:01:34 +0530
Subject: [PATCH 02/20] migrations: add downgrades, use sa.inspect, and verify
 unique constraints

- Add downgrades to 055dbd9f8bfe and 967016ee10fa to drop added columns
- Replace deprecated Inspector.from_engine(conn) with sa.inspect(conn)
- In 9f1a2b3c4d5e, verify that existing tables have the required unique
  constraints; add them via batch_alter_table if missing
- Ensure migration fails if constraints cannot be added

Addresses PR review comments #1006
---
 ...e_add_document_metadata_to_cre_and_node.py | 12 ++++----
 ...fa_add_embedding_vec_to_embeddings_for_.py |  8 ++---
 ...b3c4d5e_add_artifact_ingest_persistence.py | 29 ++++++++++++++++---
 3 files changed, 35 insertions(+), 14 deletions(-)

diff --git a/migrations/versions/055dbd9f8bfe_add_document_metadata_to_cre_and_node.py b/migrations/versions/055dbd9f8bfe_add_document_metadata_to_cre_and_node.py
index 65e67b6d6..ab1004870 100644
--- a/migrations/versions/055dbd9f8bfe_add_document_metadata_to_cre_and_node.py
+++ b/migrations/versions/055dbd9f8bfe_add_document_metadata_to_cre_and_node.py
@@ -8,7 +8,6 @@
 
 from alembic import op
 import sqlalchemy as sa
-from sqlalchemy.engine.reflection import Inspector
 
 revision = '055dbd9f8bfe'
 down_revision = 'd4e5f6a7b8c9'   # or whatever the current head is
@@ -17,7 +16,7 @@
 
 def column_exists(table, column):
     conn = op.get_bind()
-    inspector = Inspector.from_engine(conn)
+    inspector = sa.inspect(conn)
     columns = [c['name'] for c in inspector.get_columns(table)]
     return column in columns
 
@@ -37,7 +36,8 @@ def upgrade():
         print("Column document_metadata already exists in node, skipping.")
 
 def downgrade():
-    # Remove columns (optional, but we want a clean downgrade)
-    # Note: SQLite does not support DROP COLUMN directly, but you can use batch.
-    # For simplicity, we'll just raise an error or skip.
-    pass
+    # Remove document_metadata from cre and node using SQLite batch pattern
+    with op.batch_alter_table("cre") as batch_op:
+        batch_op.drop_column("document_metadata")
+    with op.batch_alter_table("node") as batch_op:
+        batch_op.drop_column("document_metadata")
diff --git a/migrations/versions/967016ee10fa_add_embedding_vec_to_embeddings_for_.py b/migrations/versions/967016ee10fa_add_embedding_vec_to_embeddings_for_.py
index 725dc1b43..e67b64883 100644
--- a/migrations/versions/967016ee10fa_add_embedding_vec_to_embeddings_for_.py
+++ b/migrations/versions/967016ee10fa_add_embedding_vec_to_embeddings_for_.py
@@ -8,7 +8,6 @@
 
 from alembic import op
 import sqlalchemy as sa
-from sqlalchemy.engine.reflection import Inspector
 
 revision = '967016ee10fa'
 down_revision = '055dbd9f8bfe'   # <-- set this to the head revision you found
@@ -17,7 +16,7 @@
 
 def column_exists(table, column):
     conn = op.get_bind()
-    inspector = Inspector.from_engine(conn)
+    inspector = sa.inspect(conn)
     columns = [c['name'] for c in inspector.get_columns(table)]
     return column in columns
 
@@ -29,5 +28,6 @@ def upgrade():
         print("Column embedding_vec already exists in embeddings, skipping.")
 
 def downgrade():
-    # SQLite does not support DROP COLUMN directly; skip or use batch if needed.
-    pass
\ No newline at end of file
+    # Remove embedding_vec from embeddings using SQLite batch pattern
+    with op.batch_alter_table("embeddings") as batch_op:
+        batch_op.drop_column("embedding_vec")
\ No newline at end of file
diff --git a/migrations/versions/9f1a2b3c4d5e_add_artifact_ingest_persistence.py b/migrations/versions/9f1a2b3c4d5e_add_artifact_ingest_persistence.py
index e56959f63..d599d5721 100644
--- a/migrations/versions/9f1a2b3c4d5e_add_artifact_ingest_persistence.py
+++ b/migrations/versions/9f1a2b3c4d5e_add_artifact_ingest_persistence.py
@@ -8,7 +8,6 @@
 
 from alembic import op
 import sqlalchemy as sa
-from sqlalchemy.engine.reflection import Inspector
 
 revision = "9f1a2b3c4d5e"
 down_revision = "a1b2c3d4e5f6"
@@ -17,9 +16,15 @@
 
 def table_exists(table_name):
     conn = op.get_bind()
-    inspector = Inspector.from_engine(conn)
+    inspector = sa.inspect(conn)
     return table_name in inspector.get_table_names()
 
+def constraint_exists(table, constraint_name):
+    conn = op.get_bind()
+    inspector = sa.inspect(conn)
+    constraints = inspector.get_unique_constraints(table)
+    return any(c['name'] == constraint_name for c in constraints)
+
 def upgrade():
     # Create artifact_ingest_event only if it doesn't exist
     if not table_exists("artifact_ingest_event"):
@@ -45,7 +50,15 @@ def upgrade():
             sa.UniqueConstraint("run_id", "artifact_id", name="uq_artifact_ingest_event_run_artifact"),
         )
     else:
-        print("Table artifact_ingest_event already exists, skipping creation.")
+        print("Table artifact_ingest_event already exists, checking constraints...")
+        # Ensure the unique constraint exists
+        if not constraint_exists("artifact_ingest_event", "uq_artifact_ingest_event_run_artifact"):
+            print("Adding missing unique constraint to artifact_ingest_event...")
+            with op.batch_alter_table("artifact_ingest_event") as batch_op:
+                batch_op.create_unique_constraint(
+                    "uq_artifact_ingest_event_run_artifact",
+                    ["run_id", "artifact_id"]
+                )
 
     # Create ingest_chunk only if it doesn't exist
     if not table_exists("ingest_chunk"):
@@ -68,7 +81,15 @@ def upgrade():
             sa.UniqueConstraint("artifact_event_id", "chunk_id", name="uq_ingest_chunk_artifact_chunk"),
         )
     else:
-        print("Table ingest_chunk already exists, skipping creation.")
+        print("Table ingest_chunk already exists, checking constraints...")
+        # Ensure the unique constraint exists
+        if not constraint_exists("ingest_chunk", "uq_ingest_chunk_artifact_chunk"):
+            print("Adding missing unique constraint to ingest_chunk...")
+            with op.batch_alter_table("ingest_chunk") as batch_op:
+                batch_op.create_unique_constraint(
+                    "uq_ingest_chunk_artifact_chunk",
+                    ["artifact_event_id", "chunk_id"]
+                )
 
 def downgrade():
     # Drop tables in reverse order; constraints are dropped automatically with the tables.

From 57284a0a81ea50ead141f7e52e306047524e5473 Mon Sep 17 00:00:00 2001
From: bornunique911 <ravirajmehta911@gmail.com>
Date: Thu, 6 Aug 2026 02:07:31 +0530
Subject: [PATCH 03/20] fixed black formatting issue

---
 ...e_add_document_metadata_to_cre_and_node.py | 17 +++++++-----
 ...fa_add_embedding_vec_to_embeddings_for_.py | 17 +++++++-----
 ...b3c4d5e_add_artifact_ingest_persistence.py | 26 ++++++++++++-------
 3 files changed, 38 insertions(+), 22 deletions(-)

diff --git a/migrations/versions/055dbd9f8bfe_add_document_metadata_to_cre_and_node.py b/migrations/versions/055dbd9f8bfe_add_document_metadata_to_cre_and_node.py
index ab1004870..448649a77 100644
--- a/migrations/versions/055dbd9f8bfe_add_document_metadata_to_cre_and_node.py
+++ b/migrations/versions/055dbd9f8bfe_add_document_metadata_to_cre_and_node.py
@@ -9,32 +9,35 @@
 from alembic import op
 import sqlalchemy as sa
 
-revision = '055dbd9f8bfe'
-down_revision = 'd4e5f6a7b8c9'   # or whatever the current head is
+revision = "055dbd9f8bfe"
+down_revision = "d4e5f6a7b8c9"  # or whatever the current head is
 branch_labels = None
 depends_on = None
 
+
 def column_exists(table, column):
     conn = op.get_bind()
     inspector = sa.inspect(conn)
-    columns = [c['name'] for c in inspector.get_columns(table)]
+    columns = [c["name"] for c in inspector.get_columns(table)]
     return column in columns
 
+
 def upgrade():
     # Add to 'cre' table if missing
-    if not column_exists('cre', 'document_metadata'):
-        op.add_column('cre', sa.Column('document_metadata', sa.Text(), nullable=True))
+    if not column_exists("cre", "document_metadata"):
+        op.add_column("cre", sa.Column("document_metadata", sa.Text(), nullable=True))
         print("Added document_metadata to cre")
     else:
         print("Column document_metadata already exists in cre, skipping.")
 
     # Add to 'node' table if missing
-    if not column_exists('node', 'document_metadata'):
-        op.add_column('node', sa.Column('document_metadata', sa.Text(), nullable=True))
+    if not column_exists("node", "document_metadata"):
+        op.add_column("node", sa.Column("document_metadata", sa.Text(), nullable=True))
         print("Added document_metadata to node")
     else:
         print("Column document_metadata already exists in node, skipping.")
 
+
 def downgrade():
     # Remove document_metadata from cre and node using SQLite batch pattern
     with op.batch_alter_table("cre") as batch_op:
diff --git a/migrations/versions/967016ee10fa_add_embedding_vec_to_embeddings_for_.py b/migrations/versions/967016ee10fa_add_embedding_vec_to_embeddings_for_.py
index e67b64883..e9df62506 100644
--- a/migrations/versions/967016ee10fa_add_embedding_vec_to_embeddings_for_.py
+++ b/migrations/versions/967016ee10fa_add_embedding_vec_to_embeddings_for_.py
@@ -9,25 +9,30 @@
 from alembic import op
 import sqlalchemy as sa
 
-revision = '967016ee10fa'
-down_revision = '055dbd9f8bfe'   # <-- set this to the head revision you found
+revision = "967016ee10fa"
+down_revision = "055dbd9f8bfe"  # <-- set this to the head revision you found
 branch_labels = None
 depends_on = None
 
+
 def column_exists(table, column):
     conn = op.get_bind()
     inspector = sa.inspect(conn)
-    columns = [c['name'] for c in inspector.get_columns(table)]
+    columns = [c["name"] for c in inspector.get_columns(table)]
     return column in columns
 
+
 def upgrade():
-    if not column_exists('embeddings', 'embedding_vec'):
-        op.add_column('embeddings', sa.Column('embedding_vec', sa.Text(), nullable=True))
+    if not column_exists("embeddings", "embedding_vec"):
+        op.add_column(
+            "embeddings", sa.Column("embedding_vec", sa.Text(), nullable=True)
+        )
         print("Added embedding_vec to embeddings")
     else:
         print("Column embedding_vec already exists in embeddings, skipping.")
 
+
 def downgrade():
     # Remove embedding_vec from embeddings using SQLite batch pattern
     with op.batch_alter_table("embeddings") as batch_op:
-        batch_op.drop_column("embedding_vec")
\ No newline at end of file
+        batch_op.drop_column("embedding_vec")
diff --git a/migrations/versions/9f1a2b3c4d5e_add_artifact_ingest_persistence.py b/migrations/versions/9f1a2b3c4d5e_add_artifact_ingest_persistence.py
index d599d5721..c0990a650 100644
--- a/migrations/versions/9f1a2b3c4d5e_add_artifact_ingest_persistence.py
+++ b/migrations/versions/9f1a2b3c4d5e_add_artifact_ingest_persistence.py
@@ -14,16 +14,19 @@
 branch_labels = None
 depends_on = None
 
+
 def table_exists(table_name):
     conn = op.get_bind()
     inspector = sa.inspect(conn)
     return table_name in inspector.get_table_names()
 
+
 def constraint_exists(table, constraint_name):
     conn = op.get_bind()
     inspector = sa.inspect(conn)
     constraints = inspector.get_unique_constraints(table)
-    return any(c['name'] == constraint_name for c in constraints)
+    return any(c["name"] == constraint_name for c in constraints)
+
 
 def upgrade():
     # Create artifact_ingest_event only if it doesn't exist
@@ -47,17 +50,20 @@ def upgrade():
                 onupdate="CASCADE",
                 ondelete="CASCADE",
             ),
-            sa.UniqueConstraint("run_id", "artifact_id", name="uq_artifact_ingest_event_run_artifact"),
+            sa.UniqueConstraint(
+                "run_id", "artifact_id", name="uq_artifact_ingest_event_run_artifact"
+            ),
         )
     else:
         print("Table artifact_ingest_event already exists, checking constraints...")
         # Ensure the unique constraint exists
-        if not constraint_exists("artifact_ingest_event", "uq_artifact_ingest_event_run_artifact"):
+        if not constraint_exists(
+            "artifact_ingest_event", "uq_artifact_ingest_event_run_artifact"
+        ):
             print("Adding missing unique constraint to artifact_ingest_event...")
             with op.batch_alter_table("artifact_ingest_event") as batch_op:
                 batch_op.create_unique_constraint(
-                    "uq_artifact_ingest_event_run_artifact",
-                    ["run_id", "artifact_id"]
+                    "uq_artifact_ingest_event_run_artifact", ["run_id", "artifact_id"]
                 )
 
     # Create ingest_chunk only if it doesn't exist
@@ -78,7 +84,9 @@ def upgrade():
                 onupdate="CASCADE",
                 ondelete="CASCADE",
             ),
-            sa.UniqueConstraint("artifact_event_id", "chunk_id", name="uq_ingest_chunk_artifact_chunk"),
+            sa.UniqueConstraint(
+                "artifact_event_id", "chunk_id", name="uq_ingest_chunk_artifact_chunk"
+            ),
         )
     else:
         print("Table ingest_chunk already exists, checking constraints...")
@@ -87,11 +95,11 @@ def upgrade():
             print("Adding missing unique constraint to ingest_chunk...")
             with op.batch_alter_table("ingest_chunk") as batch_op:
                 batch_op.create_unique_constraint(
-                    "uq_ingest_chunk_artifact_chunk",
-                    ["artifact_event_id", "chunk_id"]
+                    "uq_ingest_chunk_artifact_chunk", ["artifact_event_id", "chunk_id"]
                 )
 
+
 def downgrade():
     # Drop tables in reverse order; constraints are dropped automatically with the tables.
     op.drop_table("ingest_chunk")
-    op.drop_table("artifact_ingest_event")
\ No newline at end of file
+    op.drop_table("artifact_ingest_event")

From d4bea6fb2533284ff05a911323c8835aab8b115b Mon Sep 17 00:00:00 2001
From: bornunique911 <ravirajmehta911@gmail.com>
Date: Thu, 6 Aug 2026 02:19:19 +0530
Subject: [PATCH 04/20] migrations: improve downgrade safety and constraint
 validation

---
 ...e_add_document_metadata_to_cre_and_node.py | 15 ++++++--
 ...fa_add_embedding_vec_to_embeddings_for_.py |  4 +--
 ...b3c4d5e_add_artifact_ingest_persistence.py | 36 +++++++++++++++----
 3 files changed, 44 insertions(+), 11 deletions(-)

diff --git a/migrations/versions/055dbd9f8bfe_add_document_metadata_to_cre_and_node.py b/migrations/versions/055dbd9f8bfe_add_document_metadata_to_cre_and_node.py
index 448649a77..a3f3181ca 100644
--- a/migrations/versions/055dbd9f8bfe_add_document_metadata_to_cre_and_node.py
+++ b/migrations/versions/055dbd9f8bfe_add_document_metadata_to_cre_and_node.py
@@ -39,8 +39,17 @@ def upgrade():
 
 
 def downgrade():
-    # Remove document_metadata from cre and node using SQLite batch pattern
+    # Remove document_metadata from cre and node only if they were added by this migration
+    # (i.e., they exist but we can't tell if they were pre-existing, but we can check existence)
+    # For safety, we drop only if the column exists; if it existed before, this migration
+    # would have skipped adding it, but we can't track that. So we drop unconditionally
+    # because it's unlikely the column existed before this migration.
+    # However, to be safe, we can use batch_alter_table only if the column exists.
     with op.batch_alter_table("cre") as batch_op:
-        batch_op.drop_column("document_metadata")
+        # If the column doesn't exist, drop_column will raise an error, so we must check.
+        # We'll re-use column_exists but note that it's defined above.
+        if column_exists("cre", "document_metadata"):
+            batch_op.drop_column("document_metadata")
     with op.batch_alter_table("node") as batch_op:
-        batch_op.drop_column("document_metadata")
+        if column_exists("node", "document_metadata"):
+            batch_op.drop_column("document_metadata")
diff --git a/migrations/versions/967016ee10fa_add_embedding_vec_to_embeddings_for_.py b/migrations/versions/967016ee10fa_add_embedding_vec_to_embeddings_for_.py
index e9df62506..0feb71c2b 100644
--- a/migrations/versions/967016ee10fa_add_embedding_vec_to_embeddings_for_.py
+++ b/migrations/versions/967016ee10fa_add_embedding_vec_to_embeddings_for_.py
@@ -33,6 +33,6 @@ def upgrade():
 
 
 def downgrade():
-    # Remove embedding_vec from embeddings using SQLite batch pattern
     with op.batch_alter_table("embeddings") as batch_op:
-        batch_op.drop_column("embedding_vec")
+        if column_exists("embeddings", "embedding_vec"):
+            batch_op.drop_column("embedding_vec")
diff --git a/migrations/versions/9f1a2b3c4d5e_add_artifact_ingest_persistence.py b/migrations/versions/9f1a2b3c4d5e_add_artifact_ingest_persistence.py
index c0990a650..f5eece86c 100644
--- a/migrations/versions/9f1a2b3c4d5e_add_artifact_ingest_persistence.py
+++ b/migrations/versions/9f1a2b3c4d5e_add_artifact_ingest_persistence.py
@@ -21,11 +21,23 @@ def table_exists(table_name):
     return table_name in inspector.get_table_names()
 
 
-def constraint_exists(table, constraint_name):
+def has_unique_on_columns(table, columns):
+    """Return True if any unique constraint (named or unnamed) exists on exactly these columns."""
     conn = op.get_bind()
     inspector = sa.inspect(conn)
     constraints = inspector.get_unique_constraints(table)
-    return any(c["name"] == constraint_name for c in constraints)
+    for c in constraints:
+        if c["column_names"] == columns:
+            return True
+    return False
+
+
+def constraint_name_exists(table, constraint_name):
+    conn = op.get_bind()
+    inspector = sa.inspect(conn)
+    return any(
+        c["name"] == constraint_name for c in inspector.get_unique_constraints(table)
+    )
 
 
 def upgrade():
@@ -57,11 +69,18 @@ def upgrade():
     else:
         print("Table artifact_ingest_event already exists, checking constraints...")
         # Ensure the unique constraint exists
-        if not constraint_exists(
-            "artifact_ingest_event", "uq_artifact_ingest_event_run_artifact"
+        if not has_unique_on_columns(
+            "artifact_ingest_event", ["run_id", "artifact_id"]
         ):
             print("Adding missing unique constraint to artifact_ingest_event...")
             with op.batch_alter_table("artifact_ingest_event") as batch_op:
+                # Drop any existing named constraint with our name (if it has wrong columns)
+                if constraint_name_exists(
+                    "artifact_ingest_event", "uq_artifact_ingest_event_run_artifact"
+                ):
+                    batch_op.drop_constraint(
+                        "uq_artifact_ingest_event_run_artifact", type_="unique"
+                    )
                 batch_op.create_unique_constraint(
                     "uq_artifact_ingest_event_run_artifact", ["run_id", "artifact_id"]
                 )
@@ -90,10 +109,15 @@ def upgrade():
         )
     else:
         print("Table ingest_chunk already exists, checking constraints...")
-        # Ensure the unique constraint exists
-        if not constraint_exists("ingest_chunk", "uq_ingest_chunk_artifact_chunk"):
+        if not has_unique_on_columns("ingest_chunk", ["artifact_event_id", "chunk_id"]):
             print("Adding missing unique constraint to ingest_chunk...")
             with op.batch_alter_table("ingest_chunk") as batch_op:
+                if constraint_name_exists(
+                    "ingest_chunk", "uq_ingest_chunk_artifact_chunk"
+                ):
+                    batch_op.drop_constraint(
+                        "uq_ingest_chunk_artifact_chunk", type_="unique"
+                    )
                 batch_op.create_unique_constraint(
                     "uq_ingest_chunk_artifact_chunk", ["artifact_event_id", "chunk_id"]
                 )

From 4ea91bddbaab6a19972fd4d936c6f9ead48e773d Mon Sep 17 00:00:00 2001
From: bornunique911 <ravirajmehta911@gmail.com>
Date: Fri, 7 Aug 2026 14:56:27 +0530
Subject: [PATCH 05/20] migrations: remove document_metadata migration for cre
 and node

---
 ...e_add_document_metadata_to_cre_and_node.py | 55 -------------------
 1 file changed, 55 deletions(-)
 delete mode 100644 migrations/versions/055dbd9f8bfe_add_document_metadata_to_cre_and_node.py

diff --git a/migrations/versions/055dbd9f8bfe_add_document_metadata_to_cre_and_node.py b/migrations/versions/055dbd9f8bfe_add_document_metadata_to_cre_and_node.py
deleted file mode 100644
index a3f3181ca..000000000
--- a/migrations/versions/055dbd9f8bfe_add_document_metadata_to_cre_and_node.py
+++ /dev/null
@@ -1,55 +0,0 @@
-"""add document_metadata to cre and node
-
-Revision ID: 055dbd9f8bfe
-Revises: d4e5f6a7b8c9
-Create Date: 2026-08-06 00:13:49.191527
-
-"""
-
-from alembic import op
-import sqlalchemy as sa
-
-revision = "055dbd9f8bfe"
-down_revision = "d4e5f6a7b8c9"  # or whatever the current head is
-branch_labels = None
-depends_on = None
-
-
-def column_exists(table, column):
-    conn = op.get_bind()
-    inspector = sa.inspect(conn)
-    columns = [c["name"] for c in inspector.get_columns(table)]
-    return column in columns
-
-
-def upgrade():
-    # Add to 'cre' table if missing
-    if not column_exists("cre", "document_metadata"):
-        op.add_column("cre", sa.Column("document_metadata", sa.Text(), nullable=True))
-        print("Added document_metadata to cre")
-    else:
-        print("Column document_metadata already exists in cre, skipping.")
-
-    # Add to 'node' table if missing
-    if not column_exists("node", "document_metadata"):
-        op.add_column("node", sa.Column("document_metadata", sa.Text(), nullable=True))
-        print("Added document_metadata to node")
-    else:
-        print("Column document_metadata already exists in node, skipping.")
-
-
-def downgrade():
-    # Remove document_metadata from cre and node only if they were added by this migration
-    # (i.e., they exist but we can't tell if they were pre-existing, but we can check existence)
-    # For safety, we drop only if the column exists; if it existed before, this migration
-    # would have skipped adding it, but we can't track that. So we drop unconditionally
-    # because it's unlikely the column existed before this migration.
-    # However, to be safe, we can use batch_alter_table only if the column exists.
-    with op.batch_alter_table("cre") as batch_op:
-        # If the column doesn't exist, drop_column will raise an error, so we must check.
-        # We'll re-use column_exists but note that it's defined above.
-        if column_exists("cre", "document_metadata"):
-            batch_op.drop_column("document_metadata")
-    with op.batch_alter_table("node") as batch_op:
-        if column_exists("node", "document_metadata"):
-            batch_op.drop_column("document_metadata")

From 50e69a7f2b3cf9d26905339e27b52e21cfef2b8b Mon Sep 17 00:00:00 2001
From: bornunique911 <ravirajmehta911@gmail.com>
Date: Fri, 7 Aug 2026 14:57:05 +0530
Subject: [PATCH 06/20] migrations: enhance idempotency for document_metadata
 column addition and removal

---
 ...dd_missing_document_metadata_column_to_.py | 60 +++++++++++--------
 1 file changed, 34 insertions(+), 26 deletions(-)

diff --git a/migrations/versions/b5ac48010165_add_missing_document_metadata_column_to_.py b/migrations/versions/b5ac48010165_add_missing_document_metadata_column_to_.py
index c0f8d6b0c..57057f049 100644
--- a/migrations/versions/b5ac48010165_add_missing_document_metadata_column_to_.py
+++ b/migrations/versions/b5ac48010165_add_missing_document_metadata_column_to_.py
@@ -18,33 +18,41 @@
 depends_on = None
 
 
+def column_exists(table, column):
+    conn = op.get_bind()
+    inspector = sa.inspect(conn)
+    columns = [c["name"] for c in inspector.get_columns(table)]
+    return column in columns
+
+
 def upgrade():
-    # Defensive: some environments (e.g. production) already have this column
-    # applied out-of-band without a corresponding migration ever being
-    # committed, so this must not assume a clean "column doesn't exist" state.
-    inspector = inspect(op.get_bind())
-    node_columns = {c["name"] for c in inspector.get_columns("node")}
-    cre_columns = {c["name"] for c in inspector.get_columns("cre")}
-
-    if "document_metadata" not in node_columns:
-        with op.batch_alter_table("node", schema=None) as batch_op:
-            batch_op.add_column(
-                sa.Column("document_metadata", sa.JSON(), nullable=True)
-            )
-
-    if "document_metadata" not in cre_columns:
-        with op.batch_alter_table("cre", schema=None) as batch_op:
-            batch_op.add_column(
-                sa.Column("document_metadata", sa.JSON(), nullable=True)
-            )
+    # Add to 'cre' table if missing
+    if not column_exists("cre", "document_metadata"):
+        op.add_column("cre", sa.Column("document_metadata", sa.Text(), nullable=True))
+        print("Added document_metadata to cre")
+    else:
+        print("Column document_metadata already exists in cre, skipping.")
+
+    # Add to 'node' table if missing
+    if not column_exists("node", "document_metadata"):
+        op.add_column("node", sa.Column("document_metadata", sa.Text(), nullable=True))
+        print("Added document_metadata to node")
+    else:
+        print("Column document_metadata already exists in node, skipping.")
 
 
 def downgrade():
-    # Intentional no-op: upgrade() only adds this column where it was
-    # missing, so on environments where it pre-existed (e.g. production,
-    # applied out-of-band) this migration never created it. Unconditionally
-    # dropping it here would destroy that pre-existing data on downgrade.
-    # There is no reliable way to tell "did *this* migration add the
-    # column" apart from "did it already exist," so the safe choice is to
-    # leave the column alone rather than risk deleting real data.
-    pass
+    # Remove document_metadata from cre and node only if they were added by this migration
+    # (i.e., they exist but we can't tell if they were pre-existing, but we can check existence)
+    # For safety, we drop only if the column exists; if it existed before, this migration
+    # would have skipped adding it, but we can't track that. So we drop unconditionally
+    # because it's unlikely the column existed before this migration.
+    # However, to be safe, we can use batch_alter_table only if the column exists.
+    with op.batch_alter_table("cre") as batch_op:
+        # If the column doesn't exist, drop_column will raise an error, so we must check.
+        # We'll re-use column_exists but note that it's defined above.
+        if column_exists("cre", "document_metadata"):
+            batch_op.drop_column("document_metadata")
+    with op.batch_alter_table("node") as batch_op:
+        if column_exists("node", "document_metadata"):
+            batch_op.drop_column("document_metadata")

From 1493bfa04f3adcf1afc10135165d0221be3da78d Mon Sep 17 00:00:00 2001
From: bornunique911 <ravirajmehta911@gmail.com>
Date: Fri, 7 Aug 2026 14:58:28 +0530
Subject: [PATCH 07/20] migrations: update revision ID for embedding_vec
 migration

---
 .../967016ee10fa_add_embedding_vec_to_embeddings_for_.py        | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/migrations/versions/967016ee10fa_add_embedding_vec_to_embeddings_for_.py b/migrations/versions/967016ee10fa_add_embedding_vec_to_embeddings_for_.py
index 0feb71c2b..fe394fe39 100644
--- a/migrations/versions/967016ee10fa_add_embedding_vec_to_embeddings_for_.py
+++ b/migrations/versions/967016ee10fa_add_embedding_vec_to_embeddings_for_.py
@@ -1,7 +1,7 @@
 """add embedding_vec to embeddings for SQLite
 
 Revision ID: 967016ee10fa
-Revises: 055dbd9f8bfe
+Revises: b5ac48010165
 Create Date: 2026-08-06 00:13:49.191527
 
 """

From c5b1a56c96a2d7e5236f087e98c6880683454438 Mon Sep 17 00:00:00 2001
From: bornunique911 <ravirajmehta911@gmail.com>
Date: Fri, 7 Aug 2026 15:00:35 +0530
Subject: [PATCH 08/20] migrations: remove unused import of inspect from
 sqlalchemy

---
 .../b5ac48010165_add_missing_document_metadata_column_to_.py     | 1 -
 1 file changed, 1 deletion(-)

diff --git a/migrations/versions/b5ac48010165_add_missing_document_metadata_column_to_.py b/migrations/versions/b5ac48010165_add_missing_document_metadata_column_to_.py
index 57057f049..7b49db2c1 100644
--- a/migrations/versions/b5ac48010165_add_missing_document_metadata_column_to_.py
+++ b/migrations/versions/b5ac48010165_add_missing_document_metadata_column_to_.py
@@ -8,7 +8,6 @@
 
 from alembic import op
 import sqlalchemy as sa
-from sqlalchemy import inspect
 
 
 # revision identifiers, used by Alembic.

From f18e2ec75b35574ae672b1ef415ab3e0afd00b25 Mon Sep 17 00:00:00 2001
From: bornunique911 <ravirajmehta911@gmail.com>
Date: Fri, 7 Aug 2026 15:22:42 +0530
Subject: [PATCH 09/20] migrations: change document_metadata column type to
 JSON for cre and node

---
 .../b5ac48010165_add_missing_document_metadata_column_to_.py  | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/migrations/versions/b5ac48010165_add_missing_document_metadata_column_to_.py b/migrations/versions/b5ac48010165_add_missing_document_metadata_column_to_.py
index 7b49db2c1..dc14560cb 100644
--- a/migrations/versions/b5ac48010165_add_missing_document_metadata_column_to_.py
+++ b/migrations/versions/b5ac48010165_add_missing_document_metadata_column_to_.py
@@ -27,14 +27,14 @@ def column_exists(table, column):
 def upgrade():
     # Add to 'cre' table if missing
     if not column_exists("cre", "document_metadata"):
-        op.add_column("cre", sa.Column("document_metadata", sa.Text(), nullable=True))
+        op.add_column("cre", sa.Column("document_metadata", sa.JSON(), nullable=True))
         print("Added document_metadata to cre")
     else:
         print("Column document_metadata already exists in cre, skipping.")
 
     # Add to 'node' table if missing
     if not column_exists("node", "document_metadata"):
-        op.add_column("node", sa.Column("document_metadata", sa.Text(), nullable=True))
+        op.add_column("node", sa.Column("document_metadata", sa.JSON(), nullable=True))
         print("Added document_metadata to node")
     else:
         print("Column document_metadata already exists in node, skipping.")

From dcfafbb22611f4dce817caba9698d1777f3f9558 Mon Sep 17 00:00:00 2001
From: bornunique911 <ravirajmehta911@gmail.com>
Date: Fri, 7 Aug 2026 15:29:33 +0530
Subject: [PATCH 10/20] migrations: enhance idempotency for document_metadata
 column tracking and removal

---
 ...dd_missing_document_metadata_column_to_.py | 85 ++++++++++++++-----
 1 file changed, 64 insertions(+), 21 deletions(-)

diff --git a/migrations/versions/b5ac48010165_add_missing_document_metadata_column_to_.py b/migrations/versions/b5ac48010165_add_missing_document_metadata_column_to_.py
index dc14560cb..293f23e9f 100644
--- a/migrations/versions/b5ac48010165_add_missing_document_metadata_column_to_.py
+++ b/migrations/versions/b5ac48010165_add_missing_document_metadata_column_to_.py
@@ -9,7 +9,6 @@
 from alembic import op
 import sqlalchemy as sa
 
-
 # revision identifiers, used by Alembic.
 revision = "b5ac48010165"
 down_revision = "d4e5f6a7b8c9"
@@ -24,34 +23,78 @@ def column_exists(table, column):
     return column in columns
 
 
+def table_exists(table):
+    conn = op.get_bind()
+    inspector = sa.inspect(conn)
+    return table in inspector.get_table_names()
+
+
+def tracking_record_exists(revision, table_name, column_name):
+    conn = op.get_bind()
+    # Use a raw query because the tracking table may not exist yet in some contexts.
+    # We'll check existence first.
+    if not table_exists("_migration_tracking"):
+        return False
+    result = conn.execute(
+        sa.text(
+            "SELECT 1 FROM _migration_tracking WHERE revision = :rev AND table_name = :tbl AND column_name = :col"
+        ),
+        {"rev": revision, "tbl": table_name, "col": column_name},
+    )
+    return result.first() is not None
+
+
+def add_tracking_record(revision, table_name, column_name):
+    # Ensure the tracking table exists
+    if not table_exists("_migration_tracking"):
+        op.create_table(
+            "_migration_tracking",
+            sa.Column("revision", sa.String, primary_key=True),
+            sa.Column("table_name", sa.String, primary_key=True),
+            sa.Column("column_name", sa.String, primary_key=True),
+        )
+    op.execute(
+        sa.text(
+            "INSERT INTO _migration_tracking (revision, table_name, column_name) VALUES (:rev, :tbl, :col)"
+        ),
+        {"rev": revision, "tbl": table_name, "col": column_name},
+    )
+
+
+def remove_tracking_record(revision, table_name, column_name):
+    if table_exists("_migration_tracking"):
+        op.execute(
+            sa.text(
+                "DELETE FROM _migration_tracking WHERE revision = :rev AND table_name = :tbl AND column_name = :col"
+            ),
+            {"rev": revision, "tbl": table_name, "col": column_name},
+        )
+
+
 def upgrade():
     # Add to 'cre' table if missing
     if not column_exists("cre", "document_metadata"):
         op.add_column("cre", sa.Column("document_metadata", sa.JSON(), nullable=True))
-        print("Added document_metadata to cre")
-    else:
-        print("Column document_metadata already exists in cre, skipping.")
+        add_tracking_record(revision, "cre", "document_metadata")
 
     # Add to 'node' table if missing
     if not column_exists("node", "document_metadata"):
         op.add_column("node", sa.Column("document_metadata", sa.JSON(), nullable=True))
-        print("Added document_metadata to node")
-    else:
-        print("Column document_metadata already exists in node, skipping.")
+        add_tracking_record(revision, "node", "document_metadata")
 
 
 def downgrade():
-    # Remove document_metadata from cre and node only if they were added by this migration
-    # (i.e., they exist but we can't tell if they were pre-existing, but we can check existence)
-    # For safety, we drop only if the column exists; if it existed before, this migration
-    # would have skipped adding it, but we can't track that. So we drop unconditionally
-    # because it's unlikely the column existed before this migration.
-    # However, to be safe, we can use batch_alter_table only if the column exists.
-    with op.batch_alter_table("cre") as batch_op:
-        # If the column doesn't exist, drop_column will raise an error, so we must check.
-        # We'll re-use column_exists but note that it's defined above.
-        if column_exists("cre", "document_metadata"):
-            batch_op.drop_column("document_metadata")
-    with op.batch_alter_table("node") as batch_op:
-        if column_exists("node", "document_metadata"):
-            batch_op.drop_column("document_metadata")
+    # Drop only columns that were added by this migration (tracked)
+    # For 'cre'
+    if tracking_record_exists(revision, "cre", "document_metadata"):
+        with op.batch_alter_table("cre") as batch_op:
+            if column_exists("cre", "document_metadata"):
+                batch_op.drop_column("document_metadata")
+        remove_tracking_record(revision, "cre", "document_metadata")
+
+    # For 'node'
+    if tracking_record_exists(revision, "node", "document_metadata"):
+        with op.batch_alter_table("node") as batch_op:
+            if column_exists("node", "document_metadata"):
+                batch_op.drop_column("document_metadata")
+        remove_tracking_record(revision, "node", "document_metadata")
\ No newline at end of file

From 2c714dca327e31128bc3128f92c73312483223e5 Mon Sep 17 00:00:00 2001
From: bornunique911 <ravirajmehta911@gmail.com>
Date: Fri, 7 Aug 2026 15:31:34 +0530
Subject: [PATCH 11/20] migrations: ensure tracking record removal for
 document_metadata column in downgrade

---
 .../b5ac48010165_add_missing_document_metadata_column_to_.py    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/migrations/versions/b5ac48010165_add_missing_document_metadata_column_to_.py b/migrations/versions/b5ac48010165_add_missing_document_metadata_column_to_.py
index 293f23e9f..b12b324c8 100644
--- a/migrations/versions/b5ac48010165_add_missing_document_metadata_column_to_.py
+++ b/migrations/versions/b5ac48010165_add_missing_document_metadata_column_to_.py
@@ -97,4 +97,4 @@ def downgrade():
         with op.batch_alter_table("node") as batch_op:
             if column_exists("node", "document_metadata"):
                 batch_op.drop_column("document_metadata")
-        remove_tracking_record(revision, "node", "document_metadata")
\ No newline at end of file
+        remove_tracking_record(revision, "node", "document_metadata")

From 0ba184858d989bb60060f6b217d22f522d3aa24b Mon Sep 17 00:00:00 2001
From: bornunique911 <ravirajmehta911@gmail.com>
Date: Fri, 7 Aug 2026 15:38:57 +0530
Subject: [PATCH 12/20] migrations: refine tracking record checks and cleanup
 in downgrade

---
 ...5ac48010165_add_missing_document_metadata_column_to_.py | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/migrations/versions/b5ac48010165_add_missing_document_metadata_column_to_.py b/migrations/versions/b5ac48010165_add_missing_document_metadata_column_to_.py
index b12b324c8..4dd47d5bb 100644
--- a/migrations/versions/b5ac48010165_add_missing_document_metadata_column_to_.py
+++ b/migrations/versions/b5ac48010165_add_missing_document_metadata_column_to_.py
@@ -31,8 +31,6 @@ def table_exists(table):
 
 def tracking_record_exists(revision, table_name, column_name):
     conn = op.get_bind()
-    # Use a raw query because the tracking table may not exist yet in some contexts.
-    # We'll check existence first.
     if not table_exists("_migration_tracking"):
         return False
     result = conn.execute(
@@ -45,7 +43,6 @@ def tracking_record_exists(revision, table_name, column_name):
 
 
 def add_tracking_record(revision, table_name, column_name):
-    # Ensure the tracking table exists
     if not table_exists("_migration_tracking"):
         op.create_table(
             "_migration_tracking",
@@ -85,16 +82,14 @@ def upgrade():
 
 def downgrade():
     # Drop only columns that were added by this migration (tracked)
-    # For 'cre'
     if tracking_record_exists(revision, "cre", "document_metadata"):
         with op.batch_alter_table("cre") as batch_op:
             if column_exists("cre", "document_metadata"):
                 batch_op.drop_column("document_metadata")
         remove_tracking_record(revision, "cre", "document_metadata")
 
-    # For 'node'
     if tracking_record_exists(revision, "node", "document_metadata"):
         with op.batch_alter_table("node") as batch_op:
             if column_exists("node", "document_metadata"):
                 batch_op.drop_column("document_metadata")
-        remove_tracking_record(revision, "node", "document_metadata")
+        remove_tracking_record(revision, "node", "document_metadata")
\ No newline at end of file

From c39285032c81eb893e5d0f7de497714cdf7d957d Mon Sep 17 00:00:00 2001
From: bornunique911 <ravirajmehta911@gmail.com>
Date: Fri, 7 Aug 2026 15:39:34 +0530
Subject: [PATCH 13/20] migrations: ensure tracking record removal for node's
 document_metadata column in downgrade

---
 .../b5ac48010165_add_missing_document_metadata_column_to_.py    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/migrations/versions/b5ac48010165_add_missing_document_metadata_column_to_.py b/migrations/versions/b5ac48010165_add_missing_document_metadata_column_to_.py
index 4dd47d5bb..1e9f3b807 100644
--- a/migrations/versions/b5ac48010165_add_missing_document_metadata_column_to_.py
+++ b/migrations/versions/b5ac48010165_add_missing_document_metadata_column_to_.py
@@ -92,4 +92,4 @@ def downgrade():
         with op.batch_alter_table("node") as batch_op:
             if column_exists("node", "document_metadata"):
                 batch_op.drop_column("document_metadata")
-        remove_tracking_record(revision, "node", "document_metadata")
\ No newline at end of file
+        remove_tracking_record(revision, "node", "document_metadata")

From 7395cf8fe93a4ad8abecb19774ecfa5cf0a7aa49 Mon Sep 17 00:00:00 2001
From: bornunique911 <ravirajmehta911@gmail.com>
Date: Fri, 7 Aug 2026 15:46:50 +0530
Subject: [PATCH 14/20] migrations: remove unnecessary revision identifiers
 comment

---
 .../b5ac48010165_add_missing_document_metadata_column_to_.py     | 1 -
 1 file changed, 1 deletion(-)

diff --git a/migrations/versions/b5ac48010165_add_missing_document_metadata_column_to_.py b/migrations/versions/b5ac48010165_add_missing_document_metadata_column_to_.py
index 1e9f3b807..ffde1c617 100644
--- a/migrations/versions/b5ac48010165_add_missing_document_metadata_column_to_.py
+++ b/migrations/versions/b5ac48010165_add_missing_document_metadata_column_to_.py
@@ -9,7 +9,6 @@
 from alembic import op
 import sqlalchemy as sa
 
-# revision identifiers, used by Alembic.
 revision = "b5ac48010165"
 down_revision = "d4e5f6a7b8c9"
 branch_labels = None

From 1e31f252aa25c78d69faaf27058e437f9da68896 Mon Sep 17 00:00:00 2001
From: bornunique911 <ravirajmehta911@gmail.com>
Date: Fri, 7 Aug 2026 15:56:40 +0530
Subject: [PATCH 15/20] migrations: enhance idempotency for tracking records in
 upgrade and downgrade

---
 ...b3c4d5e_add_artifact_ingest_persistence.py | 89 +++++++++++++++----
 1 file changed, 70 insertions(+), 19 deletions(-)

diff --git a/migrations/versions/9f1a2b3c4d5e_add_artifact_ingest_persistence.py b/migrations/versions/9f1a2b3c4d5e_add_artifact_ingest_persistence.py
index f5eece86c..40b999c0b 100644
--- a/migrations/versions/9f1a2b3c4d5e_add_artifact_ingest_persistence.py
+++ b/migrations/versions/9f1a2b3c4d5e_add_artifact_ingest_persistence.py
@@ -40,6 +40,45 @@ def constraint_name_exists(table, constraint_name):
     )
 
 
+def tracking_record_exists(revision, table_name):
+    """Check if this migration created the given table."""
+    conn = op.get_bind()
+    if not table_exists("_migration_tracking"):
+        return False
+    result = conn.execute(
+        sa.text(
+            "SELECT 1 FROM _migration_tracking WHERE revision = :rev AND table_name = :tbl"
+        ),
+        {"rev": revision, "tbl": table_name},
+    )
+    return result.first() is not None
+
+
+def add_tracking_record(revision, table_name):
+    if not table_exists("_migration_tracking"):
+        op.create_table(
+            "_migration_tracking",
+            sa.Column("revision", sa.String, primary_key=True),
+            sa.Column("table_name", sa.String, primary_key=True),
+        )
+    op.execute(
+        sa.text(
+            "INSERT INTO _migration_tracking (revision, table_name) VALUES (:rev, :tbl)"
+        ),
+        {"rev": revision, "tbl": table_name},
+    )
+
+
+def remove_tracking_record(revision, table_name):
+    if table_exists("_migration_tracking"):
+        op.execute(
+            sa.text(
+                "DELETE FROM _migration_tracking WHERE revision = :rev AND table_name = :tbl"
+            ),
+            {"rev": revision, "tbl": table_name},
+        )
+
+
 def upgrade():
     # Create artifact_ingest_event only if it doesn't exist
     if not table_exists("artifact_ingest_event"):
@@ -66,24 +105,29 @@ def upgrade():
                 "run_id", "artifact_id", name="uq_artifact_ingest_event_run_artifact"
             ),
         )
+        add_tracking_record(revision, "artifact_ingest_event")
     else:
-        print("Table artifact_ingest_event already exists, checking constraints...")
         # Ensure the unique constraint exists
         if not has_unique_on_columns(
             "artifact_ingest_event", ["run_id", "artifact_id"]
         ):
-            print("Adding missing unique constraint to artifact_ingest_event...")
-            with op.batch_alter_table("artifact_ingest_event") as batch_op:
-                # Drop any existing named constraint with our name (if it has wrong columns)
-                if constraint_name_exists(
-                    "artifact_ingest_event", "uq_artifact_ingest_event_run_artifact"
-                ):
-                    batch_op.drop_constraint(
-                        "uq_artifact_ingest_event_run_artifact", type_="unique"
+            # Temporarily disable foreign keys for SQLite because we may drop the parent table
+            op.execute("PRAGMA foreign_keys=OFF")
+            try:
+                with op.batch_alter_table("artifact_ingest_event") as batch_op:
+                    if constraint_name_exists(
+                        "artifact_ingest_event",
+                        "uq_artifact_ingest_event_run_artifact",
+                    ):
+                        batch_op.drop_constraint(
+                            "uq_artifact_ingest_event_run_artifact", type_="unique"
+                        )
+                    batch_op.create_unique_constraint(
+                        "uq_artifact_ingest_event_run_artifact",
+                        ["run_id", "artifact_id"],
                     )
-                batch_op.create_unique_constraint(
-                    "uq_artifact_ingest_event_run_artifact", ["run_id", "artifact_id"]
-                )
+            finally:
+                op.execute("PRAGMA foreign_keys=ON")
 
     # Create ingest_chunk only if it doesn't exist
     if not table_exists("ingest_chunk"):
@@ -104,13 +148,14 @@ def upgrade():
                 ondelete="CASCADE",
             ),
             sa.UniqueConstraint(
-                "artifact_event_id", "chunk_id", name="uq_ingest_chunk_artifact_chunk"
+                "artifact_event_id",
+                "chunk_id",
+                name="uq_ingest_chunk_artifact_chunk",
             ),
         )
+        add_tracking_record(revision, "ingest_chunk")
     else:
-        print("Table ingest_chunk already exists, checking constraints...")
         if not has_unique_on_columns("ingest_chunk", ["artifact_event_id", "chunk_id"]):
-            print("Adding missing unique constraint to ingest_chunk...")
             with op.batch_alter_table("ingest_chunk") as batch_op:
                 if constraint_name_exists(
                     "ingest_chunk", "uq_ingest_chunk_artifact_chunk"
@@ -119,11 +164,17 @@ def upgrade():
                         "uq_ingest_chunk_artifact_chunk", type_="unique"
                     )
                 batch_op.create_unique_constraint(
-                    "uq_ingest_chunk_artifact_chunk", ["artifact_event_id", "chunk_id"]
+                    "uq_ingest_chunk_artifact_chunk",
+                    ["artifact_event_id", "chunk_id"],
                 )
 
 
 def downgrade():
-    # Drop tables in reverse order; constraints are dropped automatically with the tables.
-    op.drop_table("ingest_chunk")
-    op.drop_table("artifact_ingest_event")
+    # Drop only tables that were created by this migration
+    if tracking_record_exists(revision, "ingest_chunk"):
+        op.drop_table("ingest_chunk")
+        remove_tracking_record(revision, "ingest_chunk")
+
+    if tracking_record_exists(revision, "artifact_ingest_event"):
+        op.drop_table("artifact_ingest_event")
+        remove_tracking_record(revision, "artifact_ingest_event")

From 3dbbdc8be10a71c64e2088e7dde80cf4a2711ff7 Mon Sep 17 00:00:00 2001
From: bornunique911 <ravirajmehta911@gmail.com>
Date: Fri, 7 Aug 2026 16:31:46 +0530
Subject: [PATCH 16/20] migrations: update down_revision to correct previous
 migration reference

---
 .../967016ee10fa_add_embedding_vec_to_embeddings_for_.py        | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/migrations/versions/967016ee10fa_add_embedding_vec_to_embeddings_for_.py b/migrations/versions/967016ee10fa_add_embedding_vec_to_embeddings_for_.py
index fe394fe39..d5b3beff8 100644
--- a/migrations/versions/967016ee10fa_add_embedding_vec_to_embeddings_for_.py
+++ b/migrations/versions/967016ee10fa_add_embedding_vec_to_embeddings_for_.py
@@ -10,7 +10,7 @@
 import sqlalchemy as sa
 
 revision = "967016ee10fa"
-down_revision = "055dbd9f8bfe"  # <-- set this to the head revision you found
+down_revision = "b5ac48010165"  # <-- set this to the head revision you found
 branch_labels = None
 depends_on = None
 

From adfa5a7bb3f217eff755de16fa63cd907c176244 Mon Sep 17 00:00:00 2001
From: bornunique911 <ravirajmehta911@gmail.com>
Date: Fri, 7 Aug 2026 16:52:15 +0530
Subject: [PATCH 17/20] migrations: clean up upgrade function by removing print
 statements

---
 .../967016ee10fa_add_embedding_vec_to_embeddings_for_.py     | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/migrations/versions/967016ee10fa_add_embedding_vec_to_embeddings_for_.py b/migrations/versions/967016ee10fa_add_embedding_vec_to_embeddings_for_.py
index d5b3beff8..c76d1e1e3 100644
--- a/migrations/versions/967016ee10fa_add_embedding_vec_to_embeddings_for_.py
+++ b/migrations/versions/967016ee10fa_add_embedding_vec_to_embeddings_for_.py
@@ -10,7 +10,7 @@
 import sqlalchemy as sa
 
 revision = "967016ee10fa"
-down_revision = "b5ac48010165"  # <-- set this to the head revision you found
+down_revision = "b5ac48010165"
 branch_labels = None
 depends_on = None
 
@@ -27,9 +27,6 @@ def upgrade():
         op.add_column(
             "embeddings", sa.Column("embedding_vec", sa.Text(), nullable=True)
         )
-        print("Added embedding_vec to embeddings")
-    else:
-        print("Column embedding_vec already exists in embeddings, skipping.")
 
 
 def downgrade():

From debedcfe8988a6cad00411468a36af9b7e296cd2 Mon Sep 17 00:00:00 2001
From: bornunique911 <ravirajmehta911@gmail.com>
Date: Fri, 7 Aug 2026 16:57:33 +0530
Subject: [PATCH 18/20] migrations: add missing docstrings to improve docstring
 coverage

---
 ...016ee10fa_add_embedding_vec_to_embeddings_for_.py |  5 ++++-
 .../9f1a2b3c4d5e_add_artifact_ingest_persistence.py  | 12 +++++++++++-
 2 files changed, 15 insertions(+), 2 deletions(-)

diff --git a/migrations/versions/967016ee10fa_add_embedding_vec_to_embeddings_for_.py b/migrations/versions/967016ee10fa_add_embedding_vec_to_embeddings_for_.py
index c76d1e1e3..9eccc4126 100644
--- a/migrations/versions/967016ee10fa_add_embedding_vec_to_embeddings_for_.py
+++ b/migrations/versions/967016ee10fa_add_embedding_vec_to_embeddings_for_.py
@@ -16,6 +16,7 @@
 
 
 def column_exists(table, column):
+    """Check if a column exists in the given table."""
     conn = op.get_bind()
     inspector = sa.inspect(conn)
     columns = [c["name"] for c in inspector.get_columns(table)]
@@ -23,6 +24,7 @@ def column_exists(table, column):
 
 
 def upgrade():
+    """Add embedding_vec as a TEXT column for SQLite if it doesn't already exist."""
     if not column_exists("embeddings", "embedding_vec"):
         op.add_column(
             "embeddings", sa.Column("embedding_vec", sa.Text(), nullable=True)
@@ -30,6 +32,7 @@ def upgrade():
 
 
 def downgrade():
+    """Remove embedding_vec from embeddings if it was added by this migration."""
     with op.batch_alter_table("embeddings") as batch_op:
         if column_exists("embeddings", "embedding_vec"):
-            batch_op.drop_column("embedding_vec")
+            batch_op.drop_column("embedding_vec")
\ No newline at end of file
diff --git a/migrations/versions/9f1a2b3c4d5e_add_artifact_ingest_persistence.py b/migrations/versions/9f1a2b3c4d5e_add_artifact_ingest_persistence.py
index 40b999c0b..f40a7c088 100644
--- a/migrations/versions/9f1a2b3c4d5e_add_artifact_ingest_persistence.py
+++ b/migrations/versions/9f1a2b3c4d5e_add_artifact_ingest_persistence.py
@@ -16,6 +16,7 @@
 
 
 def table_exists(table_name):
+    """Check if a table exists in the current database."""
     conn = op.get_bind()
     inspector = sa.inspect(conn)
     return table_name in inspector.get_table_names()
@@ -33,6 +34,7 @@ def has_unique_on_columns(table, columns):
 
 
 def constraint_name_exists(table, constraint_name):
+    """Return True if a unique constraint with the given name exists on the table."""
     conn = op.get_bind()
     inspector = sa.inspect(conn)
     return any(
@@ -55,6 +57,7 @@ def tracking_record_exists(revision, table_name):
 
 
 def add_tracking_record(revision, table_name):
+    """Record that this migration created a table."""
     if not table_exists("_migration_tracking"):
         op.create_table(
             "_migration_tracking",
@@ -70,6 +73,7 @@ def add_tracking_record(revision, table_name):
 
 
 def remove_tracking_record(revision, table_name):
+    """Remove the tracking record for a table created by this migration."""
     if table_exists("_migration_tracking"):
         op.execute(
             sa.text(
@@ -80,6 +84,11 @@ def remove_tracking_record(revision, table_name):
 
 
 def upgrade():
+    """Create artifact_ingest_event and ingest_chunk tables with proper constraints.
+
+    If the tables already exist, ensure the required unique constraints are present,
+    handling SQLite batch rewrites with foreign key enforcement disabled.
+    """
     # Create artifact_ingest_event only if it doesn't exist
     if not table_exists("artifact_ingest_event"):
         op.create_table(
@@ -170,6 +179,7 @@ def upgrade():
 
 
 def downgrade():
+    """Drop only tables that were created by this migration."""
     # Drop only tables that were created by this migration
     if tracking_record_exists(revision, "ingest_chunk"):
         op.drop_table("ingest_chunk")
@@ -177,4 +187,4 @@ def downgrade():
 
     if tracking_record_exists(revision, "artifact_ingest_event"):
         op.drop_table("artifact_ingest_event")
-        remove_tracking_record(revision, "artifact_ingest_event")
+        remove_tracking_record(revision, "artifact_ingest_event")
\ No newline at end of file

From 891f1059841919bc6460043ce3f4ec9cd6c50db1 Mon Sep 17 00:00:00 2001
From: bornunique911 <ravirajmehta911@gmail.com>
Date: Fri, 7 Aug 2026 16:58:18 +0530
Subject: [PATCH 19/20] migrations: add missing docstrings to improve docstring
 coverage

---
 .../967016ee10fa_add_embedding_vec_to_embeddings_for_.py        | 2 +-
 .../versions/9f1a2b3c4d5e_add_artifact_ingest_persistence.py    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/migrations/versions/967016ee10fa_add_embedding_vec_to_embeddings_for_.py b/migrations/versions/967016ee10fa_add_embedding_vec_to_embeddings_for_.py
index 9eccc4126..6e8b38a9a 100644
--- a/migrations/versions/967016ee10fa_add_embedding_vec_to_embeddings_for_.py
+++ b/migrations/versions/967016ee10fa_add_embedding_vec_to_embeddings_for_.py
@@ -35,4 +35,4 @@ def downgrade():
     """Remove embedding_vec from embeddings if it was added by this migration."""
     with op.batch_alter_table("embeddings") as batch_op:
         if column_exists("embeddings", "embedding_vec"):
-            batch_op.drop_column("embedding_vec")
\ No newline at end of file
+            batch_op.drop_column("embedding_vec")
diff --git a/migrations/versions/9f1a2b3c4d5e_add_artifact_ingest_persistence.py b/migrations/versions/9f1a2b3c4d5e_add_artifact_ingest_persistence.py
index f40a7c088..aeafdccb3 100644
--- a/migrations/versions/9f1a2b3c4d5e_add_artifact_ingest_persistence.py
+++ b/migrations/versions/9f1a2b3c4d5e_add_artifact_ingest_persistence.py
@@ -187,4 +187,4 @@ def downgrade():
 
     if tracking_record_exists(revision, "artifact_ingest_event"):
         op.drop_table("artifact_ingest_event")
-        remove_tracking_record(revision, "artifact_ingest_event")
\ No newline at end of file
+        remove_tracking_record(revision, "artifact_ingest_event")

From 4bc02efe5b27ad97e4bcaedc785044eafbb649a8 Mon Sep 17 00:00:00 2001
From: bornunique911 <ravirajmehta911@gmail.com>
Date: Sat, 8 Aug 2026 01:23:43 +0530
Subject: [PATCH 20/20] migrations: enhance docstring coverage and improve
 upgrade/downgrade logic for document_metadata column

---
 ...5_add_missing_document_metadata_column_to_.py | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/migrations/versions/b5ac48010165_add_missing_document_metadata_column_to_.py b/migrations/versions/b5ac48010165_add_missing_document_metadata_column_to_.py
index ffde1c617..49bc3958a 100644
--- a/migrations/versions/b5ac48010165_add_missing_document_metadata_column_to_.py
+++ b/migrations/versions/b5ac48010165_add_missing_document_metadata_column_to_.py
@@ -16,6 +16,7 @@
 
 
 def column_exists(table, column):
+    """Check whether a column exists in the given table."""
     conn = op.get_bind()
     inspector = sa.inspect(conn)
     columns = [c["name"] for c in inspector.get_columns(table)]
@@ -23,12 +24,14 @@ def column_exists(table, column):
 
 
 def table_exists(table):
+    """Check whether a table exists in the current database."""
     conn = op.get_bind()
     inspector = sa.inspect(conn)
     return table in inspector.get_table_names()
 
 
 def tracking_record_exists(revision, table_name, column_name):
+    """Return True if a tracking record exists for this migration, table, and column."""
     conn = op.get_bind()
     if not table_exists("_migration_tracking"):
         return False
@@ -42,6 +45,7 @@ def tracking_record_exists(revision, table_name, column_name):
 
 
 def add_tracking_record(revision, table_name, column_name):
+    """Insert a tracking record to remember that this migration added a column."""
     if not table_exists("_migration_tracking"):
         op.create_table(
             "_migration_tracking",
@@ -58,6 +62,7 @@ def add_tracking_record(revision, table_name, column_name):
 
 
 def remove_tracking_record(revision, table_name, column_name):
+    """Remove the tracking record for a column added by this migration."""
     if table_exists("_migration_tracking"):
         op.execute(
             sa.text(
@@ -68,6 +73,12 @@ def remove_tracking_record(revision, table_name, column_name):
 
 
 def upgrade():
+    """Add the 'document_metadata' JSON column to the 'cre' and 'node' tables if missing.
+
+    This migration checks each table individually and only adds the column if it does not
+    already exist. After adding, it records the addition in the '_migration_tracking'
+    table to allow safe downgrades.
+    """
     # Add to 'cre' table if missing
     if not column_exists("cre", "document_metadata"):
         op.add_column("cre", sa.Column("document_metadata", sa.JSON(), nullable=True))
@@ -80,6 +91,11 @@ def upgrade():
 
 
 def downgrade():
+    """Remove the 'document_metadata' column from 'cre' and 'node' only if they were added by this migration.
+
+    Uses the tracking table to verify that this migration originally added the column,
+    then drops the column and removes the tracking record to keep the system consistent.
+    """
     # Drop only columns that were added by this migration (tracked)
     if tracking_record_exists(revision, "cre", "document_metadata"):
         with op.batch_alter_table("cre") as batch_op: