feat: non-blocking concurrent checkpoint (WAL rotation + MVCC snapshot)

.github/workflows/concurrent-writes-ci.yml

@@ -0,0 +1,250 @@
+name: ladybug concurrent-writes CI
+# Validates the Vela-Engineering/kuzu concurrent-write port.
+# Produces:
+#   - JSONL test results (one record per test case)
+#   - Markdown test summary → GitHub Step Summary
+#   - Before/after benchmark JSON files (both built on the same ubuntu runner)
+#   - Benchmark comparison → PR comment + GitHub Step Summary
+
+on:
+  push:
+    branches:
+      - feat/concurrent-writes
+  schedule:
+    - cron: "0 2 * * *" # 02:00 UTC nightly on feat/concurrent-writes
+  workflow_dispatch: # allow manual trigger any time
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
+
+jobs:
+  # ─── Build branch (ubuntu) ──────────────────────────────────────────────
+  build:
+    name: Build branch (ubuntu-latest)
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install dependencies
+        run: |
+          sudo apt-get update -qq
+          sudo apt-get install -y cmake ninja-build python3
+
+      - name: Cache CMake build (branch)
+        uses: actions/cache@v4
+        with:
+          path: build
+          key: build-branch-ubuntu-${{ github.sha }}
+          restore-keys: build-branch-ubuntu-
+
+      - name: Configure (CMake + Ninja)
+        run: |
+          cmake -B build/relwithdebinfo -G Ninja \
+                -DCMAKE_BUILD_TYPE=RelWithDebInfo \
+                -DBUILD_TESTS=TRUE \
+                -DENABLE_BACKTRACES=TRUE \
+                .
+
+      - name: Build
+        run: cmake --build build/relwithdebinfo --parallel 4
+
+      - name: Upload branch build
+        uses: actions/upload-artifact@v4
+        with:
+          name: ladybug-build-branch
+          path: build/relwithdebinfo
+          retention-days: 1
+
+  # ─── Build main (live baseline) ───────────────────────────────────────────
+  # Runs in parallel with the branch build on the same class of runner so the
+  # benchmark comparison is apples-to-apples. Cached by CMakeLists + header
+  # hash so it's a no-op when main hasn't changed.
+  build-main:
+    name: Build main (ubuntu-latest)
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: main
+
+      - name: Install dependencies
+        run: |
+          sudo apt-get update -qq
+          sudo apt-get install -y cmake ninja-build python3
+
+      - name: Cache CMake build (main)
+        uses: actions/cache@v4
+        with:
+          path: build
+          key: build-main-ubuntu-${{ hashFiles('CMakeLists.txt', 'src/**/*.h') }}
+          restore-keys: build-main-ubuntu-
+
+      - name: Configure (CMake + Ninja)
+        run: |
+          cmake -B build/relwithdebinfo -G Ninja \
+                -DCMAKE_BUILD_TYPE=RelWithDebInfo \
+                -DBUILD_TESTS=TRUE \
+                -DENABLE_BACKTRACES=TRUE \
+                .
+
+      - name: Build
+        run: cmake --build build/relwithdebinfo --parallel 4
+
+      - name: Upload main build
+        uses: actions/upload-artifact@v4
+        with:
+          name: ladybug-build-main
+          path: build/relwithdebinfo
+          retention-days: 1
+
+  # ─── Test (ubuntu) ───────────────────────────────────────────────────────
+  test:
+    name: Test (ubuntu-latest)
+    needs: build
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Download build
+        uses: actions/download-artifact@v4
+        with:
+          name: ladybug-build-branch
+          path: build/relwithdebinfo
+
+      - name: Restore test binary permissions
+        run: find build/relwithdebinfo/test -type f | xargs chmod +x 2>/dev/null || true
+
+      - name: Fetch dataset (shallow)
+        run: |
+          git clone --depth=1 https://github.com/ladybugdb/dataset.git dataset
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Run all tests (JSONL capture)
+        id: tests
+        run: |
+          python3 benchmarks/run_tests.py \
+            --build-dir   build/relwithdebinfo \
+            --dataset-dir dataset \
+            --output      benchmarks/test-results.jsonl \
+            --summary     benchmarks/test-summary.md \
+            --label       "ubuntu-latest / ${{ github.head_ref || github.ref_name }}"
+        continue-on-error: true
+
+      - name: Fail job if any tests failed
+        if: steps.tests.outcome == 'failure'
+        run: |
+          echo "Tests failed — see summary above and JSONL artifact for details."
+          exit 1
+
+      - name: Upload JSONL results
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: test-results
+          path: |
+            benchmarks/test-results.jsonl
+            benchmarks/test-summary.md
+          retention-days: 14
+
+  # ─── Benchmark ────────────────────────────────────────────────────────────
+  #
+  # Both main and the branch are built on ubuntu-latest (in parallel above).
+  # Running both on the same class of runner eliminates hardware variance so
+  # the comparison is genuinely apples-to-apples.
+  #
+  benchmark:
+    name: Benchmark (ubuntu-latest)
+    needs: [build, build-main]
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Download branch build
+        uses: actions/download-artifact@v4
+        with:
+          name: ladybug-build-branch
+          path: build/relwithdebinfo
+
+      - name: Download main build
+        uses: actions/download-artifact@v4
+        with:
+          name: ladybug-build-main
+          path: build-main/relwithdebinfo
+
+      - name: Restore binary permissions
+        run: |
+          find build/relwithdebinfo      -type f | xargs chmod +x 2>/dev/null || true
+          find build-main/relwithdebinfo -type f | xargs chmod +x 2>/dev/null || true
+
+      - name: Fetch dataset (shallow)
+        run: |
+          git clone --depth=1 https://github.com/ladybugdb/dataset.git dataset
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Benchmark main (live baseline)
+        run: |
+          MAIN_SHA=$(git rev-parse --short origin/main 2>/dev/null || echo "main")
+          python3 benchmarks/run_benchmarks.py \
+            --build-dir   build-main/relwithdebinfo \
+            --dataset-dir dataset \
+            --output      benchmarks/ci-baseline.json \
+            --label       "main (${MAIN_SHA})"
+
+      - name: Run benchmarks (branch)
+        run: |
+          python3 benchmarks/run_benchmarks.py \
+            --build-dir build/relwithdebinfo \
+            --dataset-dir dataset \
+            --output    benchmarks/branch-results.json \
+            --label     "${{ github.head_ref || github.ref_name }} ($(git rev-parse --short HEAD))"
+
+      - name: Compare branch vs main (same runner)
+        id: compare
+        continue-on-error: true
+        run: |
+          python3 benchmarks/compare_benchmarks.py \
+            --baseline benchmarks/ci-baseline.json \
+            --branch   benchmarks/branch-results.json \
+            --output   benchmarks/comparison.md
+          echo "has_baseline=true" >> "$GITHUB_OUTPUT"
+
+      - name: Generate history report
+        if: always()
+        run: |
+          python3 benchmarks/report.py \
+            --results-dir benchmarks/results \
+            --output benchmarks/results/report.md 2>/dev/null || true
+
+      - name: Fail CI if regressions detected
+        if: steps.compare.outcome == 'failure'
+        run: |
+          echo "Benchmark regressions detected — see comparison above."
+          exit 1
+
+      - name: Upload benchmark artifacts
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: benchmark-results
+          path: |
+            benchmarks/ci-baseline.json
+            benchmarks/branch-results.json
+            benchmarks/comparison.md
+            benchmarks/results/
+          retention-days: 30

.github/workflows/review-fixes-ci.yml

@@ -0,0 +1,142 @@
+name: PR #332 review-fixes CI
+# Validates the three fixes for adsharma's review comments on PR #332:
+#   1. lastTimestamp data race  (snapshot under mtxForSerializingPublicFunctionCalls)
+#   2. const_cast removed       (flush / scanCommitted now take const Transaction*)
+#   3. vacuumColumnIDs guarded  (now under schemaMtx; hash-index TODO added)
+#
+# Runs on every push to fix/pr332-review-comments and can be triggered manually.
+# Once green, cherry-pick or merge into feat/concurrent-writes.
+
+on:
+    push:
+        branches:
+            - fix/pr332-review-comments
+    workflow_dispatch:
+
+concurrency:
+    group: ${{ github.workflow }}-${{ github.ref }}
+    cancel-in-progress: true
+
+env:
+    FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
+
+jobs:
+    # ── Build ──────────────────────────────────────────────────────────────────
+    build:
+        name: Build (${{ matrix.os }})
+        runs-on: ${{ matrix.os }}
+        strategy:
+            fail-fast: false
+            matrix:
+                os: [ubuntu-latest, macos-latest]
+
+        steps:
+            - uses: actions/checkout@v4
+
+            - name: Install dependencies (Ubuntu)
+              if: runner.os == 'Linux'
+              run: |
+                  sudo apt-get update -qq
+                  sudo apt-get install -y cmake ninja-build python3
+
+            - name: Install dependencies (macOS)
+              if: runner.os == 'macOS'
+              run: brew install cmake ninja
+
+            - name: Configure (CMake + Ninja)
+              run: |
+                  cmake -B build/relwithdebinfo -G Ninja \
+                        -DCMAKE_BUILD_TYPE=RelWithDebInfo \
+                        -DBUILD_TESTS=TRUE \
+                        -DENABLE_BACKTRACES=TRUE \
+                        .
+
+            - name: Build
+              run: cmake --build build/relwithdebinfo --parallel 4
+
+            - name: Upload build artifacts
+              uses: actions/upload-artifact@v4
+              with:
+                  name: build-${{ matrix.os }}
+                  path: build/relwithdebinfo
+                  retention-days: 1
+
+    # ── Test ───────────────────────────────────────────────────────────────────
+    test:
+        name: Test (${{ matrix.os }})
+        needs: build
+        runs-on: ${{ matrix.os }}
+        strategy:
+            fail-fast: false
+            matrix:
+                os: [ubuntu-latest, macos-latest]
+
+        steps:
+            - uses: actions/checkout@v4
+
+            - name: Init dataset submodule
+              run: git submodule update --init --depth 1 dataset
+
+            - name: Download build artifacts
+              uses: actions/download-artifact@v4
+              with:
+                  name: build-${{ matrix.os }}
+                  path: build/relwithdebinfo
+
+            - name: Fix permissions
+              run: chmod -R +x build/relwithdebinfo/test
+
+            - name: Set test data env
+              run: echo "E2E_TEST_FILES_DIRECTORY=$GITHUB_WORKSPACE/test/test_files" >> $GITHUB_ENV
+
+            # Fix #1 – lastTimestamp race:
+            # ReviewFixesTest.CheckpointDrainWaitsForInFlightWrite is the targeted
+            # regression test (holds an in-flight write during drain).
+            # e2e_test_transaction~basic covers CheckpointWrittenDataPersistsOnReload
+            # and the existing timeout tests (CheckpointTimeoutErrorTest etc.).
+            # recovery/* and copy/* tests need the ladybugdb/dataset repo (not
+            # cloned here) and are not related to the checkpoint drain fix.
+            - name: Transaction tests (checkpoint + drain)
+              run: |
+                  ctest --test-dir build/relwithdebinfo/test \
+                    -R "e2e_test_transaction|ReviewFixesTest.CheckpointDrainWaitsForInFlightWrite" \
+                    --output-on-failure \
+                    --timeout 300
+
+            # Fix #1 continued – catalog_version() still returns the delta since the
+            # last checkpoint, which depends on the snapshot timestamp being correct.
+            - name: catalog_version() function test
+              run: |
+                  ctest --test-dir build/relwithdebinfo/test \
+                    -R "e2e_test_function.*call.*CallCatalogVersion" \
+                    --output-on-failure \
+                    --timeout 60
+
+            # Fix #2 – const_cast removed from scanAllInsertedAndVersions / flush:
+            # CheckpointInMemOnlyDataIntegrity is the targeted regression test that
+            # inserts 300 rows (in-memory only) and verifies they survive checkpoint.
+            - name: In-memory checkpoint data integrity (Fix #2)
+              run: |
+                  ctest --test-dir build/relwithdebinfo/test \
+                    -R "ReviewFixesTest.CheckpointInMemOnlyDataIntegrity|e2e_test_storage.*checkpoint|e2e_test_storage.*node_group" \
+                    --output-on-failure \
+                    --timeout 120
+
+            # Fix #3 – vacuumColumnIDs now under schemaMtx:
+            # ConcurrentReadsDuringCheckpointVacuum is the targeted regression test;
+            # ConcurrentNode*/ConcurrentRelationship* are the broader stress tests.
+            - name: Concurrent reads during checkpoint vacuum (Fix #3)
+              run: |
+                  ctest --test-dir build/relwithdebinfo/test \
+                    -R "ReviewFixesTest.ConcurrentReadsDuringCheckpointVacuum|ConcurrentNode|ConcurrentRelationship" \
+                    --output-on-failure \
+                    --timeout 180
+
+            - name: Upload test results
+              if: always()
+              uses: actions/upload-artifact@v4
+              with:
+                  name: test-results-${{ matrix.os }}
+                  path: |
+                      build/relwithdebinfo/test/Testing/
+                  retention-days: 7