Fix/step ef pipeline

.github/workflows/ci.yml

@@ -35,26 +35,33 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - name: Init CppNCorr submodule (compiled in-tree)
-        # Only CppNCorr is built. Tools/MultiDIC is a read-only MATLAB reference that
-        # lives in a separate private repo the CI runner cannot clone, so we do NOT
-        # recurse all submodules — that would fail the checkout. CppNCorr is under the
-        # same org and uses an https URL, so the persisted token can fetch it.
-        run: git submodule update --init Tools/CppNCorr
+      - name: Init submodules (CppNCorr + cuNCorr, compiled in-tree)
+        # CppNCorr and cuNCorr are public repos under the same org (https URLs), so the
+        # persisted token fetches them. Tools/MultiDIC is a separate private repo the CI
+        # runner cannot clone, so we do NOT recurse all submodules (that would fail the
+        # checkout) — we name only what cppxdic builds. Non-recursive: cuNCorr's own nested
+        # CppNCorr is not needed here (cppxdic builds ncorr; CUNCORR_BUILD_CPPNCORR=OFF).
+        run: git submodule update --init Tools/CppNCorr Tools/cuNCorr
       - name: Install dependencies
         run: |
           sudo apt-get update
           sudo apt-get install -y --no-install-recommends ${{ env.APT_DEPS }}
-      - name: Configure (camera-pairs, tests off)
+      - name: Configure (camera-pairs, tests off, cuNCorr adapter test on)
         run: >-
           cmake -S . -B build
           -DCMAKE_BUILD_TYPE=Release
           -DXDIC_MODE=camerapairs
           -DBUILD_TESTING=OFF
-      - name: Build cppxdic
-        run: cmake --build build -j"$(nproc)" --target cppxdic
+          -DBUILD_CUNCORR_ADAPTER_TEST=ON
+      - name: Build cppxdic (+ cuNCorr integration)
+        run: cmake --build build -j"$(nproc)" --target cppxdic cuncorr_adapter_test
         # NOTE: cppxdic links the concurrently-developed CppNCorr submodule. If the build
         # fails inside Tools/CppNCorr/, that is a submodule issue, not a CPPxDIC regression.
+      - name: Verify the cuNCorr -> ncorr adapter (synthetic recovery)
+        # Exercises run_cuncorr_dic end to end: builds a known-translation frame pair,
+        # runs the cuNCorr engine (CPU backend on CI), and checks the packaged
+        # ncorr::DIC_analysis_output recovers it — proving the adapter/grid mapping.
+        run: ./build/cuncorr_adapter_test
 
   test:
     name: test (unit + integration on ohtcfrp via ctest)
@@ -69,11 +76,15 @@ jobs:
           sudo apt-get update
           sudo apt-get install -y --no-install-recommends ${{ env.APT_DEPS }}
       - name: Configure with tests
+        # ncorr-only: the ohtcfrp e2e guard uses golden statistics tuned for the CppNCorr
+        # engine. cuNCorr is a different algorithm (different numbers), so keep it off here;
+        # the build job compiles + adapter-tests the cuNCorr integration separately.
         run: >-
           cmake -S . -B build
           -DCMAKE_BUILD_TYPE=Release
           -DXDIC_MODE=camerapairs
           -DBUILD_TESTING=ON
+          -DCPPXDIC_USE_CUNCORR=OFF
       - name: Build test suite
         # Build only the cppxdic_core test library + test executables (not the full cppxdic
         # binary) so the test job stays fast and is independent of the heavy main target.
@@ -143,11 +154,13 @@ jobs:
           sudo apt-get update
           sudo apt-get install -y --no-install-recommends ${{ env.APT_DEPS }} gcovr
       - name: Configure (coverage instrumentation)
+        # ncorr-only (see the test job): coverage tracks the CppNCorr golden path.
         run: >-
           cmake -S . -B build
           -DCMAKE_BUILD_TYPE=Debug
           -DXDIC_MODE=camerapairs
           -DBUILD_TESTING=ON
+          -DCPPXDIC_USE_CUNCORR=OFF
           -DCMAKE_CXX_FLAGS="--coverage -O0 -g"
           -DCMAKE_C_FLAGS="--coverage -O0 -g"
           -DCMAKE_EXE_LINKER_FLAGS="--coverage"

.gitmodules

@@ -4,3 +4,6 @@
 [submodule "Tools/MultiDIC"]
 	path = Tools/MultiDIC
 	url = git@github.com:donatiendoumont/fingertip_3dreconstruction.git
+[submodule "Tools/cuNCorr"]
+	path = Tools/cuNCorr
+	url = https://github.com/xdic-dev/cuNCorr.git

CLAUDE.md

@@ -0,0 +1,126 @@
+# CPPxDIC — cuNCorr GPU integration + CPU/GPU/MATLAB validation (READ FIRST)
+
+Working notes for the `feat/cuncorr-integration` branch family. Everything here
+was learned the hard way in July 2026; trust it before re-deriving.
+
+## Branch structure (stacked PRs on xdic-dev/CPPxDIC)
+
+1. `feat/cuncorr-integration` — cuNCorr engine + GPU image + cpugpu study harness.
+2. `feat/fixed-step-ref` (base: 1) — `fixed_step_ref` + `no_update` ncorr-params
+   options; bumps `Tools/CppNCorr` to its `feat/fixed-step-ref` branch (01b6e81).
+3. `fix/step-ef-pipeline` (base: 2) — Step-E/F segfault chain fixes +
+   `perspective_interp` wiring. **Working trees live on this tip = union of all.**
+
+Merge order: 2→1 first, then 3→2. CppNCorr PR: `feat/fixed-step-ref` → main.
+
+## Clusters: TWO separate machines, similar paths (TRAP)
+
+- **Lemaitre4** (`lm4-*`): interactive/login work. **Manneback** (`mbackf*`,
+  `ssh manneback` from Lemaitre works passwordless): where ALL cpugpu jobs run
+  (GPU partition: A100/L40s/A10/V100/RTX; `keira` = 256-core Milan CPU nodes;
+  QOS `normal,preemptible`, gpu limit ~2 jobs/user).
+- `/home/ucl/inma/jaoga` is a SYMLINK to `/home/users/j/a/jaoga` on BOTH
+  clusters, but the home filesystems are PER-CLUSTER — a file written on one is
+  NOT on the other. Sync with
+  `rsync -a --exclude='*.sif' --exclude='build/' . manneback:/home/users/j/a/jaoga/devlab/MultiDIC/CPPxDIC/`.
+  The identical absolute paths mean sbatch scripts work unchanged on both.
+- `/globalscratch` is also per-cluster and **PURGEABLE** (a purge destroyed all
+  Lemaitre results once; only logs/meta under the repo on home survived —
+  that's why the harness writes run.log/meta.txt to the repo results dir).
+- **sacct TRAP**: Manneback recycles SLURM job IDs — `sacct -j <id>` without
+  `-S <date>` can return a 2018 job with the same ID (false COMPLETED/FAILED).
+  Always add `-S <today>`.
+
+## Manneback data layout (S09)
+
+- Videos+protocol: `/globalscratch/ucl/inma/jaoga/data/S09/speckles/coating/{vid,protocol}`
+- REF masks/seeds + calib: `/globalscratch/ucl/inma/jaoga/data/analysis/S09/{coating,calib/2}`
+- The pipeline resolves videos at `<data_path>/rawdata/<subj>/speckles/<mat>/vid`;
+  satisfied by symlink `cpugpu/data_root/rawdata -> ../data` (env.sh makes it).
+- MATLAB reference results (Nov-2025 run, trial 007):
+  `/globalscratch/ucl/inma/jaoga/cpugpu/matlab_ref/loading/` (from loading.zip;
+  `unzip` needs `UNZIP_DISABLE_ZIPBOMB_DETECTION=TRUE` or use bsdtar).
+- Frame window is PROTOCOL-DERIVED per trial (idx_frame_start/end in config are
+  overridden when protocol .mat files exist). S09 trials: 5(REF),7,12,25.
+
+## The cpugpu harness (`deploy/cluster/cpugpu/`)
+
+- `env.sh` — single place for all paths. Study root
+  `/globalscratch/ucl/inma/jaoga/cpugpu/` (SIF + runs/); results+logs under the
+  repo on home (purge-safe).
+- `build_sif.sbatch` — builds `cppxdic_gpu.sif` from `deploy/cluster/cppxdic_gpu.def`
+  (nvidia/cuda 12.4.1, sm 70-89, hdf5-tools, gdb, RelWithDebInfo `cppxdic_dbg`).
+  ONE image serves CPU and GPU runs (cuNCorr falls back to CPU without `--nv`)
+  so binaries are identical for bit-comparisons. APPTAINER_TMPDIR must be on
+  globalscratch (node /tmp too small to squash the CUDA rootfs).
+- `run_one.sbatch` — one experiment in an isolated dic_path
+  (`runs/<name>`, seeded with REF/calib symlinks; the checkpoint system keys on
+  dic_path only, so shared dirs false-skip). Axes (env): TRIAL, ENGINE
+  (ncorr|cuncorr), USE_GPU, THREADS, EXACT_MATLAB, UPDATES, FIXEDSTEP, STAGES
+  (default `d`; `all` = full pipeline), REP. Every run APPENDS `dic_engine`
+  explicitly — **the compiled-in default is cuncorr**.
+- **Config-file placement TRAP**: `use_exact_matlab`, `no_update`,
+  `cutoff_corrcoef`, `fixed_step_ref`, `perspective_interp` are parsed ONLY
+  from **ncorr_params.txt** (unprefixed keys, `Config::loadFromNcorrParamsFile`);
+  appending `ncorr_*` keys to dic_params.txt is silently ignored.
+- ncorr threading needs BOTH `parallel_processing=true` AND
+  `step_d_total_threads=N` (frame-level OpenMP; OMP_NUM_THREADS alone does NOT
+  drive it).
+- `compare_bits.sh` — bit-level: h5diff on .mat (**never md5** — v7.3 userblock
+  embeds timestamps), cmp on .bin.
+- `compare_dic.cpp` — matio comparator for MATLAB↔cppxdic files (MATLAB = v5 +
+  struct-array layouts, unreadable/misaligned for h5diff). Modes: two-file walk,
+  POINTS (point-matched trajectories), DUMP/LIST probes. Build inside the SIF:
+  `g++ -O2 -o compare_dic compare_dic.cpp -lmatio`. NOTE: any matio reader of
+  v7.3 files MUST deep-load recursively (Mat_VarReadDataAll on every nested
+  struct field/cell) — deferred data otherwise reads as silently empty.
+- `debug_stepe.sbatch` — rerun selected `--stages` on an existing run dir under
+  gdb (`cppxdic_dbg`), for backtraces. RUN=<name> STAGES=<spec> LOGLEVEL=debug.
+
+## Key findings (details in deploy/cluster/cpugpu/results/REPORT.md)
+
+- **Perf (Step D, 150-frame trial)**: GPU (1×A100) ~17 min ≈ 3.7-4× faster than
+  ncorr CPU-32-thread (~65-77 min) at ~15× less RAM (3.7 vs ~55 GB). Full
+  pipeline (D+E+F): GPU 32 min, CPU 75 min. cuNCorr CPU backend is
+  single-threaded (SessionConfig.num_threads is never consumed).
+- **Bit-parity**: GPU deterministic across reps; cuncorr CPU-vs-GPU differences
+  bounded by 1e-12 (ULP noise; cams 2/3 bit-exact). ncorr↔cuncorr NOT
+  bit-comparable (different valid-point sets, ~0.2% at ROI edges).
+- **MATLAB agreement**: sub-0.1 px for the bulk; the boundary band of
+  disagreement is MATLAB's FFT-bcoef border bias (still present in MATLAB's
+  form_bcoef; fixed on our side by CppNCorr 4758ad2 recursive filter) — cppxdic
+  is the more correct side there. The interior tail traces to MATLAB's
+  fixed-step reseeding (stepanalysis step=10): reproducing it via
+  `fixed_step_ref=10` matches the structure exactly but agrees WORSE at 0.1 px
+  (boundary compositions compound) → production default stays fixed-reference.
+- **cuNCorr CANNOT do inter-camera matching** (~276 px disparity ≫ its 15 px
+  local seed search; it silently returns all-zero fields with an empty ROI).
+  Matching calls (≤2 current images) are routed to ncorr in the dispatch —
+  do not "optimize" that away.
+- **Step-E/F bug chain (all fixed 2026-07-07)**: (1) mat_writer write-then-
+  read-back matvar embedding segfaulted `Mat_VarSetStructFieldByName`;
+  (2) crash-truncated DIC3Dcombined poisoned the existence-only checkpoint —
+  saves now go `.partial`→rename; if a run crashed mid-E, DELETE the leftover
+  DIC3Dcombined before rerunning; (3) v7.3 deferred reads left Faces empty
+  ("0 faces" Step-F abort) — mat_reader now deep-loads.
+- Historical Lemaitre profiling study (threading, seed-opt, import fix) lives
+  in the `ja/profiling` branch CLAUDE.md; that OMP_NUM_THREADS-only sweeps
+  measure nothing still applies.
+
+## Routine operations
+
+```bash
+# rebuild image after code changes (from repo root ON MANNEBACK):
+sbatch deploy/cluster/cpugpu/build_sif.sbatch                # ~25-40 min warm
+# full grid (Step D comparisons):
+deploy/cluster/cpugpu/submit_all.sh                          # TRIALS/ONLY env to narrow
+# one full-pipeline run:
+TRIAL=7 ENGINE=cuncorr USE_GPU=1 THREADS=8 STAGES=all REP=9 \
+  sbatch -p gpu --gres=gpu:1 --constraint='TeslaA100|TeslaA100_80' \
+  -c 8 --mem=64G -t 12:00:00 deploy/cluster/cpugpu/run_one.sbatch
+# bit compare / MATLAB compare after runs:
+deploy/cluster/cpugpu/compare_bits.sh --all
+sbatch deploy/cluster/cpugpu/compare_matlab.sbatch
+```
+Never reuse a run NAME unless you intend to wipe it (run_one rm -rf's its own
+dic_path). Meta/logs land in `deploy/cluster/cpugpu/results/<name>/` on home.

CMakeLists.txt

@@ -15,6 +15,38 @@ include_directories(Tools/CppNCorr/include)
 # Build NCorr from source so local workflow changes are linked into cppxdic
 add_subdirectory(Tools/CppNCorr)
 
+# ----------------------------------------------------------------------------
+# cuNCorr — GPU/CPU DIC engine (cppxdic -> cuncorr; cuncorr bundles its own CppNCorr
+# for its standalone tests, disabled here since cppxdic already builds ncorr above).
+# OPTIONAL: cuNCorr is a private submodule. When present it is built as a standalone
+# static lib `cuncorr` and the adapter (src/dic/cuncorr_dic.cpp) is compiled in; when
+# absent (e.g. a CI runner without access to the private repo) the build falls back to
+# the ncorr-only path. Guarded so a checkout without Tools/cuNCorr still configures.
+#   Configure with -DCUNCORR_BACKEND=CUDA on a GPU node (default CPU everywhere).
+# ----------------------------------------------------------------------------
+option(CPPXDIC_USE_CUNCORR "Use the cuNCorr DIC engine when its submodule is present" ON)
+set(CPPXDIC_HAVE_CUNCORR OFF)
+if(CPPXDIC_USE_CUNCORR AND EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/Tools/cuNCorr/CMakeLists.txt)
+    set(CUNCORR_BUILD_TESTS    OFF CACHE BOOL "" FORCE)
+    set(CUNCORR_BUILD_APPS     OFF CACHE BOOL "" FORCE)
+    set(CUNCORR_BUILD_CPPNCORR OFF CACHE BOOL "" FORCE)  # cppxdic provides the ncorr target
+    add_subdirectory(Tools/cuNCorr)
+    include_directories(Tools/cuNCorr/include)
+    if(CUNCORR_BACKEND STREQUAL "CUDA")
+        # libcuncorr.a is consumed by executables that have no CUDA language
+        # enabled (cppxdic, singledic), so they cannot generate the device-link
+        # step that separable compilation requires (and CMake < 3.23 cannot
+        # resolve device symbols on static libs). cuNCorr's kernels are
+        # TU-local (device code lives in per-TU headers, hd.h), so rdc is
+        # unnecessary — turn it off to make libcuncorr.a self-contained.
+        set_target_properties(cuncorr PROPERTIES CUDA_SEPARABLE_COMPILATION OFF)
+    endif()
+    set(CPPXDIC_HAVE_CUNCORR ON)
+    message(STATUS "cuNCorr DIC engine: ENABLED")
+else()
+    message(STATUS "cuNCorr DIC engine: disabled (submodule absent or CPPXDIC_USE_CUNCORR=OFF) - ncorr only")
+endif()
+
 # Lightweight, dependency-free logging facility (see include/logging.h). Built as
 # a small static library and linked into every cppxdic target via link_libraries()
 # below, including the test suite and optional apps added as subdirectories after
@@ -89,6 +121,13 @@ target_link_libraries(cppxdic ${OpenCV_LIBS})
 # Add ncorr library
 target_link_libraries(cppxdic ncorr)
 
+# Add cuNCorr DIC engine (only when the submodule is present)
+if(CPPXDIC_HAVE_CUNCORR)
+    target_sources(cppxdic PRIVATE src/dic/cuncorr_dic.cpp)
+    target_compile_definitions(cppxdic PRIVATE CPPXDIC_HAVE_CUNCORR)
+    target_link_libraries(cppxdic cuncorr)
+endif()
+
 # Add OpenCV libraries explicitly (following ncorr pattern)
 find_library(OPENCV_LIBRARY1 NAMES opencv_core)
 target_link_libraries(cppxdic ${OPENCV_LIBRARY1})
@@ -392,6 +431,12 @@ if(BUILD_SINGLEDIC)
     )
     target_include_directories(singledic PRIVATE include Tools/CppNCorr/include)
     target_compile_definitions(singledic PRIVATE XDIC_MODE_CAMERAPAIRS)
+    if(CPPXDIC_HAVE_CUNCORR)
+        target_sources(singledic PRIVATE src/dic/cuncorr_dic.cpp)
+        target_include_directories(singledic PRIVATE Tools/cuNCorr/include)
+        target_compile_definitions(singledic PRIVATE CPPXDIC_HAVE_CUNCORR)
+        target_link_libraries(singledic cuncorr)
+    endif()
     target_link_libraries(singledic
         ncorr
         ${OpenCV_LIBS}
@@ -535,3 +580,19 @@ if(BUILD_GEN_SUBJECT_TRIAL)
     target_compile_options(gen_subject_trial PRIVATE -Wall -Wextra -O2)
     install(TARGETS gen_subject_trial DESTINATION bin)
 endif()
+
+# ----------------------------------------------------------------------------
+# cuNCorr adapter smoke test (opt-in). Verifies run_cuncorr_dic() packages a valid
+# ncorr::DIC_analysis_output that recovers a known synthetic translation.
+#   cmake -DBUILD_CUNCORR_ADAPTER_TEST=ON .. && cmake --build . --target cuncorr_adapter_test
+# ----------------------------------------------------------------------------
+option(BUILD_CUNCORR_ADAPTER_TEST "Build the cuNCorr->ncorr adapter smoke test" OFF)
+if(BUILD_CUNCORR_ADAPTER_TEST AND CPPXDIC_HAVE_CUNCORR)
+    add_executable(cuncorr_adapter_test tests/cuncorr_adapter_test.cpp src/dic/cuncorr_dic.cpp)
+    target_include_directories(cuncorr_adapter_test PRIVATE include Tools/CppNCorr/include Tools/cuNCorr/include)
+    target_link_libraries(cuncorr_adapter_test ncorr cuncorr
+        ${OpenCV_LIBS} ${OPENCV_LIBRARY1} ${OPENCV_LIBRARY2} ${OPENCV_LIBRARY3} ${OPENCV_LIBRARY4} ${OPENCV_LIBRARY5}
+        ${FFTW_LIBRARY}
+        ${SUITESPARSE_LIBRARY1} ${SUITESPARSE_LIBRARY2} ${SUITESPARSE_LIBRARY3} ${SUITESPARSE_LIBRARY4} ${SUITESPARSE_LIBRARY5}
+        ${LAPACK_LIBRARY} ${BLAS_LIBRARY} ${CMAKE_THREAD_LIBS_INIT})
+endif()