fix(sim): make the camera-sensor tests pass reliably on CI (isaacsim / isaacgym / mujoco)

pyproject.toml

@@ -48,6 +48,7 @@ lint.ignore=[
 "src/holosoma/holosoma/utils/warp_utils.py" = ["UP018", "RUF046"]
 "src/holosoma/holosoma/simulator/isaacsim/**/*.py" = ["ALL"]  # Disable until Jenkins has access to IsaacLab
 "src/holosoma/holosoma/train_agent.py" = ["PLC0415"]  # Disable since imports after SimApp are inherent to IsaacSim
+"src/holosoma/tests/simulators/*_assert.py" = ["PLC0415"]  # Standalone sim harnesses defer torch/isaaclab imports until after the SimApp is launched (inherent to IsaacSim)
 "src/holosoma/holosoma/utils/draw.py" = ["F401"]  # Disable since unused imports are required for the adapters
 "src/holosoma/holosoma/utils/eval_utils.py" = ["PLC0415"]  # Disable since imports after SimApp are inherent to IsaacSim
 "src/holosoma_inference/holosoma_inference/policies/base.py" = ["PLC0415"]  # Deferred imports for optional heavy deps (ROS2, joystick, keyboard providers)

src/holosoma/holosoma/simulator/isaacsim/isaacsim.py

@@ -837,7 +837,14 @@ def load_assets(self):
 
     def create_envs(self, num_envs, env_origins, base_init_state):
         self.num_envs = num_envs
-        self.env_origins = env_origins
+        # IsaacSim does NOT honor the passed env_origins: InteractiveScene clones the envs on its own
+        # env_spacing grid (built in __init__), and every internal placement (robot/object poses,
+        # terrain) uses self.scene.env_origins, not the argument. Storing the passed value here left
+        # self.env_origins disagreeing with where envs actually are (unlike mujoco/isaacgym, where the
+        # passed origins ARE the placement) — so callers reading sim.env_origins (e.g. a multi-env
+        # camera harness pinning a robot relative to its env) landed off from the grid-placed scene.
+        # Reconcile to the real grid so sim.env_origins means the same thing on every backend.
+        self.env_origins = self.scene.env_origins
         self.base_init_state = base_init_state
 
         return self.scene, self._robot
@@ -1054,14 +1061,27 @@ def simulate_at_each_physics_step(self):
 
         self.scene.write_data_to_sim()
 
+        # Render on the render-interval when the GUI or a sensor needs it, INLINE via
+        # sim.step(render=render_now). IsaacLab's self.render() only flushes fabric / drives the RTX
+        # render products when sim.render_mode >= PARTIAL_RENDERING; at NO_GUI_OR_RENDERING (-1) it is
+        # a hard no-op. render_mode is fixed at SimulationContext.__init__ from the launch flags:
+        # headless + enable_cameras => offscreen => PARTIAL (the normal camera path, incl. headless
+        # training), a GUI => FULL, but headless WITHOUT cameras — or any caller that reaches the sim
+        # before enable_cameras/headless are both set at launch — lands at -1, which is terminal
+        # (set_render_mode refuses to leave it). sim.step(render=render_now) drives the low-level
+        # render regardless of render_mode, so RTX sensors (TiledCameras) track the current poses even
+        # in that -1 state, instead of returning the stale first frame. (Equivalent to IsaacLab's
+        # canonical split step(render=False)+render() whenever render_mode is already >= PARTIAL.)
+        render_now = is_rendering and self._sim_step_counter % self.simulator_config.sim.render_interval_steps == 0
+
         # simulate
-        self.sim.step(render=False)
+        self.sim.step(render=render_now)
 
-        # Render between steps only IF the GUI or sensor need it
-        # note: we assume the render interval to be the shortest accepted rendering interval.
-        #    If a camera needs rendering at a faster frequency, this will lead to unexpected behavior.
-        if self._sim_step_counter % self.simulator_config.sim.render_interval_steps == 0 and is_rendering:
-            self.render()
+        # Debug-viz overlay (GUI only): the inline render above replaces the old self.render() call,
+        # so redraw the debug lines here when a render happened and debug viz is on.
+        if render_now and self.debug_viz_enabled:
+            self.clear_lines()
+            self.draw_debug_viz()
 
         # update buffers at sim
         self.scene.update(dt=1.0 / self.simulator_config.sim.fps)

src/holosoma/tests/simulators/camera_actor_mount_assert.py

@@ -10,6 +10,7 @@
 
 import argparse
 import dataclasses
+import os
 import sys
 
 if sys.path and sys.path[0].endswith("simulators"):
@@ -102,4 +103,13 @@ def main() -> int:
 
 
 if __name__ == "__main__":
-    sys.exit(main())
+    # IsaacSim teardown deadlocks in carbOnPluginShutdown tearing down the
+    # omni.syntheticdata/OmniGraph render-product graph a TiledCamera creates (native
+    # py-spy stack), so a normal interpreter exit hangs until the parent's subprocess
+    # timeout SIGKILLs it -- turning a PASS (verdict already written to --result-file) into
+    # a spurious timeout failure. Hard-exit past the atexit teardown, mirroring
+    # behavior_assert / scene_spawn_assert. Rendering itself is fine; only exit hangs.
+    _rc = main()
+    sys.stdout.flush()
+    sys.stderr.flush()
+    os._exit(_rc)

src/holosoma/tests/simulators/camera_assert.py

@@ -21,6 +21,7 @@
 
 import argparse
 import dataclasses
+import os
 import sys
 
 # Pop this dir off sys.path[0] so tests/simulators/isaacsim/ can't shadow the real isaacsim pkg.
@@ -184,4 +185,13 @@ def main() -> int:
 
 
 if __name__ == "__main__":
-    sys.exit(main())
+    # IsaacSim teardown deadlocks in carbOnPluginShutdown tearing down the
+    # omni.syntheticdata/OmniGraph render-product graph a TiledCamera creates (native
+    # py-spy stack), so a normal interpreter exit hangs until the parent's subprocess
+    # timeout SIGKILLs it -- turning a PASS (verdict already written to --result-file) into
+    # a spurious timeout failure. Hard-exit past the atexit teardown, mirroring
+    # behavior_assert / scene_spawn_assert. Rendering itself is fine; only exit hangs.
+    _rc = main()
+    sys.stdout.flush()
+    sys.stderr.flush()
+    os._exit(_rc)

src/holosoma/tests/simulators/camera_follow_assert.py

@@ -13,6 +13,7 @@
 
 import argparse
 import dataclasses
+import os
 import sys
 
 if sys.path and sys.path[0].endswith("simulators"):
@@ -89,11 +90,37 @@ def main() -> int:
     )
     sim.create_envs(n, env_origins, base_init)
     sim.prepare_sim()
+    # Use the origins the simulator ACTUALLY placed the envs at: IsaacSim ignores the requested
+    # env_origins and clones onto its own env_spacing grid, so pinning the robot relative to the
+    # requested spread would land it away from its (grid-placed) panel. sim.env_origins is reconciled
+    # to the real placement on every backend.
+    env_origins = sim.env_origins
 
     if not sim.get_sensor_names():
         print(f"[{args.simulator}] FAIL: no sensors created")
         return 1
 
+    _all_ids = torch.arange(n, device=device)
+    # Capture the spawn joint pose so each _place can hold the articulation rigid: the robot is
+    # un-actuated, so over the settle steps the joints sag and the pelvis (hence its mounted camera)
+    # tilts a few degrees — enough to flake a random env's frame out of tolerance in multi-env. The
+    # test wants the body to move RIGIDLY (root translate/yaw) while the camera follows it, so holding
+    # the joints at spawn is exactly the intended behavior, not a cheat.
+    _spawn_dof_pos = sim.dof_pos.clone()
+
+    def _hold_dof() -> None:
+        # Restore joints to the spawn pose with zero velocity; per-backend DOF-state tensor shapes
+        # (IsaacGym flat [n*ndof, 2], IsaacSim 3D [n, ndof, 2]). MuJoCo is unaffected by this flake.
+        ndof = sim.num_dof
+        if args.simulator == "isaacgym":
+            ds = torch.zeros(n * ndof, 2, device=device)
+            ds[:, 0] = _spawn_dof_pos.reshape(-1)
+            sim.set_dof_state_tensor_robots(_all_ids, ds)
+        elif args.simulator == "isaacsim":
+            ds = torch.zeros(n, ndof, 2, device=device)
+            ds[:, :, 0] = _spawn_dof_pos
+            sim.set_dof_state_tensor_robots(_all_ids, ds)
+
     import math
 
     move = 0.3  # meters to advance the robot toward the panel (+X base frame)
@@ -114,17 +141,29 @@ def _quat_mul_yaw(q_xyzw, yaw_rad):
             w1 * w2 - x1 * x2 - y1 * y2 - z1 * z2,
         ]
 
-    def _place(x_offset: float, rot_xyzw) -> None:
-        states = sim.get_actor_states(["robot"], torch.arange(n, device=device)).clone()
+    def _set_root(x_offset: float, rot_xyzw) -> None:
+        states = sim.get_actor_states(["robot"], _all_ids).clone()
         states[:, :3] = env_origins + torch.tensor(list(init.pos), device=device)
         states[:, 0] += x_offset
         states[:, 3:7] = torch.tensor(rot_xyzw, device=device)
         states[:, 7:] = 0.0
-        sim.set_actor_states(["robot"], torch.arange(n, device=device), states)
+        sim.set_actor_states(["robot"], _all_ids, states)
+
+    def _place(x_offset: float, rot_xyzw) -> None:
+        # Set the root pose + hold joints rigid, settle, then re-assert both right before the render:
+        # holding the DOF pose stops the un-actuated joints from sagging the pelsvis/camera over the
+        # settle, and re-setting immediately before render_sensors lands the exact pose in the frame
+        # (on IsaacSim a pose write needs a few steps to propagate to the render, which the settle
+        # provides). Robust against the multi-env drift flake.
+        _set_root(x_offset, rot_xyzw)
+        _hold_dof()
         step(sim, max(2, steps_for_seconds(sim, 0.05)))
+        _set_root(x_offset, rot_xyzw)
+        _hold_dof()
+        step(sim, 2)
         sim.render_sensors()
 
-    cam_name, cam = next(iter(config.sensor.items()))
+    cam_name, _cam = next(iter(config.sensor.items()))
     _place(0.0, base_rot)
     before = [
         _panel_median_depth(sim.get_camera_data(cam_name, "rgb")[e], sim.get_camera_data(cam_name, "depth")[e])
@@ -206,4 +245,13 @@ def _place(x_offset: float, rot_xyzw) -> None:
 
 
 if __name__ == "__main__":
-    sys.exit(main())
+    # IsaacSim teardown deadlocks in carbOnPluginShutdown tearing down the
+    # omni.syntheticdata/OmniGraph render-product graph a TiledCamera creates (native
+    # py-spy stack), so a normal interpreter exit hangs until the parent's subprocess
+    # timeout SIGKILLs it -- turning a PASS (verdict already written to --result-file) into
+    # a spurious timeout failure. Hard-exit past the atexit teardown, mirroring
+    # behavior_assert / scene_spawn_assert. Rendering itself is fine; only exit hangs.
+    _rc = main()
+    sys.stdout.flush()
+    sys.stderr.flush()
+    os._exit(_rc)

src/holosoma/tests/simulators/camera_geometry_assert.py

@@ -25,6 +25,7 @@
 import argparse
 import dataclasses
 import math
+import os
 import sys
 
 if sys.path and sys.path[0].endswith("simulators"):
@@ -148,14 +149,51 @@ def main() -> int:
 
     # Pin the robot to a known upright pose at each env origin so the (pelvis-mounted) camera aligns
     # with the panel placed ahead (IsaacGym jitters the spawn xy; other backends are already at origin).
+    # Use the origins the simulator ACTUALLY placed the envs at (IsaacSim clones onto its own grid and
+    # ignores the requested env_origins; sim.env_origins is reconciled to the real placement).
+    env_origins = sim.env_origins
     import torch as _torch
 
+    _all_ids = _torch.arange(n, device=device)
+    # Capture the spawn joint pose once so the pin can restore it: the robot is un-actuated, so
+    # holding only the ROOT still lets the JOINTS sag over the settle steps, tilting the pelvis a
+    # few degrees and shifting the pelvis-mounted camera a few px off-axis (a rare multi-env L/R
+    # asymmetry flake). Holding the joints rigid too removes the settle at its source.
+    _spawn_dof_pos = sim.dof_pos.clone()
+
+    def _hold_dof() -> None:
+        # Restore joints to the spawn pose with zero velocity. The cross-backend DOF setter takes
+        # different tensor shapes (IsaacGym flattened [n*ndof, 2], IsaacSim 3D [n, ndof, 2]); build
+        # whichever this backend expects. MuJoCo isn't affected by this flake, so only the two
+        # articulation backends are handled.
+        ndof = sim.num_dof
+        if args.simulator == "isaacgym":
+            ds = _torch.zeros(n * ndof, 2, device=device)
+            ds[:, 0] = _spawn_dof_pos.reshape(-1)
+            sim.set_dof_state_tensor_robots(_all_ids, ds)
+        elif args.simulator == "isaacsim":
+            ds = _torch.zeros(n, ndof, 2, device=device)
+            ds[:, :, 0] = _spawn_dof_pos
+            sim.set_dof_state_tensor_robots(_all_ids, ds)
+
     def _pin_robot() -> None:
-        robot_states = sim.get_actor_states(["robot"], _torch.arange(n, device=device)).clone()
-        robot_states[:, :3] = env_origins + _torch.tensor(list(init.pos), device=device)
-        robot_states[:, 3:7] = _torch.tensor(list(init.rot), device=device)
-        robot_states[:, 7:] = 0.0
-        sim.set_actor_states(["robot"], _torch.arange(n, device=device), robot_states)
+        # Set the target root pose AND zero all velocities, hold the joints rigid, then read back to
+        # verify the write landed before we rely on it: any residual root velocity, joint sag, or a
+        # dropped write lets the pelvis drift and drags its mounted camera off the panel (the
+        # multi-env flake where a random env's panel leaves frame / lands off-center). Retry until it
+        # sticks.
+        target_pos = env_origins + _torch.tensor(list(init.pos), device=device)
+        target_rot = _torch.tensor(list(init.rot), device=device)
+        for _ in range(3):
+            robot_states = sim.get_actor_states(["robot"], _all_ids).clone()
+            robot_states[:, :3] = target_pos
+            robot_states[:, 3:7] = target_rot
+            robot_states[:, 7:] = 0.0
+            sim.set_actor_states(["robot"], _all_ids, robot_states)
+            _hold_dof()
+            back = sim.get_actor_states(["robot"], _all_ids)
+            if _torch.allclose(back[:, :3], target_pos, atol=1e-3) and back[:, 7:].abs().max() < 1e-3:
+                break
 
     _pin_robot()
     step(sim, 2)
@@ -166,12 +204,15 @@ def _pin_robot() -> None:
         return 1
 
     step(sim, max(2, steps_for_seconds(sim, 0.05)))
-    # Re-pin immediately before capture: the robot is un-actuated, so the settle steps above let the
-    # pelvis drift/tilt and drag its mounted camera off the panel (an env3-only flake — the projection
-    # math is env-independent, so a real FOV bug fails all envs). Root writes are immediate on every
-    # backend and render_sensors() does its own fetch/step_graphics, so pinning here fixes the pose the
-    # camera renders from without an extra settle.
+    # Re-pin before capture, then step a few frames: the robot is un-actuated, so the settle steps
+    # above let the pelvis drift and drag its mounted camera off the panel. Re-pinning restores the
+    # exact pose, but on IsaacSim a root/joint write does NOT reach the render until physics has
+    # stepped a few times to propagate it (an immediate render_sensors() after the write still shows
+    # the pre-pin drifted pose — measured ~4px off, tripping the margin-symmetry check). Stepping ~4
+    # frames with the joints held rigid (see _pin_robot) lands the corrected, centered pose in the
+    # render on every backend without letting the robot drift again.
     _pin_robot()
+    step(sim, 4)
     sim.render_sensors()
 
     cam_name, cam = next(iter(config.sensor.items()))
@@ -205,4 +246,13 @@ def _pin_robot() -> None:
 
 
 if __name__ == "__main__":
-    sys.exit(main())
+    # IsaacSim teardown deadlocks in carbOnPluginShutdown tearing down the
+    # omni.syntheticdata/OmniGraph render-product graph a TiledCamera creates (native
+    # py-spy stack), so a normal interpreter exit hangs until the parent's subprocess
+    # timeout SIGKILLs it -- turning a PASS (verdict already written to --result-file) into
+    # a spurious timeout failure. Hard-exit past the atexit teardown, mirroring
+    # behavior_assert / scene_spawn_assert. Rendering itself is fine; only exit hangs.
+    _rc = main()
+    sys.stdout.flush()
+    sys.stderr.flush()
+    os._exit(_rc)

src/holosoma/tests/simulators/camera_multi_assert.py

@@ -9,6 +9,7 @@
 
 import argparse
 import dataclasses
+import os
 import sys
 
 if sys.path and sys.path[0].endswith("simulators"):
@@ -107,4 +108,13 @@ def main() -> int:
 
 
 if __name__ == "__main__":
-    sys.exit(main())
+    # IsaacSim teardown deadlocks in carbOnPluginShutdown tearing down the
+    # omni.syntheticdata/OmniGraph render-product graph a TiledCamera creates (native
+    # py-spy stack), so a normal interpreter exit hangs until the parent's subprocess
+    # timeout SIGKILLs it -- turning a PASS (verdict already written to --result-file) into
+    # a spurious timeout failure. Hard-exit past the atexit teardown, mirroring
+    # behavior_assert / scene_spawn_assert. Rendering itself is fine; only exit hangs.
+    _rc = main()
+    sys.stdout.flush()
+    sys.stderr.flush()
+    os._exit(_rc)

src/holosoma/tests/simulators/camera_obs_assert.py

@@ -16,6 +16,7 @@
 
 import argparse
 import dataclasses
+import os
 import sys
 
 if sys.path and sys.path[0].endswith("simulators"):
@@ -175,4 +176,13 @@ def _obs():
 
 
 if __name__ == "__main__":
-    sys.exit(main())
+    # IsaacSim teardown deadlocks in carbOnPluginShutdown tearing down the
+    # omni.syntheticdata/OmniGraph render-product graph a TiledCamera creates (native
+    # py-spy stack), so a normal interpreter exit hangs until the parent's subprocess
+    # timeout SIGKILLs it -- turning a PASS (verdict already written to --result-file) into
+    # a spurious timeout failure. Hard-exit past the atexit teardown, mirroring
+    # behavior_assert / scene_spawn_assert. Rendering itself is fine; only exit hangs.
+    _rc = main()
+    sys.stdout.flush()
+    sys.stderr.flush()
+    os._exit(_rc)

src/holosoma/tests/simulators/camera_orientation_assert.py

@@ -15,6 +15,7 @@
 
 import argparse
 import dataclasses
+import os
 import sys
 
 if sys.path and sys.path[0].endswith("simulators"):
@@ -137,4 +138,13 @@ def main() -> int:
 
 
 if __name__ == "__main__":
-    sys.exit(main())
+    # IsaacSim teardown deadlocks in carbOnPluginShutdown tearing down the
+    # omni.syntheticdata/OmniGraph render-product graph a TiledCamera creates (native
+    # py-spy stack), so a normal interpreter exit hangs until the parent's subprocess
+    # timeout SIGKILLs it -- turning a PASS (verdict already written to --result-file) into
+    # a spurious timeout failure. Hard-exit past the atexit teardown, mirroring
+    # behavior_assert / scene_spawn_assert. Rendering itself is fine; only exit hangs.
+    _rc = main()
+    sys.stdout.flush()
+    sys.stderr.flush()
+    os._exit(_rc)

src/holosoma/tests/simulators/depth_assert.py

@@ -17,6 +17,7 @@
 import argparse
 import dataclasses
 import math
+import os
 import sys
 
 if sys.path and sys.path[0].endswith("simulators"):
@@ -184,4 +185,13 @@ def main() -> int:
 
 
 if __name__ == "__main__":
-    sys.exit(main())
+    # IsaacSim teardown deadlocks in carbOnPluginShutdown tearing down the
+    # omni.syntheticdata/OmniGraph render-product graph a TiledCamera creates (native
+    # py-spy stack), so a normal interpreter exit hangs until the parent's subprocess
+    # timeout SIGKILLs it -- turning a PASS (verdict already written to --result-file) into
+    # a spurious timeout failure. Hard-exit past the atexit teardown, mirroring
+    # behavior_assert / scene_spawn_assert. Rendering itself is fine; only exit hangs.
+    _rc = main()
+    sys.stdout.flush()
+    sys.stderr.flush()
+    os._exit(_rc)