diff --git a/pyproject.toml b/pyproject.toml index 8461fe929..6e94ce6ca 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,6 +48,7 @@ lint.ignore=[ "src/holosoma/holosoma/utils/warp_utils.py" = ["UP018", "RUF046"] "src/holosoma/holosoma/simulator/isaacsim/**/*.py" = ["ALL"] # Disable until Jenkins has access to IsaacLab "src/holosoma/holosoma/train_agent.py" = ["PLC0415"] # Disable since imports after SimApp are inherent to IsaacSim +"src/holosoma/tests/simulators/*_assert.py" = ["PLC0415"] # Standalone sim harnesses defer torch/isaaclab imports until after the SimApp is launched (inherent to IsaacSim) "src/holosoma/holosoma/utils/draw.py" = ["F401"] # Disable since unused imports are required for the adapters "src/holosoma/holosoma/utils/eval_utils.py" = ["PLC0415"] # Disable since imports after SimApp are inherent to IsaacSim "src/holosoma_inference/holosoma_inference/policies/base.py" = ["PLC0415"] # Deferred imports for optional heavy deps (ROS2, joystick, keyboard providers) diff --git a/src/holosoma/holosoma/simulator/isaacsim/isaacsim.py b/src/holosoma/holosoma/simulator/isaacsim/isaacsim.py index 73bd9cf55..73c54e6e1 100644 --- a/src/holosoma/holosoma/simulator/isaacsim/isaacsim.py +++ b/src/holosoma/holosoma/simulator/isaacsim/isaacsim.py @@ -837,7 +837,14 @@ def load_assets(self): def create_envs(self, num_envs, env_origins, base_init_state): self.num_envs = num_envs - self.env_origins = env_origins + # IsaacSim does NOT honor the passed env_origins: InteractiveScene clones the envs on its own + # env_spacing grid (built in __init__), and every internal placement (robot/object poses, + # terrain) uses self.scene.env_origins, not the argument. Storing the passed value here left + # self.env_origins disagreeing with where envs actually are (unlike mujoco/isaacgym, where the + # passed origins ARE the placement) — so callers reading sim.env_origins (e.g. a multi-env + # camera harness pinning a robot relative to its env) landed off from the grid-placed scene. + # Reconcile to the real grid so sim.env_origins means the same thing on every backend. + self.env_origins = self.scene.env_origins self.base_init_state = base_init_state return self.scene, self._robot @@ -1054,14 +1061,27 @@ def simulate_at_each_physics_step(self): self.scene.write_data_to_sim() + # Render on the render-interval when the GUI or a sensor needs it, INLINE via + # sim.step(render=render_now). IsaacLab's self.render() only flushes fabric / drives the RTX + # render products when sim.render_mode >= PARTIAL_RENDERING; at NO_GUI_OR_RENDERING (-1) it is + # a hard no-op. render_mode is fixed at SimulationContext.__init__ from the launch flags: + # headless + enable_cameras => offscreen => PARTIAL (the normal camera path, incl. headless + # training), a GUI => FULL, but headless WITHOUT cameras — or any caller that reaches the sim + # before enable_cameras/headless are both set at launch — lands at -1, which is terminal + # (set_render_mode refuses to leave it). sim.step(render=render_now) drives the low-level + # render regardless of render_mode, so RTX sensors (TiledCameras) track the current poses even + # in that -1 state, instead of returning the stale first frame. (Equivalent to IsaacLab's + # canonical split step(render=False)+render() whenever render_mode is already >= PARTIAL.) + render_now = is_rendering and self._sim_step_counter % self.simulator_config.sim.render_interval_steps == 0 + # simulate - self.sim.step(render=False) + self.sim.step(render=render_now) - # Render between steps only IF the GUI or sensor need it - # note: we assume the render interval to be the shortest accepted rendering interval. - # If a camera needs rendering at a faster frequency, this will lead to unexpected behavior. - if self._sim_step_counter % self.simulator_config.sim.render_interval_steps == 0 and is_rendering: - self.render() + # Debug-viz overlay (GUI only): the inline render above replaces the old self.render() call, + # so redraw the debug lines here when a render happened and debug viz is on. + if render_now and self.debug_viz_enabled: + self.clear_lines() + self.draw_debug_viz() # update buffers at sim self.scene.update(dt=1.0 / self.simulator_config.sim.fps) diff --git a/src/holosoma/tests/simulators/camera_actor_mount_assert.py b/src/holosoma/tests/simulators/camera_actor_mount_assert.py index 11894a6f7..53fb80ced 100644 --- a/src/holosoma/tests/simulators/camera_actor_mount_assert.py +++ b/src/holosoma/tests/simulators/camera_actor_mount_assert.py @@ -10,6 +10,7 @@ import argparse import dataclasses +import os import sys if sys.path and sys.path[0].endswith("simulators"): @@ -102,4 +103,13 @@ def main() -> int: if __name__ == "__main__": - sys.exit(main()) + # IsaacSim teardown deadlocks in carbOnPluginShutdown tearing down the + # omni.syntheticdata/OmniGraph render-product graph a TiledCamera creates (native + # py-spy stack), so a normal interpreter exit hangs until the parent's subprocess + # timeout SIGKILLs it -- turning a PASS (verdict already written to --result-file) into + # a spurious timeout failure. Hard-exit past the atexit teardown, mirroring + # behavior_assert / scene_spawn_assert. Rendering itself is fine; only exit hangs. + _rc = main() + sys.stdout.flush() + sys.stderr.flush() + os._exit(_rc) diff --git a/src/holosoma/tests/simulators/camera_assert.py b/src/holosoma/tests/simulators/camera_assert.py index 9d5fdfe02..080cc7740 100644 --- a/src/holosoma/tests/simulators/camera_assert.py +++ b/src/holosoma/tests/simulators/camera_assert.py @@ -21,6 +21,7 @@ import argparse import dataclasses +import os import sys # Pop this dir off sys.path[0] so tests/simulators/isaacsim/ can't shadow the real isaacsim pkg. @@ -184,4 +185,13 @@ def main() -> int: if __name__ == "__main__": - sys.exit(main()) + # IsaacSim teardown deadlocks in carbOnPluginShutdown tearing down the + # omni.syntheticdata/OmniGraph render-product graph a TiledCamera creates (native + # py-spy stack), so a normal interpreter exit hangs until the parent's subprocess + # timeout SIGKILLs it -- turning a PASS (verdict already written to --result-file) into + # a spurious timeout failure. Hard-exit past the atexit teardown, mirroring + # behavior_assert / scene_spawn_assert. Rendering itself is fine; only exit hangs. + _rc = main() + sys.stdout.flush() + sys.stderr.flush() + os._exit(_rc) diff --git a/src/holosoma/tests/simulators/camera_follow_assert.py b/src/holosoma/tests/simulators/camera_follow_assert.py index 41cb9bac5..643ab233d 100644 --- a/src/holosoma/tests/simulators/camera_follow_assert.py +++ b/src/holosoma/tests/simulators/camera_follow_assert.py @@ -13,6 +13,7 @@ import argparse import dataclasses +import os import sys if sys.path and sys.path[0].endswith("simulators"): @@ -89,11 +90,37 @@ def main() -> int: ) sim.create_envs(n, env_origins, base_init) sim.prepare_sim() + # Use the origins the simulator ACTUALLY placed the envs at: IsaacSim ignores the requested + # env_origins and clones onto its own env_spacing grid, so pinning the robot relative to the + # requested spread would land it away from its (grid-placed) panel. sim.env_origins is reconciled + # to the real placement on every backend. + env_origins = sim.env_origins if not sim.get_sensor_names(): print(f"[{args.simulator}] FAIL: no sensors created") return 1 + _all_ids = torch.arange(n, device=device) + # Capture the spawn joint pose so each _place can hold the articulation rigid: the robot is + # un-actuated, so over the settle steps the joints sag and the pelvis (hence its mounted camera) + # tilts a few degrees — enough to flake a random env's frame out of tolerance in multi-env. The + # test wants the body to move RIGIDLY (root translate/yaw) while the camera follows it, so holding + # the joints at spawn is exactly the intended behavior, not a cheat. + _spawn_dof_pos = sim.dof_pos.clone() + + def _hold_dof() -> None: + # Restore joints to the spawn pose with zero velocity; per-backend DOF-state tensor shapes + # (IsaacGym flat [n*ndof, 2], IsaacSim 3D [n, ndof, 2]). MuJoCo is unaffected by this flake. + ndof = sim.num_dof + if args.simulator == "isaacgym": + ds = torch.zeros(n * ndof, 2, device=device) + ds[:, 0] = _spawn_dof_pos.reshape(-1) + sim.set_dof_state_tensor_robots(_all_ids, ds) + elif args.simulator == "isaacsim": + ds = torch.zeros(n, ndof, 2, device=device) + ds[:, :, 0] = _spawn_dof_pos + sim.set_dof_state_tensor_robots(_all_ids, ds) + import math move = 0.3 # meters to advance the robot toward the panel (+X base frame) @@ -114,17 +141,29 @@ def _quat_mul_yaw(q_xyzw, yaw_rad): w1 * w2 - x1 * x2 - y1 * y2 - z1 * z2, ] - def _place(x_offset: float, rot_xyzw) -> None: - states = sim.get_actor_states(["robot"], torch.arange(n, device=device)).clone() + def _set_root(x_offset: float, rot_xyzw) -> None: + states = sim.get_actor_states(["robot"], _all_ids).clone() states[:, :3] = env_origins + torch.tensor(list(init.pos), device=device) states[:, 0] += x_offset states[:, 3:7] = torch.tensor(rot_xyzw, device=device) states[:, 7:] = 0.0 - sim.set_actor_states(["robot"], torch.arange(n, device=device), states) + sim.set_actor_states(["robot"], _all_ids, states) + + def _place(x_offset: float, rot_xyzw) -> None: + # Set the root pose + hold joints rigid, settle, then re-assert both right before the render: + # holding the DOF pose stops the un-actuated joints from sagging the pelsvis/camera over the + # settle, and re-setting immediately before render_sensors lands the exact pose in the frame + # (on IsaacSim a pose write needs a few steps to propagate to the render, which the settle + # provides). Robust against the multi-env drift flake. + _set_root(x_offset, rot_xyzw) + _hold_dof() step(sim, max(2, steps_for_seconds(sim, 0.05))) + _set_root(x_offset, rot_xyzw) + _hold_dof() + step(sim, 2) sim.render_sensors() - cam_name, cam = next(iter(config.sensor.items())) + cam_name, _cam = next(iter(config.sensor.items())) _place(0.0, base_rot) before = [ _panel_median_depth(sim.get_camera_data(cam_name, "rgb")[e], sim.get_camera_data(cam_name, "depth")[e]) @@ -206,4 +245,13 @@ def _place(x_offset: float, rot_xyzw) -> None: if __name__ == "__main__": - sys.exit(main()) + # IsaacSim teardown deadlocks in carbOnPluginShutdown tearing down the + # omni.syntheticdata/OmniGraph render-product graph a TiledCamera creates (native + # py-spy stack), so a normal interpreter exit hangs until the parent's subprocess + # timeout SIGKILLs it -- turning a PASS (verdict already written to --result-file) into + # a spurious timeout failure. Hard-exit past the atexit teardown, mirroring + # behavior_assert / scene_spawn_assert. Rendering itself is fine; only exit hangs. + _rc = main() + sys.stdout.flush() + sys.stderr.flush() + os._exit(_rc) diff --git a/src/holosoma/tests/simulators/camera_geometry_assert.py b/src/holosoma/tests/simulators/camera_geometry_assert.py index 3cf6135a0..96e41e04c 100644 --- a/src/holosoma/tests/simulators/camera_geometry_assert.py +++ b/src/holosoma/tests/simulators/camera_geometry_assert.py @@ -25,6 +25,7 @@ import argparse import dataclasses import math +import os import sys if sys.path and sys.path[0].endswith("simulators"): @@ -148,14 +149,51 @@ def main() -> int: # Pin the robot to a known upright pose at each env origin so the (pelvis-mounted) camera aligns # with the panel placed ahead (IsaacGym jitters the spawn xy; other backends are already at origin). + # Use the origins the simulator ACTUALLY placed the envs at (IsaacSim clones onto its own grid and + # ignores the requested env_origins; sim.env_origins is reconciled to the real placement). + env_origins = sim.env_origins import torch as _torch + _all_ids = _torch.arange(n, device=device) + # Capture the spawn joint pose once so the pin can restore it: the robot is un-actuated, so + # holding only the ROOT still lets the JOINTS sag over the settle steps, tilting the pelvis a + # few degrees and shifting the pelvis-mounted camera a few px off-axis (a rare multi-env L/R + # asymmetry flake). Holding the joints rigid too removes the settle at its source. + _spawn_dof_pos = sim.dof_pos.clone() + + def _hold_dof() -> None: + # Restore joints to the spawn pose with zero velocity. The cross-backend DOF setter takes + # different tensor shapes (IsaacGym flattened [n*ndof, 2], IsaacSim 3D [n, ndof, 2]); build + # whichever this backend expects. MuJoCo isn't affected by this flake, so only the two + # articulation backends are handled. + ndof = sim.num_dof + if args.simulator == "isaacgym": + ds = _torch.zeros(n * ndof, 2, device=device) + ds[:, 0] = _spawn_dof_pos.reshape(-1) + sim.set_dof_state_tensor_robots(_all_ids, ds) + elif args.simulator == "isaacsim": + ds = _torch.zeros(n, ndof, 2, device=device) + ds[:, :, 0] = _spawn_dof_pos + sim.set_dof_state_tensor_robots(_all_ids, ds) + def _pin_robot() -> None: - robot_states = sim.get_actor_states(["robot"], _torch.arange(n, device=device)).clone() - robot_states[:, :3] = env_origins + _torch.tensor(list(init.pos), device=device) - robot_states[:, 3:7] = _torch.tensor(list(init.rot), device=device) - robot_states[:, 7:] = 0.0 - sim.set_actor_states(["robot"], _torch.arange(n, device=device), robot_states) + # Set the target root pose AND zero all velocities, hold the joints rigid, then read back to + # verify the write landed before we rely on it: any residual root velocity, joint sag, or a + # dropped write lets the pelvis drift and drags its mounted camera off the panel (the + # multi-env flake where a random env's panel leaves frame / lands off-center). Retry until it + # sticks. + target_pos = env_origins + _torch.tensor(list(init.pos), device=device) + target_rot = _torch.tensor(list(init.rot), device=device) + for _ in range(3): + robot_states = sim.get_actor_states(["robot"], _all_ids).clone() + robot_states[:, :3] = target_pos + robot_states[:, 3:7] = target_rot + robot_states[:, 7:] = 0.0 + sim.set_actor_states(["robot"], _all_ids, robot_states) + _hold_dof() + back = sim.get_actor_states(["robot"], _all_ids) + if _torch.allclose(back[:, :3], target_pos, atol=1e-3) and back[:, 7:].abs().max() < 1e-3: + break _pin_robot() step(sim, 2) @@ -166,12 +204,15 @@ def _pin_robot() -> None: return 1 step(sim, max(2, steps_for_seconds(sim, 0.05))) - # Re-pin immediately before capture: the robot is un-actuated, so the settle steps above let the - # pelvis drift/tilt and drag its mounted camera off the panel (an env3-only flake — the projection - # math is env-independent, so a real FOV bug fails all envs). Root writes are immediate on every - # backend and render_sensors() does its own fetch/step_graphics, so pinning here fixes the pose the - # camera renders from without an extra settle. + # Re-pin before capture, then step a few frames: the robot is un-actuated, so the settle steps + # above let the pelvis drift and drag its mounted camera off the panel. Re-pinning restores the + # exact pose, but on IsaacSim a root/joint write does NOT reach the render until physics has + # stepped a few times to propagate it (an immediate render_sensors() after the write still shows + # the pre-pin drifted pose — measured ~4px off, tripping the margin-symmetry check). Stepping ~4 + # frames with the joints held rigid (see _pin_robot) lands the corrected, centered pose in the + # render on every backend without letting the robot drift again. _pin_robot() + step(sim, 4) sim.render_sensors() cam_name, cam = next(iter(config.sensor.items())) @@ -205,4 +246,13 @@ def _pin_robot() -> None: if __name__ == "__main__": - sys.exit(main()) + # IsaacSim teardown deadlocks in carbOnPluginShutdown tearing down the + # omni.syntheticdata/OmniGraph render-product graph a TiledCamera creates (native + # py-spy stack), so a normal interpreter exit hangs until the parent's subprocess + # timeout SIGKILLs it -- turning a PASS (verdict already written to --result-file) into + # a spurious timeout failure. Hard-exit past the atexit teardown, mirroring + # behavior_assert / scene_spawn_assert. Rendering itself is fine; only exit hangs. + _rc = main() + sys.stdout.flush() + sys.stderr.flush() + os._exit(_rc) diff --git a/src/holosoma/tests/simulators/camera_multi_assert.py b/src/holosoma/tests/simulators/camera_multi_assert.py index 50171726b..f3cc6c0fa 100644 --- a/src/holosoma/tests/simulators/camera_multi_assert.py +++ b/src/holosoma/tests/simulators/camera_multi_assert.py @@ -9,6 +9,7 @@ import argparse import dataclasses +import os import sys if sys.path and sys.path[0].endswith("simulators"): @@ -107,4 +108,13 @@ def main() -> int: if __name__ == "__main__": - sys.exit(main()) + # IsaacSim teardown deadlocks in carbOnPluginShutdown tearing down the + # omni.syntheticdata/OmniGraph render-product graph a TiledCamera creates (native + # py-spy stack), so a normal interpreter exit hangs until the parent's subprocess + # timeout SIGKILLs it -- turning a PASS (verdict already written to --result-file) into + # a spurious timeout failure. Hard-exit past the atexit teardown, mirroring + # behavior_assert / scene_spawn_assert. Rendering itself is fine; only exit hangs. + _rc = main() + sys.stdout.flush() + sys.stderr.flush() + os._exit(_rc) diff --git a/src/holosoma/tests/simulators/camera_obs_assert.py b/src/holosoma/tests/simulators/camera_obs_assert.py index e10472ee0..27a40ca7f 100644 --- a/src/holosoma/tests/simulators/camera_obs_assert.py +++ b/src/holosoma/tests/simulators/camera_obs_assert.py @@ -16,6 +16,7 @@ import argparse import dataclasses +import os import sys if sys.path and sys.path[0].endswith("simulators"): @@ -175,4 +176,13 @@ def _obs(): if __name__ == "__main__": - sys.exit(main()) + # IsaacSim teardown deadlocks in carbOnPluginShutdown tearing down the + # omni.syntheticdata/OmniGraph render-product graph a TiledCamera creates (native + # py-spy stack), so a normal interpreter exit hangs until the parent's subprocess + # timeout SIGKILLs it -- turning a PASS (verdict already written to --result-file) into + # a spurious timeout failure. Hard-exit past the atexit teardown, mirroring + # behavior_assert / scene_spawn_assert. Rendering itself is fine; only exit hangs. + _rc = main() + sys.stdout.flush() + sys.stderr.flush() + os._exit(_rc) diff --git a/src/holosoma/tests/simulators/camera_orientation_assert.py b/src/holosoma/tests/simulators/camera_orientation_assert.py index de20ba17c..60f79a69c 100644 --- a/src/holosoma/tests/simulators/camera_orientation_assert.py +++ b/src/holosoma/tests/simulators/camera_orientation_assert.py @@ -15,6 +15,7 @@ import argparse import dataclasses +import os import sys if sys.path and sys.path[0].endswith("simulators"): @@ -137,4 +138,13 @@ def main() -> int: if __name__ == "__main__": - sys.exit(main()) + # IsaacSim teardown deadlocks in carbOnPluginShutdown tearing down the + # omni.syntheticdata/OmniGraph render-product graph a TiledCamera creates (native + # py-spy stack), so a normal interpreter exit hangs until the parent's subprocess + # timeout SIGKILLs it -- turning a PASS (verdict already written to --result-file) into + # a spurious timeout failure. Hard-exit past the atexit teardown, mirroring + # behavior_assert / scene_spawn_assert. Rendering itself is fine; only exit hangs. + _rc = main() + sys.stdout.flush() + sys.stderr.flush() + os._exit(_rc) diff --git a/src/holosoma/tests/simulators/depth_assert.py b/src/holosoma/tests/simulators/depth_assert.py index 5b01dac7c..bf7dcc099 100644 --- a/src/holosoma/tests/simulators/depth_assert.py +++ b/src/holosoma/tests/simulators/depth_assert.py @@ -17,6 +17,7 @@ import argparse import dataclasses import math +import os import sys if sys.path and sys.path[0].endswith("simulators"): @@ -184,4 +185,13 @@ def main() -> int: if __name__ == "__main__": - sys.exit(main()) + # IsaacSim teardown deadlocks in carbOnPluginShutdown tearing down the + # omni.syntheticdata/OmniGraph render-product graph a TiledCamera creates (native + # py-spy stack), so a normal interpreter exit hangs until the parent's subprocess + # timeout SIGKILLs it -- turning a PASS (verdict already written to --result-file) into + # a spurious timeout failure. Hard-exit past the atexit teardown, mirroring + # behavior_assert / scene_spawn_assert. Rendering itself is fine; only exit hangs. + _rc = main() + sys.stdout.flush() + sys.stderr.flush() + os._exit(_rc) diff --git a/src/holosoma/tests/simulators/world_camera_assert.py b/src/holosoma/tests/simulators/world_camera_assert.py index 262cefb50..8730bc095 100644 --- a/src/holosoma/tests/simulators/world_camera_assert.py +++ b/src/holosoma/tests/simulators/world_camera_assert.py @@ -21,6 +21,7 @@ import argparse import dataclasses +import os import sys if sys.path and sys.path[0].endswith("simulators"): @@ -227,4 +228,13 @@ def _reds(): if __name__ == "__main__": - sys.exit(main()) + # IsaacSim teardown deadlocks in carbOnPluginShutdown tearing down the + # omni.syntheticdata/OmniGraph render-product graph a TiledCamera creates (native + # py-spy stack), so a normal interpreter exit hangs until the parent's subprocess + # timeout SIGKILLs it -- turning a PASS (verdict already written to --result-file) into + # a spurious timeout failure. Hard-exit past the atexit teardown, mirroring + # behavior_assert / scene_spawn_assert. Rendering itself is fine; only exit hangs. + _rc = main() + sys.stdout.flush() + sys.stderr.flush() + os._exit(_rc)