diff --git a/Cargo.lock b/Cargo.lock index f6e7aad..3440089 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1998,6 +1998,7 @@ dependencies = [ "http-body-util", "hyper", "hyper-util", + "hyperlight-host", "log", "proptest", "serde", diff --git a/src/hyperlight_sandbox/Cargo.toml b/src/hyperlight_sandbox/Cargo.toml index a73a542..f7450f5 100644 --- a/src/hyperlight_sandbox/Cargo.toml +++ b/src/hyperlight_sandbox/Cargo.toml @@ -16,6 +16,7 @@ bytes = "1" cap-std = "4.0.3" http = "1" http-body-util = "0.1" +hyperlight-host = { workspace = true, features = ["kvm", "mshv3", "hvf"] } hyper = { version = "1", features = ["server", "http1"] } hyper-util = { version = "0.1", features = ["tokio"] } log = "0.4" diff --git a/src/hyperlight_sandbox/src/lib.rs b/src/hyperlight_sandbox/src/lib.rs index a6b5f27..c316568 100644 --- a/src/hyperlight_sandbox/src/lib.rs +++ b/src/hyperlight_sandbox/src/lib.rs @@ -17,6 +17,7 @@ pub use cap_fs::{ CapFs, DescriptorFlags, DescriptorStat, DescriptorType, Dir, DirPerms, FilePerms, FilesystemLimits, FsError, OpenFlags, }; +pub use hyperlight_host::HyperlightError; pub use network::{HttpMethod, MethodFilter, NetworkPermission, NetworkPermissions}; use serde::{Deserialize, Serialize}; pub use tools::{ArgType, ToolRegistry, ToolSchema}; @@ -106,6 +107,30 @@ impl Snapshot { // Guest traits // --------------------------------------------------------------------------- +/// An independently owned handle for interrupting guest execution. +/// +/// Host callbacks are not interrupted. After an interruption, restore a +/// snapshot before reusing the sandbox. +#[derive(Clone)] +pub struct InterruptHandle { + inner: std::sync::Arc, +} + +impl InterruptHandle { + pub fn new(inner: std::sync::Arc) -> Self { + Self { inner } + } + + /// Request interruption, returning whether the vCPU was running. + /// + /// This may block until the running vCPU stops. A false return during a + /// host callback can still leave cancellation pending until it returns. + /// A request just before guest entry can be missed; callers can retry. + pub fn kill(&self) -> bool { + self.inner.kill() + } +} + pub trait Guest: Sized { type Sandbox: GuestSandbox; fn build( @@ -129,6 +154,10 @@ pub trait GuestSandbox: Send { fn snapshot(&mut self) -> Result>; /// Restore a previously captured guest runtime state. fn restore(&mut self, snapshot: &Snapshot) -> Result<()>; + /// Obtain an independently owned interruption handle. + fn interrupt_handle(&self) -> Result { + anyhow::bail!("guest backend does not support interruption") + } } // --------------------------------------------------------------------------- @@ -173,6 +202,10 @@ impl Sandbox { self.inner.run(code) } + pub fn interrupt_handle(&self) -> Result { + self.inner.interrupt_handle() + } + pub fn snapshot(&mut self) -> Result::SnapshotData>> { self.inner.snapshot() } @@ -373,6 +406,12 @@ where mod tests { use super::*; + #[test] + fn interrupt_handle_is_send_and_sync() { + fn assert_send_sync() {} + assert_send_sync::(); + } + struct TestGuest; struct TestGuestSandbox; diff --git a/src/javascript_sandbox/src/lib.rs b/src/javascript_sandbox/src/lib.rs index 7f7ba37..36fb2dc 100644 --- a/src/javascript_sandbox/src/lib.rs +++ b/src/javascript_sandbox/src/lib.rs @@ -7,8 +7,8 @@ use hyperlight_js::{ }; use hyperlight_sandbox::runtime::BlockOn; use hyperlight_sandbox::{ - CapFs, ExecutionResult, Guest, GuestSandbox, NetworkPermissions, SandboxConfig, Snapshot, - ToolRegistry, http as sandbox_http, + CapFs, ExecutionResult, Guest, GuestSandbox, InterruptHandle, NetworkPermissions, + SandboxConfig, Snapshot, ToolRegistry, http as sandbox_http, }; use serde::Deserialize; @@ -393,6 +393,15 @@ impl JsGuestSandbox { exit_code: response.exit_code, }) } + Err(error) + if matches!( + error, + hyperlight_sandbox::HyperlightError::ExecutionCanceledByHost() + | hyperlight_sandbox::HyperlightError::PoisonedSandbox + ) => + { + Err(error.into()) + } Err(error) => { let mut stderr = error.to_string(); let stdout = match self.stdout.lock() { @@ -415,6 +424,11 @@ impl JsGuestSandbox { impl GuestSandbox for JsGuestSandbox { type SnapshotData = JsSnapshot; + fn interrupt_handle(&self) -> Result { + let handle = self.sandbox.interrupt_handle(); + Ok(InterruptHandle::new(handle)) + } + fn run(&mut self, code: &str) -> Result { self.run_impl(code) } diff --git a/src/sdk/dotnet/README.md b/src/sdk/dotnet/README.md index 375ac03..54329fc 100644 --- a/src/sdk/dotnet/README.md +++ b/src/sdk/dotnet/README.md @@ -266,7 +266,8 @@ just dotnet copilot-sdk-example # Run Copilot SDK example | Method | Description | |--------|-------------| | `Run(string code)` | Execute guest code, returns `ExecutionResult` | -| `RunAsync(string, CancellationToken)` | Async version on thread pool | +| `Run(string, CancellationToken)` | Execute guest code with interruption on cancellation | +| `RunAsync(string, CancellationToken)` | Execute on the thread pool with interruption on cancellation | | `RegisterTool(name, handler)` | Register typed tool | | `RegisterTool(name, Func)` | Register raw JSON tool | | `AllowDomain(target, methods?)` | Add domain to network allowlist | @@ -275,6 +276,13 @@ just dotnet copilot-sdk-example # Run Copilot SDK example | `Snapshot()` | Capture sandbox state | | `Restore(snapshot)` | Restore to captured state | +Cancellation stops running guest execution on both backends and throws +`OperationCanceledException` only after native execution and interruption callbacks +have finished. Runtime initialization and host tool callbacks cannot be interrupted. +Capture a snapshot before execution and restore it after cancellation before reusing +the sandbox. Use a `CancellationTokenSource` with a finite timeout to bound guest +execution; host tools must enforce their own cancellation policy. + ### `ExecutionResult` | Property | Type | Description | diff --git a/src/sdk/dotnet/core/Api/Sandbox.cs b/src/sdk/dotnet/core/Api/Sandbox.cs index 1a1b98b..df4a3aa 100644 --- a/src/sdk/dotnet/core/Api/Sandbox.cs +++ b/src/sdk/dotnet/core/Api/Sandbox.cs @@ -319,6 +319,21 @@ public void RegisterToolAsync(string name, Func> handler) /// permissions). Subsequent calls reuse the initialized runtime. /// public ExecutionResult Run(string code) + => Run(code, CancellationToken.None); + + /// Executes guest code with cancellation of running guest execution. + /// The code to execute. + /// Token used to interrupt guest execution. + /// The execution result. + /// + /// Runtime initialization and host callbacks cannot be interrupted. + /// Cancellation waits for native execution to stop before returning. + /// Completed results and unrelated failures are preserved if cancellation arrives late. + /// Restore a snapshot before reusing an interrupted sandbox. + /// + /// Thrown before execution if cancellation is + /// requested, or when native execution reports cancellation for this token. + public ExecutionResult Run(string code, CancellationToken cancellationToken) { lock (_gate) { @@ -333,9 +348,43 @@ public ExecutionResult Run(string code) nameof(code)); } - var result = SafeNativeMethods.hyperlight_sandbox_run(_handle, code); - GC.KeepAlive(this); - result.ThrowIfError(); + cancellationToken.ThrowIfCancellationRequested(); + using var completed = new ManualResetEventSlim(); + using var interruptHandle = cancellationToken.CanBeCanceled ? GetInterruptHandle() : null; + Task? interruption = null; + using var registration = cancellationToken.Register(() => + { + // kill() can miss an idle guest just before native execution starts. + // Retry off the callback thread, which may itself be a host tool. + interruption = Task.Factory.StartNew(() => + { + while (!completed.IsSet) + { + if (SafeNativeMethods.hyperlight_sandbox_interrupt(interruptHandle!)) + { + break; + } + + completed.Wait(10); + } + }, CancellationToken.None, TaskCreationOptions.LongRunning, TaskScheduler.Default); + }); + + FFIResult result; + try + { + cancellationToken.ThrowIfCancellationRequested(); + result = SafeNativeMethods.hyperlight_sandbox_run(_handle, code); + } + finally + { + completed.Set(); + registration.Dispose(); + interruption?.GetAwaiter().GetResult(); + GC.KeepAlive(this); + } + + result.ThrowIfError(cancellationToken); var json = FFIResult.StringFromPtr(result.value); if (json == null) @@ -359,14 +408,25 @@ public ExecutionResult Run(string code) /// For ASP.NET Core, prefer calling directly. /// /// - /// The cancellation token prevents scheduling of the task but cannot - /// cancel an in-progress FFI call. Once starts - /// executing in the native layer, it will run to completion. + /// Cancellation interrupts running guest execution, but not runtime + /// initialization or host callbacks. Restore a snapshot after interruption. /// /// The code to execute. - /// Token to prevent scheduling (does not cancel in-progress execution). + /// Token to prevent scheduling or interrupt running guest execution. public Task RunAsync(string code, CancellationToken cancellationToken = default) - => Task.Run(() => Run(code), cancellationToken); + => Task.Run(() => Run(code, cancellationToken), cancellationToken); + + private InterruptSafeHandle GetInterruptHandle() + { + var result = SafeNativeMethods.hyperlight_sandbox_get_interrupt_handle(_handle); + result.ThrowIfError(); + if (result.value == IntPtr.Zero) + { + throw new SandboxException("Native interruption handle was null."); + } + + return new InterruptSafeHandle(result.value); + } // ----------------------------------------------------------------------- // Network diff --git a/src/sdk/dotnet/core/PInvoke/FFIErrorCode.cs b/src/sdk/dotnet/core/PInvoke/FFIErrorCode.cs index c2f1dae..3cf0d3f 100644 --- a/src/sdk/dotnet/core/PInvoke/FFIErrorCode.cs +++ b/src/sdk/dotnet/core/PInvoke/FFIErrorCode.cs @@ -33,4 +33,7 @@ internal enum FFIErrorCode : uint /// Filesystem I/O error. IoError = 7, + + /// Guest execution was interrupted by the host. + Cancelled = 8, } diff --git a/src/sdk/dotnet/core/PInvoke/FFIResult.cs b/src/sdk/dotnet/core/PInvoke/FFIResult.cs index 29ff008..0acfae9 100644 --- a/src/sdk/dotnet/core/PInvoke/FFIResult.cs +++ b/src/sdk/dotnet/core/PInvoke/FFIResult.cs @@ -59,7 +59,7 @@ internal struct FFIResult /// Maps values to specific exception types /// for structured error handling in the API layer. /// - public void ThrowIfError() + public void ThrowIfError(CancellationToken cancellationToken = default) { if (is_success) { @@ -71,6 +71,8 @@ public void ThrowIfError() throw code switch { + FFIErrorCode.Cancelled when cancellationToken.IsCancellationRequested => + new OperationCanceledException(errorMessage, cancellationToken), FFIErrorCode.Timeout => new SandboxTimeoutException(errorMessage), FFIErrorCode.Poisoned => diff --git a/src/sdk/dotnet/core/PInvoke/SafeHandles.cs b/src/sdk/dotnet/core/PInvoke/SafeHandles.cs index a14f5b0..ed796ba 100644 --- a/src/sdk/dotnet/core/PInvoke/SafeHandles.cs +++ b/src/sdk/dotnet/core/PInvoke/SafeHandles.cs @@ -103,3 +103,25 @@ protected override bool ReleaseHandle() return true; } } + +/// Owns a native interruption handle independently of sandbox state. +internal sealed class InterruptSafeHandle : SafeHandle +{ + public InterruptSafeHandle(IntPtr handle) : base(IntPtr.Zero, ownsHandle: true) + { + SetHandle(handle); + } + + public override bool IsInvalid => handle == IntPtr.Zero; + + protected override bool ReleaseHandle() + { + IntPtr oldHandle = Interlocked.Exchange(ref handle, IntPtr.Zero); + if (oldHandle != IntPtr.Zero) + { + SafeNativeMethods.hyperlight_sandbox_free_interrupt_handle(oldHandle); + } + + return true; + } +} diff --git a/src/sdk/dotnet/core/PInvoke/SafeNativeMethods.cs b/src/sdk/dotnet/core/PInvoke/SafeNativeMethods.cs index 7e005cf..07dfac1 100644 --- a/src/sdk/dotnet/core/PInvoke/SafeNativeMethods.cs +++ b/src/sdk/dotnet/core/PInvoke/SafeNativeMethods.cs @@ -214,6 +214,19 @@ internal static partial FFIResult hyperlight_sandbox_run( SandboxSafeHandle handle, [MarshalAs(UnmanagedType.LPUTF8Str)] string code); + [LibraryImport(LibName)] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] + internal static partial FFIResult hyperlight_sandbox_get_interrupt_handle(SandboxSafeHandle handle); + + [LibraryImport(LibName)] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] + [return: MarshalAs(UnmanagedType.I1)] + internal static partial bool hyperlight_sandbox_interrupt(InterruptSafeHandle handle); + + [LibraryImport(LibName)] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] + internal static partial void hyperlight_sandbox_free_interrupt_handle(IntPtr handle); + // ----------------------------------------------------------------------- // Filesystem // ----------------------------------------------------------------------- diff --git a/src/sdk/dotnet/core/Tests/HyperlightSandbox.Tests/IntegrationTests.cs b/src/sdk/dotnet/core/Tests/HyperlightSandbox.Tests/IntegrationTests.cs index 70984d0..3a5b520 100644 --- a/src/sdk/dotnet/core/Tests/HyperlightSandbox.Tests/IntegrationTests.cs +++ b/src/sdk/dotnet/core/Tests/HyperlightSandbox.Tests/IntegrationTests.cs @@ -64,6 +64,41 @@ public IntegrationTests(ITestOutputHelper output) // Basic execution // ----------------------------------------------------------------------- + [Fact] + public async Task Integration_RunAsync_CancellationStopsGuestAndAllowsRestoreAsync() + { + var guestPath = FindPythonGuest(); + if (guestPath == null) return; + using var cancellation = new CancellationTokenSource(); + using var sandbox = new SandboxBuilder().WithModulePath(guestPath).Build(); + sandbox.RegisterTool("cancel_run", (string _) => + { + cancellation.CancelAfter(TimeSpan.FromMilliseconds(100)); + return "{}"; + }); + var warmup = await sandbox.RunAsync("print('ready')").ConfigureAwait(false); + Assert.True(warmup.Success, warmup.Stderr); + using var snapshot = await sandbox.SnapshotAsync().ConfigureAwait(false); + var elapsed = System.Diagnostics.Stopwatch.StartNew(); + + await Assert.ThrowsAnyAsync(() => sandbox.RunAsync(""" + import time + call_tool('cancel_run') + end = time.monotonic() + 5 + while time.monotonic() < end: + pass + print('finished') + """, cancellation.Token)).ConfigureAwait(false); + + Assert.True(elapsed.Elapsed < TimeSpan.FromSeconds(2), "Guest execution was not interrupted promptly."); + await sandbox.RestoreAsync(snapshot).ConfigureAwait(false); + using var recovery = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + var result = await sandbox.RunAsync("print('recovered')", recovery.Token) + .WaitAsync(TimeSpan.FromSeconds(10)).ConfigureAwait(false); + Assert.True(result.Success, result.Stderr); + Assert.Contains("recovered", result.Stdout, StringComparison.Ordinal); + } + [Fact] public void Integration_BasicExecution_ReturnsStdout() diff --git a/src/sdk/dotnet/core/Tests/HyperlightSandbox.Tests/JavaScriptBackendTests.cs b/src/sdk/dotnet/core/Tests/HyperlightSandbox.Tests/JavaScriptBackendTests.cs index dd57cd9..630c857 100644 --- a/src/sdk/dotnet/core/Tests/HyperlightSandbox.Tests/JavaScriptBackendTests.cs +++ b/src/sdk/dotnet/core/Tests/HyperlightSandbox.Tests/JavaScriptBackendTests.cs @@ -66,6 +66,89 @@ public void WithBackend_UpdatesBackendProperty() // Lifecycle // ----------------------------------------------------------------------- + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task JavaScript_RunAsync_CancellationDuringExecution_StopsGuestAsync(bool delayedCancellation) + { + using var cancellation = new CancellationTokenSource(); + using var sandbox = new SandboxBuilder() + .WithBackend(SandboxBackend.JavaScript) + .Build(); + sandbox.RegisterTool("cancel_run", (string _) => + { + if (delayedCancellation) + { + cancellation.CancelAfter(TimeSpan.FromMilliseconds(100)); + } + else + { + cancellation.Cancel(); + } + + return "{}"; + }); + Assert.True((await sandbox.RunAsync("console.log('ready');").ConfigureAwait(false)).Success); + using var snapshot = await sandbox.SnapshotAsync().ConfigureAwait(false); + var elapsed = System.Diagnostics.Stopwatch.StartNew(); + + await Assert.ThrowsAnyAsync(() => sandbox.RunAsync(""" + call_tool('cancel_run'); + const end = Date.now() + 5000; + while (Date.now() < end) {} + console.log('finished'); + """, cancellation.Token)).ConfigureAwait(false); + + Assert.True(elapsed.Elapsed < TimeSpan.FromSeconds(2), "Guest execution was not interrupted promptly."); + await sandbox.RestoreAsync(snapshot).ConfigureAwait(false); + using var recovery = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + var result = await sandbox.RunAsync("console.log('recovered');", recovery.Token) + .WaitAsync(TimeSpan.FromSeconds(10)).ConfigureAwait(false); + Assert.True(result.Success, result.Stderr); + Assert.Contains("recovered", result.Stdout, StringComparison.Ordinal); + } + + [Fact] + public void JavaScript_Run_PreCanceledTokenDoesNotExecuteGuest() + { + using var cancellation = new CancellationTokenSource(); + using var sandbox = new SandboxBuilder() + .WithBackend(SandboxBackend.JavaScript) + .Build(); + var invoked = false; + sandbox.RegisterTool("mark_run", (string _) => + { + invoked = true; + return "{}"; + }); + cancellation.Cancel(); + + Assert.ThrowsAny( + () => sandbox.Run("call_tool('mark_run');", cancellation.Token)); + Assert.False(invoked); + using var recovery = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + Assert.True(sandbox.Run("console.log('recovered');", recovery.Token).Success); + } + + [Fact] + public void JavaScript_Run_CompletedCancellationDoesNotInterruptNextRun() + { + using var sandbox = new SandboxBuilder() + .WithBackend(SandboxBackend.JavaScript) + .Build(); + for (int i = 0; i < 5; i++) + { + using var completed = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + Assert.True(sandbox.Run("console.log('first');", completed.Token).Success); + completed.Cancel(); + + using var next = new CancellationTokenSource(TimeSpan.FromSeconds(5)); + var result = sandbox.Run("console.log('next');", next.Token); + Assert.True(result.Success, result.Stderr); + Assert.Contains("next", result.Stdout, StringComparison.Ordinal); + } + } + [Fact] public void JavaScript_CreateAndDispose_NoLeak() { diff --git a/src/sdk/dotnet/core/Tests/HyperlightSandbox.Tests/PInvokeLayerTests.cs b/src/sdk/dotnet/core/Tests/HyperlightSandbox.Tests/PInvokeLayerTests.cs index bcd95b2..5b6412d 100644 --- a/src/sdk/dotnet/core/Tests/HyperlightSandbox.Tests/PInvokeLayerTests.cs +++ b/src/sdk/dotnet/core/Tests/HyperlightSandbox.Tests/PInvokeLayerTests.cs @@ -25,6 +25,7 @@ public class PInvokeLayerTests [InlineData(5u, 5u)] // GuestError [InlineData(6u, 6u)] // InvalidArgument [InlineData(7u, 7u)] // IoError + [InlineData(8u, 8u)] // Cancelled public void FFIErrorCode_Values_MatchRust(uint codeValue, uint expected) { var code = (FFIErrorCode)codeValue; @@ -49,6 +50,36 @@ public void FFIResult_Success_DoesNotThrow() result.ThrowIfError(); } + [Fact] + public void FFIResult_Success_PreservesResultAfterCancellation() + { + var result = new FFIResult { is_success = true }; + result.ThrowIfError(new CancellationToken(canceled: true)); + } + + [Fact] + public void FFIResult_UnrelatedFailure_PreservesErrorAfterCancellation() + { + var result = new FFIResult { error_code = (uint)FFIErrorCode.IoError }; + Assert.Throws(() => result.ThrowIfError(new CancellationToken(canceled: true))); + } + + [Fact] + public void FFIResult_NativeCancellation_WithRequestedToken_ThrowsCancellation() + { + var result = new FFIResult { error_code = (uint)FFIErrorCode.Cancelled }; + var token = new CancellationToken(canceled: true); + var exception = Assert.Throws(() => result.ThrowIfError(token)); + Assert.Equal(token, exception.CancellationToken); + } + + [Fact] + public void FFIResult_NativeCancellation_WithoutRequestedToken_PreservesNativeError() + { + var result = new FFIResult { error_code = (uint)FFIErrorCode.Cancelled }; + Assert.Throws(() => result.ThrowIfError(CancellationToken.None)); + } + [Fact] public void FFIResult_Timeout_ThrowsOperationCanceledException() { diff --git a/src/sdk/dotnet/ffi/src/lib.rs b/src/sdk/dotnet/ffi/src/lib.rs index 79f498c..8791539 100644 --- a/src/sdk/dotnet/ffi/src/lib.rs +++ b/src/sdk/dotnet/ffi/src/lib.rs @@ -27,7 +27,7 @@ use anyhow::Result; use hyperlight_javascript_sandbox::HyperlightJs; use hyperlight_sandbox::{ DEFAULT_HEAP_SIZE, DEFAULT_STACK_SIZE, DirPerms, FilePerms, FilesystemLimits, GuestSandbox, - HttpMethod, Sandbox, SandboxBuilder, SandboxConfig, ToolRegistry, ToolSchema, + HttpMethod, InterruptHandle, Sandbox, SandboxBuilder, SandboxConfig, ToolRegistry, ToolSchema, }; use hyperlight_wasm_sandbox::Wasm; use log::{debug, error}; @@ -60,6 +60,8 @@ pub enum FFIErrorCode { InvalidArgument = 6, /// Filesystem I/O error. IoError = 7, + /// Guest execution was interrupted by the host. + Cancelled = 8, } // --------------------------------------------------------------------------- @@ -270,6 +272,17 @@ fn safe_cstring>>(t: T) -> CString { /// Uses `downcast_ref` for concrete types where possible, falling back /// to string matching for untyped `anyhow` errors. fn classify_error(err: &anyhow::Error) -> FFIErrorCode { + if let Some(error) = err.downcast_ref::() { + match error { + hyperlight_sandbox::HyperlightError::ExecutionCanceledByHost() => { + return FFIErrorCode::Cancelled; + } + hyperlight_sandbox::HyperlightError::PoisonedSandbox => { + return FFIErrorCode::Poisoned; + } + _ => {} + } + } // Try concrete type downcasts first (more reliable than string matching). if err.downcast_ref::>().is_some() { return FFIErrorCode::Poisoned; @@ -282,12 +295,7 @@ fn classify_error(err: &anyhow::Error) -> FFIErrorCode { let msg = err.to_string().to_lowercase(); if msg.contains("poisoned") || msg.contains("mutex") { FFIErrorCode::Poisoned - } else if msg.contains("timeout") - || msg.contains("cancelled") - || msg.contains("canceled") - || msg.contains("timed out") - || msg.contains("deadline") - { + } else if msg.contains("timeout") || msg.contains("timed out") || msg.contains("deadline") { FFIErrorCode::Timeout } else if msg.contains("permission") || msg.contains("not allowed") || msg.contains("denied") { FFIErrorCode::PermissionDenied @@ -1063,6 +1071,61 @@ fn ensure_initialized(state: &mut SandboxState) -> Result<()> { Ok(()) } +/// Obtain an independently owned interruption handle, initializing the sandbox +/// if necessary. The successful value is an opaque handle, not a string. +/// +/// # Safety +/// +/// `handle` must be valid and exclusively accessible during this call. +/// Free the returned handle with `hyperlight_sandbox_free_interrupt_handle`. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn hyperlight_sandbox_get_interrupt_handle( + handle: *mut SandboxState, +) -> FFIResult { + let state = match unsafe { deref_handle_mut(handle, "sandbox") } { + Ok(state) => state, + Err(error) => return error, + }; + if let Err(error) = ensure_initialized(state) { + return error_result(error); + } + let Some(sandbox) = state.inner.as_ref() else { + return error_result(anyhow::anyhow!( + "Sandbox initialization produced no backend" + )); + }; + match with_sandbox!(sandbox, sb => sb.interrupt_handle()) { + Ok(handle) => FFIResult::success(Box::into_raw(Box::new(handle)).cast()), + Err(error) => error_result(error), + } +} + +/// Interrupt guest execution without accessing mutable sandbox state. +/// +/// # Safety +/// +/// `handle` must be a live interruption handle. Do not free it during this call. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn hyperlight_sandbox_interrupt(handle: *const InterruptHandle) -> bool { + let Some(handle) = NonNull::new(handle.cast_mut()) else { + error!("Null pointer passed for interruption handle"); + return false; + }; + unsafe { handle.as_ref() }.kill() +} + +/// Free an independently owned interruption handle. Null is a no-op. +/// +/// # Safety +/// +/// `handle` must be null or a live handle that is not in use. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn hyperlight_sandbox_free_interrupt_handle(handle: *mut InterruptHandle) { + if !handle.is_null() { + drop(unsafe { Box::from_raw(handle) }); + } +} + /// Executes guest code in the sandbox. /// /// The first call triggers lazy initialization (building the sandbox, registering @@ -1424,9 +1487,24 @@ mod tests { } #[test] - fn classify_error_cancelled() { + fn classify_error_untyped_cancellation_is_not_timeout() { let err = anyhow::anyhow!("operation was cancelled by host"); - assert_eq!(classify_error(&err), FFIErrorCode::Timeout); + assert_eq!(classify_error(&err), FFIErrorCode::Unknown); + } + + #[test] + fn classify_error_preserves_native_cancellation_with_context() { + let err = + anyhow::Error::new(hyperlight_sandbox::HyperlightError::ExecutionCanceledByHost()) + .context("guest execution failed"); + assert_eq!(classify_error(&err), FFIErrorCode::Cancelled); + } + + #[test] + fn classify_error_preserves_native_poison_with_context() { + let err = anyhow::Error::new(hyperlight_sandbox::HyperlightError::PoisonedSandbox) + .context("guest execution failed"); + assert_eq!(classify_error(&err), FFIErrorCode::Poisoned); } #[test] diff --git a/src/sdk/python/core/hyperlight_sandbox/__init__.py b/src/sdk/python/core/hyperlight_sandbox/__init__.py index 2287515..c911b02 100644 --- a/src/sdk/python/core/hyperlight_sandbox/__init__.py +++ b/src/sdk/python/core/hyperlight_sandbox/__init__.py @@ -62,6 +62,7 @@ def _load_backend(backend: str): __all__ = [ "CodeExecutionTool", "ExecutionResult", + "InterruptHandle", "Sandbox", "SandboxEnvironment", "__version__", @@ -104,6 +105,24 @@ class SandboxEnvironment: max_file_count: int | None = None +class InterruptHandle: + """An independently owned handle that can interrupt a guest from any thread.""" + + def __init__(self, inner: Any) -> None: + self._inner = inner + + def kill(self) -> bool: + """Interrupt a running guest and wait for it to stop. + + Returns whether the vCPU was running. False during a host callback can + still leave cancellation pending until that callback returns. + A request just before guest entry can be missed; callers can retry. + Host callbacks are not interrupted. Restore a snapshot before reusing + an interrupted sandbox. The handle can outlive its sandbox. + """ + return self._inner.kill() + + class Sandbox: """Stable Python API over swappable Hyperlight backends. @@ -199,6 +218,7 @@ def register_tool(self, name_or_tool: Any, callback: Any | None = None) -> None: MAX_CODE_SIZE: int = 10 * 1024 * 1024 def run(self, code: str) -> ExecutionResult: + """Execute code, raising InterruptedError if a handle interrupts the guest.""" if len(code) > self.MAX_CODE_SIZE: raise ValueError(f"code exceeds maximum size ({len(code)} > {self.MAX_CODE_SIZE} bytes)") native_result = self._inner.run(code) @@ -208,6 +228,14 @@ def run(self, code: str) -> ExecutionResult: exit_code=native_result.exit_code, ) + def interrupt_handle(self) -> InterruptHandle: + """Initialize the runtime if needed and return a thread-safe interrupt handle. + + Register tools before obtaining the handle. Use the sandbox itself only + on its creating thread; the independent handle can be shared. + """ + return InterruptHandle(self._inner.interrupt_handle()) + def get_output_files(self) -> list[str]: """List filenames written by the guest to the output directory. diff --git a/src/sdk/python/core/tests/test_interruption.py b/src/sdk/python/core/tests/test_interruption.py new file mode 100644 index 0000000..55c9dbb --- /dev/null +++ b/src/sdk/python/core/tests/test_interruption.py @@ -0,0 +1,143 @@ +"""Cross-thread guest interruption for both Python SDK backends.""" + +import threading +import time +import unittest + +from hyperlight_sandbox import Sandbox + + +class TestInterruption(unittest.TestCase): + def test_idle_handle_and_handle_after_sandbox_drop(self): + for backend in ("wasm", "hyperlight-js"): + with self.subTest(backend=backend): + sandbox = Sandbox(backend=backend) + handle = sandbox.interrupt_handle() + self.assertFalse(handle.kill()) + del sandbox + self.assertFalse(handle.kill()) + + def test_interrupt_from_another_thread_and_restore(self): + for backend in ("wasm", "hyperlight-js"): + with self.subTest(backend=backend): + self._interrupt_and_restore(backend) + + def test_pending_host_call_interruption_and_restore(self): + for backend in ("wasm", "hyperlight-js"): + with self.subTest(backend=backend): + attempts = self._interrupt_and_restore(backend, interrupt_during_host_call=True) + self.assertEqual(attempts, [False]) + + def _interrupt_and_restore(self, backend, interrupt_during_host_call=False): + sandbox = Sandbox(backend=backend) + started = threading.Event() + finished = threading.Event() + release = threading.Event() + attempts = [] + + def ready() -> bool: + started.set() + if interrupt_during_host_call: + return release.wait(5) + return True + + sandbox.register_tool("ready", ready) + handle = sandbox.interrupt_handle() + warmup = "print('ready')" if backend == "wasm" else "console.log('ready');" + self.assertTrue(sandbox.run(warmup).success) + snapshot = sandbox.snapshot() + + def interrupt(): + if not started.wait(5): + return + if finished.wait(0.05): + return + while not finished.is_set(): + attempts.append(handle.kill()) + release.set() + if attempts[-1] or interrupt_during_host_call: + return + finished.wait(0.01) + + worker = threading.Thread(target=interrupt, daemon=True) + worker.start() + code = ( + "import time\ncall_tool('ready')\nend = time.monotonic() + 5\n" + "while time.monotonic() < end:\n" + " for counter in range(100_000):\n value = counter * counter" + if backend == "wasm" + else "call_tool('ready'); const end = Date.now() + 5000; let value = 0;" + "while (Date.now() < end) {" + "for (let counter = 0; counter < 100000; counter++) { value = counter * counter; }}" + ) + start = time.monotonic() + try: + with self.assertRaises(InterruptedError): + sandbox.run(code) + finally: + finished.set() + release.set() + worker.join(5) + self.assertFalse(worker.is_alive()) + self.assertTrue(attempts) + self.assertLess(time.monotonic() - start, 3) + with self.assertRaises(RuntimeError): + sandbox.run(warmup) + sandbox.restore(snapshot) + self.assertTrue(sandbox.run(warmup).success) + return attempts + + def test_interruption_does_not_abort_host_callback(self): + for backend in ("wasm", "hyperlight-js"): + with self.subTest(backend=backend): + self._interrupt_host_callback(backend) + + def _interrupt_host_callback(self, backend): + sandbox = Sandbox(backend=backend) + entered = threading.Event() + release = threading.Event() + finished = threading.Event() + callback_completed = [] + attempts = [] + + def blocked() -> bool: + entered.set() + released = release.wait(5) + callback_completed.append(released) + return released + + sandbox.register_tool("blocked", blocked) + handle = sandbox.interrupt_handle() + + def interrupt(): + try: + if entered.wait(5): + attempts.append(handle.kill()) + finally: + release.set() + while not finished.is_set(): + if handle.kill(): + return + finished.wait(0.01) + + worker = threading.Thread(target=interrupt, daemon=True) + worker.start() + code = ( + "import time\ncall_tool('blocked')\nend = time.monotonic() + 5\nwhile time.monotonic() < end:\n pass" + if backend == "wasm" + else "call_tool('blocked'); const end = Date.now() + 5000; while (Date.now() < end) {}" + ) + try: + with self.assertRaises(InterruptedError): + sandbox.run(code) + finally: + finished.set() + release.set() + worker.join(5) + self.assertFalse(worker.is_alive()) + self.assertEqual(attempts, [False]) + self.assertEqual(callback_completed, [True]) + + +if __name__ == "__main__": + unittest.main() diff --git a/src/sdk/python/hyperlight_js_backend/src/lib.rs b/src/sdk/python/hyperlight_js_backend/src/lib.rs index 64c4744..11acd10 100644 --- a/src/sdk/python/hyperlight_js_backend/src/lib.rs +++ b/src/sdk/python/hyperlight_js_backend/src/lib.rs @@ -6,8 +6,8 @@ use hyperlight_sandbox::{ MethodFilter, Sandbox, SandboxBuilder, SandboxConfig, }; use hyperlight_sandbox_pyo3_common::{ - PyExecutionResult, build_tool_registry, parse_filesystem_limits, parse_size, - parse_tool_registration, + PyExecutionResult, PyInterruptHandle, build_tool_registry, execution_error, + parse_filesystem_limits, parse_size, parse_tool_registration, }; use pyo3::exceptions::PyRuntimeError; use pyo3::prelude::*; @@ -34,6 +34,45 @@ pub struct JSSandbox { filesystem_limits: Option, } +impl JSSandbox { + fn ensure_initialized(&mut self, py: Python<'_>) -> PyResult<()> { + if self.inner.is_none() { + let registry = build_tool_registry(py, &mut self.tools)?; + let mut builder = SandboxBuilder::new() + .module_path(&self.config.module_path) + .heap_size(self.config.heap_size) + .stack_size(self.config.stack_size) + .with_tools(registry) + .guest(HyperlightJs); + if let Some(limits) = self.filesystem_limits { + builder = builder.filesystem_limits(limits); + } + if let Some(ref dir) = self.input_dir { + builder = builder.input_dir(dir); + } + if let Some(ref dir) = self.output_dir { + builder = builder.output_dir( + dir, + DirPerms::READ | DirPerms::MUTATE, + FilePerms::READ | FilePerms::WRITE, + ); + } else if self.temp_output { + builder = builder.temp_output(); + } + let mut sandbox = builder + .build() + .map_err(|e| PyRuntimeError::new_err(format!("Failed to create sandbox: {e:#}")))?; + for (target, methods) in std::mem::take(&mut self.pending_networks) { + sandbox + .allow_domain(&target, methods) + .map_err(|e| PyRuntimeError::new_err(format!("{e}")))?; + } + self.inner = Some(sandbox); + } + Ok(()) + } +} + #[pymethods] impl JSSandbox { #[new] @@ -101,45 +140,24 @@ impl JSSandbox { Ok(()) } + fn interrupt_handle(&mut self, py: Python<'_>) -> PyResult { + self.ensure_initialized(py)?; + let inner = self + .inner + .as_ref() + .unwrap() + .interrupt_handle() + .map_err(|error| { + PyRuntimeError::new_err(format!("Failed to get interrupt handle: {error:#}")) + })?; + Ok(PyInterruptHandle { inner }) + } + #[pyo3(signature = (code))] - fn run(&mut self, py: Python<'_>, code: &str) -> PyResult { - if self.inner.is_none() { - let registry = build_tool_registry(py, &mut self.tools)?; - let mut builder = SandboxBuilder::new() - .module_path(&self.config.module_path) - .heap_size(self.config.heap_size) - .stack_size(self.config.stack_size) - .with_tools(registry) - .guest(HyperlightJs); - if let Some(limits) = self.filesystem_limits { - builder = builder.filesystem_limits(limits); - } - if let Some(ref dir) = self.input_dir { - builder = builder.input_dir(dir); - } - if let Some(ref dir) = self.output_dir { - builder = builder.output_dir( - dir, - DirPerms::READ | DirPerms::MUTATE, - FilePerms::READ | FilePerms::WRITE, - ); - } else if self.temp_output { - builder = builder.temp_output(); - } - let mut sandbox = builder - .build() - .map_err(|e| PyRuntimeError::new_err(format!("Failed to create sandbox: {e:#}")))?; - for (target, methods) in std::mem::take(&mut self.pending_networks) { - sandbox - .allow_domain(&target, methods) - .map_err(|e| PyRuntimeError::new_err(format!("{e}")))?; - } - self.inner = Some(sandbox); - } + fn run(&mut self, py: Python<'_>, code: String) -> PyResult { + self.ensure_initialized(py)?; let sandbox = self.inner.as_mut().unwrap(); - let result = sandbox - .run(code) - .map_err(|e| PyRuntimeError::new_err(format!("Execution failed: {e}")))?; + let result = py.detach(|| sandbox.run(&code)).map_err(execution_error)?; Ok(PyExecutionResult { stdout: result.stdout, stderr: result.stderr, @@ -210,6 +228,7 @@ fn _native_js(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; + m.add_class::()?; m.add("__version__", "0.1.0")?; Ok(()) } diff --git a/src/sdk/python/pyo3_common/src/lib.rs b/src/sdk/python/pyo3_common/src/lib.rs index 11c3afd..a8ef7a2 100644 --- a/src/sdk/python/pyo3_common/src/lib.rs +++ b/src/sdk/python/pyo3_common/src/lib.rs @@ -1,8 +1,33 @@ -use hyperlight_sandbox::{ArgType, FilesystemLimits, ToolRegistry, ToolSchema}; -use pyo3::exceptions::{PyRuntimeError, PyTypeError, PyValueError}; +use hyperlight_sandbox::{ + ArgType, FilesystemLimits, HyperlightError, InterruptHandle, ToolRegistry, ToolSchema, +}; +use pyo3::exceptions::{PyInterruptedError, PyRuntimeError, PyTypeError, PyValueError}; use pyo3::prelude::*; use pyo3::types::{PyDict, PyModule}; +#[pyclass(frozen, name = "InterruptHandle")] +pub struct PyInterruptHandle { + pub inner: InterruptHandle, +} + +#[pymethods] +impl PyInterruptHandle { + fn kill(&self, py: Python<'_>) -> bool { + py.detach(|| self.inner.kill()) + } +} + +pub fn execution_error(error: anyhow::Error) -> PyErr { + if matches!( + error.downcast_ref::(), + Some(HyperlightError::ExecutionCanceledByHost()) + ) { + PyInterruptedError::new_err(format!("Execution interrupted: {error:#}")) + } else { + PyRuntimeError::new_err(format!("Execution failed: {error:#}")) + } +} + /// Convert a human-readable size string (e.g. `"200Mi"`) to bytes. pub fn parse_size(size: &str) -> PyResult { parse_size_value(size).map_err(PyRuntimeError::new_err) diff --git a/src/sdk/python/wasm_backend/src/lib.rs b/src/sdk/python/wasm_backend/src/lib.rs index ad54527..3b32dbd 100644 --- a/src/sdk/python/wasm_backend/src/lib.rs +++ b/src/sdk/python/wasm_backend/src/lib.rs @@ -5,8 +5,8 @@ use hyperlight_sandbox::{ MethodFilter, Sandbox, SandboxBuilder, SandboxConfig, }; use hyperlight_sandbox_pyo3_common::{ - PyExecutionResult, build_tool_registry, parse_filesystem_limits, parse_size, - parse_tool_registration, + PyExecutionResult, PyInterruptHandle, build_tool_registry, execution_error, + parse_filesystem_limits, parse_size, parse_tool_registration, }; use hyperlight_wasm_sandbox::Wasm; use pyo3::exceptions::PyRuntimeError; @@ -34,6 +34,45 @@ pub struct WasmSandbox { filesystem_limits: Option, } +impl WasmSandbox { + fn ensure_initialized(&mut self, py: Python<'_>) -> PyResult<()> { + if self.inner.is_none() { + let registry = build_tool_registry(py, &mut self.tools)?; + let mut builder = SandboxBuilder::new() + .module_path(&self.config.module_path) + .heap_size(self.config.heap_size) + .stack_size(self.config.stack_size) + .with_tools(registry) + .guest(Wasm); + if let Some(limits) = self.filesystem_limits { + builder = builder.filesystem_limits(limits); + } + if let Some(ref dir) = self.input_dir { + builder = builder.input_dir(dir); + } + if let Some(ref dir) = self.output_dir { + builder = builder.output_dir( + dir, + DirPerms::READ | DirPerms::MUTATE, + FilePerms::READ | FilePerms::WRITE, + ); + } else if self.temp_output { + builder = builder.temp_output(); + } + let mut sandbox = builder + .build() + .map_err(|e| PyRuntimeError::new_err(format!("Failed to create sandbox: {e:#}")))?; + for (target, methods) in std::mem::take(&mut self.pending_networks) { + sandbox + .allow_domain(&target, methods) + .map_err(|e| PyRuntimeError::new_err(format!("{e}")))?; + } + self.inner = Some(sandbox); + } + Ok(()) + } +} + #[pymethods] impl WasmSandbox { #[new] @@ -95,45 +134,24 @@ impl WasmSandbox { Ok(()) } + fn interrupt_handle(&mut self, py: Python<'_>) -> PyResult { + self.ensure_initialized(py)?; + let inner = self + .inner + .as_ref() + .unwrap() + .interrupt_handle() + .map_err(|error| { + PyRuntimeError::new_err(format!("Failed to get interrupt handle: {error:#}")) + })?; + Ok(PyInterruptHandle { inner }) + } + #[pyo3(signature = (code))] - fn run(&mut self, py: Python<'_>, code: &str) -> PyResult { - if self.inner.is_none() { - let registry = build_tool_registry(py, &mut self.tools)?; - let mut builder = SandboxBuilder::new() - .module_path(&self.config.module_path) - .heap_size(self.config.heap_size) - .stack_size(self.config.stack_size) - .with_tools(registry) - .guest(Wasm); - if let Some(limits) = self.filesystem_limits { - builder = builder.filesystem_limits(limits); - } - if let Some(ref dir) = self.input_dir { - builder = builder.input_dir(dir); - } - if let Some(ref dir) = self.output_dir { - builder = builder.output_dir( - dir, - DirPerms::READ | DirPerms::MUTATE, - FilePerms::READ | FilePerms::WRITE, - ); - } else if self.temp_output { - builder = builder.temp_output(); - } - let mut sandbox = builder - .build() - .map_err(|e| PyRuntimeError::new_err(format!("Failed to create sandbox: {e:#}")))?; - for (target, methods) in std::mem::take(&mut self.pending_networks) { - sandbox - .allow_domain(&target, methods) - .map_err(|e| PyRuntimeError::new_err(format!("{e}")))?; - } - self.inner = Some(sandbox); - } + fn run(&mut self, py: Python<'_>, code: String) -> PyResult { + self.ensure_initialized(py)?; let sandbox = self.inner.as_mut().unwrap(); - let result = sandbox - .run(code) - .map_err(|e| PyRuntimeError::new_err(format!("Execution failed: {e}")))?; + let result = py.detach(|| sandbox.run(&code)).map_err(execution_error)?; Ok(PyExecutionResult { stdout: result.stdout, stderr: result.stderr, @@ -204,6 +222,7 @@ fn _native_wasm(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; + m.add_class::()?; m.add("__version__", "0.1.0")?; Ok(()) } diff --git a/src/wasm_sandbox/src/lib.rs b/src/wasm_sandbox/src/lib.rs index 6ae2486..6ddc5c0 100644 --- a/src/wasm_sandbox/src/lib.rs +++ b/src/wasm_sandbox/src/lib.rs @@ -7,8 +7,8 @@ use std::sync::{Arc, Mutex}; use anyhow::{Context, Result}; use hyperlight_sandbox::{ - CapFs, ExecutionResult, Guest, GuestSandbox, NetworkPermissions, SandboxConfig, Snapshot, - ToolRegistry, + CapFs, ExecutionResult, Guest, GuestSandbox, InterruptHandle, NetworkPermissions, + SandboxConfig, Snapshot, ToolRegistry, }; use hyperlight_wasm::{ LoadedWasmSandbox, SandboxBuilder as HyperlightSandboxBuilder, Snapshot as WasmSnapshot, @@ -282,8 +282,10 @@ impl WasmComponentSandbox { match result { Ok(wit_result) => { - let wit_result = - wit_result.map_err(|e| anyhow::anyhow!("guest execution failed: {e}"))?; + let wit_result = wit_result.map_err(|error| { + let message = format!("guest execution failed: {error}"); + error.context(message) + })?; Ok(ExecutionResult { stdout: wit_result.stdout, stderr: wit_result.stderr, @@ -317,6 +319,11 @@ impl WasmComponentSandbox { impl GuestSandbox for WasmComponentSandbox { type SnapshotData = WasmSnapshot; + fn interrupt_handle(&self) -> Result { + let handle = self.sandbox.sb.interrupt_handle()?; + Ok(InterruptHandle::new(handle)) + } + fn run(&mut self, code: &str) -> Result { self.run_impl(code) }