From fe62349084418a2d43ee5b9a77e30bb1b90aef99 Mon Sep 17 00:00:00 2001 From: maverick123123 Date: Thu, 29 Oct 2026 00:02:12 -0700 Subject: [PATCH 1/2] fix: defer Ascend dependencies for container preload Resolve CANN, ACL and DCMI APIs on demand so ordinary processes can start before the NPU runtime is available. Defer logger initialization, remove the global sigaction interposer, and keep unconfigured processes passive. Add preload regressions, reject eager SDK dependencies during release verification, and document container preload deployment. Signed-off-by: maverick123123 --- Cargo.lock | 11 ---- README.md | 24 ++++++-- README.zh-CN.md | 24 ++++++-- crates/hook/src/lib.rs | 38 ++++++++---- crates/hook/src/ownership.rs | 6 +- crates/hook/src/signal_compat.rs | 32 ---------- crates/hook/src/spatial.rs | 10 ++-- crates/limiter/Cargo.toml | 1 - crates/limiter/build.rs | 13 +--- crates/limiter/src/config.rs | 17 ++++-- crates/limiter/src/dynamic.rs | 95 ++++++++++++++++++++++++++++++ crates/limiter/src/externed_api.rs | 48 ++++++++++----- crates/limiter/src/lib.rs | 11 +--- crates/limiter/src/manager.rs | 45 ++++++++------ crates/limiter/src/supervisor.rs | 9 ++- dockerfiles/Dockerfile | 2 +- docs/configuration.md | 12 ++-- docs/configuration.zh-CN.md | 12 ++-- docs/testing.md | 39 ++++++------ scripts/ci.sh | 15 +---- scripts/ci/link-stubs.c | 62 ------------------- scripts/ci/verify-release.py | 15 ++--- test/manager_cpu/src/lib.rs | 2 + test/memory_cpu/src/lib.rs | 2 + test/preload/probe.c | 70 ++++++++++++++++++++++ test/preload/run.py | 84 ++++++++++++++++++++++++++ test/preload/runtime.c | 22 +++++++ test/spatial_cpu/run.py | 3 +- test/worker_cpu/src/lib.rs | 2 + 29 files changed, 474 insertions(+), 252 deletions(-) delete mode 100644 crates/hook/src/signal_compat.rs create mode 100644 crates/limiter/src/dynamic.rs delete mode 100644 scripts/ci/link-stubs.c create mode 100644 test/preload/probe.c create mode 100644 test/preload/run.py create mode 100644 test/preload/runtime.c diff --git a/Cargo.lock b/Cargo.lock index 7175d5e..7be8499 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -67,16 +67,6 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" -[[package]] -name = "ctor" -version = "0.2.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a2785755761f3ddc1492979ce1e48d2c00d09311c39e4466429188f3dd6501" -dependencies = [ - "quote", - "syn", -] - [[package]] name = "env_filter" version = "1.0.1" @@ -149,7 +139,6 @@ checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" name = "limiter" version = "0.1.0" dependencies = [ - "ctor", "env_logger", "libc", "log", diff --git a/README.md b/README.md index a5fd8e3..5cd4669 100644 --- a/README.md +++ b/README.md @@ -33,12 +33,12 @@ make build-in-docker cd dist && sha256sum -c SHA256SUMS ``` -This builds the complete release library against real CANN runtime/DCMI libraries -and exports `dist/libvnpu.so`, `dist/build-info.json` and `dist/SHA256SUMS`. +This builds the complete release library, verifies its APIs against real CANN +runtime/DCMI libraries, and exports `dist/libvnpu.so`, `dist/build-info.json` and `dist/SHA256SUMS`. The Dockerfile uses the official `quay.io/ascend/cann:8.5.2-910b-ubuntu22.04-py3.11` image, pinned by digest, and Rust **1.94.0**. It downloads the official **HDK 25.5.1** package, verifies its SHA256 and extracts the real driver user libraries for -linking; it does not install a kernel driver. The build environment is +API verification; it does not install a kernel driver. The build environment is **CANN 8.5.2 on Ubuntu 22.04** and supports **Linux AMD64 and ARM64**. The Make target selects the host architecture by default. Select a target and separate output directories explicitly when building both: @@ -72,7 +72,18 @@ libraries available. Mount a writable host directory at `/hami-shared-region` in each participating container on the node, and mount the built library at `/usr/local/lib/libvnpu.so`. -Inside the workload container: +Mount a container-specific file at `/etc/ld.so.preload` with this content: + +```text +/usr/local/lib/libvnpu.so +``` + +Mount this file only into workload containers. The library resolves CANN/runtime +and DCMI on first NPU use, so ordinary commands such as `bash` and `ls` can start +before the CANN environment is prepared. NPU applications still need compatible +runtime/driver libraries. The supervisor and logger also start on first NPU use. + +Inside the workload container, launch the framework with its normal command: ```sh # Shared file for workloads coordinating on physical card 7; use a different file for another card. @@ -82,9 +93,12 @@ export NPU_LOCAL_SHM_PATH=/tmp/vnpu-local export NPU_MEMORY_LIMIT_MB=10240 export NPU_COMPUTE_LIMIT=30 export NPU_SCHED_POLICY=spatial # Default policy; shown explicitly for clarity. -LD_PRELOAD=/usr/local/lib/libvnpu.so python3 your_model.py +python3 your_model.py ``` +For local diagnostics, `LD_PRELOAD=/usr/local/lib/libvnpu.so python3 your_model.py` +is also supported; it is not required with the container preload file. + The card number identifies the physical coordination domain; the application still selects its container-local logical device, commonly `0`. `NPU_GLOBAL_SHM_PATH` remains required for manager configuration, including diff --git a/README.zh-CN.md b/README.zh-CN.md index 19f5022..12c8294 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -30,11 +30,11 @@ make build-in-docker cd dist && sha256sum -c SHA256SUMS ``` -该流程使用真实的 CANN runtime/DCMI 库完整构建 release 版本，导出 +该流程完整构建 release 版本，并对照真实 CANN runtime/DCMI 库验证 API，导出 `dist/libvnpu.so`、`dist/build-info.json` 和 `dist/SHA256SUMS`。 Dockerfile 使用官方 `quay.io/ascend/cann:8.5.2-910b-ubuntu22.04-py3.11` 镜像并固定摘要，Rust 固定为 **1.94.0**；下载官方 **HDK 25.5.1** 包并验证 SHA256， -只提取真实驱动用户态库用于链接，不安装内核驱动。 +只提取真实驱动用户态库用于 API 校验，不安装内核驱动。 构建环境为 **CANN 8.5.2、Ubuntu 22.04**，支持 **Linux AMD64 和 ARM64**，Make 默认选择当前主机架构。构建两种架构时，显式选择目标并使用不同的产物目录： @@ -50,8 +50,7 @@ HDK 驱动用户态库版本只说明构建环境，宿主机驱动应与业务 编译成功不能直接确定最低兼容 CANN 版本，也不能代替各策略的真机验证。 产物来源、检查内容和验证范围见[测试与 CI](docs/testing.md)。 -如需在已安装真实 CANN/DCMI 开发库的 Linux 主机上直接构建，准备 Rust 1.94.0 -和 C 编译器，按安装包说明初始化 CANN 环境后执行 +也可在 Linux 主机上直接构建，准备 Rust 1.94.0 和 C 编译器后执行 `cargo build --release --workspace --locked`，产物为 `target/release/libvnpu.so`。 非标准 SDK 路径见[配置参考](docs/configuration.zh-CN.md)。 @@ -61,7 +60,17 @@ HDK 驱动用户态库版本只说明构建环境，宿主机驱动应与业务 将同一个可写的宿主机共享目录挂载到参与容器的 `/hami-shared-region`， 将构建的库挂载到 `/usr/local/lib/libvnpu.so`。 -在业务容器内执行： +将仅用于该业务容器的预加载文件挂载到 `/etc/ld.so.preload`，内容为： + +```text +/usr/local/lib/libvnpu.so +``` + +此文件只挂载到业务容器。库在首次 NPU 使用时按需解析 CANN runtime 和 DCMI， +因此 `bash`、`ls` 等普通命令可以在 CANN 环境准备好之前启动；NPU 应用仍需提供 +兼容的 runtime/driver 库。supervisor 和 logger 也在首次 NPU 使用时启动。 + +在业务容器内使用框架原有命令启动： ```sh # 同一物理卡 7 上需要协调的负载共享此文件，其他物理卡使用不同文件。 @@ -71,9 +80,12 @@ export NPU_LOCAL_SHM_PATH=/tmp/vnpu-local export NPU_MEMORY_LIMIT_MB=10240 export NPU_COMPUTE_LIMIT=30 export NPU_SCHED_POLICY=spatial # 默认策略，此处显式写出。 -LD_PRELOAD=/usr/local/lib/libvnpu.so python3 your_model.py +python3 your_model.py ``` +本地排障也可以使用 `LD_PRELOAD=/usr/local/lib/libvnpu.so python3 your_model.py`； +使用容器预加载文件时无需设置该变量。 + 文件名中的卡号表示物理设备协调范围；业务仍选择容器内的逻辑设备，通常为 `0`。 包括默认空分策略在内，manager 配置仍要求设置 `NPU_GLOBAL_SHM_PATH`。 diff --git a/crates/hook/src/lib.rs b/crates/hook/src/lib.rs index a1de5ce..7cebec5 100644 --- a/crates/hook/src/lib.rs +++ b/crates/hook/src/lib.rs @@ -18,6 +18,7 @@ struct Clients { // PID-aware storage avoids inherited locks after fork. Each physical card owns // its own quota, manager election, timing contexts and allocation handles. static CLIENTS: process::ProcessLocal = process::ProcessLocal::new(); +static STUB: process::ProcessLocal = process::ProcessLocal::new(); fn fatal(reason: &str) -> ! { eprintln!("[hami-vnpu-core] {reason}"); @@ -83,6 +84,9 @@ fn ensure_supervisor(entry: &DeviceClient, physical: u32) { } pub fn npu_limiter() -> &'static SchedulerClient { + if !limiter::config::vnpu_configuration_requested() { + return STUB.get_or_init(std::process::id(), SchedulerClient::stub); + } // Apply spatial budgets before resource creation or operator submission. spatial::ensure_current_device(); client_for_physical(current_physical_device()) @@ -91,6 +95,9 @@ pub fn npu_limiter() -> &'static SchedulerClient { /// Physical-memory properties can target a card other than the current card. /// Resolve the target without changing the application's current context. pub fn npu_limiter_for_device(logical: i32) -> &'static SchedulerClient { + if !limiter::config::vnpu_configuration_requested() { + return STUB.get_or_init(std::process::id(), SchedulerClient::stub); + } // Physical memory can target a device before it has a compute context. // Apply core budgets when the application selects that context or launches. if spatial::enabled() { @@ -163,19 +170,29 @@ pub(crate) fn retire_context(context: u64) { } } +fn missing_runtime_symbol(name: &str) -> u64 { + if limiter::config::vnpu_configuration_requested() { + fatal(&format!("cannot find original function: {name}")); + } + log::debug!("optional Ascend runtime symbol `{name}` is unavailable"); + u64::MAX +} + macro_rules! passthrough { ($name:expr, ($($sig:tt)*), $($arg:expr),*) => { { - static REAL: ::std::sync::LazyLock u64> = - ::std::sync::LazyLock::new(|| unsafe { - let ptr = libc::dlsym(libc::RTLD_NEXT, concat!($name, "\0").as_ptr() as *const libc::c_char); - if ptr.is_null() { - panic!("cannot find original function: {}", $name); - } - std::mem::transmute(ptr) - }); - // println!("in func {:?}", $name); - (*REAL)($($arg),*) + type Real = unsafe extern "C" fn($($sig)*) -> u64; + static REAL: ::std::sync::atomic::AtomicPtr = + ::std::sync::atomic::AtomicPtr::new(::std::ptr::null_mut()); + let name = ::std::ffi::CStr::from_bytes_with_nul(concat!($name, "\0").as_bytes()) + .expect("static hook symbol"); + match limiter::dynamic::resolve_sdk_symbol(&REAL, name) { + Some(address) => { + let real: Real = unsafe { ::std::mem::transmute(address) }; + unsafe { real($($arg),*) } + } + None => crate::missing_runtime_symbol($name), + } } }; } @@ -187,5 +204,4 @@ mod memory_extra; mod memory_physical; mod ownership; mod process; -mod signal_compat; pub mod spatial; diff --git a/crates/hook/src/ownership.rs b/crates/hook/src/ownership.rs index e29dc3a..f1b7046 100644 --- a/crates/hook/src/ownership.rs +++ b/crates/hook/src/ownership.rs @@ -88,10 +88,8 @@ fn records() -> MutexGuard<'static, Records> { } unsafe fn symbol(name: &CStr) -> T { - let address = unsafe { libc::dlsym(libc::RTLD_NEXT, name.as_ptr()) }; - if address.is_null() { - crate::spatial::fatal(name.to_str().unwrap_or("missing CANN lifecycle API")); - } + let address = limiter::dynamic::sdk_symbol(name) + .unwrap_or_else(|| crate::spatial::fatal(name.to_str().unwrap_or("missing CANN symbol"))); assert_eq!(std::mem::size_of::(), std::mem::size_of_val(&address)); unsafe { std::mem::transmute_copy(&address) } } diff --git a/crates/hook/src/signal_compat.rs b/crates/hook/src/signal_compat.rs deleted file mode 100644 index 64a3925..0000000 --- a/crates/hook/src/signal_compat.rs +++ /dev/null @@ -1,32 +0,0 @@ -use std::sync::LazyLock; - -static REAL_SIGACTION: LazyLock< - extern "C" fn(libc::c_int, *const libc::sigaction, *mut libc::sigaction) -> libc::c_int, -> = LazyLock::new(|| unsafe { - let ptr = libc::dlsym(libc::RTLD_NEXT, c"sigaction".as_ptr()); - if ptr.is_null() { - panic!("Cannot find original sigaction function!"); - } - std::mem::transmute(ptr) -}); - -// workaround to solve python error -#[unsafe(no_mangle)] -pub extern "C" fn sigaction( - signum: libc::c_int, - act: *const libc::sigaction, - oldact: *mut libc::sigaction, -) -> libc::c_int { - let ret = REAL_SIGACTION(signum, act, oldact); - - if ret == 0 && !oldact.is_null() { - unsafe { - let handler = (*oldact).sa_sigaction; - if handler != libc::SIG_DFL && handler != libc::SIG_IGN { - (*oldact).sa_sigaction = libc::SIG_DFL; - } - } - } - - ret -} diff --git a/crates/hook/src/spatial.rs b/crates/hook/src/spatial.rs index be62031..2d2d0b9 100644 --- a/crates/hook/src/spatial.rs +++ b/crates/hook/src/spatial.rs @@ -47,8 +47,9 @@ fn policy() -> &'static Policy { let pid = std::process::id(); let policy = POLICY.get_or_init(pid, || { // Other policies must not require any of the optional resource APIs. - let spatial = - SchedPolicy::from_env().unwrap_or_else(|error| fatal(&error)) == SchedPolicy::Spatial; + let spatial = limiter::config::vnpu_configuration_requested() + && SchedPolicy::from_env().unwrap_or_else(|error| fatal(&error)) + == SchedPolicy::Spatial; let percent = if spatial { let cfg = ManagerConfig::from_env().unwrap_or_else(|error| fatal(&error)); if cfg.sched_policy != SchedPolicy::Spatial { @@ -127,10 +128,7 @@ unsafe fn symbol(name: &CStr) -> T { } unsafe fn optional_symbol(name: &CStr) -> Option { - let address = unsafe { libc::dlsym(libc::RTLD_NEXT, name.as_ptr()) }; - if address.is_null() { - return None; - } + let address = limiter::dynamic::sdk_symbol(name)?; // Each call site supplies the exact function-pointer type from acl_rt.h. assert_eq!(std::mem::size_of::(), std::mem::size_of_val(&address)); Some(unsafe { std::mem::transmute_copy(&address) }) diff --git a/crates/limiter/Cargo.toml b/crates/limiter/Cargo.toml index 9939dca..7a75c9f 100644 --- a/crates/limiter/Cargo.toml +++ b/crates/limiter/Cargo.toml @@ -7,4 +7,3 @@ edition.workspace = true libc = { workspace = true } log = "0.4" env_logger = "0.11" -ctor = "0.2" \ No newline at end of file diff --git a/crates/limiter/build.rs b/crates/limiter/build.rs index b579a6d..a4dfabb 100644 --- a/crates/limiter/build.rs +++ b/crates/limiter/build.rs @@ -1,13 +1,4 @@ -// Link the Ascend DCMI (+ CANN runtime) libs. Paths default to the standard Ascend -// install and can be overridden via env (CI, a relocated SDK, etc.) rather than hardcoded. fn main() { - let toolkit = std::env::var("ASCEND_TOOLKIT_HOME") - .unwrap_or_else(|_| "/usr/local/Ascend/ascend-toolkit/latest".into()); - let driver = - std::env::var("ASCEND_DRIVER_HOME").unwrap_or_else(|_| "/usr/local/Ascend/driver".into()); - println!("cargo:rustc-link-search=native={toolkit}/lib64"); - println!("cargo:rustc-link-search=native={driver}/lib64/driver"); - println!("cargo:rustc-link-lib=dcmi"); - println!("cargo:rerun-if-env-changed=ASCEND_TOOLKIT_HOME"); - println!("cargo:rerun-if-env-changed=ASCEND_DRIVER_HOME"); + // A system-preloaded libvnpu must also load in ordinary shells without CANN. + // Resolve runtime/DCMI entry points lazily instead of adding DT_NEEDED entries. } diff --git a/crates/limiter/src/config.rs b/crates/limiter/src/config.rs index db73afd..d5cee4f 100644 --- a/crates/limiter/src/config.rs +++ b/crates/limiter/src/config.rs @@ -213,9 +213,12 @@ impl ManagerConfig { } } -/// Validate explicit resource settings before a worker can enter the stub path. -pub fn validate_requested_limits() -> Result<(), String> { - let requested = [ +/// System preload alone does not opt an unrelated process into NPU control. +/// Explicit settings, including invalid or legacy values, must still fail closed. +pub fn vnpu_configuration_requested() -> bool { + [ + ENV_GLOBAL_SHM_PATH, + ENV_LOCAL_SHM_PATH, ENV_MEMORY_LIMIT_MB, ENV_COMPUTE_LIMIT, ENV_SCHED_POLICY, @@ -225,8 +228,12 @@ pub fn validate_requested_limits() -> Result<(), String> { "NPU_OVERDRAFT_ENABLED", ] .iter() - .any(|name| std::env::var_os(name).is_some()); - if requested { + .any(|name| std::env::var_os(name).is_some()) +} + +/// Validate explicit resource settings before a worker can enter the stub path. +pub fn validate_requested_limits() -> Result<(), String> { + if vnpu_configuration_requested() { ManagerConfig::from_env().map(|_| ()) } else { Ok(()) diff --git a/crates/limiter/src/dynamic.rs b/crates/limiter/src/dynamic.rs new file mode 100644 index 0000000..55406a6 --- /dev/null +++ b/crates/limiter/src/dynamic.rs @@ -0,0 +1,95 @@ +//! Ascend libraries are optional until an intercepted NPU call actually needs them. + +use std::ffi::{CStr, c_void}; +use std::ptr; +use std::sync::atomic::{AtomicPtr, Ordering}; + +struct Library { + name: &'static CStr, + handle: AtomicPtr, +} + +impl Library { + const fn new(name: &'static CStr) -> Self { + Self { + name, + handle: AtomicPtr::new(ptr::null_mut()), + } + } + + fn symbol(&self, name: &CStr) -> Option<*mut c_void> { + let mut handle = self.handle.load(Ordering::Acquire); + if handle.is_null() { + let opened = + unsafe { libc::dlopen(self.name.as_ptr(), libc::RTLD_NOW | libc::RTLD_LOCAL) }; + if opened.is_null() { + return None; + } + handle = match self.handle.compare_exchange( + ptr::null_mut(), + opened, + Ordering::AcqRel, + Ordering::Acquire, + ) { + Ok(_) => opened, + Err(existing) => { + unsafe { libc::dlclose(opened) }; + existing + } + }; + } + // Retain the handle: the driver/runtime may keep callbacks and threads alive. + let address = unsafe { libc::dlsym(handle, name.as_ptr()) }; + (!address.is_null()).then_some(address) + } +} + +static RUNTIME: Library = Library::new(c"libruntime.so"); +static ACL: Library = Library::new(c"libascendcl.so"); +static DCMI: Library = Library::new(c"libdcmi.so"); + +/// Prefer the application's backend, including fake backends used by CPU tests. +/// An explicit local handle also supports SDKs loaded with `RTLD_LOCAL`. +pub fn sdk_symbol(name: &CStr) -> Option<*mut c_void> { + let address = unsafe { libc::dlsym(libc::RTLD_NEXT, name.as_ptr()) }; + if !address.is_null() { + return Some(address); + } + if name.to_bytes().starts_with(b"acl") { + ACL.symbol(name) + } else { + RUNTIME.symbol(name) + } +} + +fn cached_symbol( + cache: &AtomicPtr, + resolve: impl FnOnce() -> Option<*mut c_void>, +) -> Option<*mut c_void> { + let address = cache.load(Ordering::Acquire); + if !address.is_null() { + return Some(address); + } + // Do not cache a miss: a later dlopen can make an optional symbol available. + let address = resolve()?; + Some( + match cache.compare_exchange( + ptr::null_mut(), + address, + Ordering::AcqRel, + Ordering::Acquire, + ) { + Ok(_) => address, + Err(existing) => existing, + }, + ) +} + +/// Cache successful SDK lookups only; retry a missing backend on the next call. +pub fn resolve_sdk_symbol(cache: &AtomicPtr, name: &CStr) -> Option<*mut c_void> { + cached_symbol(cache, || sdk_symbol(name)) +} + +pub(crate) fn resolve_dcmi_symbol(cache: &AtomicPtr, name: &CStr) -> Option<*mut c_void> { + cached_symbol(cache, || DCMI.symbol(name)) +} diff --git a/crates/limiter/src/externed_api.rs b/crates/limiter/src/externed_api.rs index 14964cc..01bd900 100644 --- a/crates/limiter/src/externed_api.rs +++ b/crates/limiter/src/externed_api.rs @@ -1,17 +1,37 @@ -#[link(name = "runtime")] -unsafe extern "C" { - pub fn rtStreamCreate(stream: *mut u64, priority: i32) -> i32; - pub fn rtEventCreate(evt: *mut u64) -> usize; - pub fn rtEventRecord(evt: u64, stm: u64) -> usize; - pub fn rtStreamWaitEvent(stream: u64, event: u64) -> usize; - pub fn rtStreamSynchronize(stream: u64) -> i32; - pub fn rtEventElapsedTime(time_interval: *mut f32, start_event: u64, end_event: u64) -> usize; - pub fn rtSetDevice(device: i32) -> i32; - pub fn rtGetDevice(device: *mut i32) -> i32; - pub fn rtGetDevicePhyIdByIndex(device: u32, physical: *mut u32) -> i32; - pub fn rtCtxGetCurrent(context: *mut u64) -> i32; - pub fn rtDeviceSynchronize() -> i32; - pub fn rtStreamGetCaptureInfo(stream: u64, status: *mut u32, model: *mut u64) -> i32; +use std::ffi::CStr; +use std::sync::atomic::AtomicPtr; + +macro_rules! runtime_apis { + ($($name:ident($($arg:ident: $ty:ty),*) -> $ret:ty;)+) => {$( + /// # Safety + /// Arguments and handles must satisfy the corresponding CANN runtime API. + #[allow(non_snake_case)] + pub unsafe fn $name($($arg: $ty),*) -> $ret { + type Real = unsafe extern "C" fn($($ty),*) -> $ret; + static REAL: AtomicPtr = AtomicPtr::new(std::ptr::null_mut()); + let name = CStr::from_bytes_with_nul(concat!(stringify!($name), "\0").as_bytes()) + .expect("static runtime symbol"); + let address = crate::dynamic::resolve_sdk_symbol(&REAL, name) + .unwrap_or_else(|| panic!("cannot find Ascend runtime symbol: {}", name.to_string_lossy())); + let real: Real = unsafe { std::mem::transmute(address) }; + unsafe { real($($arg),*) } + } + )+}; +} + +runtime_apis! { + rtStreamCreate(stream: *mut u64, priority: i32) -> i32; + rtEventCreate(evt: *mut u64) -> usize; + rtEventRecord(evt: u64, stm: u64) -> usize; + rtStreamWaitEvent(stream: u64, event: u64) -> usize; + rtStreamSynchronize(stream: u64) -> i32; + rtEventElapsedTime(time_interval: *mut f32, start_event: u64, end_event: u64) -> usize; + rtSetDevice(device: i32) -> i32; + rtGetDevice(device: *mut i32) -> i32; + rtGetDevicePhyIdByIndex(device: u32, physical: *mut u32) -> i32; + rtCtxGetCurrent(context: *mut u64) -> i32; + rtDeviceSynchronize() -> i32; + rtStreamGetCaptureInfo(stream: u64, status: *mut u32, model: *mut u64) -> i32; } // RT ERROR CODE diff --git a/crates/limiter/src/lib.rs b/crates/limiter/src/lib.rs index 0e340ab..6215e85 100644 --- a/crates/limiter/src/lib.rs +++ b/crates/limiter/src/lib.rs @@ -1,4 +1,5 @@ pub mod config; +pub mod dynamic; pub mod externed_api; pub mod manager; pub mod overdraft; @@ -6,16 +7,6 @@ pub mod shmem; pub mod supervisor; pub mod worker; -use ctor::ctor; - -#[ctor] -fn init_logger() { - let _ = env_logger::builder() - .filter_level(log::LevelFilter::Info) - .parse_default_env() - .try_init(); -} - #[macro_export] macro_rules! check_rts { ($call:expr) => {{ diff --git a/crates/limiter/src/manager.rs b/crates/limiter/src/manager.rs index a386280..c6af49c 100644 --- a/crates/limiter/src/manager.rs +++ b/crates/limiter/src/manager.rs @@ -1,5 +1,5 @@ use std::collections::HashMap; -use std::sync::atomic::Ordering; +use std::sync::atomic::{AtomicPtr, Ordering}; use std::thread; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; @@ -18,20 +18,27 @@ struct DcmiProcMemInfo { proc_mem_usage: u64, } -unsafe extern "C" { - fn dcmi_init() -> i32; - fn dcmi_get_card_list(card_num: *mut i32, card_list: *mut i32, list_len: i32) -> i32; - fn dcmi_get_card_id_device_id_from_phyid( - card_id: *mut i32, - device_id: *mut i32, - physical_device_id: u32, - ) -> i32; - fn dcmi_get_device_resource_info( - card_id: i32, - device_id: i32, - proc_info: *mut DcmiProcMemInfo, - proc_num: *mut i32, - ) -> i32; +macro_rules! dcmi_apis { + ($($name:ident($($arg:ident: $ty:ty),*);)+) => {$( + unsafe fn $name($($arg: $ty),*) -> i32 { + type Real = unsafe extern "C" fn($($ty),*) -> i32; + static REAL: AtomicPtr = AtomicPtr::new(std::ptr::null_mut()); + let name = std::ffi::CStr::from_bytes_with_nul(concat!(stringify!($name), "\0").as_bytes()) + .expect("static DCMI symbol"); + let Some(address) = crate::dynamic::resolve_dcmi_symbol(&REAL, name) else { + return -1; + }; + let real: Real = unsafe { std::mem::transmute(address) }; + unsafe { real($($arg),*) } + } + )+}; +} + +dcmi_apis! { + dcmi_init(); + dcmi_get_card_list(card_num: *mut i32, card_list: *mut i32, list_len: i32); + dcmi_get_card_id_device_id_from_phyid(card_id: *mut i32, device_id: *mut i32, physical_device_id: u32); + dcmi_get_device_resource_info(card_id: i32, device_id: i32, proc_info: *mut DcmiProcMemInfo, proc_num: *mut i32); } fn physical_device_location(physical_device_id: u32) -> Option<(i32, i32)> { @@ -95,13 +102,13 @@ fn observe_physical_device( /// Discover (card_id, device_id) pairs for all present NPU logic devices. fn discover_npu_devices() -> Vec<(i32, i32)> { - unsafe { - dcmi_init(); + if unsafe { dcmi_init() } != 0 { + return Vec::new(); } let mut card_num: i32 = 0; let mut card_list: [i32; 8] = [0; 8]; - unsafe { - dcmi_get_card_list(&mut card_num, card_list.as_mut_ptr(), 8); + if unsafe { dcmi_get_card_list(&mut card_num, card_list.as_mut_ptr(), 8) } != 0 { + return Vec::new(); } if card_num <= 0 || card_num > 8 { return Vec::new(); diff --git a/crates/limiter/src/supervisor.rs b/crates/limiter/src/supervisor.rs index d7e9989..d06b69e 100644 --- a/crates/limiter/src/supervisor.rs +++ b/crates/limiter/src/supervisor.rs @@ -1,8 +1,7 @@ //! In-process manager supervisor. //! -//! The manager daemon no longer runs as a separate binary. Instead every process -//! that loads `libvnpu.so` spawns a lightweight supervisor thread (see the ctor in -//! the `hook` crate). Supervisors in the same physical-card domain +//! A process starts a lightweight supervisor on its first intercepted NPU use, +//! after leaving the dynamic loader. Supervisors in the same physical-card domain //! race to become the *single* manager via a CAS on `LocalContainerShmem::manager_pid`. //! The winner runs `ContainerManager::run()`; the losers stand by and watch, ready to //! take over if the current manager dies. This guarantees one manager per Pod/card @@ -47,6 +46,10 @@ pub fn run_for_device(physical_device_id: u32) { } fn run_configured(physical_device_id: Option) { + let _ = env_logger::builder() + .filter_level(log::LevelFilter::Info) + .parse_default_env() + .try_init(); block_all_signals(); let my_pid = std::process::id() as i32; diff --git a/dockerfiles/Dockerfile b/dockerfiles/Dockerfile index 3de291e..151bfcb 100644 --- a/dockerfiles/Dockerfile +++ b/dockerfiles/Dockerfile @@ -20,7 +20,7 @@ RUN case "${TARGETARCH}/$(uname -m)" in \ && apt-get install -y --no-install-recommends ca-certificates curl gcc libc6-dev python3 binutils \ && rm -rf /var/lib/apt/lists/* -# Extract only real HDK user libraries for linking. No kernel driver is installed. +# Extract only real HDK user libraries for API verification. No kernel driver is installed. # Official download reference: Ascend/docs, _static/package_info.js. RUN case "${TARGETARCH}" in \ amd64) driver_arch=x86-64; driver_sha256=f64012342cdf72d44f527efdef1304af41aedc73b202bf0d91243d21765555f7 ;; \ diff --git a/docs/configuration.md b/docs/configuration.md index 5a6f861..1bd0c76 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -71,12 +71,14 @@ The virtual HBM free-memory report reserves 128 MiB of overhead. This internal c | Variable | Scope | Meaning | | --- | --- | --- | -| `LD_PRELOAD` | Application's Linux dynamic loader | Absolute path to `libvnpu.so`, for example `/opt/hami/libvnpu.so`. Set it when launching every participating application process. | +| `LD_PRELOAD` | Application's Linux dynamic loader | Optional alternative to a container-specific `/etc/ld.so.preload` file. Use the absolute path to `libvnpu.so`, for example `/opt/hami/libvnpu.so`. | | `LD_LIBRARY_PATH` | Application/build loader | Normal Linux dependency search path. Make the matching CANN runtime and Ascend driver libraries available; this is a loader setting rather than a quota control. | -| `ASCEND_TOOLKIT_HOME` | Build only | Toolkit root, default `/usr/local/Ascend/ascend-toolkit/latest`; the build adds `/lib64` as a native library search path. | -| `ASCEND_DRIVER_HOME` | Build only | Driver root, default `/usr/local/Ascend/driver`; the build adds `/lib64/driver` and links `dcmi`. | +| `ASCEND_TOOLKIT_HOME` | Release verification | Toolkit root, default `/usr/local/Ascend/ascend-toolkit/latest`; verification inspects `/lib64`. | +| `ASCEND_DRIVER_HOME` | Release verification | Driver root, default `/usr/local/Ascend/driver`; verification inspects `/lib64/driver`. | -The library also links the CANN `runtime` library. Build against the intended Linux architecture and install compatible runtime/driver dependencies in the deployment. Setting the two build roots does not install those dependencies or configure the application's runtime library lookup. +`libvnpu.so` has no link-time CANN/runtime/DCMI dependency. It looks up SDK symbols only when used, including SDKs loaded with `RTLD_LOCAL`; missing optional symbols are retried after a later `dlopen`. Without resource configuration, preloading stays passive and runtime hooks pass through. Explicit resource settings still require valid manager configuration. + +Mount a container-specific `/etc/ld.so.preload` containing the absolute library path to inject it into child processes without changing framework commands. Keep this mount within workload containers. Prepare compatible CANN/driver libraries and their search paths before launching NPU applications. Setting the two verification roots does not install dependencies or configure runtime lookup. ## Migration from older variables @@ -98,4 +100,4 @@ Remove the old variables completely, even if set to an empty string or alongside file path to select the local accounting group; setting only a name does not override `/tmp/vnpu_local_shmem`. -Sources: [resource parser](../crates/limiter/src/config.rs), [manager tuning](../crates/limiter/src/manager.rs), [supervisor](../crates/limiter/src/supervisor.rs), [worker and HBM accounting](../crates/limiter/src/worker.rs), [logger](../crates/limiter/src/lib.rs), [build paths](../crates/limiter/build.rs). +Sources: [resource parser](../crates/limiter/src/config.rs), [manager tuning](../crates/limiter/src/manager.rs), [supervisor](../crates/limiter/src/supervisor.rs), [worker and HBM accounting](../crates/limiter/src/worker.rs), [logger](../crates/limiter/src/supervisor.rs), [lazy loading](../crates/limiter/src/dynamic.rs). diff --git a/docs/configuration.zh-CN.md b/docs/configuration.zh-CN.md index b604145..8fcc701 100644 --- a/docs/configuration.zh-CN.md +++ b/docs/configuration.zh-CN.md @@ -72,12 +72,14 @@ SHM 字段偏移保持不变，但文件名现在表示物理卡域。监控程 | 环境变量 | 生效范围 | 含义 | | --- | --- | --- | -| `LD_PRELOAD` | 应用的 Linux 动态加载器 | `libvnpu.so` 的绝对路径，例如 `/opt/hami/libvnpu.so`。启动每个参与控制的应用进程时设置。 | +| `LD_PRELOAD` | 应用的 Linux 动态加载器 | 容器专用 `/etc/ld.so.preload` 文件的可选替代方式；指定 `libvnpu.so` 的绝对路径，例如 `/opt/hami/libvnpu.so`。 | | `LD_LIBRARY_PATH` | 应用/构建加载器 | Linux 通用依赖搜索路径。需能找到匹配的 CANN runtime 和 Ascend driver 库；它是加载器配置，不是配额控制。 | -| `ASCEND_TOOLKIT_HOME` | 仅构建时 | Toolkit 根目录，默认 `/usr/local/Ascend/ascend-toolkit/latest`；构建添加 `/lib64` 原生库搜索路径。 | -| `ASCEND_DRIVER_HOME` | 仅构建时 | Driver 根目录，默认 `/usr/local/Ascend/driver`；构建添加 `/lib64/driver` 并链接 `dcmi`。 | +| `ASCEND_TOOLKIT_HOME` | 发布产物校验 | Toolkit 根目录，默认 `/usr/local/Ascend/ascend-toolkit/latest`；校验读取 `/lib64`。 | +| `ASCEND_DRIVER_HOME` | 发布产物校验 | Driver 根目录，默认 `/usr/local/Ascend/driver`；校验读取 `/lib64/driver`。 | -库还会链接 CANN `runtime`。请面向目标 Linux 架构构建，并在部署中提供兼容的 runtime/driver 依赖。设置两个构建目录变量不会自动安装这些依赖，也不会设置应用的运行时库搜索路径。 +库不在链接时依赖 CANN runtime/DCMI，只在实际使用时解析 SDK 符号，也支持通过 `RTLD_LOCAL` 加载的 SDK；可选符号缺失后，后续 `dlopen` 成功可再次查找。未配置资源时，预加载保持被动、runtime hook 透传；显式资源配置仍要求有效的 manager 配置。 + +在业务容器内挂载专用的 `/etc/ld.so.preload` 文件，内容为库的绝对路径，即可让子进程继承注入，无需修改框架启动命令。挂载范围仅限业务容器。启动 NPU 应用前仍需准备兼容的 CANN/driver 库及其搜索路径。上述两个校验目录变量不会安装依赖或设置应用运行时搜索路径。 ## 旧变量迁移 @@ -97,4 +99,4 @@ SHM 字段偏移保持不变，但文件名现在表示物理卡域。监控程 库不会读取 `NPU_LOCAL_SHM_NAME`。应通过 `NPU_LOCAL_SHM_PATH` 指定文件路径， 选择本地记账组；只设置名称不会覆盖默认的 `/tmp/vnpu_local_shmem`。 -实现来源：[资源解析](../crates/limiter/src/config.rs)、[manager 参数](../crates/limiter/src/manager.rs)、[supervisor](../crates/limiter/src/supervisor.rs)、[worker 与 HBM 记账](../crates/limiter/src/worker.rs)、[日志](../crates/limiter/src/lib.rs)、[构建路径](../crates/limiter/build.rs)。 +实现来源：[资源解析](../crates/limiter/src/config.rs)、[manager 参数](../crates/limiter/src/manager.rs)、[supervisor](../crates/limiter/src/supervisor.rs)、[worker 与 HBM 记账](../crates/limiter/src/worker.rs)、[日志](../crates/limiter/src/supervisor.rs)、[懒加载](../crates/limiter/src/dynamic.rs)。 diff --git a/docs/testing.md b/docs/testing.md index c030952..97e9cf6 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -5,7 +5,7 @@ ### Build the release library The [release jobs](../.github/workflows/ci.yml) build the complete Rust workspace -with real CANN runtime and DCMI libraries. They use the same +and verify SDK exports against real CANN runtime and DCMI libraries. They use the same [Dockerfile](../dockerfiles/Dockerfile) and [Make target](../Makefile) as local builds, following the container-build approach used by [HAMi-core](https://github.com/Project-HAMi/HAMi-core/blob/main/.github/workflows/build-src.yml). @@ -57,8 +57,8 @@ It does not install a kernel driver. HDK 25.5.x is listed in the The build uses real CANN libraries under `/usr/local/Ascend/cann-8.5.2/lib64` and DCMI under `/usr/local/Ascend/driver/lib64/driver`, with driver dependencies -under `lib64/driver` and `lib64/common`. Release verification checks that the -selected real libraries are resolved. The driver version above describes the +under `lib64/driver` and `lib64/common`. Release verification checks the selected real SDK exports and rejects +eager CANN/runtime/DCMI dependencies in `libvnpu.so`. The driver version above describes the user libraries present at build time; choose the deployed host driver according to the workload's CANN and hardware compatibility requirements. Set `OUT_DIR` to change the export location, for example `make build-in-docker OUT_DIR=out`. @@ -73,7 +73,8 @@ The exported directory contains: Before export, [release verification](../scripts/ci/verify-release.py) checks the ELF architecture and shared-object type, required hook and SDK exports, and -`ldd -r` resolution against the selected real runtime/DCMI libraries. It requires +`ldd -r` resolution of system dependencies. CANN/runtime/DCMI symbols are +resolved lazily and must not appear in `DT_NEEDED`. It requires and records the spatial-policy APIs, because spatial is the default policy. These checks do not initialize CANN or DCMI and do not execute NPU kernels. @@ -103,17 +104,16 @@ python3 scripts/ci/check-doc-links.py ``` Formatting checks the root workspace's two production crates and does not need -Linux or the SDK. The CPU script prepares test-only link dependencies, then runs +Linux or the SDK. The CPU script needs no SDK/driver link stubs and runs `cargo clippy --workspace --all-targets --locked -- -D warnings` for the root workspace, including its test targets. The standalone CPU test workspaces are run separately below; they are not included in `--workspace`. The CPU script builds in a temporary directory and removes it on exit. It never -replaces an installed library. Its `libvnpu.so` is linked against test-only -runtime/DCMI dependencies and must not be deployed. Any unexpected call into -those link dependencies aborts; the spatial runner supplies its own fake ACL -backend for the calls under test. Clippy uses this test link environment; it does -not build or validate the deployable release artifact. +replaces an installed library. Preload tests start ordinary commands without +CANN, verify signal handlers and thread counts, and exercise missing-symbol +retry with global/local backends. Other integration runners supply their own +fake SDKs. These CPU artifacts have not passed real-SDK release verification. | Check | Coverage | | --- | --- | @@ -122,6 +122,7 @@ not build or validate the deployable release artifact. | `test/manager_cpu` | Production manager arithmetic, fixed rest calculation, report aggregation and spatial exclusion from time rotation. The unused process scan is a panic-on-call test double. | | `test/memory_cpu` | Production memory scan/quota paths, rollback, shared-memory ABI and bounded fork cleanup. Procfs reads and CANN calls use explicit test doubles; Linux futexes are real. | | `test/worker_cpu/run.py` | Source-linked worker and hook admission cases with observed lock/futex boundaries; Cached/configured/DVPP/physical allocation admission, failure rollback, allocation/free alias deduplication, physical retain lifetimes, concurrent free/address reuse, per-card ownership, threads and fork. Each case runs in a separate process with a timeout. | +| `test/preload/run.py` | No eager SDK dependencies; ordinary shell/tool startup without CANN, preserved signal handlers, no preload-time supervisor, passive hooks and successful retry after global/local `dlopen`. | | `test/process_local.sh` | Actual per-process cache, concurrent initialization and real fork cases. | | `test/spatial_cpu/run.py` | Actual built Rust hook library preloaded into a C client with fake ACL: ceil budgets, resets, API failures, device/context switching and fork handling. | | Documentation links | Local Markdown link destinations in the root READMEs and `docs/`; no external requests or rendered-anchor checks. | @@ -188,7 +189,7 @@ Dockerfile 下载匹配目标架构的官方 HDK 包，并核对其中固定的 不安装内核驱动；官方配套表包含 CANN 8.5.2 与 HDK 25.5.x，镜像源码、下载地址及 配套表链接见英文部分。构建使用 `/usr/local/Ascend/cann-8.5.2/lib64` 下的真实 runtime 库，以及 `/usr/local/Ascend/driver/lib64/driver` 下的 DCMI 库和依赖， -产物检查会核对实际解析路径。这里的驱动版本只说明构建时的用户态库；宿主机驱动 +产物检查会核对真实 SDK 导出，并拒绝 `libvnpu.so` 的 CANN/runtime/DCMI 硬依赖。这里的驱动版本只说明构建时的用户态库；宿主机驱动 应与业务 CANN 和硬件配套。通过 `make build-in-docker OUT_DIR=out` 可修改导出目录。 导出产物包括： @@ -200,7 +201,8 @@ runtime 库，以及 `/usr/local/Ascend/driver/lib64/driver` 下的 DCMI 库和 | `dist/SHA256SUMS` | 库和构建信息的校验和。 | 导出前，[产物检查脚本](../scripts/ci/verify-release.py) 核对 ELF 架构与共享库类型、 -hook 与 SDK 导出符号，以及 `ldd -r` 是否正确解析到指定的真实 runtime/DCMI 库， +hook 与 SDK 导出符号，以及 `ldd -r` 是否能解析系统依赖；CANN/runtime/DCMI +按需解析，不能出现在 `DT_NEEDED` 中， 要求并记录 CANN 提供空分所需 API，因为空分已是默认策略。 这些检查不会初始化 CANN/DCMI 或执行 NPU 算子。 @@ -227,15 +229,14 @@ python3 scripts/ci/check-doc-links.py ``` 这些检查无需 CANN SDK、Ascend 驱动、NPU、Docker 服务或集群凭据。 -格式检查覆盖根工作区的两个生产 crate，不依赖 Linux 或 SDK。CPU 脚本先准备测试 -链接库，再执行 `cargo clippy --workspace --all-targets --locked -- -D warnings`， +格式检查覆盖根工作区的两个生产 crate，不依赖 Linux 或 SDK。CPU 脚本无需 SDK/driver +链接桩，直接执行 `cargo clippy --workspace --all-targets --locked -- -D warnings`， 覆盖根工作区及其测试目标，警告作为错误。独立 CPU 测试工作区在后续步骤运行， 不属于这里的 `--workspace` 范围。 -CPU 脚本在临时目录构建，退出时清理，不替换已安装的库。脚本生成的 `libvnpu.so` -依赖测试用 runtime/DCMI 链接库，不能部署到业务环境；意外进入这些链接桩函数会 -立即中止测试。空分测试使用独立的 fake ACL 后端。 -Clippy 使用这套测试链接环境，不构建或验证可部署的发布产物。 +CPU 脚本在临时目录构建，退出时清理，不替换已安装的库。预加载测试在没有 CANN +时启动普通命令，核对信号处理器和线程数，并用 global/local 后端验证缺失符号的 +重试。其他集成测试提供独立的 fake SDK。这些 CPU 构建产物未通过真实 SDK 发布校验。 - 格式检查：`cargo fmt --all -- --check` 检查根工作区的两个生产 crate。 - 工作区 Clippy 和单元测试：对两个 crate 的所有目标运行 Clippy，将警告视为错误； @@ -244,6 +245,8 @@ Clippy 使用这套测试链接环境，不构建或验证可部署的发布产 - Memory：内存扫描、配额、回滚、共享内存 ABI、带超时的 fork 清理；procfs 和 CANN 使用 明确的测试替身，Linux futex 使用真实实现。 - Worker：源码链接的准入与 hook 路径，观测锁和 futex 边界；覆盖 Cached/配置/DVPP/物理分配的准入、失败回滚、分配及释放转调去重、物理句柄 retain 生命周期、并发释放时地址复用、每卡归属、线程及 fork。每个用例独立进程并限制超时。 +- Preload：无 SDK 硬依赖、无 CANN 的普通命令启动、保留信号处理器、加载期无 supervisor、 + 未配置时透传，以及 global/local `dlopen` 后的符号重试。 - Process-local：真实缓存实现的并发初始化和 fork。 - Spatial：预加载实际构建的 Rust 库，通过 fake ACL 验证向上取整、重置、API 错误、 设备/上下文切换及 fork 行为。 diff --git a/scripts/ci.sh b/scripts/ci.sh index d9454dd..b5abc13 100644 --- a/scripts/ci.sh +++ b/scripts/ci.sh @@ -11,20 +11,10 @@ work_dir="$(mktemp -d "${TMPDIR:-/tmp}/hami-core-ci.XXXXXX")" trap 'rm -rf "$work_dir"' EXIT cd "$repo_dir" -# Keep test-linked artifacts separate from deployable builds. No CANN SDK or -# driver is used: every unexpected call into these link stubs aborts the test. +# No SDK/driver link stubs are needed: libvnpu resolves backends on first use. +# Keep CPU-test artifacts separate from verified release builds. export CARGO_TARGET_DIR="$work_dir/target" -export ASCEND_TOOLKIT_HOME="$work_dir/toolkit" -export ASCEND_DRIVER_HOME="$work_dir/driver" unset HAMI_SOURCE_ROOT -runtime_dir="$ASCEND_TOOLKIT_HOME/lib64" -driver_dir="$ASCEND_DRIVER_HOME/lib64/driver" -mkdir -p "$runtime_dir" "$driver_dir" -cc -std=c11 -Wall -Wextra -Werror -shared -fPIC \ - scripts/ci/link-stubs.c -DSTUB_RUNTIME -o "$runtime_dir/libruntime.so" -cc -std=c11 -Wall -Wextra -Werror -shared -fPIC \ - scripts/ci/link-stubs.c -DSTUB_DCMI -o "$driver_dir/libdcmi.so" -export LD_LIBRARY_PATH="$runtime_dir:$driver_dir${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" rustc --version cargo --version @@ -35,5 +25,6 @@ cargo test --manifest-path test/memory_cpu/Cargo.toml --locked -- --test-threads python3 test/worker_cpu/run.py bash test/process_local.sh cargo build --release --locked -p hook +python3 test/preload/run.py "$CARGO_TARGET_DIR/release/libvnpu.so" python3 test/spatial_cpu/run.py "$CARGO_TARGET_DIR/release/libvnpu.so" HAMI_ARTIFACT_DIR="$CARGO_TARGET_DIR/release" bash test/host_args.sh diff --git a/scripts/ci/link-stubs.c b/scripts/ci/link-stubs.c deleted file mode 100644 index 4fb74d2..0000000 --- a/scripts/ci/link-stubs.c +++ /dev/null @@ -1,62 +0,0 @@ -/* Link-only test dependencies. These are never a CANN implementation. - * Spatial tests resolve their exercised ACL/RT calls from fake_acl.c; any call - * reaching one of these fallback symbols must fail rather than fake success. - */ -#include -#include -#include - -static _Noreturn void unexpected(const char *name) { - fprintf(stderr, "CI link stub called unexpectedly: %s\n", name); - abort(); -} - -#if defined(STUB_RUNTIME) -int rtStreamCreate(uint64_t *stream, int priority) { - (void)stream; (void)priority; unexpected(__func__); -} -uintptr_t rtEventCreate(uint64_t *event) { - (void)event; unexpected(__func__); -} -uintptr_t rtEventRecord(uint64_t event, uint64_t stream) { - (void)event; (void)stream; unexpected(__func__); -} -uintptr_t rtStreamWaitEvent(uint64_t stream, uint64_t event) { - (void)stream; (void)event; unexpected(__func__); -} -int rtStreamSynchronize(uint64_t stream) { - (void)stream; unexpected(__func__); -} -uintptr_t rtEventElapsedTime(float *elapsed, uint64_t start, uint64_t end) { - (void)elapsed; (void)start; (void)end; unexpected(__func__); -} -int rtSetDevice(int device) { - (void)device; unexpected(__func__); -} -int rtGetDevice(int *device) { - (void)device; unexpected(__func__); -} -int rtGetDevicePhyIdByIndex(uint32_t device, uint32_t *physical) { - (void)device; (void)physical; unexpected(__func__); -} -int rtCtxGetCurrent(uint64_t *context) { - (void)context; unexpected(__func__); -} -int rtDeviceSynchronize(void) { unexpected(__func__); } -int rtStreamGetCaptureInfo(uint64_t stream, uint32_t *status, uint64_t *model) { - (void)stream; (void)status; (void)model; unexpected(__func__); -} -#elif defined(STUB_DCMI) -int dcmi_init(void) { unexpected(__func__); } -int dcmi_get_card_list(int *count, int *cards, int capacity) { - (void)count; (void)cards; (void)capacity; unexpected(__func__); -} -int dcmi_get_card_id_device_id_from_phyid(int *card, int *device, unsigned physical) { - (void)card; (void)device; (void)physical; unexpected(__func__); -} -int dcmi_get_device_resource_info(int card, int device, void *processes, int *count) { - (void)card; (void)device; (void)processes; (void)count; unexpected(__func__); -} -#else -#error "Select STUB_RUNTIME or STUB_DCMI for this CPU-only link dependency" -#endif diff --git a/scripts/ci/verify-release.py b/scripts/ci/verify-release.py index 8fe1837..e60e968 100644 --- a/scripts/ci/verify-release.py +++ b/scripts/ci/verify-release.py @@ -140,12 +140,16 @@ def validate(args): dynamic = run("readelf", "--dynamic", "--wide", str(library), env=env) needed = re.findall(r"\(NEEDED\).*?\[(.*?)\]", dynamic) - missing = {"libruntime.so", "libdcmi.so", "libc.so.6"} - set(needed) + missing = {"libc.so.6"} - set(needed) if missing: raise ValueError("missing real SDK/system dependencies: " + ", ".join(sorted(missing))) sonames = re.findall(r"\(SONAME\).*?\[(.*?)\]", dynamic) if sonames and sonames != ["libvnpu.so"]: raise ValueError("unexpected SONAME: " + ", ".join(sonames)) + # Ordinary processes must start even when CANN/driver libraries are absent. + eager = {"libruntime.so", "libascendcl.so", "libdcmi.so"}.intersection(needed) + if eager: + raise ValueError("unexpected eager SDK dependencies: " + ", ".join(sorted(eager))) require_symbols(library, RUNTIME_HOOKS | ACL_HOOKS, env) require_symbols(runtime, RUNTIME_LINK_APIS | { "rtCtxSetCurrent", "rtStreamCreateWithFlags", @@ -162,13 +166,6 @@ def validate(args): linkage = run("ldd", "-r", str(library), env=env) if re.search(r"not found|undefined symbol|symbol lookup error", linkage, re.I): raise ValueError("unresolved release dependencies:\n" + linkage) - # Ensure ldd actually loaded the SDK selected above, rather than another SDK - # or the CPU suite's link stubs earlier in the inherited search path. - for name, expected in {"libruntime.so": runtime, "libdcmi.so": dcmi}.items(): - match = re.search(r"^\s*" + re.escape(name) + r"\s+=>\s+(\S+)", linkage, re.M) - if not match or Path(match.group(1)).resolve() != expected: - raise ValueError("ldd did not resolve {} to {}".format(name, expected)) - return { "schema_version": 1, "architecture": args.arch, @@ -189,7 +186,7 @@ def validate(args): "cargo": run("cargo", "--version", env=env).strip()}, "source_commit": args.source_commit or run("git", "rev-parse", "HEAD", env=env).strip(), "dynamic_linkage": linkage, - "validation": "ELF, SDK exports and dynamic relocations; no NPU initialized", + "validation": "ELF, SDK exports and dynamic relocations; lazy CANN/DCMI resolution; no NPU initialized", } diff --git a/test/manager_cpu/src/lib.rs b/test/manager_cpu/src/lib.rs index a6a1825..2194a65 100644 --- a/test/manager_cpu/src/lib.rs +++ b/test/manager_cpu/src/lib.rs @@ -1,4 +1,6 @@ #![allow(dead_code)] +#[path = "../../../crates/limiter/src/dynamic.rs"] +pub mod dynamic; #[path = "../../../crates/limiter/src/config.rs"] pub mod config; #[path = "../../../crates/limiter/src/overdraft.rs"] diff --git a/test/memory_cpu/src/lib.rs b/test/memory_cpu/src/lib.rs index 4aa67e1..bd0af4d 100644 --- a/test/memory_cpu/src/lib.rs +++ b/test/memory_cpu/src/lib.rs @@ -1,4 +1,6 @@ #![allow(dead_code, non_snake_case)] +#[path = "../../../crates/limiter/src/dynamic.rs"] +pub mod dynamic; include!(concat!(env!("OUT_DIR"), "/source.rs")); pub mod shmem { include!(concat!(env!("OUT_DIR"), "/shared.rs")); } #[macro_export] diff --git a/test/preload/probe.c b/test/preload/probe.c new file mode 100644 index 0000000..0898435 --- /dev/null +++ b/test/preload/probe.c @@ -0,0 +1,70 @@ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include + +static volatile sig_atomic_t delivered; + +static void handler(int signal) { + delivered = signal; +} + +static void check_passive(void) { + DIR *tasks = opendir("/proc/self/task"); + assert(tasks != NULL); + unsigned count = 0; + struct dirent *entry; + while ((entry = readdir(tasks)) != NULL) { + if (entry->d_name[0] != '.') { + count++; + } + } + closedir(tasks); + assert(count == 1); + + struct sigaction action = {0}, observed = {0}; + action.sa_handler = handler; + sigemptyset(&action.sa_mask); + assert(sigaction(SIGUSR1, &action, NULL) == 0); + assert(sigaction(SIGUSR1, NULL, &observed) == 0); + assert(observed.sa_handler == handler); + assert(raise(SIGUSR1) == 0); + assert(delivered == SIGUSR1); +} + +int main(int argc, char **argv) { + assert(argc >= 2); + check_passive(); + if (strcmp(argv[1], "idle") == 0) { + return 0; + } + typedef uint64_t (*Free)(uint64_t); + Free release = (Free)dlsym(RTLD_DEFAULT, "rtFree"); + assert(release != NULL); + // A configured process must exit here if its required backend is missing. + assert(release(0) == UINT64_MAX); + if (strcmp(argv[1], "missing") == 0) { + return 0; + } + assert(argc == 4); + int visibility = strcmp(argv[2], "global") == 0 ? RTLD_GLOBAL : RTLD_LOCAL; + void *backend = dlopen(argv[3], RTLD_NOW | visibility); + if (backend == NULL) { + fprintf(stderr, "%s\n", dlerror()); + return 1; + } + // A previous failed lookup must not poison subsequent hook calls. + assert(release(0) == 73); + typedef uint64_t (*Malloc)(uint64_t, uint64_t, uint32_t, uint16_t); + Malloc allocate = (Malloc)dlsym(RTLD_DEFAULT, "rtMalloc"); + assert(allocate != NULL); + uint64_t address = 0; + assert(allocate((uint64_t)(uintptr_t)&address, 4096, 0, 0) == 0); + assert(address == 0x1234); + check_passive(); + return 0; +} diff --git a/test/preload/run.py b/test/preload/run.py new file mode 100644 index 0000000..c0c9efb --- /dev/null +++ b/test/preload/run.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +"""Preload the built library without CANN, then load a backend after a miss.""" + +import os +from pathlib import Path +import platform +import re +import subprocess +import sys +import tempfile + + +def main(): + if platform.system() != "Linux" or len(sys.argv) != 2: + raise SystemExit("Usage on Linux: run.py /absolute/path/to/libvnpu.so") + library = Path(sys.argv[1]).resolve(strict=True) + source = Path(__file__).resolve().parent + dynamic = subprocess.check_output(["readelf", "-dW", str(library)], text=True) + needed = set(re.findall(r"\(NEEDED\).*?\[(.*?)\]", dynamic)) + assert not needed.intersection({"libruntime.so", "libascendcl.so", "libdcmi.so"}), dynamic + symbols = subprocess.check_output(["readelf", "--dyn-syms", "--wide", str(library)], text=True) + assert not re.search(r"\bGLOBAL\s+DEFAULT\s+(?!UND\b)\S+\s+sigaction\b", symbols), symbols + + with tempfile.TemporaryDirectory(prefix="hami-preload-") as directory: + work = Path(directory) + probe = work / "probe" + subprocess.run(["cc", "-std=c11", "-Wall", "-Wextra", "-Werror", + str(source / "probe.c"), "-ldl", "-o", str(probe)], check=True) + headers = subprocess.check_output(["readelf", "-lW", str(probe)], text=True, + env=dict(os.environ, LC_ALL="C")) + interpreter = re.search(r"Requesting program interpreter: ([^\]]+)\]", headers) + assert interpreter, headers + # Developer hosts can have an SDK or old link stubs registered in ld.so.cache. + # Absent-backend cases must not accidentally open those instead of the fixture. + isolated_probe = [interpreter.group(1), "--inhibit-cache", str(probe)] + forbidden = work / "forbidden" + forbidden.mkdir() + for name in ("libruntime.so", "libascendcl.so", "libdcmi.so"): + subprocess.run(["cc", "-shared", "-fPIC", "-DFORBID_LOAD", + str(source / "runtime.c"), "-o", str(forbidden / name)], check=True) + backend = work / "libruntime.so" + subprocess.run(["cc", "-shared", "-fPIC", "-Wl,-soname,libruntime.so", + str(source / "runtime.c"), "-o", str(backend)], check=True) + clean = {key: value for key, value in os.environ.items() + if not key.startswith("NPU_") and key not in {"LD_PRELOAD", "LD_AUDIT", "LD_LIBRARY_PATH"}} + clean["LD_PRELOAD"] = str(library) + + def case(name, command, settings=None, code=0): + env = dict(clean, **(settings or {})) + run = subprocess.run(command, env=env, text=True, capture_output=True, timeout=15) + assert run.returncode == code, (name, run.returncode, run.stdout, run.stderr) + assert not list(work.glob("*.shm*")), (name, list(work.iterdir())) + print("PASS " + name, flush=True) + + for label, settings in [ + ("no-sdk", {}), + ("sdk-must-stay-unloaded", {"LD_LIBRARY_PATH": str(forbidden)}), + ("configured-but-unused", { + "LD_LIBRARY_PATH": str(forbidden), + "NPU_GLOBAL_SHM_PATH": str(work / "global.shm"), + "NPU_LOCAL_SHM_PATH": str(work / "local.shm"), + "NPU_MEMORY_LIMIT_MB": "1024", + }), + ]: + for name, command in [ + ("true", ["/bin/true"]), + ("ls", ["/bin/ls", str(work)]), + ("bash", ["/bin/bash", "--noprofile", "--norc", "-c", ":"]), + ("signals-and-threads", [str(probe), "idle"]), + ]: + case(label + "-" + name, command, settings) + case("missing-optional-runtime", [*isolated_probe, "missing"]) + for visibility in ("global", "local"): + case("retry-after-" + visibility + "-dlopen", + [*isolated_probe, "late", visibility, str(backend)]) + case("configured-missing-runtime-fails-closed", [*isolated_probe, "missing"], { + "NPU_GLOBAL_SHM_PATH": str(work / "global.shm"), + "NPU_LOCAL_SHM_PATH": str(work / "local.shm"), + "NPU_SCHED_POLICY": "dynamic", + }, code=78) + + +if __name__ == "__main__": + main() diff --git a/test/preload/runtime.c b/test/preload/runtime.c new file mode 100644 index 0000000..ae26a6d --- /dev/null +++ b/test/preload/runtime.c @@ -0,0 +1,22 @@ +#include +#include + +#ifdef FORBID_LOAD +__attribute__((constructor)) static void forbid_load(void) { + // Loading an SDK/driver during an ordinary command is itself a failure. + abort(); +} +#endif + +uint64_t rtFree(uint64_t address) { + (void)address; + return 73; +} + +uint64_t rtMalloc(uint64_t out, uint64_t size, uint32_t type, uint16_t module) { + (void)size; + (void)type; + (void)module; + *(uint64_t *)(uintptr_t)out = 0x1234; + return 0; +} diff --git a/test/spatial_cpu/run.py b/test/spatial_cpu/run.py index d79a04f..4a8a273 100644 --- a/test/spatial_cpu/run.py +++ b/test/spatial_cpu/run.py @@ -2,8 +2,7 @@ """Exercise the actual built Rust libvnpu.so against a fake ACL, with no NPU. Usage: python3 test/spatial_cpu/run.py target/release/libvnpu.so -The library's normal runtime/DCMI dependencies must be resolvable in the build -environment; no CANN resource call reaches those libraries in these tests. +No CANN runtime or DCMI library is required; calls resolve from the fake backend. """ import json import os diff --git a/test/worker_cpu/src/lib.rs b/test/worker_cpu/src/lib.rs index eff3baa..d437dfa 100644 --- a/test/worker_cpu/src/lib.rs +++ b/test/worker_cpu/src/lib.rs @@ -1,6 +1,8 @@ #![allow(dead_code, non_snake_case)] mod wait_observer; extern crate self as limiter; +#[path = "../../../crates/limiter/src/dynamic.rs"] +pub mod dynamic; #[macro_export] macro_rules! check_rts { ($call:expr) => {{ ($call) as u32 }}; } macro_rules! passthrough { From 90c1fce4958dc228d9f1a05622b2753ba74731f1 Mon Sep 17 00:00:00 2001 From: maverick123123 Date: Thu, 29 Oct 2026 00:37:38 -0700 Subject: [PATCH 2/2] fix: retain SDK backends for cached symbols Keep a reference to the library owning each cached address so application dlclose calls cannot invalidate runtime or HostArgs entry points. Reuse the retained resolver for HostArgs and cover concurrent first lookup and global/local handle release. Correct the English direct-build prerequisites to match the lazy loading implementation. Signed-off-by: maverick123123 --- README.md | 5 ++--- crates/hook/src/host_args.rs | 27 +++++++++------------------ crates/limiter/src/dynamic.rs | 28 ++++++++++++++++++++++++++-- docs/testing.md | 6 ++++-- test/host_args/probe.c | 6 +++++- test/preload/probe.c | 29 ++++++++++++++++++++++++++++- test/preload/run.py | 4 ++-- 7 files changed, 76 insertions(+), 29 deletions(-) diff --git a/README.md b/README.md index 5cd4669..10ce08c 100644 --- a/README.md +++ b/README.md @@ -59,9 +59,8 @@ A successful build does not establish the minimum supported CANN version or hardware support for every policy. See [testing and CI](docs/testing.md) for artifact provenance, verification and hardware-validation scope. -To build directly on a Linux host with Rust 1.94.0, a C compiler and real Ascend -CANN/DCMI development libraries, initialize the installed CANN environment and -run `cargo build --release --workspace --locked`. The library is then at +To build directly on a Linux host with Rust 1.94.0 and a C compiler, run +`cargo build --release --workspace --locked`. The library is then at `target/release/libvnpu.so`. Nonstandard SDK paths are covered in the [configuration reference](docs/configuration.md). diff --git a/crates/hook/src/host_args.rs b/crates/hook/src/host_args.rs index 6ccd63c..c2731b6 100644 --- a/crates/hook/src/host_args.rs +++ b/crates/hook/src/host_args.rs @@ -2,7 +2,7 @@ use std::cell::Cell; use std::ffi::c_void; -use std::sync::LazyLock; +use std::sync::atomic::AtomicPtr; thread_local! { // PID rather than a boolean: a fork child must not inherit an admission bypass. @@ -51,25 +51,16 @@ type HostArgsFn = unsafe extern "C" fn( usize, ) -> i32; -static REAL: LazyLock = LazyLock::new(|| unsafe { - let name = c"aclrtLaunchKernelWithHostArgs"; - let mut symbol = libc::dlsym(libc::RTLD_NEXT, name.as_ptr()); - if symbol.is_null() { - // Frameworks can load ACL with RTLD_LOCAL. Resolve its public entry point - // explicitly in that case and retain the handle for the process lifetime. - let handle = libc::dlopen( - c"libascendcl.so".as_ptr(), - libc::RTLD_NOW | libc::RTLD_LOCAL, - ); - assert!(!handle.is_null(), "cannot load CANN libascendcl.so"); - symbol = libc::dlsym(handle, name.as_ptr()); - } +fn real() -> HostArgsFn { + static REAL: AtomicPtr = AtomicPtr::new(std::ptr::null_mut()); + let symbol = limiter::dynamic::resolve_sdk_symbol(&REAL, c"aclrtLaunchKernelWithHostArgs") + .expect("cannot resolve CANN aclrtLaunchKernelWithHostArgs"); assert!( - !symbol.is_null() && symbol != aclrtLaunchKernelWithHostArgs as *const () as *mut c_void, + symbol != aclrtLaunchKernelWithHostArgs as *const () as *mut c_void, "cannot resolve CANN aclrtLaunchKernelWithHostArgs" ); - std::mem::transmute(symbol) -}); + unsafe { std::mem::transmute(symbol) } +} // CANN acl_rt.h ABI: aclError is a signed C int; argument sizes/counts are size_t. // Opaque pointers are forwarded without inspecting their pointed-to layouts. @@ -85,7 +76,7 @@ pub unsafe extern "C" fn aclrtLaunchKernelWithHostArgs( placeholders: *mut c_void, placeholder_count: usize, ) -> i32 { - let real = *REAL; + let real = real(); let scope = HostArgsScope::enter(); if scope.outermost { let mut capture_status = 0; diff --git a/crates/limiter/src/dynamic.rs b/crates/limiter/src/dynamic.rs index 55406a6..1b3efa5 100644 --- a/crates/limiter/src/dynamic.rs +++ b/crates/limiter/src/dynamic.rs @@ -62,6 +62,23 @@ pub fn sdk_symbol(name: &CStr) -> Option<*mut c_void> { } } +/// Take a reference to the actual backend before caching one of its addresses. +/// `RTLD_NEXT` itself does not keep an application's dlopen handle alive. +fn retain_symbol_library(address: *mut c_void) -> *mut c_void { + let mut info: libc::Dl_info = unsafe { std::mem::zeroed() }; + if unsafe { libc::dladdr(address, &mut info) } == 0 || info.dli_fname.is_null() { + return ptr::null_mut(); + } + // NOLOAD avoids initializing another SDK or selecting a different backend. + // A main-executable symbol needs no extra reference and may return null here. + unsafe { + libc::dlopen( + info.dli_fname, + libc::RTLD_LAZY | libc::RTLD_LOCAL | libc::RTLD_NOLOAD, + ) + } +} + fn cached_symbol( cache: &AtomicPtr, resolve: impl FnOnce() -> Option<*mut c_void>, @@ -72,6 +89,7 @@ fn cached_symbol( } // Do not cache a miss: a later dlopen can make an optional symbol available. let address = resolve()?; + let retained = retain_symbol_library(address); Some( match cache.compare_exchange( ptr::null_mut(), @@ -80,12 +98,18 @@ fn cached_symbol( Ordering::Acquire, ) { Ok(_) => address, - Err(existing) => existing, + Err(existing) => { + if !retained.is_null() { + unsafe { libc::dlclose(retained) }; + } + existing + } }, ) } -/// Cache successful SDK lookups only; retry a missing backend on the next call. +/// Cache successful SDK lookups and retain their backend for the process lifetime. +/// Retry a missing backend on the next call. pub fn resolve_sdk_symbol(cache: &AtomicPtr, name: &CStr) -> Option<*mut c_void> { cached_symbol(cache, || sdk_symbol(name)) } diff --git a/docs/testing.md b/docs/testing.md index 97e9cf6..54e3a0d 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -122,9 +122,10 @@ fake SDKs. These CPU artifacts have not passed real-SDK release verification. | `test/manager_cpu` | Production manager arithmetic, fixed rest calculation, report aggregation and spatial exclusion from time rotation. The unused process scan is a panic-on-call test double. | | `test/memory_cpu` | Production memory scan/quota paths, rollback, shared-memory ABI and bounded fork cleanup. Procfs reads and CANN calls use explicit test doubles; Linux futexes are real. | | `test/worker_cpu/run.py` | Source-linked worker and hook admission cases with observed lock/futex boundaries; Cached/configured/DVPP/physical allocation admission, failure rollback, allocation/free alias deduplication, physical retain lifetimes, concurrent free/address reuse, per-card ownership, threads and fork. Each case runs in a separate process with a timeout. | -| `test/preload/run.py` | No eager SDK dependencies; ordinary shell/tool startup without CANN, preserved signal handlers, no preload-time supervisor, passive hooks and successful retry after global/local `dlopen`. | +| `test/preload/run.py` | No eager SDK dependencies; ordinary shell/tool startup without CANN, preserved signal handlers, no preload-time supervisor, passive hooks, successful retry after global/local `dlopen`, and cached calls surviving the application's `dlclose`. | | `test/process_local.sh` | Actual per-process cache, concurrent initialization and real fork cases. | | `test/spatial_cpu/run.py` | Actual built Rust hook library preloaded into a C client with fake ACL: ceil budgets, resets, API failures, device/context switching and fork handling. | +| `test/host_args.sh` | HostArgs ABI, capture, admission, nested delegation, per-card routing, threads/fork, and cached launch calls after the application releases its ACL handle. | | Documentation links | Local Markdown link destinations in the root READMEs and `docs/`; no external requests or rendered-anchor checks. | The [workflow](../.github/workflows/ci.yml) runs on pull requests, pushes to `main` @@ -246,10 +247,11 @@ CPU 脚本在临时目录构建，退出时清理，不替换已安装的库。 明确的测试替身，Linux futex 使用真实实现。 - Worker：源码链接的准入与 hook 路径，观测锁和 futex 边界；覆盖 Cached/配置/DVPP/物理分配的准入、失败回滚、分配及释放转调去重、物理句柄 retain 生命周期、并发释放时地址复用、每卡归属、线程及 fork。每个用例独立进程并限制超时。 - Preload：无 SDK 硬依赖、无 CANN 的普通命令启动、保留信号处理器、加载期无 supervisor、 - 未配置时透传，以及 global/local `dlopen` 后的符号重试。 + 未配置时透传、global/local `dlopen` 后的符号重试，以及应用 `dlclose` 后缓存调用仍有效。 - Process-local：真实缓存实现的并发初始化和 fork。 - Spatial：预加载实际构建的 Rust 库，通过 fake ACL 验证向上取整、重置、API 错误、 设备/上下文切换及 fork 行为。 +- HostArgs：ABI、capture、准入、嵌套转调、按卡路由、线程/fork，以及应用释放 ACL 句柄后的缓存调用。 - 文档：检查根目录 README 与 `docs/` 的本地 Markdown 链接目标；不访问外部链接， 不检查渲染后的锚点。 diff --git a/test/host_args/probe.c b/test/host_args/probe.c index edb226a..049136e 100644 --- a/test/host_args/probe.c +++ b/test/host_args/probe.c @@ -102,6 +102,10 @@ int main(int argc, char **argv) { usleep(50000); assert(!atomic_load(&completed)); atomic_store(tokens, 1); assert(!pthread_join(threads[0], NULL)); assert(atomic_load(&completed) && atomic_load(tokens) == 0); - printf("PASS %s: ABI, capture, errors, nested delegation, replay, per-card streams, threads, fork, quota wait\n", argv[1]); + // Releasing the application's ACL handle must not invalidate the cached launch. + mode(0); capture(1, 0); + assert(dlclose(acl) == 0); + call(); + printf("PASS %s: ABI, capture, errors, nested delegation, replay, per-card streams, threads, fork, quota wait, SDK lifetime\n", argv[1]); return 0; } diff --git a/test/preload/probe.c b/test/preload/probe.c index 0898435..6a915bc 100644 --- a/test/preload/probe.c +++ b/test/preload/probe.c @@ -2,6 +2,7 @@ #include #include #include +#include #include #include #include @@ -9,6 +10,19 @@ static volatile sig_atomic_t delivered; +typedef uint64_t (*Free)(uint64_t); +struct ConcurrentCall { + Free release; + pthread_barrier_t barrier; +}; + +static void *concurrent_free(void *argument) { + struct ConcurrentCall *call = argument; + pthread_barrier_wait(&call->barrier); + assert(call->release(0) == 73); + return NULL; +} + static void handler(int signal) { delivered = signal; } @@ -42,7 +56,6 @@ int main(int argc, char **argv) { if (strcmp(argv[1], "idle") == 0) { return 0; } - typedef uint64_t (*Free)(uint64_t); Free release = (Free)dlsym(RTLD_DEFAULT, "rtFree"); assert(release != NULL); // A configured process must exit here if its required backend is missing. @@ -58,6 +71,17 @@ int main(int argc, char **argv) { return 1; } // A previous failed lookup must not poison subsequent hook calls. + // Race first resolution so losing cache publishers also release their references. + struct ConcurrentCall call = {.release = release}; + pthread_t threads[8]; + assert(pthread_barrier_init(&call.barrier, NULL, 8) == 0); + for (unsigned index = 0; index < 8; index++) { + assert(pthread_create(&threads[index], NULL, concurrent_free, &call) == 0); + } + for (unsigned index = 0; index < 8; index++) { + assert(pthread_join(threads[index], NULL) == 0); + } + assert(pthread_barrier_destroy(&call.barrier) == 0); assert(release(0) == 73); typedef uint64_t (*Malloc)(uint64_t, uint64_t, uint32_t, uint16_t); Malloc allocate = (Malloc)dlsym(RTLD_DEFAULT, "rtMalloc"); @@ -66,5 +90,8 @@ int main(int argc, char **argv) { assert(allocate((uint64_t)(uintptr_t)&address, 4096, 0, 0) == 0); assert(address == 0x1234); check_passive(); + // Cached addresses must survive the application's release of its SDK handle. + assert(dlclose(backend) == 0); + assert(release(0) == 73); return 0; } diff --git a/test/preload/run.py b/test/preload/run.py index c0c9efb..b0b5dfd 100644 --- a/test/preload/run.py +++ b/test/preload/run.py @@ -25,7 +25,7 @@ def main(): work = Path(directory) probe = work / "probe" subprocess.run(["cc", "-std=c11", "-Wall", "-Wextra", "-Werror", - str(source / "probe.c"), "-ldl", "-o", str(probe)], check=True) + str(source / "probe.c"), "-ldl", "-pthread", "-o", str(probe)], check=True) headers = subprocess.check_output(["readelf", "-lW", str(probe)], text=True, env=dict(os.environ, LC_ALL="C")) interpreter = re.search(r"Requesting program interpreter: ([^\]]+)\]", headers) @@ -71,7 +71,7 @@ def case(name, command, settings=None, code=0): case(label + "-" + name, command, settings) case("missing-optional-runtime", [*isolated_probe, "missing"]) for visibility in ("global", "local"): - case("retry-after-" + visibility + "-dlopen", + case("retry-after-" + visibility + "-dlopen-and-close", [*isolated_probe, "late", visibility, str(backend)]) case("configured-missing-runtime-fails-closed", [*isolated_probe, "missing"], { "NPU_GLOBAL_SHM_PATH": str(work / "global.shm"),