diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..c7f83c2 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,2 @@ +target +.git diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md new file mode 100644 index 0000000..48f9d10 --- /dev/null +++ b/.github/copilot-instructions.md @@ -0,0 +1,88 @@ +# Copilot Instructions for KernelPort + +## Repository Summary +- KernelPort is a Rust-based inference server/orchestrator that serves TensorFlow, PyTorch (TorchScript), ONNX Runtime, and TensorRT models with native kernels. +- Primary runtime is Rust; Dockerfiles provide CPU/GPU container images. Small helper scripts are in `scripts/` (Python + shell). +- Repo size is moderate (Rust workspace with multiple crates). Main targets: server (`kernelportd`), ONNX Runtime backend, and gRPC API. + +## High-Level Project Info +- Languages: Rust (core/runtime/server), shell/Python (container entrypoint + model fetch). +- Build system: Cargo workspace (`Cargo.toml` at repo root). +- gRPC stack: tonic 0.14 + prost 0.14; protobufs in `crates/kernelport-proto/src/inference.proto`. +- Containers: `Dockerfile.cpu` (Debian bookworm runtime), `Dockerfile.gpu` (CUDA runtime on Ubuntu 22.04). + +## Build / Validate (validated commands) +Always prefer these sequences and avoid ad-hoc commands unless needed. + +Bootstrap (local dev): +- Install Rust stable and components: `rustup component add rustfmt clippy` +- Optional but useful: `pipx install pre-commit` then `pre-commit install` + +Format (validated): +- `cargo fmt --all` +- Pre-commit hook mirrors this (`.pre-commit-config.yaml`). + +Lint (validated): +- `cargo clippy --all-targets --all-features -- -D warnings` + +Test (validated): +- `cargo test --all` +- Targeted ORT test (validated): `cargo test -p kernelport-backend-ort --test identity` + +Build (validated): +- `cargo build --all` +- If a build runs without network access, Cargo may fail to download crates; ensure network access is available or dependencies are cached. + +Run (validated via docs): +- Local server (CPU): `cargo run -p kernelport-server -- serve --device cpu` +- CUDA local dev (Linux + NVIDIA): set `ORT_DYLIB_PATH` to a CUDA-enabled `libonnxruntime.so` and run with `--features ort-cuda --device cuda:N`. +- Validation via containers is documented in `docs/validation.md`. + +Containers (validated builds): +- `docker build -f Dockerfile.cpu -t kernelport:cpu .` +- `docker build -f Dockerfile.gpu -t kernelport:gpu .` +- GPU runtime requires NVIDIA drivers + `--gpus` and a mounted CUDA-enabled `libonnxruntime.so`. + +Known pitfalls and workarounds: +- gRPC reflection is enabled; use `grpcurl` without `-protoset` after starting the server. +- The sample ONNX model expects input name `x` with shape `[1,1,2,2]` and outputs `y`; see `docs/validation.md`. +- `clippy` enforces `-D warnings`; do not leave warnings in `kernelport-backend-ort`. + +## CI / Checks (must match locally) +`.github/workflows/ci.yml` runs: +- `cargo fmt --all -- --check` +- `cargo clippy --all-targets --all-features -- -D warnings` +- `cargo test --all` +- `cargo build --all` +- Extra CPU ORT identity test: `cargo test -p kernelport-backend-ort --test identity` + +## Project Layout & Architecture +Root layout (files): +- `Cargo.toml`, `Cargo.lock` (workspace), `Makefile` (fmt/clippy/test/build/check targets) +- `Dockerfile.cpu`, `Dockerfile.gpu`, `.pre-commit-config.yaml`, `.dockerignore` +- `README.md` (concepts, dev + container usage, pre-commit) +- `docs/` (model ingestion, manifest, validation) +- `models/` (sample ONNX bundle + example manifest) +- `scripts/` (entrypoint + HF model fetcher) +- `.github/workflows/ci.yml` + +Key Rust crates: +- `crates/kernelport-server/`: + - `src/main.rs` (server entry; batcher/scheduler/worker wiring; reflection) + - `src/grpc.rs` (gRPC Infer handler, DType mapping) + - `src/cli.rs` (CLI flags: `--device`, `--model-path`, `--grpc-addr`, `--log`) + - `src/registry.rs` (model loading via ORT, output name capture) +- `crates/kernelport-backend-ort/`: + - ONNX Runtime backend; CPU and optional CUDA feature. +- `crates/kernelport-proto/`: + - `src/inference.proto` (gRPC API, `Infer` RPC, `DType` enum) + - `build.rs` (generates descriptor set for reflection) +- `crates/kernelport-core/` and `crates/kernelport-runtime/` (tensor types, batching, scheduler/worker). + +Docs and validated validation flow: +- `docs/validation.md` (CPU/GPU run + grpcurl example). +- `docs/model-ingestion.md` and `docs/model-manifest.md` (HF model ingestion plan). + +## Read This First (reduce exploration) +- Assume `kernelportd` is the server binary. +- Trust these instructions and only search if information here is missing or incorrect. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d2fc913..3575880 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,6 +16,8 @@ jobs: components: rustfmt, clippy - uses: Swatinem/rust-cache@v2 + - name: install protoc + run: sudo apt-get update && sudo apt-get install -y protobuf-compiler - name: fmt run: cargo fmt --all -- --check @@ -39,6 +41,8 @@ jobs: components: rustfmt, clippy - uses: Swatinem/rust-cache@v2 + - name: install protoc + run: sudo apt-get update && sudo apt-get install -y protobuf-compiler - name: ort identity (cpu) run: cargo test -p kernelport-backend-ort --test identity diff --git a/.gitignore b/.gitignore index c7827c0..8f0a58b 100644 --- a/.gitignore +++ b/.gitignore @@ -23,3 +23,6 @@ target # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. #.idea/ + +# Snyk Security Extension - AI Rules (auto-generated) +.cursor/rules/snyk_rules.mdc diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..6238b31 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,9 @@ +repos: + - repo: local + hooks: + - id: cargo-fmt + name: cargo fmt + entry: cargo fmt --all + language: system + types: [rust] + pass_filenames: false diff --git a/Cargo.lock b/Cargo.lock index 4fba0fe..3861191 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -73,28 +73,6 @@ version = "1.0.100" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61" -[[package]] -name = "async-stream" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476" -dependencies = [ - "async-stream-impl", - "futures-core", - "pin-project-lite", -] - -[[package]] -name = "async-stream-impl" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "async-trait" version = "0.1.89" @@ -120,11 +98,10 @@ checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" [[package]] name = "axum" -version = "0.7.9" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f" +checksum = "8b52af3cb4058c895d37317bb27508dccc8e5f2d39454016b297bf4a400597b8" dependencies = [ - "async-trait", "axum-core", "bytes", "futures-util", @@ -137,29 +114,26 @@ dependencies = [ "mime", "percent-encoding", "pin-project-lite", - "rustversion", - "serde", + "serde_core", "sync_wrapper", - "tower 0.5.2", + "tower", "tower-layer", "tower-service", ] [[package]] name = "axum-core" -version = "0.4.5" +version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199" +checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" dependencies = [ - "async-trait", "bytes", - "futures-util", + "futures-core", "http", "http-body", "http-body-util", "mime", "pin-project-lite", - "rustversion", "sync_wrapper", "tower-layer", "tower-service", @@ -398,6 +372,12 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + [[package]] name = "foreign-types" version = "0.3.2" @@ -462,17 +442,6 @@ dependencies = [ "version_check", ] -[[package]] -name = "getrandom" -version = "0.2.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592" -dependencies = [ - "cfg-if", - "libc", - "wasi", -] - [[package]] name = "getrandom" version = "0.3.4" @@ -497,7 +466,7 @@ dependencies = [ "futures-core", "futures-sink", "http", - "indexmap 2.12.1", + "indexmap", "slab", "tokio", "tokio-util", @@ -506,9 +475,12 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.12.3" +version = "0.15.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "foldhash", +] [[package]] name = "hashbrown" @@ -618,22 +590,12 @@ dependencies = [ "hyper", "libc", "pin-project-lite", - "socket2 0.6.1", + "socket2", "tokio", "tower-service", "tracing", ] -[[package]] -name = "indexmap" -version = "1.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" -dependencies = [ - "autocfg", - "hashbrown 0.12.3", -] - [[package]] name = "indexmap" version = "2.12.1" @@ -690,7 +652,8 @@ version = "0.0.0" dependencies = [ "prost", "tonic", - "tonic-build", + "tonic-prost", + "tonic-prost-build", ] [[package]] @@ -717,6 +680,7 @@ dependencies = [ "kernelport-runtime", "tokio", "tonic", + "tonic-reflection", "tracing", "tracing-subscriber", ] @@ -767,9 +731,9 @@ dependencies = [ [[package]] name = "matchit" -version = "0.7.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" +checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" [[package]] name = "matrixmultiply" @@ -986,12 +950,13 @@ checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" [[package]] name = "petgraph" -version = "0.7.1" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" +checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" dependencies = [ "fixedbitset", - "indexmap 2.12.1", + "hashbrown 0.15.5", + "indexmap", ] [[package]] @@ -1047,15 +1012,6 @@ dependencies = [ "portable-atomic", ] -[[package]] -name = "ppv-lite86" -version = "0.2.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" -dependencies = [ - "zerocopy", -] - [[package]] name = "prettyplease" version = "0.2.37" @@ -1077,9 +1033,9 @@ dependencies = [ [[package]] name = "prost" -version = "0.13.5" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" +checksum = "d2ea70524a2f82d518bce41317d0fae74151505651af45faf1ffbd6fd33f0568" dependencies = [ "bytes", "prost-derive", @@ -1087,19 +1043,20 @@ dependencies = [ [[package]] name = "prost-build" -version = "0.13.5" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" +checksum = "343d3bd7056eda839b03204e68deff7d1b13aba7af2b2fd16890697274262ee7" dependencies = [ "heck", "itertools", "log", "multimap", - "once_cell", "petgraph", "prettyplease", "prost", "prost-types", + "pulldown-cmark", + "pulldown-cmark-to-cmark", "regex", "syn", "tempfile", @@ -1107,9 +1064,9 @@ dependencies = [ [[package]] name = "prost-derive" -version = "0.13.5" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" +checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b" dependencies = [ "anyhow", "itertools", @@ -1120,57 +1077,47 @@ dependencies = [ [[package]] name = "prost-types" -version = "0.13.5" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52c2c1bf36ddb1a1c396b3601a3cec27c2462e45f07c386894ec3ccf5332bd16" +checksum = "8991c4cbdb8bc5b11f0b074ffe286c30e523de90fee5ba8132f1399f23cb3dd7" dependencies = [ "prost", ] [[package]] -name = "quote" -version = "1.0.42" +name = "pulldown-cmark" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a338cc41d27e6cc6dce6cefc13a0729dfbb81c262b1f519331575dd80ef3067f" +checksum = "1e8bbe1a966bd2f362681a44f6edce3c2310ac21e4d5067a6e7ec396297a6ea0" dependencies = [ - "proc-macro2", + "bitflags", + "memchr", + "unicase", ] [[package]] -name = "r-efi" -version = "5.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" - -[[package]] -name = "rand" -version = "0.8.5" +name = "pulldown-cmark-to-cmark" +version = "22.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +checksum = "50793def1b900256624a709439404384204a5dc3a6ec580281bfaac35e882e90" dependencies = [ - "libc", - "rand_chacha", - "rand_core", + "pulldown-cmark", ] [[package]] -name = "rand_chacha" -version = "0.3.1" +name = "quote" +version = "1.0.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +checksum = "a338cc41d27e6cc6dce6cefc13a0729dfbb81c262b1f519331575dd80ef3067f" dependencies = [ - "ppv-lite86", - "rand_core", + "proc-macro2", ] [[package]] -name = "rand_core" -version = "0.6.4" +name = "r-efi" +version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" -dependencies = [ - "getrandom 0.2.16", -] +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" [[package]] name = "rawpointer" @@ -1238,12 +1185,6 @@ dependencies = [ "zeroize", ] -[[package]] -name = "rustversion" -version = "1.0.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" - [[package]] name = "schannel" version = "0.1.28" @@ -1276,15 +1217,6 @@ dependencies = [ "libc", ] -[[package]] -name = "serde" -version = "1.0.228" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" -dependencies = [ - "serde_core", -] - [[package]] name = "serde_core" version = "1.0.228" @@ -1355,16 +1287,6 @@ version = "2.0.0-alpha.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51d44cfb396c3caf6fbfd0ab422af02631b69ddd96d2eff0b0f0724f9024051b" -[[package]] -name = "socket2" -version = "0.5.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678" -dependencies = [ - "libc", - "windows-sys 0.52.0", -] - [[package]] name = "socket2" version = "0.6.1" @@ -1427,7 +1349,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "655da9c7eb6305c55742045d5a8d2037996d61d8de95806335c7c86ce0f82e9c" dependencies = [ "fastrand", - "getrandom 0.3.4", + "getrandom", "once_cell", "rustix", "windows-sys 0.61.2", @@ -1452,7 +1374,7 @@ dependencies = [ "libc", "mio", "pin-project-lite", - "socket2 0.6.1", + "socket2", "tokio-macros", "windows-sys 0.61.2", ] @@ -1494,11 +1416,10 @@ dependencies = [ [[package]] name = "tonic" -version = "0.12.3" +version = "0.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877c5b330756d856ffcc4553ab34a5684481ade925ecc54bcd1bf02b1d0d4d52" +checksum = "eb7613188ce9f7df5bfe185db26c5814347d110db17920415cf2fbcad85e7203" dependencies = [ - "async-stream", "async-trait", "axum", "base64", @@ -1512,11 +1433,11 @@ dependencies = [ "hyper-util", "percent-encoding", "pin-project", - "prost", - "socket2 0.5.10", + "socket2", + "sync_wrapper", "tokio", "tokio-stream", - "tower 0.4.13", + "tower", "tower-layer", "tower-service", "tracing", @@ -1524,9 +1445,32 @@ dependencies = [ [[package]] name = "tonic-build" -version = "0.12.3" +version = "0.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9557ce109ea773b399c9b9e5dca39294110b74f1f342cb347a80d1fce8c26a11" +checksum = "4c40aaccc9f9eccf2cd82ebc111adc13030d23e887244bc9cfa5d1d636049de3" +dependencies = [ + "prettyplease", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tonic-prost" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66bd50ad6ce1252d87ef024b3d64fe4c3cf54a86fb9ef4c631fdd0ded7aeaa67" +dependencies = [ + "bytes", + "prost", + "tonic", +] + +[[package]] +name = "tonic-prost-build" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4a16cba4043dc3ff43fcb3f96b4c5c154c64cbd18ca8dce2ab2c6a451d058a2" dependencies = [ "prettyplease", "proc-macro2", @@ -1534,26 +1478,22 @@ dependencies = [ "prost-types", "quote", "syn", + "tempfile", + "tonic-build", ] [[package]] -name = "tower" -version = "0.4.13" +name = "tonic-reflection" +version = "0.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +checksum = "34da53e8387581d66db16ff01f98a70b426b091fdf76856e289d5c1bd386ed7b" dependencies = [ - "futures-core", - "futures-util", - "indexmap 1.9.3", - "pin-project", - "pin-project-lite", - "rand", - "slab", + "prost", + "prost-types", "tokio", - "tokio-util", - "tower-layer", - "tower-service", - "tracing", + "tokio-stream", + "tonic", + "tonic-prost", ] [[package]] @@ -1564,10 +1504,15 @@ checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" dependencies = [ "futures-core", "futures-util", + "indexmap", "pin-project-lite", + "slab", "sync_wrapper", + "tokio", + "tokio-util", "tower-layer", "tower-service", + "tracing", ] [[package]] @@ -1655,6 +1600,12 @@ version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" +[[package]] +name = "unicase" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142" + [[package]] name = "unicode-ident" version = "1.0.22" @@ -1782,22 +1733,13 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" -[[package]] -name = "windows-sys" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" -dependencies = [ - "windows-targets 0.52.6", -] - [[package]] name = "windows-sys" version = "0.60.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" dependencies = [ - "windows-targets 0.53.5", + "windows-targets", ] [[package]] @@ -1809,22 +1751,6 @@ dependencies = [ "windows-link", ] -[[package]] -name = "windows-targets" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" -dependencies = [ - "windows_aarch64_gnullvm 0.52.6", - "windows_aarch64_msvc 0.52.6", - "windows_i686_gnu 0.52.6", - "windows_i686_gnullvm 0.52.6", - "windows_i686_msvc 0.52.6", - "windows_x86_64_gnu 0.52.6", - "windows_x86_64_gnullvm 0.52.6", - "windows_x86_64_msvc 0.52.6", -] - [[package]] name = "windows-targets" version = "0.53.5" @@ -1832,106 +1758,58 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" dependencies = [ "windows-link", - "windows_aarch64_gnullvm 0.53.1", - "windows_aarch64_msvc 0.53.1", - "windows_i686_gnu 0.53.1", - "windows_i686_gnullvm 0.53.1", - "windows_i686_msvc 0.53.1", - "windows_x86_64_gnu 0.53.1", - "windows_x86_64_gnullvm 0.53.1", - "windows_x86_64_msvc 0.53.1", + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", ] -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" - [[package]] name = "windows_aarch64_gnullvm" version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" -[[package]] -name = "windows_aarch64_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" - [[package]] name = "windows_aarch64_msvc" version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" -[[package]] -name = "windows_i686_gnu" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" - [[package]] name = "windows_i686_gnu" version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" -[[package]] -name = "windows_i686_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" - [[package]] name = "windows_i686_gnullvm" version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" -[[package]] -name = "windows_i686_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" - [[package]] name = "windows_i686_msvc" version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" -[[package]] -name = "windows_x86_64_gnu" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" - [[package]] name = "windows_x86_64_gnu" version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" - [[package]] name = "windows_x86_64_gnullvm" version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" -[[package]] -name = "windows_x86_64_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" - [[package]] name = "windows_x86_64_msvc" version = "0.53.1" @@ -1954,26 +1832,6 @@ dependencies = [ "rustix", ] -[[package]] -name = "zerocopy" -version = "0.8.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd74ec98b9250adb3ca554bdde269adf631549f51d8a8f8f0a10b50f1cb298c3" -dependencies = [ - "zerocopy-derive", -] - -[[package]] -name = "zerocopy-derive" -version = "0.8.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8a8d209fdf45cf5138cbb5a506f6b52522a25afccc534d1475dad8e31105c6a" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "zeroize" version = "1.8.2" diff --git a/Dockerfile.cpu b/Dockerfile.cpu new file mode 100644 index 0000000..15633a4 --- /dev/null +++ b/Dockerfile.cpu @@ -0,0 +1,31 @@ +FROM rust:1.85-bookworm AS build + +WORKDIR /app +COPY . . +RUN apt-get update \ + && apt-get install -y --no-install-recommends protobuf-compiler \ + && rm -rf /var/lib/apt/lists/* +RUN cargo build -p kernelport-server --release + +FROM debian:bookworm-slim + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + ca-certificates \ + libstdc++6 \ + python3 \ + python3-pip \ + && rm -rf /var/lib/apt/lists/* + +RUN python3 -m pip install --no-cache-dir --break-system-packages \ + huggingface_hub==0.20.3 \ + pyyaml==6.0.1 + +WORKDIR /app +COPY --from=build /app/target/release/kernelportd /app/kernelportd +COPY scripts /app/scripts + +RUN chmod +x /app/scripts/entrypoint.sh + +ENTRYPOINT ["/app/scripts/entrypoint.sh"] +CMD ["--device", "cpu"] diff --git a/Dockerfile.gpu b/Dockerfile.gpu new file mode 100644 index 0000000..d3f5176 --- /dev/null +++ b/Dockerfile.gpu @@ -0,0 +1,31 @@ +FROM rust:1.85-bookworm AS build + +WORKDIR /app +COPY . . +RUN apt-get update \ + && apt-get install -y --no-install-recommends protobuf-compiler \ + && rm -rf /var/lib/apt/lists/* +RUN cargo build -p kernelport-server --release --features ort-cuda + +FROM nvidia/cuda:12.2.0-runtime-ubuntu22.04 + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + ca-certificates \ + libstdc++6 \ + python3 \ + python3-pip \ + && rm -rf /var/lib/apt/lists/* + +RUN python3 -m pip install --no-cache-dir \ + huggingface_hub==0.20.3 \ + pyyaml==6.0.1 + +WORKDIR /app +COPY --from=build /app/target/release/kernelportd /app/kernelportd +COPY scripts /app/scripts + +RUN chmod +x /app/scripts/entrypoint.sh + +ENTRYPOINT ["/app/scripts/entrypoint.sh"] +CMD ["--device", "cuda:0"] diff --git a/README.md b/README.md index 302e6e4..d95756c 100644 --- a/README.md +++ b/README.md @@ -104,3 +104,94 @@ KernelPort supports CUDA via ONNX Runtime. For GPU inference: - Set `ORT_DYLIB_PATH` to the CUDA-enabled `libonnxruntime.so`. - Build with the CUDA feature and select a device: - `cargo run -p kernelport-server --features ort-cuda -- serve --device cuda:0` + +--- + +## Containers (CPU and GPU) + +KernelPort ships with separate Dockerfiles for CPU and GPU runtime environments: + +- `Dockerfile.cpu` builds a CPU-only image intended for local dev or CPU deployments. +- `Dockerfile.gpu` builds a GPU-ready image that expects CUDA + TensorRT on the host. + - Base image: `nvidia/cuda:12.2.0-runtime-ubuntu22.04` + +The GPU image is designed for "bring your own kernel" by letting you mount a +CUDA-enabled ONNX Runtime shared library at runtime: + +- Mount your `libonnxruntime.so` and set `ORT_DYLIB_PATH`. +- Run with `--gpus` and choose `--device cuda:N`. + +### Recommended NVIDIA stack + +- Driver: 535+ (or newer) +- CUDA: 12.2 +- cuDNN: 8.9 +- TensorRT: 8.6 + +### Build + +```bash +docker build -f Dockerfile.cpu -t kernelport:cpu . +docker build -f Dockerfile.gpu -t kernelport:gpu . +``` + +### Run (CPU) + +```bash +docker run --rm -p 8080:8080 kernelport:cpu --device cpu +``` + +### Run (CPU + HF pull) + +```bash +docker run --rm -p 8080:8080 \ + -e MODEL_MANIFEST_PATH=/app/models/manifest.yaml \ + -e HF_TOKEN=your_hf_token \ + -v /path/to/manifest.yaml:/app/models/manifest.yaml:ro \ + -v /path/to/model-cache:/models \ + kernelport:cpu --device cpu --model-path /models/bert-base-uncased/model.onnx +``` + +### Run (GPU) + +```bash +docker run --rm --gpus all \ + -e ORT_DYLIB_PATH=/opt/ort/libonnxruntime.so \ + -v /path/to/ort/libonnxruntime.so:/opt/ort/libonnxruntime.so:ro \ + -p 8080:8080 \ + kernelport:gpu --device cuda:0 --model-path /models/bert-base-uncased/model.onnx +``` + +### Run (GPU + HF pull) + +```bash +docker run --rm --gpus all \ + -e ORT_DYLIB_PATH=/opt/ort/libonnxruntime.so \ + -e MODEL_MANIFEST_PATH=/app/models/manifest.yaml \ + -e HF_TOKEN=your_hf_token \ + -v /path/to/ort/libonnxruntime.so:/opt/ort/libonnxruntime.so:ro \ + -v /path/to/manifest.yaml:/app/models/manifest.yaml:ro \ + -v /path/to/model-cache:/models \ + -p 8080:8080 \ + kernelport:gpu --device cuda:0 --model-path /models/bert-base-uncased/model.onnx +``` + +See `docs/model-ingestion.md` for the HuggingFace model ingestion plan and +`docs/model-manifest.md` for the proposed manifest schema. + +Validation steps are in `docs/validation.md`. + +## Pre-commit (local) + +Install pre-commit and enable the hook: + +```bash +pipx install pre-commit +pre-commit install +``` + +Run on demand: + +```bash +pre-commit run --all-files +``` diff --git a/crates/kernelport-backend-ort/src/lib.rs b/crates/kernelport-backend-ort/src/lib.rs index 7808e19..7071e3a 100644 --- a/crates/kernelport-backend-ort/src/lib.rs +++ b/crates/kernelport-backend-ort/src/lib.rs @@ -147,9 +147,7 @@ fn configure_cuda(builder: SessionBuilder, device_id: u32) -> Result) -> Result { match *ty { TensorElementType::Float32 => { let array = value.try_extract_array::()?; - let slice = array - .as_slice() - .context("non-contiguous output tensor")?; + let slice = array.as_slice().context("non-contiguous output tensor")?; Ok(Tensor::from_cpu_bytes( DType::F32, kernel_shape, @@ -257,9 +253,7 @@ fn ort_value_to_tensor(value: &ort::value::ValueRef<'_>) -> Result { } TensorElementType::Int64 => { let array = value.try_extract_array::()?; - let slice = array - .as_slice() - .context("non-contiguous output tensor")?; + let slice = array.as_slice().context("non-contiguous output tensor")?; Ok(Tensor::from_cpu_bytes( DType::I64, kernel_shape, @@ -268,9 +262,7 @@ fn ort_value_to_tensor(value: &ort::value::ValueRef<'_>) -> Result { } TensorElementType::Int32 => { let array = value.try_extract_array::()?; - let slice = array - .as_slice() - .context("non-contiguous output tensor")?; + let slice = array.as_slice().context("non-contiguous output tensor")?; Ok(Tensor::from_cpu_bytes( DType::I32, kernel_shape, @@ -279,9 +271,7 @@ fn ort_value_to_tensor(value: &ort::value::ValueRef<'_>) -> Result { } TensorElementType::Uint8 => { let array = value.try_extract_array::()?; - let slice = array - .as_slice() - .context("non-contiguous output tensor")?; + let slice = array.as_slice().context("non-contiguous output tensor")?; Ok(Tensor::from_cpu_bytes( DType::U8, kernel_shape, @@ -293,33 +283,27 @@ fn ort_value_to_tensor(value: &ort::value::ValueRef<'_>) -> Result { } } +#[allow(clippy::manual_is_multiple_of)] fn bytes_to_f32(bytes: &Bytes) -> Result> { - ensure!( - bytes.len().is_multiple_of(4), - "f32 input has invalid byte length" - ); + ensure!(bytes.len() % 4 == 0, "f32 input has invalid byte length"); Ok(bytes .chunks_exact(4) .map(|b| f32::from_le_bytes([b[0], b[1], b[2], b[3]])) .collect()) } +#[allow(clippy::manual_is_multiple_of)] fn bytes_to_i64(bytes: &Bytes) -> Result> { - ensure!( - bytes.len().is_multiple_of(8), - "i64 input has invalid byte length" - ); + ensure!(bytes.len() % 8 == 0, "i64 input has invalid byte length"); Ok(bytes .chunks_exact(8) .map(|b| i64::from_le_bytes([b[0], b[1], b[2], b[3], b[4], b[5], b[6], b[7]])) .collect()) } +#[allow(clippy::manual_is_multiple_of)] fn bytes_to_i32(bytes: &Bytes) -> Result> { - ensure!( - bytes.len().is_multiple_of(4), - "i32 input has invalid byte length" - ); + ensure!(bytes.len() % 4 == 0, "i32 input has invalid byte length"); Ok(bytes .chunks_exact(4) .map(|b| i32::from_le_bytes([b[0], b[1], b[2], b[3]])) diff --git a/crates/kernelport-backend-ort/tests/identity.rs b/crates/kernelport-backend-ort/tests/identity.rs index 88c6f40..6aca7c9 100644 --- a/crates/kernelport-backend-ort/tests/identity.rs +++ b/crates/kernelport-backend-ort/tests/identity.rs @@ -9,8 +9,7 @@ use kernelport_core::{ #[test] fn ort_identity_cpu() -> Result<()> { - let model_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("../../models/identity.onnx"); + let model_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../models/identity.onnx"); let backend = OrtBackend::new(); let mut model = backend.load(&ModelArtifact::OnnxPath(model_path), Device::Cpu)?; @@ -33,7 +32,11 @@ fn ort_identity_cpu() -> Result<()> { let numel = shape.iter().product::().max(1); let data: Vec = (0..numel).map(|i| i as f32).collect(); - let input = Tensor::from_cpu_bytes(DType::F32, Shape::from_slice(&shape), bytes_from_slice(&data)); + let input = Tensor::from_cpu_bytes( + DType::F32, + Shape::from_slice(&shape), + bytes_from_slice(&data), + ); let outputs = model.infer(vec![input])?; let out = outputs.first().context("missing model output")?; diff --git a/crates/kernelport-proto/Cargo.toml b/crates/kernelport-proto/Cargo.toml index f3583f0..6e20085 100644 --- a/crates/kernelport-proto/Cargo.toml +++ b/crates/kernelport-proto/Cargo.toml @@ -5,8 +5,9 @@ edition = "2021" build = "build.rs" [dependencies] -prost = "0.13" -tonic = "0.12" +prost = "0.14" +tonic = "0.14" +tonic-prost = "0.14" [build-dependencies] -tonic-build = "0.12" +tonic-prost-build = "0.14" diff --git a/crates/kernelport-proto/build.rs b/crates/kernelport-proto/build.rs index 953a835..9070667 100644 --- a/crates/kernelport-proto/build.rs +++ b/crates/kernelport-proto/build.rs @@ -1,4 +1,9 @@ fn main() -> Result<(), Box> { - tonic_build::compile_protos("src/inference.proto")?; + let out_dir = std::env::var("OUT_DIR")?; + let descriptor_path = std::path::Path::new(&out_dir).join("kernelport_descriptor.bin"); + + tonic_prost_build::configure() + .file_descriptor_set_path(descriptor_path) + .compile_protos(&["src/inference.proto"], &["src"])?; Ok(()) } diff --git a/crates/kernelport-proto/src/inference.proto b/crates/kernelport-proto/src/inference.proto index d406667..4f51bbe 100644 --- a/crates/kernelport-proto/src/inference.proto +++ b/crates/kernelport-proto/src/inference.proto @@ -4,17 +4,26 @@ package kernelport.v1; message Tensor { // Minimal v0 tensor format (CPU bytes only) string name = 1; - string dtype = 2; // "F32", "I64", etc. + DType dtype = 2; repeated int64 shape = 3; bytes data = 4; } -message PredictRequest { +enum DType { + DTYPE_UNSPECIFIED = 0; + F32 = 1; + F16 = 2; + I64 = 3; + I32 = 4; + U8 = 5; +} + +message InferRequest { string model = 1; repeated Tensor inputs = 2; } -message PredictResponse { +message InferResponse { repeated Tensor outputs = 1; uint64 queued_us = 2; uint64 batched_us = 3; @@ -22,6 +31,5 @@ message PredictResponse { } service InferenceService { - rpc Predict(PredictRequest) returns (PredictResponse); + rpc Infer(InferRequest) returns (InferResponse); } - diff --git a/crates/kernelport-proto/src/lib.rs b/crates/kernelport-proto/src/lib.rs index 920fb25..5ceb20c 100644 --- a/crates/kernelport-proto/src/lib.rs +++ b/crates/kernelport-proto/src/lib.rs @@ -3,3 +3,5 @@ pub mod kernelport { tonic::include_proto!("kernelport.v1"); } } + +pub const FILE_DESCRIPTOR_SET: &[u8] = tonic::include_file_descriptor_set!("kernelport_descriptor"); diff --git a/crates/kernelport-server/Cargo.toml b/crates/kernelport-server/Cargo.toml index d0d3521..383cb90 100644 --- a/crates/kernelport-server/Cargo.toml +++ b/crates/kernelport-server/Cargo.toml @@ -12,9 +12,10 @@ anyhow = "1" bytes = "1" clap = { version = "4", features = ["derive"] } tokio = { version = "1", features = ["macros", "rt-multi-thread", "sync", "time"] } -tonic = "0.12" +tonic = "0.14" tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["env-filter"] } +tonic-reflection = "0.14" kernelport-core = { path = "../kernelport-core" } kernelport-runtime = { path = "../kernelport-runtime" } diff --git a/crates/kernelport-server/src/cli.rs b/crates/kernelport-server/src/cli.rs index 82e2c95..061264f 100644 --- a/crates/kernelport-server/src/cli.rs +++ b/crates/kernelport-server/src/cli.rs @@ -22,5 +22,9 @@ pub enum Command { /// Device for inference (cpu or cuda:N) #[arg(long, default_value = "cpu")] device: String, + + /// Path to ONNX model file + #[arg(long, default_value = "models/demo.onnx")] + model_path: String, }, } diff --git a/crates/kernelport-server/src/grpc.rs b/crates/kernelport-server/src/grpc.rs index a3b8182..044e92e 100644 --- a/crates/kernelport-server/src/grpc.rs +++ b/crates/kernelport-server/src/grpc.rs @@ -1,4 +1,4 @@ -use anyhow::Result; +use anyhow::{Context, Result}; use bytes::Bytes; use kernelport_core::{DType, IOName, Shape, Tensor}; use kernelport_proto::kernelport::v1 as pb; @@ -13,16 +13,16 @@ pub struct GrpcSvc { #[tonic::async_trait] impl pb::inference_service_server::InferenceService for GrpcSvc { - async fn predict( + async fn infer( &self, - req: Request, - ) -> std::result::Result, Status> { + req: Request, + ) -> std::result::Result, Status> { let req = req.into_inner(); let mut inputs = Vec::with_capacity(req.inputs.len()); for t in req.inputs { let dtype = - parse_dtype(&t.dtype).map_err(|e| Status::invalid_argument(e.to_string()))?; + parse_dtype(t.dtype).map_err(|e| Status::invalid_argument(e.to_string()))?; let shape_usize: Vec = t .shape .into_iter() @@ -59,13 +59,13 @@ impl pb::inference_service_server::InferenceService for GrpcSvc { }; pb_outs.push(pb::Tensor { name: name.0, - dtype: format!("{:?}", t.desc.dtype), + dtype: to_proto_dtype(t.desc.dtype) as i32, shape: t.desc.shape.0.iter().map(|d| *d as i64).collect(), data: data.to_vec(), }); } - Ok(Response::new(pb::PredictResponse { + Ok(Response::new(pb::InferResponse { outputs: pb_outs, queued_us: timings.queued_us, batched_us: timings.batched_us, @@ -74,13 +74,24 @@ impl pb::inference_service_server::InferenceService for GrpcSvc { } } -fn parse_dtype(s: &str) -> Result { - Ok(match s { - "F32" => DType::F32, - "F16" => DType::F16, - "I64" => DType::I64, - "I32" => DType::I32, - "U8" => DType::U8, - _ => anyhow::bail!("unknown dtype: {}", s), +fn parse_dtype(raw: i32) -> Result { + let dtype = pb::DType::try_from(raw).context("unknown dtype enum value")?; + Ok(match dtype { + pb::DType::F32 => DType::F32, + pb::DType::F16 => DType::F16, + pb::DType::I64 => DType::I64, + pb::DType::I32 => DType::I32, + pb::DType::U8 => DType::U8, + pb::DType::DtypeUnspecified => anyhow::bail!("dtype is unspecified"), }) } + +fn to_proto_dtype(dtype: DType) -> pb::DType { + match dtype { + DType::F32 => pb::DType::F32, + DType::F16 => pb::DType::F16, + DType::I64 => pb::DType::I64, + DType::I32 => pb::DType::I32, + DType::U8 => pb::DType::U8, + } +} diff --git a/crates/kernelport-server/src/main.rs b/crates/kernelport-server/src/main.rs index c5c3e5c..8941b03 100644 --- a/crates/kernelport-server/src/main.rs +++ b/crates/kernelport-server/src/main.rs @@ -8,6 +8,7 @@ use cli::{Cli, Command}; use kernelport_proto::kernelport::v1::inference_service_server::InferenceServiceServer; use kernelport_runtime::{BatchPolicy, Batcher, Scheduler, Worker}; use tokio::sync::mpsc; +use tonic_reflection::server::Builder as ReflectionBuilder; use tracing_subscriber::EnvFilter; use grpc::GrpcSvc; @@ -21,14 +22,20 @@ async fn main() -> Result<()> { grpc_addr, log, device, + model_path, } => { let device = parse_device(&device)?; - serve(grpc_addr, log, device).await + serve(grpc_addr, log, device, model_path.into()).await } } } -async fn serve(grpc_addr: String, log: String, device: kernelport_core::Device) -> Result<()> { +async fn serve( + grpc_addr: String, + log: String, + device: kernelport_core::Device, + model_path: std::path::PathBuf, +) -> Result<()> { std::env::set_var("RUST_LOG", &log); tracing_subscriber::fmt() .with_env_filter(EnvFilter::from_default_env()) @@ -52,12 +59,7 @@ async fn serve(grpc_addr: String, log: String, device: kernelport_core::Device) // Model registry let mut reg = registry::ModelRegistry::new(); - reg.load_onnx( - "demo", - std::path::PathBuf::from("models/demo.onnx"), - device, - ) - .ok(); // allow startup without the file for now + reg.load_onnx("demo", model_path, device).ok(); // allow startup without the file for now let loaded = reg.get("demo"); let worker_model = DemoWorkerModel { loaded }; @@ -90,8 +92,14 @@ async fn serve(grpc_addr: String, log: String, device: kernelport_core::Device) let svc = GrpcSvc { batcher_tx }; tracing::info!(%addr, "kernelportd gRPC listening"); + let reflection = ReflectionBuilder::configure() + .register_encoded_file_descriptor_set(kernelport_proto::FILE_DESCRIPTOR_SET) + .build_v1() + .map_err(|e| anyhow::anyhow!("reflection build failed: {e}"))?; + tonic::transport::Server::builder() .add_service(InferenceServiceServer::new(svc)) + .add_service(reflection) .serve(addr) .await?; @@ -111,7 +119,6 @@ fn parse_device(raw: &str) -> Result { anyhow::bail!("unsupported device: {raw} (expected cpu or cuda:N)"); } -use anyhow::anyhow; use kernelport_runtime::{BatchJob, WorkerModel}; struct DemoWorkerModel { @@ -120,25 +127,61 @@ struct DemoWorkerModel { impl WorkerModel for DemoWorkerModel { fn infer_batch(&mut self, job: BatchJob) -> anyhow::Result<()> { - let model = self - .loaded - .as_ref() - .ok_or_else(|| anyhow!("model not loaded (demo)"))?; + let model = match self.loaded.as_ref() { + Some(model) => model, + None => { + tracing::error!("model not loaded (demo)"); + for req in job.requests { + let _ = req.resp_tx.send(kernelport_runtime::InferenceResponse { + outputs: Vec::new(), + timings: kernelport_runtime::Timings { + queued_us: 0, + batched_us: 0, + backend_us: 0, + }, + }); + } + return Ok(()); + } + }; // v0: no real batching; call model once and fan-out same output let mut guard = model.model.lock().unwrap(); let t0 = std::time::Instant::now(); - let outputs = guard.infer(job.merged_inputs.into_iter().map(|(_, t)| t).collect())?; + let outputs = match guard.infer(job.merged_inputs.into_iter().map(|(_, t)| t).collect()) { + Ok(outputs) => outputs, + Err(err) => { + tracing::error!(error = ?err, "model inference failed (demo)"); + for req in job.requests { + let _ = req.resp_tx.send(kernelport_runtime::InferenceResponse { + outputs: Vec::new(), + timings: kernelport_runtime::Timings { + queued_us: 0, + batched_us: 0, + backend_us: 0, + }, + }); + } + return Ok(()); + } + }; let backend_us = t0.elapsed().as_micros() as u64; + let output_names = model.output_names.clone(); for req in job.requests { let _ = req.resp_tx.send(kernelport_runtime::InferenceResponse { outputs: outputs .iter() .cloned() .enumerate() - .map(|(i, t)| (kernelport_core::IOName(format!("out{}", i)), t)) + .map(|(i, t)| { + let name = output_names + .get(i) + .cloned() + .unwrap_or_else(|| kernelport_core::IOName(format!("out{}", i))); + (name, t) + }) .collect(), timings: kernelport_runtime::Timings { queued_us: 0, diff --git a/crates/kernelport-server/src/registry.rs b/crates/kernelport-server/src/registry.rs index d1e7a6e..ddd3172 100644 --- a/crates/kernelport-server/src/registry.rs +++ b/crates/kernelport-server/src/registry.rs @@ -3,10 +3,11 @@ use std::sync::{Arc, Mutex}; use anyhow::Result; use kernelport_backend_ort::OrtBackend; -use kernelport_core::{Backend, BackendModel, Device, ModelArtifact, Tensor}; +use kernelport_core::{Backend, BackendModel, Device, IOName, ModelArtifact, Tensor}; pub struct LoadedModel { pub model: Mutex>, + pub output_names: Vec, } pub trait BackendModelAdapter: Send { @@ -41,8 +42,15 @@ impl ModelRegistry { let artifact = ModelArtifact::OnnxPath(path); let model = backend.load(&artifact, device)?; + let output_names = model + .spec() + .outputs + .iter() + .map(|spec| spec.name.clone()) + .collect(); let loaded = LoadedModel { model: Mutex::new(Box::new(model)), + output_names, }; self.models.insert(name.to_string(), Arc::new(loaded)); diff --git a/docs/model-ingestion.md b/docs/model-ingestion.md new file mode 100644 index 0000000..7c58363 --- /dev/null +++ b/docs/model-ingestion.md @@ -0,0 +1,50 @@ +# HuggingFace Model Ingestion (Plan) + +KernelPort will support two model delivery modes so users can trade off +performance vs. flexibility. + +## Modes + +1) Baked into image (maximum performance, reproducible) + - Artifacts are copied into the container at build time. + - Images are per-model (or per-bundle). + +2) Pulled at startup (flexible, cacheable) + - Container pulls from HuggingFace into a mounted cache volume. + - Runtime loads from the cache directory. + +## Formats (initial) + +- ONNX +- TorchScript +- TensorRT (requires CUDA + TensorRT toolchain) + +## Proposed Flow + +1) Provide a manifest (see `docs/model-manifest.md`). +2) An entrypoint or sidecar fetches artifacts into `/models`. +3) KernelPort loads the artifacts from the manifest path. + +## Loading from the cache + +The server currently accepts a direct model path: + +```bash +kernelport-server serve --model-path /models//model.onnx +``` + +## HuggingFace Authentication + +Set `HF_TOKEN` (or `HUGGINGFACE_HUB_TOKEN`) when pulling from private repos. + +## Container Integration + +- Bake mode: copy artifacts into the image under `/models`. +- Pull mode: mount a volume at `/models` and populate it at startup. +- GPU images should mount a CUDA-enabled `libonnxruntime.so` via `ORT_DYLIB_PATH` + when using ONNX Runtime with CUDA. + +## Examples + +- Sample bundle: `models/sample/model.onnx` +- Sample manifest: `models/sample-manifest.yaml` diff --git a/docs/model-manifest.md b/docs/model-manifest.md new file mode 100644 index 0000000..9a7b035 --- /dev/null +++ b/docs/model-manifest.md @@ -0,0 +1,35 @@ +# Model Manifest (Proposed) + +This document defines a proposed manifest used to fetch, cache, and load models +from HuggingFace or other sources. It is not yet implemented in the runtime. + +## Goals + +- Keep model source info declarative. +- Support ONNX, TorchScript, and TensorRT artifacts. +- Allow a "fetch at startup" flow without forcing new images per model. + +## Schema (YAML) + +```yaml +id: bert-base-uncased +source: + kind: huggingface + repo: bert-base-uncased + revision: main +format: onnx # onnx | torchscript | tensorrt +files: + - model.onnx +cache: + dir: /models +runtime: + device: cuda:0 +``` + +## Notes + +- `files` lists the expected artifact filenames once fetched or generated. +- `cache.dir` is where the container stores artifacts when pulling at startup. +- The runtime may later accept a `conversion` block for ONNX or TensorRT builds. +- For private repos, set `HF_TOKEN` (or `HUGGINGFACE_HUB_TOKEN`) in the container. +- See `models/sample-manifest.yaml` for a concrete example. diff --git a/docs/validation.md b/docs/validation.md new file mode 100644 index 0000000..deb963a --- /dev/null +++ b/docs/validation.md @@ -0,0 +1,56 @@ +# Validation + +This document covers quick CPU/GPU validation and a minimal gRPC test. + +## 1) CPU: run the sample model + +```bash +docker run --rm -p 50051:50051 \ + -v "$PWD/models/sample:/models/sample:ro" \ + kernelport:cpu \ + --device cpu \ + --model-path /models/sample/model.onnx +``` + +## 2) GPU: run with CUDA + ORT dylib + +```bash +docker run --rm --gpus all -p 50051:50051 \ + -e ORT_DYLIB_PATH=/opt/ort/libonnxruntime.so \ + -v /path/to/ort/libonnxruntime.so:/opt/ort/libonnxruntime.so:ro \ + -v "$PWD/models/sample:/models/sample:ro" \ + kernelport:gpu \ + --device cuda:0 \ + --model-path /models/sample/model.onnx +``` + +## 3) Minimal gRPC test + +This uses `grpcurl` and assumes the server is on `localhost:50051`. + +```bash +grpcurl -plaintext -d '{ + "model": "demo", + "inputs": [ + { "name": "x", "dtype": "F32", "shape": [1, 1, 2, 2], "data": "AACAPwAAAEAAAEBAAACAQA==" } + ] +}' localhost:50051 kernelport.v1.InferenceService/Infer +``` + +Expected response (identity model): + +```json +{ + "outputs": [ + { "name": "y", "dtype": "F32", "shape": ["1","1","2","2"], "data": "AACAPwAAAEAAAEBAAACAQA==" } + ], + "queuedUs": "0", + "batchedUs": "0", + "backendUs": "..." +} +``` + +Notes: +- `data` is raw bytes, base64-encoded (`[1.0,2.0,3.0,4.0]` = `AACAPwAAAEAAAEBAAACAQA==`). +- Input/output names must match the model (sample identity model uses `x` -> `y`). +- `dtype` is an enum; grpcurl accepts the symbolic name (`F32`). diff --git a/models/sample-manifest.yaml b/models/sample-manifest.yaml new file mode 100644 index 0000000..5e7a0e0 --- /dev/null +++ b/models/sample-manifest.yaml @@ -0,0 +1,12 @@ +id: bert-base-uncased +source: + kind: huggingface + repo: bert-base-uncased + revision: main +format: onnx +files: + - model.onnx +cache: + dir: /models +runtime: + device: cuda:0 diff --git a/models/sample/README.md b/models/sample/README.md new file mode 100644 index 0000000..b2a38d1 --- /dev/null +++ b/models/sample/README.md @@ -0,0 +1,4 @@ +This is a sample model bundle intended for baked-image demos. + +Contents: +- model.onnx: simple identity ONNX model diff --git a/scripts/entrypoint.sh b/scripts/entrypoint.sh new file mode 100644 index 0000000..f481511 --- /dev/null +++ b/scripts/entrypoint.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env sh +set -eu + +if [ -n "${MODEL_MANIFEST_PATH:-}" ]; then + python3 /app/scripts/model_fetch.py "$MODEL_MANIFEST_PATH" +fi + +exec /app/kernelportd serve "$@" diff --git a/scripts/model_fetch.py b/scripts/model_fetch.py new file mode 100644 index 0000000..4895c89 --- /dev/null +++ b/scripts/model_fetch.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +import os +import sys +from pathlib import Path +from typing import Optional + +import yaml +from huggingface_hub import snapshot_download + + +def load_manifest(path: Path) -> dict: + with path.open("r", encoding="utf-8") as handle: + return yaml.safe_load(handle) + + +def resolve_token() -> Optional[str]: + return os.environ.get("HF_TOKEN") or os.environ.get("HUGGINGFACE_HUB_TOKEN") + + +def main() -> int: + if len(sys.argv) != 2: + print("usage: model_fetch.py ", file=sys.stderr) + return 2 + + manifest_path = Path(sys.argv[1]).resolve() + manifest = load_manifest(manifest_path) + + source = manifest.get("source", {}) + if source.get("kind") != "huggingface": + raise RuntimeError("manifest source.kind must be 'huggingface'") + + repo = source.get("repo") + revision = source.get("revision", "main") + if not repo: + raise RuntimeError("manifest source.repo is required") + + cache_dir = Path(manifest.get("cache", {}).get("dir", "/models")) + model_id = manifest.get("id", repo.replace("/", "__")) + local_dir = cache_dir / model_id + local_dir.mkdir(parents=True, exist_ok=True) + + files = manifest.get("files") + allow_patterns = files if isinstance(files, list) else None + + snapshot_download( + repo_id=repo, + revision=revision, + allow_patterns=allow_patterns, + local_dir=str(local_dir), + local_dir_use_symlinks=False, + token=resolve_token(), + ) + + if allow_patterns: + missing = [name for name in allow_patterns if not (local_dir / name).exists()] + if missing: + raise RuntimeError(f"missing expected files: {missing}") + + print(f"model ready at {local_dir}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())